NFS-Ganesha : Do not empty global config file when cluster is torn

The global config file will need new blocks to support client lock recovery. Current cleanup function empties the entire file. Deleting only "include" lines in the config file. Change-Id: I21f09e30a738d2ba01861ce480ecf906667d887b BUG: 1218854 Signed-off-by: Meghana Madhusudhan <mmadhusu@redhat.com> Reviewed-on: http://review.gluster.org/10592 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: soumya k <skoduri@redhat.com> Reviewed-by: jiffin tony Thottan <jthottan@redhat.com> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
author: Meghana Madhusudhan <mmadhusu@redhat.com> 2015-05-06 09:58:11 +0530
committer: Kaleb KEITHLEY <kkeithle@redhat.com> 2015-05-06 05:34:56 -0700
commit: e79edf35178f2558d6afb262444294380b76d740 (patch)
tree: 22a5e069388e141f8e0a64fc5e3497c5ef1169ed /tests/basic
parent: 403a74da5d3287489829c629d42643b8b320ee62 (diff)
0 files changed, 0 insertions, 0 deletions
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/tests/distaf/__init__.py b/.github/PULL_REQUEST_TEMPLATE
index e69de29bb2d..e69de29bb2d 100644
--- a/tests/distaf/__init__.py
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index 45400ac449d..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@ Makefile
 Makefile.in
 missing
 stamp-h1
+stamp-h2
 test-driver
 *compile
 *.gcda
@@ -28,10 +29,12 @@ test-driver
 *.patch
 .libs
 .deps
+.dirstamp
 
 # Softlinks to test and log
 log
 *.vol
+.clang-format
 
 # cmocka unit tests
 *.log
@@ -39,14 +42,13 @@ log
 *_unittest
 
 # Generated files
+site.h
 *.py[co]
 api/examples/__init__.py
 api/examples/setup.py
 api/src/gfapi.map
 cli/src/gluster
-contrib/argp-standalone/libargp.a
 contrib/fuse-util/fusermount-glusterfs
-contrib/uuid/uuid_types.h
 extras/geo-rep/gsync-sync-gfid
 extras/geo-rep/schedule_georep.py
 extras/snap_scheduler/conf.py
@@ -59,8 +61,11 @@ extras/ocf/glusterd
 extras/ocf/volume
 extras/run-gluster.tmpfiles
 extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
 extras/who-wrote-glusterfs/gitdm
 geo-replication/.tox
+geo-replication/gsyncd.conf
 geo-replication/src/gsyncd
 geo-replication/src/peer_gsec_create
 geo-replication/src/peer_mountbroker
@@ -79,17 +84,20 @@ glusterfs.spec
 glusterfsd/src/glusterd
 glusterfsd/src/glusterfs
 glusterfsd/src/glusterfsd
+glusterfsd/src/gf_attach
 heal/src/glfsheal
 libgfchangelog.pc
-libgfdb.pc
 libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
 libglusterfs/src/defaults.c
-libglusterfs/src/glusterfs-fops.h
 libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
 libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
 run-tests.sh
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
 tests/env.rc
 tests/utils/arequal-checksum
 xlators/features/glupy/src/__init__.py
@@ -100,16 +108,18 @@ extras/peer_add_secret_pub
 tools/gfind_missing_files/gcrawler
 tools/glusterfind/glusterfind
 tools/glusterfind/src/tool.conf
-# Generated by fdl xlator
-xlators/experimental/fdl/src/fdl.c
-xlators/experimental/fdl/src/gf_logdump
-xlators/experimental/fdl/src/gf_recon
-xlators/experimental/fdl/src/libfdl.c
-xlators/experimental/fdl/src/librecon.c
-xlators/experimental/jbr-client/src/jbrc-cg.c
-xlators/experimental/jbr-server/src/jbr-cg.c
 # Eventing
 events/src/eventsapiconf.py
 extras/systemd/glustereventsd.service
 events/src/eventtypes.py
 libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
index d4a13b40065..141a1667ffa 100644
--- a/.mailmap
+++ b/.mailmap
@@ -10,15 +10,20 @@ Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
 Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
 Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
 Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
 Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
 Justin Clift <justin@gluster.org> <jclift@redhat.com>
-Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
 Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
 Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
 Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
 Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
 Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
 M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
 Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
 Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
 Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
@@ -27,9 +32,12 @@ Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
 Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
 Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
 Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
 Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
 Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
 Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
 Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
 Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
 shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 7bccd88d7e5..00000000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-        Developer's Certificate of Origin 1.1
-
-        By making a contribution to this project, I certify that:
-
-        (a) The contribution was created in whole or in part by me and I
-            have the right to submit it under the open source license
-            indicated in the file; or
-
-        (b) The contribution is based upon previous work that, to the best
-            of my knowledge, is covered under an appropriate open source
-            license and I have the right under that license to submit that
-            work with modifications, whether created in whole or in part
-            by me, under the same open source license (unless I am
-            permitted to submit under a different license), as indicated
-            in the file; or
-
-        (c) The contribution was provided directly to me by some other
-            person who certified (a), (b) or (c) and I have not modified
-            it.
-
-        (d) I understand and agree that this project and the contribution
-            are public and that a record of the contribution (including all
-            personal information I submit with it, including my sign-off) is
-            maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/INSTALL b/INSTALL
index 76beb978775..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -44,4 +44,4 @@ Installation completed :-)
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
 
-For more information on GlusterFS installation refer# http://gluster.org/community/documentation/index.php/Building_GlusterFS
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
index 93faf264042..953e8755fd9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -11,7 +11,7 @@ consult gluster-devel@gluster.org and not any specific individual privately.
 
 Descriptions of section entries:
 
-	M: Mail patches to: FullName <address@domain>
+	M: Main contact that knows and takes care of this area
 	L: Mailing list that is relevant to this area
 	W: Web-page with status/info
 	Q: Patchwork web based patch tracking system site
@@ -46,169 +46,304 @@ Descriptions of section entries:
 	      matches patches or files that contain one or more of the words
 	      printk, pr_info or pr_err
 	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
 
 
 General Project Architects
 --------------------------
-M: Anand Avati <avati@redhat.com>
-M: Jeff Darcy <jdarcy@redhat.com>
-M: Kaleb S. Keithley <kkeithle@redhat.com>
-M: Vijay Bellur <vbellur@redhat.com>
-
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
 
 xlators:
 --------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
 Automatic File Replication (AFR)
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
 S: Maintained
 F: xlators/cluster/afr/
 
-Block Device
-S: Orphan
-F: xlators/storage/bd/
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
 
 BitRot
-M: Raghavendra Bhat <rabhat@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
 S: Maintained
 F: xlators/features/bit-rot/
 
 Changelog
 M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
 S: Maintained
 F: xlators/features/changelog/
 
-Changetimerecorder
-M: Dan Lambright <dlambrig@redhat.com>
-S: Maintained
-F: xlators/features/changetimerecorder/
-
 Distributed Hashing Table (DHT)
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
-M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Susant Palai <spalai@redhat.com>
 S: Maintained
 F: xlators/cluster/dht/
 
 Erasure Coding
-M: Pranith Karampuri <pkarampu@redhat.com>
-M: Xavier Hernandez  <xhernandez@datalab.es>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
 S: Maintained
 F: xlators/cluster/ec/
 
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
 FUSE Bridge
-M: Niels de Vos <ndevos@redhat.com>
-M: Raghavendra Bhat <rabhat@redhat.com>
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
 S: Maintained
 F: xlators/mount/
 
 Index
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
 S: Maintained
 F: xlators/features/index/
 
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
 IO threads
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
 S: Maintained
 F: xlators/performance/io-threads/
 
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
 Locks
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
 S: Maintained
 F: xlators/features/locks/
 
 Marker
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
 S: Maintained
 F: xlators/features/marker/
 
-NFS
-M: Niels de Vos <ndevos@redhat.com>
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/features/meta/
+
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/performance/md-cache/
+
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
 S: Maintained
+F: xlators/performance/nl-cache/
+
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
 F: xlators/nfs/server/
 
-Performance
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+Open-behind
 S: Maintained
-F: xlators/performance/
+F: xlators/performance/open-behind/
 
 Posix:
-M: Pranith Karampuri <pkarampu@redhat.com>
 M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
 S: Maintained
 F: xlators/storage/posix/
 
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
 Quota
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
 S: Maintained
 F: xlators/features/quota/
 
-Tiering
-M: Dan Lambright <dlambrig@redhat.com>
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
 S: Maintained
-F: xlators/cluster/dht/src/tier.c
-F: xlators/features/changetimerecorder
-F: libglusterfs/src/gfdb
-W: http://www.gluster.org/community/documentation/index.php/Features/data-classification
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
+S: Maintained
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
 
 Upcall
-M: Niels de Vos <ndevos@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
 S: Maintained
 F: xlators/features/upcall/
 
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
+
 Other bits of code:
 -------------------
 
 Doc
 M: Humble Chirammal <hchiramm@redhat.com>
 M: Raghavendra Talur <rtalur@redhat.com>
-M: Prashanth Pai <ppai@redhat.com>
 S: Maintained
 F: doc/
 
 Geo Replication
 M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
 S: Maintained
 F: geo-replication/
 
-
-Glupy
-S: Orphan
-F: xlators/features/glupy/
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: tools/glusterfind/
 
 libgfapi
 M: Niels de Vos <ndevos@redhat.com>
-M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 F: api/
 
-libgfdb
-M: Dan Lambright <dlambrig@redhat.com>
-S: Maintained
-F: libglusterfs/src/gfdb/
-
 libglusterfs
-M: Niels de Vos <ndevos@redhat.com>
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
 S: Maintained
 F: libglusterfs/
 
-Management Daemon
-M: Kaushal Madappa <kmadapp@redhat.com>
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
 M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
 S: Maintained
 F: cli/
-F: xlators/mgmt/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
 
 Remote Procedure Call subsystem
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
 S: Maintained
-F: rpc/
+F: rpc/rpc-lib/
+F: rpc/xdr/
 
 Snapshot
-M: Rajesh Joseph <rjoseph@redhat.com>
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
 S: Maintained
 F: xlators/mgmt/glusterd/src/glusterd-snap*
 F: extras/snap-scheduler.py
 
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-transport/socket/
+
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: tests/
+
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
 Events APIs
 M: Aravinda VK <avishwan@redhat.com>
 S: Maintained
@@ -220,57 +355,67 @@ F: extras/systemd/glustereventsd*
 Distribution Specific:
 ----------------------
 Build:
-M: Kaleb Keithley <kkeithle@redhat.com>
 M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
 S: Maintained
 
-Debian Packaging
-M: Patrick Matthäi <pmatthaei@debian.org>
-M: Louis Zuckerman <me@louiszuckerman.com>
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
-W: http://packages.qa.debian.org/g/glusterfs.html
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
 
-Fedora Packaging
-M: glusterfs-owner@fedoraproject.org
-M: Humble Chirammal <hchiramm@redhat.com>
+OpenSuSE
+M: packaging@gluster.org
 M: Kaleb Keithley <kkeithle@redhat.com>
-M: Niels de Vos <ndevos@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
-W: https://apps.fedoraproject.org/packages/glusterfs
-T: http://pkgs.fedoraproject.org/git/glusterfs.git
-
-FreeBSD port
-S: Orphan
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
 
-MacOS X port
-S: Orphan
-
-NetBSD port
-M: Emmanuel Dreyfus <manu@netbsd.org>
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
 S: Maintained
-W: http://pkgsrc.se/filesystems/glusterfs
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
 
-Ubuntu Packaging
-M: Louis Zuckerman <me@louiszuckerman.com>
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: https://launchpad.net/~gluster
 W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
-
+T: https://github.com/gluster/glusterfs-debian.git
 
 Related projects
 ----------------
-Gluster Openstack Swift
-M: Luis Pabon <lpabon@redhat.com>
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
 S: Maintained
-T: https://github.com/gluster/gluster-swift.git
+T: https://github.com/gluster/gluster-block.git
 
-GlusterFS Hadoop HCFS plugin
-M: Jay Vyas <jvyas@redhat.com>
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
 S: Maintained
-W: https://github.com/gluster/glusterfs-hadoop/wiki
-T: https://github.com/gluster/glusterfs-hadoop.git
+T: https://github.com/gluster/glusterfs-coreutils.git
 
 NFS-Ganesha FSAL plugin
+M: Jiffin Tony Thottan <jthottan@redhat.com>
 M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 T: git://github.com/nfs-ganesha/nfs-ganesha.git
 F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
@@ -283,13 +428,27 @@ T: git://git.qemu.org/qemu.git
 F: block/gluster.c
 
 Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
 M: Raghavendra Talur <rtalur@redhat.com>
-M: Jose Rivera <jrivera@redhat.com>
-M: Ira Cooper <icooper@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
 S: Maintained
 T: git://git.samba.org/samba.git
 F: source3/modules/vfs_glusterfs.c
 
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
 Wireshark dissectors
 M: Niels de Vos <ndevos@redhat.com>
 S: Maintained
@@ -297,25 +456,55 @@ W: https://forge.gluster.org/wireshark
 T: http://code.wireshark.org/git/wireshark
 F: epan/dissectors/packet-gluster*
 
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
 Special Thanks
 --------------
 
 GlusterFS would not be possible without the contributions of:
 
-M: Amar Tumballi <amarts@gmail.com>
-M: Chris Hertel <chertel@redhat.com>
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
 M: M. Mohan Kumar <mohan@in.ibm.com>
 M: Shishir Gowda <gowda.shishir@gmail.com>
 M: Brian Foster <bfoster@redhat.com>
-M: Csaba Henk <chenk@redhat.com>
-M: Anand Avati <avati@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
 M: Dennis Schafroth <dennis@schafroth.com>
 M: Harshavardhana <harsha@harshavardhana.net>
-M: Krishnan Parthasarathi <kparthas@redhat.com>
+M: Krishnan Parthasarathi
 M: Justin Clift <justin@gluster.org>
 M: Venky Shankar <vshankar@redhat.com>
 M: Shravan Chandrashekar <shravantc99@gmail.com>
-M: Joseph Fernandes <josferna@redhat.com>
-M: Vijaikumar Mallikarjuna <vmallika@redhat.com>
-M: Anand Subramanian <ansubram@redhat.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
 M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index bc4627e9f54..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,48 +1,40 @@
+SOURCES = site.h
+
 EXTRA_DIST = autogen.sh \
-	COPYING-GPLV2 COPYING-LGPLV3 \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
 	INSTALL README.md AUTHORS THANKS NEWS \
-	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in libgfdb.pc.in \
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
 	run-tests.sh \
 	build-aux/pkg-version \
-        contrib/argp-standalone \
-        contrib/umountd \
-        contrib/uuid \
+	contrib/umountd \
 	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = $(ARGP_STANDALONE_DIR) rpc/xdr/src libglusterfs rpc api xlators \
-	glusterfsd $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
 	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
 
 pkgconfigdir = @pkgconfigdir@
 pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
-if USE_GFDB
-pkgconfig_DATA += libgfdb.pc
-endif
 
-CLEANFILES = glusterfs-api.pc libgfchangelog.pc libgfdb.pc \
-	tests/env.rc contrib/umountd/Makefile{,.in}
-CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	-rm -fr $(CONTRIBDIR)/argp-standalone/autom4te.cache
-	-rm -f $(CONTRIBDIR)/argp-standalone/aclocal.m4
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.h.in
-	-rm -f $(CONTRIBDIR)/argp-standalone/configure
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.status
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.log
-	-rm -f $(CONTRIBDIR)/argp-standalone/depcomp
-	-rm -fr $(CONTRIBDIR)/argp-standalone/.deps
-	-rm -f $(CONTRIBDIR)/argp-standalone/install-sh
-	-rm -f $(CONTRIBDIR)/argp-standalone/missing
 
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
 dist-hook: gen-VERSION gen-ChangeLog
-	-rm -fr $(distdir)/contrib/argp-standalone/autom4te.cache
-	-rm -fr $(distdir)/contrib/argp-standalone/.deps
 	-rm -fr $(distdir)/contrib/umountd/.deps
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
 	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
 	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
 
@@ -61,4 +53,3 @@ gen-VERSION:
 	  ./build-aux/pkg-version --full                \
 	    > $(abs_top_builddir)/$(distdir)/VERSION;   \
 	fi
-
diff --git a/README.md b/README.md
index 8ff6c83265a..9d68e033782 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,46 @@
-For information about contributing to GlusterFS, please follow [Contributing to GlusterFS community](http://www.gluster.org/community/documentation/index.php/Main_Page#Contributing_to_the_Gluster_Community).
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
 
-*GlusterFS does not follow the [GitHub: Fork & pull](https://help.github.com/articles/using-pull-requests/) workflow but use [Gerrit](http://review.gluster.org) for code review.*
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
 
-The development guidelines are detailed in [Development Workflow](http://www.gluster.org/community/documentation/index.php/Simplified_dev_workflow).
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
 
-The GlusterFS documentation can be found at [our Read the Docs site](http://gluster.readthedocs.io/en/latest).
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
 
-For more info, please visit [gluster.org](http://www.gluster.org/).
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
index 184586c632d..3b2c8ab5a15 100755
--- a/api/examples/getvolfile.py
+++ b/api/examples/getvolfile.py
@@ -1,43 +1,45 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import ctypes
 import ctypes.util
 
-api = ctypes.CDLL(ctypes.util.find_library("gfapi"))
+api = ctypes.CDLL("libgfapi.so")
 api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
                                  ctypes.c_void_p,
                                  ctypes.c_ulong]
-api.glfs_get_volfile.restype = ctypes.c_long;
+api.glfs_get_volfile.restype = ctypes.c_long
 
-def get_volfile (host, volume):
-	# This is set to a large value to exercise the "buffer not big enough"
-	# path.  More realistically, you'd just start with a huge buffer.
-	BUF_LEN = 0
-	fs = api.glfs_new(volume)
-	#api.glfs_set_logging(fs,"/dev/stderr",7)
-	api.glfs_set_volfile_server(fs,"tcp",host,24007)
-	api.glfs_init(fs)
-	vbuf = ctypes.create_string_buffer(BUF_LEN)
-	vlen = api.glfs_get_volfile(fs,vbuf,BUF_LEN)
-	if vlen < 0:
-		vlen = BUF_LEN - vlen
-		vbuf = ctypes.create_string_buffer(vlen)
-		vlen = api.glfs_get_volfile(fs,vbuf,vlen)
-	api.glfs_fini(fs)
-	if vlen <= 0:
-		return vlen
-	return vbuf.value[:vlen]
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
 
 if __name__ == "__main__":
-	import sys
+    import sys
 
-	try:
-		res = apply(get_volfile,sys.argv[1:3])
-	except:
-		print "fetching volfile failed (volume not started?)"
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
 
-	try:
-		for line in res.split('\n'):
-			print line
-	except:
-		print "bad return value %s" % res
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 7ff3f0eb7ee..a55616ef739 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,1598 +1,1811 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
-#include "api/glfs.h"
-#include "api/glfs-handles.h"
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
 #include <string.h>
 #include <time.h>
 
+#define TEST_STR_LEN 2048
 
 int
-test_dirops (glfs_t *fs)
+test_dirops(glfs_t *fs)
 {
-        glfs_fd_t *fd = NULL;
-        char buf[512];
-        struct dirent *entry = NULL;
-
-        fd = glfs_opendir (fs, "/");
-        if (!fd) {
-                fprintf (stderr, "/: %s\n", strerror (errno));
-                return -1;
-        }
-
-        fprintf (stderr, "Entries:\n");
-        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
-                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
-        }
-
-        glfs_closedir (fd);
-        return 0;
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
 }
 
-
 int
-test_xattr (glfs_t *fs)
+test_xattr(glfs_t *fs)
 {
-        char *filename = "/filename2";
-        char buf[512];
-        char *ptr;
-        int ret;
-
-        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_listxattr (fs, filename, buf, 512);
-        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-        if (ret < 0)
-                return -1;
-
-        for (ptr = buf; ptr < buf + ret; ptr++) {
-                printf ("key=%s\n", ptr);
-                ptr += strlen (ptr);
-        }
-
-        return 0;
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
 }
 
-
 int
-test_chdir (glfs_t *fs)
+test_chdir(glfs_t *fs)
 {
-        int ret = -1;
-        char *topdir = "/topdir";
-        char *linkdir = "/linkdir";
-        char *subdir = "./subdir";
-        char *respath = NULL;
-        char pathbuf[4096];
-
-        ret = glfs_mkdir (fs, topdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        ret = glfs_symlink (fs, topdir, linkdir);
-        if (ret) {
-                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
-                return -1;
-        }
-
-        ret = glfs_chdir (fs, linkdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (respath) {
-                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
-                return -1;
-        }
-
-        ret = glfs_mkdir (fs, subdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
-        }
-
-        ret = glfs_chdir (fs, subdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
-        }
-
-        return 0;
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
 }
 
 #ifdef DEBUG
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        printf ("Dumping stat information:\n");
-        printf ("File type:                ");
-
-        switch (sb->st_mode & S_IFMT) {
-                case S_IFBLK:  printf ("block device\n");            break;
-                case S_IFCHR:  printf ("character device\n");        break;
-                case S_IFDIR:  printf ("directory\n");               break;
-                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
-                case S_IFLNK:  printf ("symlink\n");                 break;
-                case S_IFREG:  printf ("regular file\n");            break;
-                case S_IFSOCK: printf ("socket\n");                  break;
-                default:       printf ("unknown?\n");                break;
-        }
-
-        printf ("I-node number:            %ld\n", (long) sb->st_ino);
-
-        printf ("Mode:                     %lo (octal)\n",
-                (unsigned long) sb->st_mode);
-
-        printf ("Link count:               %ld\n", (long) sb->st_nlink);
-        printf ("Ownership:                UID=%ld   GID=%ld\n",
-                (long) sb->st_uid, (long) sb->st_gid);
-
-        printf ("Preferred I/O block size: %ld bytes\n",
-                (long) sb->st_blksize);
-        printf ("File size:                %lld bytes\n",
-                (long long) sb->st_size);
-        printf ("Blocks allocated:         %lld\n",
-                (long long) sb->st_blocks);
-
-        printf ("Last status change:       %s", ctime(&sb->st_ctime));
-        printf ("Last file access:         %s", ctime(&sb->st_atime));
-        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
-
-        return;
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
 }
 
 static void
-peek_handle (unsigned char *glid)
+peek_handle(unsigned char *glid)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
-        {
-                printf (":%02x:", glid[i]);
-        }
-        printf ("\n");
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
 }
-#else /* DEBUG */
+#else  /* DEBUG */
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        return;
+    return;
 }
 
 static void
-peek_handle (unsigned char *id)
+peek_handle(unsigned char *id)
 {
-        return;
+    return;
 }
 #endif /* DEBUG */
 
-glfs_t    *fs = NULL;
-char      *full_parent_name = "/testdir", *parent_name = "testdir";
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
 
 void
-test_h_unlink (void)
+test_h_unlink(void)
 {
-        char               *my_dir = "unlinkdir";
-        char               *my_file = "file.txt";
-        char               *my_subdir = "dir1";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
-                           *subdir = NULL, *subleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf ("glfs_h_unlink tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
-        if (subdir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
-        if (subleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non empty directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
-                         my_subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, subdir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s: invalid errno ,%d, %s\n",
-                         my_file, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s:  invalid errno ,%d, %s\n",
-                         my_subdir, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, parent, my_dir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_unlink tests: PASSED\n");
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
 
 out:
-        if (dir)
-                glfs_h_close (dir);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (subdir)
-                glfs_h_close (subdir);
-        if (subleaf)
-                glfs_h_close (subleaf);
-        if (parent)
-                glfs_h_close (parent);
-
-        return;
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
 }
 
 void
-test_h_getsetattrs (void)
+test_h_getsetattrs(void)
 {
-        char               *my_dir = "attrdir";
-        char               *my_file = "attrfile.txt";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb, retsb;
-        int                 ret, valid;
-        struct timespec     timestamp;
-
-        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_getattrs (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-        /* TODO: Compare stat information */
-
-        retsb.st_mode = 00666;
-        retsb.st_uid = 1000;
-        retsb.st_gid = 1001;
-        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        retsb.st_atim = timestamp;
-        retsb.st_mtim = timestamp;
-        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
-        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
-        peek_stat (&retsb);
-
-        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-
-        memset(&retsb, 0, sizeof (struct stat));
-        ret = glfs_h_stat (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-
-        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
 out:
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dir)
-                glfs_h_close (dir);
-
-        return;
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
 }
 
 void
-test_h_truncate (void)
+test_h_truncate(void)
 {
-        char               *my_dir = "truncatedir";
-        char               *my_file = "file.txt";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
-        struct stat         sb;
-        glfs_fd_t          *fd = NULL;
-        char                buf[32];
-        off_t               offset = 0;
-        int                 ret = 0;
-
-        printf("glfs_h_truncate tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         my_file, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, buf, 32, 0);
-
-        /* run tests */
-        /* truncate lower */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate higher */
-        offset = 32;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate equal */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_truncate tests: PASSED\n");
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
 out:
-        if (fd)
-                glfs_close (fd);
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return;
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
 }
 
 void
-test_h_links (void)
+test_h_links(void)
 {
-        char               *my_dir = "linkdir";
-        char               *my_file = "file.txt";
-        char               *my_symlnk = "slnk.txt";
-        char               *my_lnk = "lnk.txt";
-        char               *linksrc_dir = "dir1";
-        char               *linktgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct glfs_object *ln1 = NULL;
-        struct stat         sb;
-        int                 ret;
-        char               *buf = NULL;
-
-        printf("glfs_h_link(s) tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linksrc_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linktgt_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
-        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
-        if (ln1 == NULL) {
-                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
-                         my_symlnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        buf = calloc (1024, sizeof(char));
-        if (buf == NULL) {
-                fprintf (stderr, "Error allocating memory\n");
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        ret = glfs_h_readlink (fs, ln1, buf, 1024);
-        if (ret <= 0) {
-                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
-                         my_symlnk, ln1, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
-                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
-                         my_symlnk, buf);
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
-        ret = glfs_h_link (fs, leaf, parent, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
-        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, dirtgt, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        printf ("glfs_h_link(s) tests: PASSED\n");
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-        if (ln1)
-                glfs_h_close (ln1);
-        if (buf)
-                free (buf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
 }
 
 void
-test_h_rename (void)
+test_h_rename(void)
 {
-        char               *my_dir = "renamedir";
-        char               *my_file = "file.txt";
-        char               *src_dir = "dir1";
-        char               *tgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf("glfs_h_rename tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         src_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         tgt_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* Rename file.txt -> file1.txt */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file1.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1/file.txt -> file.txt */
-        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
-                         src_dir, "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file1.txt -> file.txt (exists) */
-        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1 -> dir3 */
-        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir1", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir2 ->dir3 (exists) */
-        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir2", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file.txt -> dir3 (fail) */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "file.txt", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir3 -> file.txt (fail) */
-        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "dir3", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_rename tests: PASSED\n");
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
 }
 
 void
-assimilatetime (struct timespec *ts, struct timespec ts_st,
-                struct timespec ts_ed)
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
 {
-        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
-                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
-        } else {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
-                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
-        }
-
-        if (ts->tv_nsec > 1000000000) {
-                ts->tv_nsec = ts->tv_nsec - 1000000000;
-                ts->tv_sec += 1;
-        }
-
-        return;
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
 }
 
 #define MAX_FILES_CREATE 10
-#define MAXPATHNAME      512
+#define MAXPATHNAME 512
 void
-test_h_performance (void)
+test_h_performance(void)
 {
-        char               *my_dir = "perftest",
-                           *full_dir_path="/testdir/perftest";
-        char               *my_file = "file_", my_file_name[MAXPATHNAME];
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb;
-        int                 ret, i;
-        struct glfs_fd     *fd;
-        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
-        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
-
-        printf("glfs_h_performance tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* create performance */
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s%d", my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb, 0);
-                if (leaf != NULL) {
-                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
-                if (leaf == NULL) {
-                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_h_close (leaf); leaf = NULL;
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
-
-        /* create using path */
-        c_ts.tv_sec = o_ts.tv_sec = 0;
-        c_ts.tv_nsec = o_ts.tv_nsec = 0;
-
-        sprintf (my_file_name, "%s1", full_dir_path);
-        ret = glfs_mkdir (fs, my_file_name, 0644);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
         if (ret != 0) {
-                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = glfs_stat (fs, my_file_name, &sb);
-                if (ret == 0) {
-                        fprintf (stderr, "glfs_stat: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
-                if (fd == NULL) {
-                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_close (fd);
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (path based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
 out:
-        return;
+    return;
 }
 
 int
-test_handleops (int argc, char *argv[])
+test_handleops(int argc, char *argv[])
 {
-        int                 ret = 0;
-        glfs_fd_t          *fd = NULL;
-        struct stat         sb = {0, };
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *tmp = NULL;
-        char                readbuf[32], writebuf[32];
-        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
-
-        char *full_leaf_name = "/testdir/testfile.txt",
-             *leaf_name = "testfile.txt",
-             *relative_leaf_name = "testdir/testfile.txt";
-        char *leaf_name1 = "testfile1.txt";
-        char *full_newparent_name = "/testdir/dir1",
-             *newparent_name = "dir1";
-        char *full_newnod_name = "/testdir/nod1",
-             *newnod_name = "nod1";
-
-        /* Initialize test area */
-        ret = glfs_mkdir (fs, full_parent_name, 0644);
-        if (ret != 0 && errno != EEXIST) {
-                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        else if (ret != 0) {
-                printf ("Found test directory %s to be existing\n",
-                        full_parent_name);
-                printf ("Cleanup test directory and restart tests\n");
-                goto out;
-        }
-
-        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
-        if (fd == NULL) {
-                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        glfs_close (fd);
-
-        printf ("Initialized the test area, within volume %s\n", argv[1]);
-
-        /* Handle based APIs test area */
-
-        /* glfs_lookupat test */
-        printf ("glfs_h_lookupat tests: In Progress\n");
-        /* start at root of the volume */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a parent within root */
-        parent = glfs_h_lookupat (fs, root, parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a leaf/child within the parent */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check absolute paths */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (leaf); leaf = NULL;
-
-        /* check multiple component paths */
-        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         relative_leaf_name, parent, strerror (errno));
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check symlinks in path */
-
-        /* TODO: -ve test cases */
-        /* parent invalid
-        * path invalid
-        * path does not exist after some components
-        * no parent, but relative path
-        * parent and full path? -ve?
-        */
-
-        printf ("glfs_h_lookupat tests: PASSED\n");
-
-        /* glfs_openat test */
-        printf ("glfs_h_open tests: In Progress\n");
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", readbuf,
-                        writebuf);
-                glfs_close (fd);
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_close (fd);
-
-        printf ("glfs_h_open tests: PASSED\n");
-
-        /* Create tests */
-        printf ("glfs_h_creat tests: In Progress\n");
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
-                            &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
-
-        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
-                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close (tmp); tmp = NULL;
-                }
-        }
-
-        /* TODO: Other combinations and -ve cases as applicable */
-        printf ("glfs_h_creat tests: PASSED\n");
-
-        /* extract handle and create from handle test */
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
-        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
-         * that needs to be fixed */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_extract_handle (leaf, leaf_handle,
-                                             GFAPI_HANDLE_LENGTH);
-        if (ret < 0) {
-                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_handle (leaf_handle);
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
 
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
-                                          &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
-                         leaf_name1, leaf_handle, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", writebuf,
-                        writebuf);
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                glfs_close (fd);
-                goto out;
-        }
-
-        glfs_close (fd);
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
-
-        /* Mkdir tests */
-        printf ("glfs_h_mkdir tests: In Progress\n");
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
+}
 
-        ret = glfs_rmdir (fs, full_newparent_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
-                         full_newparent_name, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+int
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
+{
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newparent_name, parent, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    char *filename = "/filename2";
+    int ret;
 
-        glfs_h_close (parent); parent = NULL;
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        printf ("glfs_h_mkdir tests: PASSED\n");
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        /* Mknod tests */
-        printf ("glfs_h_mknod tests: In Progress\n");
-        ret = glfs_unlink (fs, full_newnod_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
-                         full_newnod_name, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+    glfs_lseek(fd2, 0, SEEK_SET);
 
-        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newnod_name, parent, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* TODO: creat op on a FIFO node hangs, need to check and fix
-        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || errno != EINVAL) {
-                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
-                        tmp, strerror (errno));
-                printf ("glfs_h_creat/mknod tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close(tmp); tmp = NULL;
-                }
-        } */
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    ret = glfs_read(fd2, readbuf, 32, 0);
 
-        glfs_h_close (parent); parent = NULL;
+    printf("read %d, %s", ret, readbuf);
 
-        printf ("glfs_h_mknod tests: PASSED\n");
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
 
-        /* unlink tests */
-        test_h_unlink ();
+    ret = glfs_access(fs, filename, R_OK);
 
-        /* TODO: opendir tests */
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
 
-        /* getattr tests */
-        test_h_getsetattrs ();
+    glfs_close(fd);
+    glfs_close(fd2);
 
-        /* TODO: setattr tests */
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* truncate tests */
-        test_h_truncate();
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* link tests */
-        test_h_links ();
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        /* rename tests */
-        test_h_rename ();
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
 
-        /* performance tests */
-        test_h_performance ();
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* END: New APIs test area */
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-out:
-        /* Cleanup glfs handles */
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return ret;
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
 }
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glfs_t    *fs2 = NULL;
-        int        ret = 0;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        struct stat sb = {0, };
-        char       readbuf[32];
-        char       writebuf[32];
-
-        char      *filename = "/filename2";
-
-        if (argc != 3) {
-                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
-                return -1;
-        }
-
-        fs = glfs_new (argv[1]);
-        if (!fs) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
-
-//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
-
-        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        ret = glfs_init (fs);
+    char *filename = "/filename2";
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
 
-        sleep (2);
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        fs2 = glfs_new (argv[1]);
-        if (!fs2) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-
-//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
 
-        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
 
-        ret = glfs_init (fs2);
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs);
 
-        fd = glfs_creat (fs, filename, O_RDWR, 0644);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-        fd2 = glfs_open (fs2, filename, O_RDWR);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    if (ret)
+        goto out;
 
-        sprintf (writebuf, "hi there\n");
-        ret = glfs_write (fd, writebuf, 32, 0);
+    sleep(2);
 
-        glfs_lseek (fd2, 0, SEEK_SET);
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        ret = glfs_read (fd2, readbuf, 32, 0);
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-        printf ("read %d, %s", ret, readbuf);
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
 
-        glfs_close (fd);
-        glfs_close (fd2);
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
 
-        filename = "/filename3";
-        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs2);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
+    test_metadata_ops(fs, fs2);
 
-        ret = glfs_rename (fs, filename, "/filename4");
-        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
-                 strerror (errno));
+    test_dirops(fs);
 
-        ret = glfs_unlink (fs, "/filename4");
-        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
-                 strerror (errno));
+    test_xattr(fs);
 
-        filename = "/dirname2";
-        ret = glfs_mkdir (fs, filename, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    test_chdir(fs);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_handleops(argc, argv);
+    // done
 
-        ret = glfs_rmdir (fs, filename);
-        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+    /* Test some extra apis */
+    test_write_apis(fs);
 
-        test_dirops (fs);
+    glfs_statvfs(fs, "/", &sfs);
 
-        test_xattr (fs);
+    glfs_fini(fs);
+    glfs_fini(fs2);
 
-        test_chdir (fs);
-
-        test_handleops (argc, argv);
-        // done
-
-        glfs_fini (fs);
-        glfs_fini (fs2);
-
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index 625183c8dbc..7f9a7d17b35 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -9,21 +9,20 @@ libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
 	glfs-handleops.c
 libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
+
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
         -I$(top_srcdir)/rpc/rpc-lib/src \
         -I$(top_srcdir)/rpc/xdr/src \
         -I$(top_builddir)/rpc/xdr/src \
         -DDATADIR=\"$(localstatedir)\" \
-        -D__USE_FILE_OFFSET64
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
-libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) \
-	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
-
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
 # workaround for broken parallel install support in automake with LTLIBRARIES
@@ -33,9 +32,12 @@ $(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
 api_la_DEPENDENCIES = libgfapi.la
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
 api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
 api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/api/src/libgfapi.la
-
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
index d5cdedd826b..b6ec95f9311 100644
--- a/api/src/README.Symbol_Versions
+++ b/api/src/README.Symbol_Versions
@@ -1,3 +1,3 @@
 
-See .../doc/gfapi-symbol-versions/gfapi-symbol-versions.md
+See ../../doc/developer-guide/gfapi-symbol-versions.md
 
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
index 5deef15e3ae..b9223940416 100644
--- a/api/src/gfapi-messages.h
+++ b/api/src/gfapi-messages.h
@@ -1,5 +1,5 @@
 /*
- *   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
  *   This file is part of GlusterFS.
  *
  *   This file is licensed to you under your choice of the GNU Lesser
@@ -11,91 +11,137 @@
 #ifndef _GFAPI_MESSAGES_H__
 #define _GFAPI_MESSAGES_H__
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
-/*! \file gfapi-messages.h
- *  \brief libgfapi log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
 
-#define GLFS_GFAPI_BASE             GLFS_MSGID_COMP_API
-#define GLFS_NUM_MESSAGES           48
-#define GLFS_MSGID_END              (GLFS_GFAPI_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messages with message IDs */
-#define glfs_msg_start_x GLFS_GFAPI_BASE, "Invalid: Start of messages"
-/*------------*/
-
-#define API_MSG_MEM_ACCT_INIT_FAILED            (GLFS_GFAPI_BASE + 1)
-#define API_MSG_MASTER_XLATOR_INIT_FAILED       (GLFS_GFAPI_BASE + 2)
-#define API_MSG_GFAPI_XLATOR_INIT_FAILED        (GLFS_GFAPI_BASE + 3)
-#define API_MSG_VOLFILE_OPEN_FAILED             (GLFS_GFAPI_BASE + 4)
-#define API_MSG_VOL_SPEC_FILE_ERROR             (GLFS_GFAPI_BASE + 5)
-#define API_MSG_GLFS_FSOBJ_NULL                 (GLFS_GFAPI_BASE + 6)
-#define API_MSG_INVALID_ENTRY                   (GLFS_GFAPI_BASE + 7)
-#define API_MSG_FSMUTEX_LOCK_FAILED             (GLFS_GFAPI_BASE + 8)
-#define API_MSG_COND_WAIT_FAILED                (GLFS_GFAPI_BASE + 9)
-#define API_MSG_FSMUTEX_UNLOCK_FAILED           (GLFS_GFAPI_BASE + 10)
-#define API_MSG_INODE_REFRESH_FAILED            (GLFS_GFAPI_BASE + 11)
-#define API_MSG_GRAPH_CONSTRUCT_FAILED          (GLFS_GFAPI_BASE + 12)
-#define API_MSG_FUSE_XLATOR_ERROR               (GLFS_GFAPI_BASE + 13)
-#define API_MSG_XDR_PAYLOAD_FAILED              (GLFS_GFAPI_BASE + 14)
-#define API_MSG_GET_VOLINFO_CBK_FAILED          (GLFS_GFAPI_BASE + 15)
-#define API_MSG_FETCH_VOLUUID_FAILED            (GLFS_GFAPI_BASE + 16)
-#define API_MSG_INSUFF_SIZE                     (GLFS_GFAPI_BASE + 17)
-#define API_MSG_FRAME_CREAT_FAILED              (GLFS_GFAPI_BASE + 18)
-#define API_MSG_DICT_SET_FAILED                 (GLFS_GFAPI_BASE + 19)
-#define API_MSG_XDR_DECODE_FAILED               (GLFS_GFAPI_BASE + 20)
-#define API_MSG_GET_VOLFILE_FAILED              (GLFS_GFAPI_BASE + 21)
-#define API_MSG_WRONG_OPVERSION                 (GLFS_GFAPI_BASE + 22)
-#define API_MSG_DICT_SERIALIZE_FAILED           (GLFS_GFAPI_BASE + 23)
-#define API_MSG_REMOTE_HOST_CONN_FAILED         (GLFS_GFAPI_BASE + 24)
-#define API_MSG_VOLFILE_SERVER_EXHAUST          (GLFS_GFAPI_BASE + 25)
-#define API_MSG_CREATE_RPC_CLIENT_FAILED        (GLFS_GFAPI_BASE + 26)
-#define API_MSG_REG_NOTIFY_FUNC_FAILED          (GLFS_GFAPI_BASE + 27)
-#define API_MSG_REG_CBK_FUNC_FAILED             (GLFS_GFAPI_BASE + 28)
-#define API_MSG_GET_CWD_FAILED                  (GLFS_GFAPI_BASE + 29)
-#define API_MSG_FGETXATTR_FAILED                (GLFS_GFAPI_BASE + 30)
-#define API_MSG_LOCKINFO_KEY_MISSING            (GLFS_GFAPI_BASE + 31)
-#define API_MSG_FSETXATTR_FAILED                (GLFS_GFAPI_BASE + 32)
-#define API_MSG_FSYNC_FAILED                    (GLFS_GFAPI_BASE + 33)
-#define API_MSG_FDCREATE_FAILED                 (GLFS_GFAPI_BASE + 34)
-#define API_MSG_INODE_PATH_FAILED               (GLFS_GFAPI_BASE + 35)
-#define API_MSG_SYNCOP_OPEN_FAILED              (GLFS_GFAPI_BASE + 36)
-#define API_MSG_LOCK_MIGRATE_FAILED             (GLFS_GFAPI_BASE + 37)
-#define API_MSG_OPENFD_SKIPPED                  (GLFS_GFAPI_BASE + 38)
-#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED       (GLFS_GFAPI_BASE + 39)
-#define API_MSG_CWD_GRAPH_REF_FAILED            (GLFS_GFAPI_BASE + 40)
-#define API_MSG_SWITCHED_GRAPH                  (GLFS_GFAPI_BASE + 41)
-#define API_MSG_XDR_RESPONSE_DECODE_FAILED      (GLFS_GFAPI_BASE + 42)
-#define API_MSG_VOLFILE_INFO                    (GLFS_GFAPI_BASE + 43)
-#define API_MSG_VOLFILE_CONNECTING              (GLFS_GFAPI_BASE + 44)
-#define API_MSG_NEW_GRAPH                       (GLFS_GFAPI_BASE + 45)
-#define API_MSG_ALLOC_FAILED                    (GLFS_GFAPI_BASE + 46)
-#define API_MSG_CREATE_HANDLE_FAILED            (GLFS_GFAPI_BASE + 47)
-#define API_MSG_INODE_LINK_FAILED               (GLFS_GFAPI_BASE + 48)
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
 
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
 
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
 #endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
index 74ef4d3442d..bc639e6b99f 100644
--- a/api/src/gfapi.aliases
+++ b/api/src/gfapi.aliases
@@ -18,31 +18,24 @@ _pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
 _pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
 _pub_glfs_read _glfs_read$GFAPI_3.4.0
 _pub_glfs_write _glfs_write$GFAPI_3.4.0
-_pub_glfs_read_async _glfs_read_async$GFAPI_3.4.0
-_pub_glfs_write_async _glfs_write_async$GFAPI_3.4.0
 _pub_glfs_readv _glfs_readv$GFAPI_3.4.0
 _pub_glfs_writev _glfs_writev$GFAPI_3.4.0
-_pub_glfs_readv_async _glfs_readv_async$GFAPI_3.4.0
-_pub_glfs_writev_async _glfs_writev_async$GFAPI_3.4.0
-_pub_glfs_pread _glfs_pread$GFAPI_3.4.0
-_pub_glfs_pwrite _glfs_pwrite$GFAPI_3.4.0
-_pub_glfs_pread_async _glfs_pread_async$GFAPI_3.4.0
-_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
 _pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
 _pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
-_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_3.4.0
-_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_3.4.0
 _pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
-_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
-_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_3.4.0
-_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
 _pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
 _pub_glfs_stat _glfs_stat$GFAPI_3.4.0
 _pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
-_pub_glfs_fsync _glfs_fsync$GFAPI_3.4.0
-_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_3.4.0
-_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_3.4.0
-_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
 _pub_glfs_access _glfs_access$GFAPI_3.4.0
 _pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
 _pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
@@ -113,9 +106,8 @@ _pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
 _pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
 _pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
 _pub_glfs_discard _glfs_discard$GFAPI_3.5.0
-_pub_glfs_discard_async _glfs_discard_async$GFAPI_3.5.0
 _pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
-_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
 _pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
 
 _pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
@@ -125,12 +117,10 @@ _pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
 _pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
 _pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
 
-_pub_glfs_ipc37 _glfs_ipc$GFAPI_3.7.0
-_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
 _pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
 _pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
 _pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
-
 _pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
 _pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
 
@@ -141,10 +131,13 @@ _priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
 
 _pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
 
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
 _pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
 _pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
 _pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
-_pub_glfs_upcall_inode_get_event _glfs_upcall_inode_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
@@ -153,6 +146,56 @@ _pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
 _pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
 _pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
 
-_pub_glfs_ipc _glfs_ipc$GFAPI_4.0.0
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
index 964db76f84a..228ac47c084 100644
--- a/api/src/gfapi.map
+++ b/api/src/gfapi.map
@@ -97,7 +97,6 @@ GFAPI_3.4.2 {
 		glfs_setfsuid;
 		glfs_setfsgid;
 		glfs_setfsgroups;
-		glfs_h_lookupat;
 		glfs_h_creat;
 		glfs_h_mkdir;
 		glfs_h_mknod;
@@ -115,6 +114,7 @@ GFAPI_3.4.2 {
 		glfs_h_create_from_handle;
 		glfs_h_opendir;
 		glfs_h_open;
+		glfs_h_lookupat;
 } GFAPI_3.4.0;
 
 GFAPI_3.5.0 {
@@ -146,7 +146,6 @@ GFAPI_3.6.0 {
 
 GFAPI_3.7.0 {
 	global:
-		glfs_ipc;
 		glfs_h_poll_upcall;
 		glfs_h_acl_set;
 		glfs_h_acl_get;
@@ -175,9 +174,10 @@ GFAPI_3.7.15 {
 
 GFAPI_3.7.16 {
 	global:
+		glfs_free;
 		glfs_upcall_get_fs;
 		glfs_upcall_get_reason;
-		glfs_upcall_inode_get_event;
+		glfs_upcall_get_event;
 		glfs_upcall_inode_get_object;
 		glfs_upcall_inode_get_flags;
 		glfs_upcall_inode_get_stat;
@@ -186,6 +186,7 @@ GFAPI_3.7.16 {
 		glfs_upcall_inode_get_pstat;
 		glfs_upcall_inode_get_oldpobject;
 		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
 } GFAPI_3.7.15;
 
 GFAPI_3.7.17 {
@@ -193,7 +194,90 @@ GFAPI_3.7.17 {
 		glfs_realpath;
 } GFAPI_3.7.16;
 
-GFAPI_4.0.0 {
+GFAPI_3.10.0 {
 	global:
-		glfs_ipc;
+		glfs_sysrq;
 } GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 0a59a8dcf1c..6aa3c5602d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -1,6 +1,6 @@
 
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -18,12 +18,13 @@
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "gfapi-messages.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include <limits.h>
 #include "glusterfs3.h"
+#include <glusterfs/iatt.h>
 
 #ifdef NAME_MAX
 #define GF_NAME_MAX NAME_MAX
@@ -31,22 +32,33 @@
 #define GF_NAME_MAX 255
 #endif
 
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
+
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
+
 /*
  * This function will mark glfd for deletion and decrement its refcount.
  */
 int
-glfs_mark_glfd_for_deletion (struct glfs_fd *glfd)
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
 {
+    LOCK(&glfd->lock);
+    {
         glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
 
-        GF_REF_PUT (glfd);
+    GF_REF_PUT(glfd);
 
-        return 0;
+    return 0;
 }
 
-/* This function is usefull for all async fops. There is chance that glfd is
+/* This function is useful for all async fops. There is chance that glfd is
  * closed before async fop is completed. When glfd is closed we change the
  * state to GLFD_CLOSE.
  *
@@ -54,12 +66,33 @@ glfs_mark_glfd_for_deletion (struct glfs_fd *glfd)
  * _gf_false.
  */
 gf_boolean_t
-glfs_is_glfd_still_valid (struct glfs_fd *glfd)
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
 {
+    gf_boolean_t ret = _gf_false;
+
+    LOCK(&glfd->lock);
+    {
         if (glfd->state != GLFD_CLOSE)
-                return _gf_true;
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
+
+    return ret;
+}
+
+void
+glfd_set_state_bind(struct glfs_fd *glfd)
+{
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
+
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
 
-        return _gf_false;
+    return;
 }
 
 /*
@@ -70,4542 +103,6343 @@ glfs_is_glfd_still_valid (struct glfs_fd *glfd)
  * maintained by gfapi.
  */
 int
-glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
-                                    struct gf_upcall *from_up_data)
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
 {
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
 
-        struct gf_upcall_cache_invalidation *ca_data = NULL;
-        struct gf_upcall_cache_invalidation *f_ca_data = NULL;
-        int                                 ret      = -1;
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-        GF_VALIDATE_OR_GOTO (THIS->name, to_up_data, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, from_up_data, out);
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-        f_ca_data = from_up_data->data;
-        GF_VALIDATE_OR_GOTO (THIS->name, f_ca_data, out);
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-        ca_data = GF_CALLOC (1, sizeof(*ca_data),
-                            glfs_mt_upcall_entry_t);
-
-        if (!ca_data) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        API_MSG_ALLOC_FAILED,
-                        "Upcall entry allocation failed.");
-                goto out;
-        }
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-        to_up_data->data = ca_data;
+    to_up_data->data = ca_data;
 
-        ca_data->flags      = f_ca_data->flags;
-        ca_data->expire_time_attr = f_ca_data->expire_time_attr;
-        ca_data->stat       = f_ca_data->stat;
-        ca_data->p_stat     = f_ca_data->p_stat;
-        ca_data->oldp_stat  = f_ca_data->oldp_stat;
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glfs_loc_link (loc_t *loc, struct iatt *iatt)
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
 {
-	int ret = -1;
-        inode_t *old_inode = NULL;
-        uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
 
-	if (!loc->inode) {
-		errno = EINVAL;
-		return -1;
-	}
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-        old_inode = loc->inode;
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-        /* If the inode already exists in the cache, the inode
-         * returned here points to the existing one. We need
-         * to update loc.inode accordingly.
-         */
-	loc->inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
-	if (loc->inode) {
-                inode_ctx_set (loc->inode, THIS, &ctx_value);
-		inode_lookup (loc->inode);
-                inode_unref (old_inode);
-		ret = 0;
-	} else {
-		ret = -1;
-	}
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
+
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	return ret;
+    to_up_data->data = ca_data;
+
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
+
+    ret = 0;
+out:
+    return ret;
 }
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
 
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
 
 void
-glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
 {
-	iatt_to_stat (iatt, stat);
-	stat->st_dev = fs->dev_id;
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
 }
 
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
+{
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
+}
 
-int
-glfs_loc_unlink (loc_t *loc)
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
 {
-	inode_unlink (loc->inode, loc->parent, loc->name);
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
+}
 
-	return 0;
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
 }
 
+int
+glfs_loc_unlink(loc_t *loc)
+{
+    inode_unlink(loc->inode, loc->parent, loc->name);
+
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
+
+    return 0;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
 struct glfs_fd *
-pub_glfs_open (struct glfs *fs, const char *path, int flags)
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
 {
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+    if (ret)
+        goto out;
 
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret)
-		goto out;
-
-	if (IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
-
-	if (!IA_ISREG (iatt.ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-        glfd->fd->flags = flags;
-
-	ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-                GF_REF_PUT (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->state = GLFD_OPEN;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
 int
-pub_glfs_close (struct glfs_fd *glfd)
+pub_glfs_close(struct glfs_fd *glfd)
 {
-	xlator_t  *subvol = NULL;
-	int        ret = -1;
-	fd_t      *fd = NULL;
-	struct glfs *fs = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_flush (subvol, fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	fs = glfd->fs;
+    fs = glfd->fs;
 
-        if (fd)
-                fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        glfs_mark_glfd_for_deletion (glfd);
-	glfs_subvol_done (fs, subvol);
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
 int
-pub_glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
 int
-pub_glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0);
-
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
 int
-pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct iatt      iatt = {0, };
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        GF_REF_GET (glfd);
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
+out:
+    loc_wipe(&loc);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    glfs_subvol_done(fs, subvol);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	ret = syncop_fstat (subvol, fd, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+invalid_fs:
+    return ret;
+}
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
 struct glfs_fd *
-pub_glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	/* This must be glfs_resolve() and NOT glfs_lresolve().
-	   That is because open("name", O_CREAT) where "name"
-	   is a danging symlink must create the dangling
-	   destinataion.
-	*/
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
-
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
-
-	if (loc.inode) {
-		if (flags & O_EXCL) {
-			ret = -1;
-			errno = EEXIST;
-			goto out;
-		}
-
-		if (IA_ISDIR (iatt.ia_type)) {
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-
-		if (!IA_ISREG (iatt.ia_type)) {
-			ret = -1;
-			errno = EINVAL;
-			goto out;
-		}
-	}
-
-	if (ret == -1 && errno == ENOENT) {
-		loc.inode = inode_new (loc.parent->table);
-		if (!loc.inode) {
-			ret = -1;
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-        glfd->fd->flags = flags;
-
-	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-	} else {
-		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-				     &iatt, xattr_req, NULL);
-                DECODE_SYNCOP_ERR (ret);
-	}
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	if (ret && glfd) {
-                GF_REF_PUT (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->state = GLFD_OPEN;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0);
-
 #ifdef HAVE_SEEK_HOLE
 static int
-glfs_seek (struct glfs_fd *glfd, off_t offset, int whence)
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
 {
-        int               ret             = -1;
-        xlator_t         *subvol          = NULL;
-        fd_t             *fd              = NULL;
-        gf_seek_what_t    what            = 0;
-        off_t             off             = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
 
-        switch (whence) {
+    switch (whence) {
         case SEEK_DATA:
-                what = GF_SEEK_DATA;
-                break;
+            what = GF_SEEK_DATA;
+            break;
         case SEEK_HOLE:
-                what = GF_SEEK_HOLE;
-                break;
+            what = GF_SEEK_HOLE;
+            break;
         default:
-                /* other SEEK_* do not make sense, all operations get an offset
-                 * and the position in the fd is not tracked */
-                errno = EINVAL;
-                goto out;
-        }
-
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto done;
-        }
-
-        ret = syncop_seek (subvol, fd, offset, what, NULL, &off);
-        DECODE_SYNCOP_ERR (ret);
-
-        if (ret != -1)
-                glfd->offset = off;
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
+
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret != -1)
+        glfd->offset = off;
 
 done:
-        if (fd)
-                fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-        glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
 out:
-        return ret;
+    return ret;
 }
 #endif
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
 off_t
-pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
-{
-	struct stat sb  = {0, };
-	int         ret = -1;
-        off_t       off = -1;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	switch (whence) {
-	case SEEK_SET:
-		glfd->offset = offset;
-                ret = 0;
-		break;
-	case SEEK_CUR:
-		glfd->offset += offset;
-                ret = 0;
-		break;
-	case SEEK_END:
-		ret = pub_glfs_fstat (glfd, &sb);
-		if (ret) {
-			/* seek cannot fail :O */
-			break;
-		}
-		glfd->offset = sb.st_size + offset;
-		break;
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
 #ifdef HAVE_SEEK_HOLE
         case SEEK_DATA:
         case SEEK_HOLE:
-                ret = glfs_seek (glfd, offset, whence);
-                break;
+            ret = glfs_seek(glfd, offset, whence);
+            break;
 #endif
         default:
-                errno = EINVAL;
-	}
+            errno = EINVAL;
+    }
 
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        if (ret != -1)
-                off = glfd->offset;
+    if (ret != -1)
+        off = glfd->offset;
 
-        return off;
+    return off;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0);
-
-
-ssize_t
-pub_glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                 off_t offset, int flags)
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
 {
-	xlator_t       *subvol = NULL;
-	ssize_t         ret = -1;
-	ssize_t         size = -1;
-	struct iovec   *iov = NULL;
-	int             cnt = 0;
-	struct iobref  *iobref = NULL;
-	fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	size = iov_length (iovec, iovcnt);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref,
-                            NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret <= 0)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+invalid_fs:
+    return ret;
+}
 
-	glfd->offset = (offset + size);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
+ssize_t
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
+{
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
+}
 
-	ret = size;
-out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
+ssize_t
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	glfs_subvol_done (glfd->fs, subvol);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-        __GLFS_EXIT_FS;
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
 
-invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
 ssize_t
-pub_glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
 ssize_t
-pub_glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-                int flags)
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, offset, flags);
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
 ssize_t
-pub_glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                int flags)
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
 {
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv (glfd, iov, count, glfd->offset, flags);
-
-	return ret;
-}
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0);
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
 
+    return ret;
+}
 
 struct glfs_io {
-	struct glfs_fd      *glfd;
-	int                  op;
-	off_t                offset;
-	struct iovec        *iov;
-	int                  count;
-	int                  flags;
-	glfs_io_cbk          fn;
-	void                *data;
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
 };
 
-
 static int
-glfs_io_async_cbk (int op_ret, int op_errno, call_frame_t *frame,
-                   void *cookie, struct iovec *iovec, int count)
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
-        struct glfs_io *gio = NULL;
-        xlator_t       *subvol = NULL;
-        struct glfs    *fs = NULL;
-        struct glfs_fd *glfd = NULL;
-        int             ret  = -1;
-
-        GF_VALIDATE_OR_GOTO ("gfapi", frame, inval);
-        GF_VALIDATE_OR_GOTO ("gfapi", cookie, inval);
-
-        gio = frame->local;
-        frame->local = NULL;
-        subvol = cookie;
-        glfd = gio->glfd;
-        fs = glfd->fs;
-
-        if (!glfs_is_glfd_still_valid (glfd))
-                goto err;
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
 
-        if (op_ret <= 0) {
-                goto out;
-        } else if (gio->op == GF_FOP_READ) {
-                if (!iovec) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
 
-                op_ret = iov_copy (gio->iov, gio->count, iovec, count);
-                glfd->offset = gio->offset + op_ret;
-        } else if (gio->op == GF_FOP_WRITE) {
-                glfd->offset = gio->offset + gio->iov->iov_len;
+out:
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
         }
 
-out:
-        errno = op_errno;
-        gio->fn (gio->glfd, op_ret, gio->data);
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
 
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
 err:
-        fd_unref (glfd->fd);
-        /* Since the async operation is complete
-         * release the ref taken during the start
-         * of async operation
-         */
-        GF_REF_PUT (glfd);
-
-        GF_FREE (gio->iov);
-        GF_FREE (gio);
-        STACK_DESTROY (frame->root);
-        glfs_subvol_done (fs, subvol);
-
-        ret = 0;
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
 inval:
-        return ret;
+    return ret;
 }
 
 static int
-glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, struct iovec *iovec,
-                       int count, struct iatt *stbuf, struct iobref *iobref,
-                       dict_t *xdata)
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, iovec, count);
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
 
-	return 0;
+    return 0;
 }
 
-
-int
-pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                       int count, off_t offset, int flags, glfs_io_cbk fn,
-                       void *data)
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-	call_frame_t   *frame = NULL;
-	xlator_t       *subvol = NULL;
-	glfs_t         *fs = NULL;
-	fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	fs = glfd->fs;
-
-	frame = syncop_create_frame (THIS);
-	if (!frame) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_READ;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	frame->local = gio;
-
-	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
-			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
 
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                if (gio) {
-                        GF_FREE (gio->iov);
-                        GF_FREE (gio);
-                }
-                if (frame) {
-                        STACK_DESTROY (frame->root);
-                }
-		glfs_subvol_done (fs, subvol);
-	}
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-	return ret;
+    return ret;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
+int
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
 int
-pub_glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
-                     glfs_io_cbk fn, void *data)
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
+int
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
 int
-pub_glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count,
-                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
 int
-pub_glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
-				  fn, data);
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 3.4.0);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
 
+    return ret;
+}
 
-static int
-glfs_buf_copy (xlator_t *subvol, const struct iovec *iovec_src, int iovcnt,
-               struct iobref **iobref, struct iobuf **iobuf,
-               struct iovec *iov_dst)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
+int
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-        size_t         size = -1;
-        int            ret  = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        size = iov_length (iovec_src, iovcnt);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-        *iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-        if (!(*iobuf)) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
 
-        *iobref = iobref_new ();
-        if (!(*iobref)) {
-                iobuf_unref (*iobuf);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
-
-        ret = iobref_add (*iobref, *iobuf);
-        if (ret) {
-                iobuf_unref (*iobuf);
-                iobref_unref (*iobref);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        iov_unload (iobuf_ptr (*iobuf), iovec_src, iovcnt);  /* FIXME!!! */
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
+int
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
 
-        iov_dst->iov_base = iobuf_ptr (*iobuf);
-        iov_dst->iov_len = size;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-out:
-        return ret;
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
 }
 
-
-ssize_t
-pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                  off_t offset, int flags)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
+int
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	struct iobref  *iobref = NULL;
-	struct iobuf   *iobuf = NULL;
-	struct iovec    iov = {0, };
-	fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-        ret = glfs_buf_copy (subvol, iovec, iovcnt, &iobref, &iobuf, &iov);
-        if (ret)
-                goto out;
+    ssize_t ret = 0;
 
-	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags, NULL,
-                             NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret <= 0)
-		goto out;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	glfd->offset = (offset + iov.iov_len);
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
+}
 
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
 out:
-        if (iobuf)
-                iobuf_unref (iobuf);
-        if (iobref)
-                iobref_unref (iobref);
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
 ssize_t
-pub_glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
-
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+out:
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	return ret;
-}
+    glfs_subvol_done(glfd_in->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
 ssize_t
-pub_glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                 int flags)
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
 {
-	ssize_t      ret = 0;
-
-	ret = pub_glfs_pwritev (glfd, iov, count, glfd->offset, flags);
-
-	return ret;
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
 ssize_t
-pub_glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count,
-                 off_t offset, int flags)
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
-
-	ret = pub_glfs_pwritev (glfd, &iov, 1, offset, flags);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
-extern glfs_t *pub_glfs_from_glfd (glfs_fd_t *);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
+ssize_t
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
+{
+    ssize_t ret = 0;
 
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-static int
-glfs_pwritev_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
-{
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, NULL, 0);
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
 
-        return 0;
+    return ret;
 }
 
-int
-pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                        int count, off_t offset, int flags, glfs_io_cbk fn,
-                        void *data)
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
+ssize_t
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
-        call_frame_t   *frame = NULL;
-        xlator_t       *subvol = NULL;
-        fd_t           *fd = NULL;
-        struct iobref  *iobref = NULL;
-        struct iobuf   *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        /* Need to take explicit ref so that the fd
-         * is not destroyed before the fop is complete
-         */
-        GF_REF_GET (glfd);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
 
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    return ret;
+}
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
+ssize_t
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                goto out;
-        }
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        gio->op     = GF_FOP_WRITE;
-        gio->glfd   = glfd;
-        gio->offset = offset;
-        gio->flags  = flags;
-        gio->fn     = fn;
-        gio->data   = data;
-        gio->count  = 1;
-        gio->iov = GF_CALLOC (gio->count, sizeof (*(gio->iov)),
-                              gf_common_mt_iovec);
-        if (!gio->iov) {
-                errno = ENOMEM;
-                goto out;
-        }
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
 
-        ret = glfs_buf_copy (subvol, iovec, count, &iobref, &iobuf, gio->iov);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        frame = syncop_create_frame (THIS);
-        if (!frame) {
-                errno = ENOMEM;
-                goto out;
-        }
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
 
-        frame->local = gio;
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-        STACK_WIND_COOKIE (frame, glfs_pwritev_async_cbk, subvol, subvol,
-                           subvol->fops->writev, fd, gio->iov,
-                           gio->count, offset, flags, iobref, NULL);
+    return 0;
+}
 
-        ret = 0;
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                GF_FREE (gio);
-                if (frame)
-                        STACK_DESTROY (frame->root);
-
-                glfs_subvol_done (glfd->fs, subvol);
-        }
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (iobuf)
-                iobuf_unref (iobuf);
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
+int
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
 int
-pub_glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
+int
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
 int
-pub_glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
-                       off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
-
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
 int
-pub_glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                       int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
-				   fn, data);
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 3.4.0);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
 int
-pub_glfs_fsync (struct glfs_fd *glfd)
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        GF_REF_GET (glfd);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsync (subvol, fd, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    return ret;
+}
 
-	glfs_subvol_done (glfd->fs, subvol);
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
+int
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
 
-        __GLFS_EXIT_FS;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-invalid_fs:
-	return ret;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 3.4.0);
-
-static int
-glfs_fsync_async_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret,
-                      int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
 {
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, NULL, 0);
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-        return 0;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
 }
 
 static int
-glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
-			 int dataonly)
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-        call_frame_t   *frame = NULL;
-        xlator_t       *subvol = NULL;
-        fd_t           *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        /* Need to take explicit ref so that the fd
-         * is not destroyed before the fop is complete
-         */
-        GF_REF_GET (glfd);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                ret = -1;
-                errno = EBADFD;
-                goto out;
-        }
+invalid_fs:
+    return ret;
+}
 
-        frame = syncop_create_frame (THIS);
-        if (!frame) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
+int
+pub_glfs_fsync34(struct glfs_fd *glfd)
+{
+    return glfs_fsync_common(glfd, NULL, NULL);
+}
 
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
+int
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
+{
+    return glfs_fsync_common(glfd, prestat, poststat);
+}
 
-	gio->op     = GF_FOP_FSYNC;
-	gio->glfd   = glfd;
-	gio->flags  = dataonly;
-	gio->fn     = fn;
-	gio->data   = data;
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-        frame->local = gio;
+    return 0;
+}
 
-        STACK_WIND_COOKIE (frame, glfs_fsync_async_cbk, subvol, subvol,
-                           subvol->fops->fsync, fd, dataonly, NULL);
+static int
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
 
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                GF_FREE (gio);
-                if (frame)
-                        STACK_DESTROY (frame->root);
-                glfs_subvol_done (glfd->fs, subvol);
-        }
-
-        return ret;
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
 int
-pub_glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        ret = glfs_fsync_async_common (glfd, fn, data, 0);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
 int
-pub_glfs_fdatasync (struct glfs_fd *glfd)
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        GF_REF_GET (glfd);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_fsync (subvol, fd, 1, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
+int
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
+{
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
 int
-pub_glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ret = glfs_fsync_async_common (glfd, fn, data, 1);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
 int
-pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        GF_REF_GET (glfd);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	ret = syncop_ftruncate (subvol, fd, offset, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
 int
-pub_glfs_truncate (struct glfs *fs, const char *path, off_t length)
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-        int              reval = 0;
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
 
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
+int
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-        ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-        ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = syncop_truncate (subvol, &loc, length, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15);
-
-
 static int
-glfs_ftruncate_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *prebuf, struct iatt *postbuf,
-                          dict_t *xdata)
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
 {
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, NULL, 0);
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-        return 0;
+    return 0;
 }
 
-int
-pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
-                          void *data)
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
-        call_frame_t   *frame = NULL;
-        xlator_t       *subvol = NULL;
-        fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        /* Need to take explicit ref so that the fd
-         * is not destroyed before the fop is complete
-         */
-        GF_REF_GET (glfd);
-
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
-        }
-
-        frame = syncop_create_frame (THIS);
-        if (!frame) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_FTRUNCATE;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->fn     = fn;
-	gio->data   = data;
-
-        frame->local = gio;
-
-        STACK_WIND_COOKIE (frame, glfs_ftruncate_async_cbk, subvol, subvol,
-                           subvol->fops->ftruncate, fd, offset, NULL);
-
-        ret = 0;
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
 
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                GF_FREE (gio);
-                if (frame)
-                        STACK_DESTROY (frame->root);
-                glfs_subvol_done (glfd->fs, subvol);
-        }
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
+int
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
 int
-pub_glfs_access (struct glfs *fs, const char *path, int mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_access (subvol, &loc, mode, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
 int
-pub_glfs_symlink (struct glfs *fs, const char *data, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_symlink (subvol, &loc, data, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
 int
-pub_glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-	char            *linkval = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFLNK) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret > 0) {
-		memcpy (buf, linkval, ret);
-		GF_FREE (linkval);
-	}
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
 int
-pub_glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mknod (subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
 int
-pub_glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mkdir (subvol, &loc, mode, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
 int
-pub_glfs_unlink (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_unlink(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_unlink (subvol, &loc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
 int
-pub_glfs_rmdir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_rmdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ret = syncop_rmdir (subvol, &loc, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
 int
-pub_glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                    (newiatt.ia_type == IA_IFDIR)) {
-                       /* Either both old and new must be dirs,
-                        * or both must be non-dirs. Else, fail.
-                        */
-                       ret = -1;
-                       errno = EISDIR;
-                       goto out;
-                }
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
         }
+    }
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
 
-	ret = syncop_rename (subvol, &oldloc, &newloc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		if (reval < DEFAULT_REVAL_COUNT) {
-			reval++;
-			loc_wipe (&oldloc);
-			loc_wipe (&newloc);
-			goto retry;
-		}
-	}
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
 
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
 int
-pub_glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    glfs_subvol_done(fs, subvol);
 
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+    __GLFS_EXIT_FS;
 
-	if (ret == 0) {
-		ret = -1;
-		errno = EEXIST;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-	if (oldiatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-        /* Filling the inode of the hard link to be same as that of the
-           original file
-        */
-	if (newloc.inode) {
-		inode_unref (newloc.inode);
-		newloc.inode = NULL;
-	}
-        newloc.inode = inode_ref (oldloc.inode);
-
-	ret = syncop_link (subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret == -1 && errno == ESTALE) {
-		loc_wipe (&oldloc);
-		loc_wipe (&newloc);
-		if (reval--)
-			goto retry;
-	}
-
-	if (ret == 0)
-		ret = glfs_loc_link (&newloc, &newiatt);
-out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	glfs_subvol_done (fs, subvol);
+    if (ret)
+        goto out;
 
-        __GLFS_EXIT_FS;
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-invalid_fs:
-	return ret;
-}
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0);
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-struct glfs_fd *
-pub_glfs_opendir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	INIT_LIST_HEAD (&glfd->entries);
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret)
-		goto out;
-
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	ret = syncop_opendir (subvol, &loc, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		GF_REF_PUT (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-                glfd->state = GLFD_OPEN;
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
 int
-pub_glfs_closedir (struct glfs_fd *glfd)
+pub_glfs_closedir(struct glfs_fd *glfd)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
 
-        glfs_mark_glfd_for_deletion (glfd);
+    glfs_mark_glfd_for_deletion(glfd);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        ret = 0;
+    ret = 0;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
 long
-pub_glfs_telldir (struct glfs_fd *fd)
+pub_glfs_telldir(struct glfs_fd *fd)
 {
-	return fd->offset;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0);
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    return fd->offset;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
 void
-pub_glfs_seekdir (struct glfs_fd *fd, long offset)
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
 
-	if (fd->offset == offset)
-		return;
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
 
-	fd->offset = offset;
-	fd->next = NULL;
+    if (fd->offset == offset)
+        return;
 
-	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
-		if (entry->d_off != offset)
-			continue;
+    fd->offset = offset;
+    fd->next = NULL;
 
-		if (&tmp->list != &fd->entries) {
-			/* found! */
-			fd->next = tmp;
-			return;
-		}
-	}
-	/* could not find entry at requested offset in the cache.
-	   next readdir_r() will result in glfd_entry_refresh()
-	*/
-}
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0);
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
 
 static int
-glfs_discard_async_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, NULL, 0);
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-        return 0;
+    return 0;
 }
 
-int
-pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
-                        glfs_io_cbk fn, void *data)
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
-        call_frame_t   *frame = NULL;
-        xlator_t       *subvol = NULL;
-        fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        /* Need to take explicit ref so that the fd
-         * is not destroyed before the fop is complete
-         */
-        GF_REF_GET (glfd);
-
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
-        }
-
-        frame = syncop_create_frame (THIS);
-        if (!frame) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_DISCARD;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->count  = len;
-	gio->fn     = fn;
-	gio->data   = data;
-
-        frame->local = gio;
-
-        STACK_WIND_COOKIE (frame, glfs_discard_async_cbk, subvol, subvol,
-                           subvol->fops->discard, fd, offset, len, NULL);
-
-        ret = 0;
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                GF_FREE (gio);
-                if (frame)
-                        STACK_DESTROY (frame->root);
-                glfs_subvol_done (glfd->fs, subvol);
-        }
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 3.5.0);
-
-
-static int
-glfs_zerofill_async_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, struct iatt *preop_stbuf,
-                         struct iatt *postop_stbuf, dict_t *xdata)
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
+int
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
 {
-        glfs_io_async_cbk (op_ret, op_errno, frame, cookie, NULL, 0);
-
-        return 0;
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
 int
-pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
-                         glfs_io_cbk fn, void *data)
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
 {
-        struct glfs_io *gio  = NULL;
-        int             ret  = -1;
-        call_frame_t   *frame = NULL;
-        xlator_t       *subvol = NULL;
-        fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        /* Need to take explicit ref so that the fd
-         * is not destroyed before the fop is complete
-         */
-        GF_REF_GET (glfd);
-
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
-        }
-
-        frame = syncop_create_frame (THIS);
-        if (!frame) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                goto out;
-        }
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
-        gio->op     = GF_FOP_ZEROFILL;
-        gio->glfd   = glfd;
-        gio->offset = offset;
-        gio->count  = len;
-        gio->fn     = fn;
-        gio->data   = data;
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-        frame->local = gio;
+    return 0;
+}
 
-        STACK_WIND_COOKIE (frame, glfs_zerofill_async_cbk, subvol, subvol,
-                           subvol->fops->zerofill, fd, offset, len, NULL);
-        ret = 0;
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
 out:
+    if (ret) {
         if (fd)
-                fd_unref (fd);
-
-        if (ret) {
-                if (glfd)
-                        GF_REF_PUT (glfd);
-                GF_FREE (gio);
-                if (frame)
-                        STACK_DESTROY (frame->root);
-                glfs_subvol_done (glfd->fs, subvol);
-        }
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 3.5.0);
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
 void
-gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
 {
-	dirent->d_ino = gf_dirent->d_ino;
+    dirent->d_ino = gf_dirent->d_ino;
 
 #ifdef _DIRENT_HAVE_D_OFF
-	dirent->d_off = gf_dirent->d_off;
+    dirent->d_off = gf_dirent->d_off;
 #endif
 
 #ifdef _DIRENT_HAVE_D_TYPE
-	dirent->d_type = gf_dirent->d_type;
+    dirent->d_type = gf_dirent->d_type;
 #endif
 
 #ifdef _DIRENT_HAVE_D_NAMLEN
-	dirent->d_namlen = strlen (gf_dirent->d_name);
+    dirent->d_namlen = strlen(gf_dirent->d_name);
 #endif
 
-	strncpy (dirent->d_name, gf_dirent->d_name, NAME_MAX);
-	dirent->d_name[NAME_MAX] = 0;
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
 }
 
-
 int
-glfd_entry_refresh (struct glfs_fd *glfd, int plus)
-{
-	xlator_t        *subvol = NULL;
-	gf_dirent_t      entries;
-	gf_dirent_t      old;
-        gf_dirent_t     *entry = NULL;
-	int              ret = -1;
-	fd_t            *fd = NULL;
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	if (fd->inode->ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = EBADF;
-		goto out;
-	}
-
-	INIT_LIST_HEAD (&entries.list);
-	INIT_LIST_HEAD (&old.list);
-
-	if (plus)
-		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
-				       &entries, NULL, NULL);
-	else
-		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
-				      &entries, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret >= 0) {
-		if (plus) {
-                        /**
-                         * Set inode_needs_lookup flag before linking the
-                         * inode. Doing it later post linkage might lead
-                         * to a race where a fop comes after inode link
-                         * but before setting need_lookup flag.
-                         */
-                        list_for_each_entry (entry, &entries.list, list) {
-                                if (entry->inode)
-                                        inode_set_need_lookup (entry->inode, THIS);
-                                else if (!IA_ISDIR (entry->d_stat.ia_type)) {
-                                        /* entry->inode for directories will be
-                                         * always set to null to force a lookup
-                                         * on the dentry. Also we will have
-                                         * proper stat if directory present on
-                                         * hashed subvolume.
-                                         */
-                                        gf_fill_iatt_for_dirent (entry,
-                                                                 fd->inode,
-                                                                 subvol);
-                                }
-                        }
-
-			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
+{
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
                 }
+            }
 
-		list_splice_init (&glfd->entries, &old.list);
-		list_splice_init (&entries.list, &glfd->entries);
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
 
-		/* spurious errno is dangerous for glfd_entry_next() */
-		errno = 0;
-	}
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
 
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
 
-	if (ret > 0)
-		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
 
-	gf_dirent_free (&old);
+    gf_dirent_free(&old);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    return ret;
 }
 
-
 gf_dirent_t *
-glfd_entry_next (struct glfs_fd *glfd, int plus)
+glfd_entry_next(struct glfs_fd *glfd, int plus)
 {
-	gf_dirent_t     *entry = NULL;
-	int              ret = -1;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
 
-	if (!glfd->offset || !glfd->next) {
-		ret = glfd_entry_refresh (glfd, plus);
-		if (ret < 0)
-			return NULL;
-	}
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
 
-	entry = glfd->next;
-	if (!entry)
-		return NULL;
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
 
-	if (&entry->next->list == &glfd->entries)
-		glfd->next = NULL;
-	else
-		glfd->next = entry->next;
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
 
-	glfd->offset = entry->d_off;
+    glfd->offset = entry->d_off;
 
-	return entry;
+    return entry;
 }
 
-
-static struct dirent *
-glfs_readdirbuf_get (struct glfs_fd *glfd)
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
 {
-        struct dirent *buf = NULL;
-
-        LOCK (&glfd->fd->lock);
-        {
-                buf = glfd->readdirbuf;
-                if (buf) {
-                        memset (buf, 0, READDIRBUF_SIZE);
-                        goto unlock;
-                }
-
-                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
-                if (!buf) {
-                        errno = ENOMEM;
-                        goto unlock;
-                }
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
 
-                glfd->readdirbuf = buf;
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
         }
+
+        glfd->readdirbuf = buf;
+    }
 unlock:
-        UNLOCK (&glfd->fd->lock);
+    UNLOCK(&glfd->fd->lock);
 
-        return buf;
+    return buf;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
 int
-pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
-                        struct dirent *ext, struct dirent **res)
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
 {
-	int              ret = 0;
-	gf_dirent_t     *entry = NULL;
-	struct dirent   *buf = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        GF_REF_GET (glfd);
+    GF_REF_GET(glfd);
 
-	errno = 0;
+    errno = 0;
 
-	if (ext)
-		buf = ext;
-	else
-		buf = glfs_readdirbuf_get (glfd);
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-	if (!buf) {
-		errno = ENOMEM;
-                ret = -1;
-                goto out;
-	}
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-	entry = glfd_entry_next (glfd, !!stat);
-	if (errno)
-		ret = -1;
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
 
-	if (res) {
-		if (entry)
-			*res = buf;
-		else
-			*res = NULL;
-	}
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
 
-	if (entry) {
-		gf_dirent_to_dirent (entry, buf);
-		if (stat)
-			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
-	}
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
 
 out:
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-	return ret;
+    return ret;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
 int
-pub_glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf,
-                    struct dirent **res)
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
 {
-	return pub_glfs_readdirplus_r (glfd, 0, buf, res);
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
 struct dirent *
-pub_glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
 {
-        struct dirent *res = NULL;
-        int ret = -1;
+    struct dirent *res = NULL;
+    int ret = -1;
 
-        ret = pub_glfs_readdirplus_r (glfd, stat, NULL, &res);
-        if (ret)
-                return NULL;
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
 
-        return res;
+    return res;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
 struct dirent *
-pub_glfs_readdir (struct glfs_fd *glfd)
+pub_glfs_readdir(struct glfs_fd *glfd)
 {
-        return pub_glfs_readdirplus (glfd, NULL);
+    return pub_glfs_readdirplus(glfd, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
 int
-pub_glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_statfs (subvol, &loc, buf, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
 int
-glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
-	      int valid, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      riatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
+{
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
 int
-glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
 int
-pub_glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
 int
-pub_glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
 int
-pub_glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
 int
-pub_glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-	        iatt.ia_uid = uid;
-	        valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
 int
-pub_glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_fsetattr (glfd, &iatt, valid);
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
 int
-pub_glfs_utimens (struct glfs *fs, const char *path,
-                  const struct timespec times[2])
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
 int
-pub_glfs_lutimens (struct glfs *fs, const char *path,
-                   const struct timespec times[2])
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
 int
-pub_glfs_futimens (struct glfs_fd *glfd, const struct timespec times[2])
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0);
-
-
 int
-glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-		       const char *name)
-{
-	data_t *data = NULL;
-	int     ret = -1;
-
-	data = dict_get (xattr, (char *)name);
-	if (!data) {
-		errno = ENODATA;
-		ret = -1;
-		goto out;
-	}
-
-	ret = data->len;
-	if (!value || !size)
-		goto out;
-
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	memcpy (value, data->data, ret);
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
 out:
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
-		      void *value, size_t size, int follow)
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
 ssize_t
-pub_glfs_getxattr (struct glfs *fs, const char *path, const char *name,
-                   void *value, size_t size)
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 1);
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
 ssize_t
-pub_glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
-                    void *value, size_t size)
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 0);
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
 ssize_t
-pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
-                    size_t size)
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
-        if (xattr)
-                dict_unref (xattr);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0);
-
-
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!xattr)
-		goto out;
+    if (!xattr)
+        goto out;
 
-	ret = dict_keys_join (NULL, 0, xattr, NULL);
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
 
-        if (!value || !size)
-                goto out;
+    if (!value || !size)
+        goto out;
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-	} else {
-		dict_keys_join (value, size, xattr, NULL);
-	}
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
-		       size_t size, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_listxattr_process (value, size, xattr);
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
 ssize_t
-pub_glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 1);
+    return glfs_listxattr_common(fs, path, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
 ssize_t
-pub_glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 0);
+    return glfs_listxattr_common(fs, path, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
 ssize_t
-pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	ret = glfs_listxattr_process (value, size, xattr);
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
-        if (xattr)
-                dict_unref (xattr);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0);
-
 int
-glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
-		      const void *value, size_t size, int flags, int follow)
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
 
-	ret = syncop_setxattr (subvol, &loc, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
 int
-pub_glfs_setxattr (struct glfs *fs, const char *path, const char *name,
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
                    const void *value, size_t size, int flags)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
 int
-pub_glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
-                    const void *value, size_t size, int flags)
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
 int
-pub_glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
-                    size_t size, int flags)
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        GF_REF_GET (glfd);
+    if (ret)
+        goto out;
 
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (subvol, fd, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
 
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    glfs_subvol_done(fs, subvol);
 
-	glfs_subvol_done (glfd->fs, subvol);
-
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
+int
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
 int
-glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
-			 int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 0);
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
+int
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	if (ret)
-		goto out;
+    glfs_subvol_done(glfd->fs, subvol);
 
-	ret = syncop_removexattr (subvol, &loc, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    __GLFS_EXIT_FS;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
+int
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
 int
-pub_glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
 {
-	return glfs_removexattr_common (fs, path, name, 1);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0);
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
 int
-pub_glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
 {
-	return glfs_removexattr_common (fs, path, name, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
 int
-pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+pub_glfs_chdir(struct glfs *fs, const char *path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        GF_REF_GET (glfd);
+    if (ret)
+        goto out;
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    glfs_cwd_set(fs, loc.inode);
 
-	ret = syncop_fremovexattr (subvol, fd, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
 int
-pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+pub_glfs_fchdir(struct glfs_fd *glfd)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
+{
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-        GF_REF_GET (glfd);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
 
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref(fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
 
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return retpath;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0);
-
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
 
-int
-pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
+}
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
 
-        GF_REF_GET (glfd);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    inode = glfs_cwd_get(fs);
 
-	ret = syncop_discard (subvol, fd, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
 out:
-	if (fd)
-		fd_unref(fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    GF_FREE(path);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    if (inode)
+        inode_unref(inode);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    if (ret < 0)
+        return NULL;
+
+    return buf;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0);
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
+}
 
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
 
-int
-pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
 {
-        int               ret             = -1;
-        xlator_t         *subvol          = NULL;
-        fd_t             *fd              = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+        if (ret)
+            goto out;
+    }
 
-        GF_REF_GET (glfd);
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
 
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
+
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
         }
+    }
 
-        ret = syncop_zerofill (subvol, fd, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
 out:
-        if (fd)
-                fd_unref(fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-        glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
 
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
 
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
 int
-pub_glfs_chdir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+    return glfs_lock_common(glfd, cmd, flock, NULL);
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
+int
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
+{
+    int ret = -1;
 
-	if (ret)
-		goto out;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
 
-	glfs_cwd_set (fs, loc.inode);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-out:
-	loc_wipe (&loc);
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfd->lk_owner.len = len;
 
-        __GLFS_EXIT_FS;
+    memcpy(glfd->lk_owner.data, data, len);
 
-invalid_fs:
-	return ret;
-}
+    ret = 0;
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0);
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
 
-int
-pub_glfs_fchdir (struct glfs_fd *glfd)
-{
-	int       ret = -1;
-	inode_t  *inode = NULL;
-	xlator_t *subvol = NULL;
-	fd_t     *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	inode = fd->inode;
-
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
-
-	glfs_cwd_set (glfd->fs, inode);
-	ret = 0;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return dupfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0);
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    upcall_entry *u_list = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
 
-static gf_boolean_t warn_realpath = _gf_true; /* log once */
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
 
-static char *
-glfs_realpath_common (struct glfs *fs, const char *path, char *resolved_path,
-                      gf_boolean_t warn_deprecated)
-{
-	int              ret = -1;
-	char            *retpath = NULL;
-	char            *allocpath = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	if (resolved_path)
-		retpath = resolved_path;
-        else if (warn_deprecated) {
-                retpath = allocpath = malloc (PATH_MAX + 1);
-                if (warn_realpath) {
-                        warn_realpath = _gf_false;
-                        gf_log (THIS->name, GF_LOG_WARNING, "this application "
-                                "is compiled against an old version of "
-                                "libgfapi, it should use glfs_free() to "
-                                "release the path returned by "
-                                "glfs_realpath()");
-                }
-        } else {
-                retpath = allocpath = GF_CALLOC (1, PATH_MAX + 1,
-                                                 glfs_mt_realpath_t);
-        }
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	if (!retpath) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    INIT_LIST_HEAD(&u_list->upcall_list);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
 
-	if (ret)
-		goto out;
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
 
-	if (loc.path) {
-		strncpy (retpath, loc.path, PATH_MAX);
-		retpath[PATH_MAX] = 0;
-	}
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    ret = 0;
 
 out:
-	loc_wipe (&loc);
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
 
-	if (ret == -1) {
-                if (warn_deprecated && allocpath)
-                        free (allocpath);
-                else if (allocpath)
-                        GF_FREE (allocpath);
-                retpath = NULL;
-	}
+static void
+glfs_free_upcall_lease(void *to_free)
+{
+    struct glfs_upcall_lease *arg = to_free;
 
-	glfs_subvol_done (fs, subvol);
+    if (!arg)
+        return;
 
-        __GLFS_EXIT_FS;
+    if (arg->object)
+        glfs_h_close(arg->object);
 
-invalid_fs:
-	return retpath;
+    GF_FREE(arg);
 }
 
-
-char *
-pub_glfs_realpath34 (struct glfs *fs, const char *path, char *resolved_path)
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
 {
-        return glfs_realpath_common (fs, path, resolved_path, _gf_true);
-}
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
 
-GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0);
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
 
-char *
-pub_glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
-{
-        return glfs_realpath_common (fs, path, resolved_path, _gf_false);
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
+        }
+    }
+
+out:
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17);
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
 
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
 
-char *
-pub_glfs_getcwd (struct glfs *fs, char *buf, size_t n)
-{
-	int              ret = -1;
-	inode_t         *inode = NULL;
-	char            *path = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	if (!buf || n < 2) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
-
-	inode = glfs_cwd_get (fs);
-
-	if (!inode) {
-		strncpy (buf, "/", n);
-		ret = 0;
-		goto out;
-	}
-
-	ret = inode_path (inode, 0, &path);
-	if (n <= ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	strncpy (buf, path, n);
-	ret = 0;
-out:
-	GF_FREE (path);
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
 
-	if (inode)
-		inode_unref (inode);
+    up_lease_arg->lease_type = recall_lease->lease_type;
 
-        __GLFS_EXIT_FS;
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
 
-invalid_fs:
-	if (ret < 0)
-		return NULL;
+    ret = 0;
 
-	return buf;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0);
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
 
-static void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-	flock->l_type   = gf_flock->l_type;
-	flock->l_whence = gf_flock->l_whence;
-	flock->l_start  = gf_flock->l_start;
-	flock->l_len    = gf_flock->l_len;
-	flock->l_pid    = gf_flock->l_pid;
-}
+    struct upcall_syncop_args *args = opaque;
 
+    (void)upcall_syncop_args_free(args);
 
-static void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
-{
-	gf_flock->l_type   = flock->l_type;
-	gf_flock->l_whence = flock->l_whence;
-	gf_flock->l_start  = flock->l_start;
-	gf_flock->l_len    = flock->l_len;
-	gf_flock->l_pid    = flock->l_pid;
+    return 0;
 }
 
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
 
-int
-pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct gf_flock  gf_flock = {0, };
-	struct gf_flock  saved_flock = {0, };
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	gf_flock_from_flock (&gf_flock, flock);
-	gf_flock_from_flock (&saved_flock, flock);
-	ret = syncop_lk (subvol, fd, cmd, &gf_flock, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	gf_flock_to_flock (&gf_flock, flock);
-
-	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
-		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
 out:
-	if (fd)
-		fd_unref (fd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    return ret;
+}
 
-	glfs_subvol_done (glfd->fs, subvol);
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
 
-        __GLFS_EXIT_FS;
+    if (!src)
+        goto out;
 
-invalid_fs:
-	return ret;
-}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0);
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
 
-struct glfs_fd *
-pub_glfs_dup (struct glfs_fd *glfd)
-{
-	xlator_t  *subvol = NULL;
-	fd_t      *fd = NULL;
-	glfs_fd_t *dupfd = NULL;
-	struct glfs *fs = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        GF_REF_GET (glfd);
-
-	fs = glfd->fs;
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (fs, subvol, glfd);
-	if (!fd) {
-		errno = EBADFD;
-		goto out;
-	}
-
-	dupfd = glfs_fd_new (fs);
-	if (!dupfd) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	dupfd->fd = fd_ref (fd);
-        dupfd->state = glfd->state;
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
 out:
-	if (fd)
-		fd_unref (fd);
-	if (dupfd)
-		glfs_fd_bind (dupfd);
-        if (glfd)
-                GF_REF_PUT (glfd);
+    return NULL;
+}
 
-	glfs_subvol_done (fs, subvol);
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
+{
+    struct gf_upcall_recall_lease *dst = NULL;
 
-        __GLFS_EXIT_FS;
+    if (!src)
+        goto out;
 
-invalid_fs:
-	return dupfd;
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
+
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
+
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
+out:
+    return NULL;
+}
+
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
+
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
+
+    args = upcall_syncop_args_init(fs, upcall_data);
+
+    if (!args)
+        goto out;
+
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
+
+out:
+    return;
+}
 
 /*
  * This routine is called in case of any notification received
  * from the server. All the upcall events are queued up in a list
  * to be read by the applications.
  *
- * XXX: Applications may register a cbk function for each 'fs'
- * which then needs to be called by this routine incase of any
- * event received. The cbk fn is responsible for notifying the
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
  * applications the way it desires for each event queued (for eg.,
  * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
  */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
 void
-priv_glfs_process_upcall_event (struct glfs *fs, void *data)
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
 {
-        int                ret             = -1;
-        upcall_entry       *u_list         = NULL;
-        glusterfs_ctx_t    *ctx            = NULL;
-        struct gf_upcall   *upcall_data    = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-        gf_msg_debug (THIS->name, 0,
-                      "Upcall gfapi callback is called");
+    DECLARE_OLD_THIS;
 
-        if (!fs || !data)
-                goto out;
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
 
-        /* Unlike in I/O path, "glfs_fini" would not have freed
-         * 'fs' by the time we take lock as it waits for all epoll
-         * threads to exit including this
-         */
-        pthread_mutex_lock (&fs->mutex);
-        {
-                ctx = fs->ctx;
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
 
-                /* if we're not interested in upcalls (anymore), skip them */
-                if (ctx->cleanup_started || !fs->cache_upcalls) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto out;
-                }
+    if (!data)
+        goto out;
 
-                fs->pin_refcnt++;
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
         }
-        pthread_mutex_unlock (&fs->mutex);
 
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        upcall_data = (struct gf_upcall *)data;
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
 
-        gf_msg_trace (THIS->name, 0, "Upcall gfapi gfid = %s"
-                      "ret = %d", (char *)(upcall_data->gfid), ret);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
 
-        u_list = GF_CALLOC (1, sizeof(*u_list),
-                            glfs_mt_upcall_entry_t);
+out:
 
-        if (!u_list) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
-                        "Upcall entry allocation failed.");
-                goto out;
-        }
+    if (fd)
+        fd_unref(fd);
 
-        INIT_LIST_HEAD (&u_list->upcall_list);
+    if (inode)
+        inode_unref(inode);
 
-        gf_uuid_copy (u_list->upcall_data.gfid, upcall_data->gfid);
-        u_list->upcall_data.event_type = upcall_data->event_type;
+    glfs_subvol_done(fs, subvol);
 
-        switch (upcall_data->event_type) {
-        case GF_UPCALL_CACHE_INVALIDATION:
-                ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
-                                                          upcall_data);
-                break;
-        default:
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        API_MSG_INVALID_ENTRY,
-                        "Upcall entry validation failed.");
-                goto out;
-        }
+invalid_fs:
+    return ret;
+}
 
-        pthread_mutex_lock (&fs->upcall_list_mutex);
-        {
-                list_add_tail (&u_list->upcall_list,
-                               &fs->upcall_list);
-        }
-        pthread_mutex_unlock (&fs->upcall_list_mutex);
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
 
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs->pin_refcnt--;
-        }
-        pthread_mutex_unlock (&fs->mutex);
+    if (inode)
+        inode_unref(inode);
 
-        ret = 0;
-out:
-        if (ret && u_list) {
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE(u_list);
-        }
-        return;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0);
+static void
+glfs_release_xreaddirp_stat(void *ptr)
+{
+    struct glfs_xreaddirp_stat *to_free = ptr;
 
-ssize_t
-glfs_anonymous_pwritev (struct glfs *fs, struct glfs_object *object,
-                        const struct iovec *iovec, int iovcnt,
-                        off_t offset, int flags)
-{
-        xlator_t        *subvol = NULL;
-        struct iobref   *iobref = NULL;
-        struct iobuf    *iobuf  = NULL;
-        struct iovec    iov     = {0, };
-        inode_t         *inode  = NULL;
-        fd_t            *fd     = NULL;
-        int             ret     = -1;
-        size_t          size    = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                ret = -1;
-                errno = ESTALE;
-                goto out;
-        }
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
+int
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
+{
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
 
-        fd = fd_anonymous (inode);
-        if (!fd) {
-                ret = -1;
-                gf_msg ("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED,
-                        "Allocating anonymous fd failed");
-                errno = ENOMEM;
-                goto out;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        size = iov_length (iovec, iovcnt);
+    GF_REF_GET(glfd);
 
-        iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-        if (!iobuf) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
 
-        iobref = iobref_new ();
-        if (!iobref) {
-                iobuf_unref (iobuf);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    errno = 0;
 
-        ret = iobref_add (iobref, iobuf);
-        if (ret) {
-                iobuf_unref (iobuf);
-                iobref_unref (iobref);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-        iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);
+    if (!buf)
+        goto out;
 
-        iov.iov_base = iobuf_ptr (iobuf);
-        iov.iov_len = size;
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
 
-        ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags,
-                             NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (!xstat)
+        goto out;
 
-        iobuf_unref (iobuf);
-        iobref_unref (iobref);
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
 
-        if (ret <= 0)
-                goto out;
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
+
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
+
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
 
 out:
+    GF_REF_PUT(glfd);
 
-        if (fd)
-                fd_unref(fd);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
+
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        __GLFS_EXIT_FS;
+    return ret;
 
 invalid_fs:
-        return ret;
+    return -1;
 }
 
-ssize_t
-glfs_anonymous_preadv (struct glfs *fs,  struct glfs_object *object,
-                       const struct iovec *iovec, int iovcnt,
-                       off_t offset, int flags)
-{
-        xlator_t        *subvol = NULL;
-        struct iovec    *iov    = NULL;
-        struct iobref   *iobref = NULL;
-        inode_t         *inode  = NULL;
-        fd_t            *fd     = NULL;
-        int             cnt     = 0;
-        ssize_t         ret     = -1;
-        ssize_t         size    = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
+
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
+
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
+{
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
                 ret = -1;
-                errno = EIO;
+                errno = EINVAL;
                 goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
                 ret = -1;
-                errno = ESTALE;
+                errno = EINVAL;
                 goto out;
-        }
-
-        fd = fd_anonymous (inode);
-        if (!fd) {
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
                 ret = -1;
-                gf_msg ("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED,
-                        "Allocating anonymous fd failed");
-                errno = ENOMEM;
+                errno = EINVAL;
                 goto out;
-        }
+            }
+            break;
+    }
 
-        size = iov_length (iovec, iovcnt);
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
 
-	ret = syncop_readv (subvol, fd, size, offset, flags, &iov, &cnt,
-                            &iobref, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-        if (ret <= 0)
-                goto out;
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        size = iov_copy (iovec, iovcnt, iov, cnt);
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
 
-        ret = size;
 out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
-        if (fd)
-                fd_unref(fd);
 
-        glfs_subvol_done (fs, subvol);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index 84dba5b82c5..53c2ee896f9 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
  *  This file is part of GlusterFS.
  *
  *  This file is licensed to you under your choice of the GNU Lesser
@@ -8,1982 +8,2156 @@
  *  cases as published by the Free Software Foundation.
  */
 
-
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "glfs-handles.h"
 #include "gfapi-messages.h"
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr);
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
 
-static void
-glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
-                     int *glvalid)
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
 {
-        /* validate in args */
-        if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
-                errno = EINVAL;
-                return;
-        }
-
-        *glvalid = 0;
-
-        if (valid & GFAPI_SET_ATTR_MODE) {
-                iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
-                *glvalid |= GF_SET_ATTR_MODE;
-        }
-
-        if (valid & GFAPI_SET_ATTR_UID) {
-                iatt->ia_uid = stat->st_uid;
-                *glvalid |= GF_SET_ATTR_UID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_GID) {
-                iatt->ia_gid = stat->st_gid;
-                *glvalid |= GF_SET_ATTR_GID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_ATIME) {
-                iatt->ia_atime = stat->st_atime;
-                iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_ATIME;
-        }
-
-        if (valid & GFAPI_SET_ATTR_MTIME) {
-                iatt->ia_mtime = stat->st_mtime;
-                iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_MTIME;
-        }
-
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
         return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
 struct glfs_object *
-pub_glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
-                     const char *path, struct stat *stat, int follow)
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
 {
-        int                      ret = 0;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-        struct glfs_object      *object = NULL;
-        loc_t                    loc = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if (path == NULL) {
-                errno = EINVAL;
-                return NULL;
-        }
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        if (parent) {
-                inode = glfs_resolve_inode (fs, subvol, parent);
-                if (!inode) {
-                        errno = ESTALE;
-                        goto out;
-                }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
         }
+    }
 
-        /* fop/op */
-        ret = glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
-                                    follow, 0);
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
 
-        /* populate out args */
-        if (!ret) {
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4);
-
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
 struct glfs_object *
-pub_glfs_h_lookupat34 (struct glfs *fs, struct glfs_object *parent,
-                       const char *path, struct stat *stat)
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
 {
-        return pub_glfs_h_lookupat (fs, parent, path, stat, 0);
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
 }
 
-GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
 int
-pub_glfs_h_statfs (struct glfs *fs, struct glfs_object *object,
-                   struct statvfs *statvfs)
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* fop/op */
-        ret = syncop_statfs (subvol, &loc, statvfs, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
 out:
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
 int
-pub_glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_stat (subvol, &loc, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
 int
-pub_glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat)
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
 {
-        int                      ret = -1;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                ret = 0;
-                errno = ESTALE;
-                goto out;
-        }
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
 
-        /* fop/op */
-        ret = glfs_resolve_base (fs, subvol, inode, &iatt);
-
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 
 out:
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2);
-
-
 int
-glfs_h_getxattrs_common (struct glfs *fs, struct glfs_object *object,
-                         dict_t **xattr, const char *name,
-                         gf_boolean_t is_listxattr)
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
 {
-        int                 ret = 0;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        if (!is_listxattr) {
-                if (!name || *name == '\0') {
-                        errno = EINVAL;
-                        return -1;
-                }
-
-                if (strlen(name) > GF_XATTR_NAME_MAX) {
-                        errno = ENAMETOOLONG;
-                        return -1;
-                }
-        }
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
+    if (!is_listxattr) {
+        if (!name || *name == '\0') {
+            errno = EINVAL;
+            return -1;
         }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        ret = syncop_getxattr (subvol, &loc, xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+        if (strlen(name) > GF_XATTR_NAME_MAX) {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
 int
-pub_glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, void *value, size_t size)
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
 {
-        int                    ret   = -1;
-        dict_t                *xattr = NULL;
+    int ret = -1;
+    dict_t *xattr = NULL;
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = glfs_h_getxattrs_common (fs, object, &xattr, name,
-                                       (name == NULL));
-        if (ret)
-                goto out;
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
 
-        /* If @name is NULL, means get all the xattrs (i.e listxattr). */
-        if (name)
-                ret = glfs_getxattr_process (value, size, xattr, name);
-        else
-                ret = glfs_listxattr_process (value, size, xattr);
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
 
 out:
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
 int
-pub_glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat, int valid)
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-        int              glvalid = 0;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* map valid masks from in args */
-        glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
 int
-pub_glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, const void *value, size_t size,
-                      int flags)
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t          *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) ||
-                 (name == NULL) || (value == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        if (!name || *name == '\0') {
-                errno = EINVAL;
-                return -1;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                errno = ENAMETOOLONG;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr = dict_for_key_value (name, value, size);
-        if (!xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* fop/op */
-        ret = syncop_setxattr (subvol, &loc, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
 int
-pub_glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-                         const char *name)
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* fop/op */
-        ret = syncop_removexattr (subvol, &loc, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
 struct glfs_fd *
-pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
 {
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* check types to open */
-        if (IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
-
-        if (!IA_ISREG (inode->ia_type)) {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        glfd = glfs_fd_new (fs);
-        if (!glfd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-        glfd->fd->flags = flags;
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        glfd->fd->flags = flags;
-        fd_bind (glfd->fd);
-        glfs_fd_bind (glfd);
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (ret && glfd) {
-                GF_REF_PUT (glfd);
-                glfd = NULL;
-        } else if (glfd) {
-                glfd->state = GLFD_OPEN;
-        }
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
 struct glfs_object *
-pub_glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  int flags, mode_t mode, struct stat *stat)
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
 {
-        int                 ret = -1;
-        fd_t               *fd = NULL;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                ret = -1;
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-        fd = fd_create (loc.inode, getpid());
-        if (!fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
-        fd->flags = flags;
-
-        /* fop/op */
-        ret = syncop_create (subvol, &loc, flags, mode, fd, &iatt,
-                             xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        if (ret && object != NULL) {
-                /* Release the held reference */
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (fd)
-                fd_unref(fd);
+    if (fd)
+        fd_unref(fd);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
 struct glfs_object *
-pub_glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, struct stat *stat)
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
 
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* fop/op */
-        ret = syncop_mkdir (subvol, &loc, mode, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        /* populate out args */
-        if ( ret == 0 )  {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+    loc_wipe(&loc);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+    if (inode)
+        inode_unref(inode);
 
-out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        loc_wipe(&loc);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (inode)
-                inode_unref (inode);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    glfs_subvol_done(fs, subvol);
 
-        glfs_subvol_done (fs, subvol);
-
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
 struct glfs_object *
-pub_glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, dev_t dev, struct stat *stat)
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        /* fop/op */
-        ret = syncop_mknod (subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    glfs_subvol_done(fs, subvol);
 
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+    __GLFS_EXIT_FS;
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+invalid_fs:
+    return object;
+}
 
-                ret = glfs_create_object (&loc, &object);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
 int
-pub_glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        ret = glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
         if (ret != 0) {
-                goto out;
+            goto out;
         }
-
-        if (!IA_ISDIR(loc.inode->ia_type)) {
-                ret = syncop_unlink (subvol, &loc, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
-        } else {
-                ret = syncop_rmdir (subvol, &loc, 0, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
         }
+    }
 
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
 struct glfs_fd *
-pub_glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
 {
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        if (!IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = ENOTDIR;
-                goto out;
-        }
-
-        glfd = glfs_fd_new (fs);
-        if (!glfd)
-                goto out;
-
-        INIT_LIST_HEAD (&glfd->entries);
-
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+out:
+    loc_wipe(&loc);
 
-        /* fop/op */
-        ret = syncop_opendir (subvol, &loc, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (inode)
+        inode_unref(inode);
 
-out:
-        loc_wipe (&loc);
-
-        if (inode)
-                inode_unref (inode);
-
-        if (ret && glfd) {
-                GF_REF_PUT (glfd);
-                glfd = NULL;
-        } else if (glfd) {
-                glfd->state = GLFD_OPEN;
-                fd_bind (glfd->fd);
-                glfs_fd_bind (glfd);
-        }
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
 int
-pub_glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask)
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-
-	DECLARE_OLD_THIS;
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return ret;
-	}
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
 
-	/* get the active volume */
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-	GLFS_LOC_FILL_INODE (inode, loc, out);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-	/* fop/op */
+    /* fop/op */
 
-	ret = syncop_access (subvol, &loc, mask, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (inode)
-		inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
+    glfs_subvol_done(fs, subvol);
 
-	glfs_subvol_done (fs, subvol);
-
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
 ssize_t
-pub_glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
-                           int len)
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
 {
-        ssize_t ret = -1;
+    ssize_t ret = -1;
 
-        /* validate in args */
-        if (object == NULL) {
-                errno = EINVAL;
-                goto out;
-        }
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
 
-        if (!handle || !len) {
-                ret = GFAPI_HANDLE_LENGTH;
-                goto out;
-        }
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
 
-        if (len < GFAPI_HANDLE_LENGTH)
-        {
-                errno = ERANGE;
-                goto out;
-        }
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
 
-        ret = GFAPI_HANDLE_LENGTH;
+    ret = GFAPI_HANDLE_LENGTH;
 
 out:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
 struct glfs_object *
-pub_glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
-                               struct stat *stat)
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        struct iatt         iatt = {0, };
-        inode_t            *newinode = NULL;
-        xlator_t           *subvol = NULL;
-        struct glfs_object *object = NULL;
-        uint64_t            ctx_value = LOOKUP_NOT_NEEDED;
-
-        /* validate in args */
-        if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* make sure the gfid received is valid */
-        GF_VALIDATE_OR_GOTO ("glfs_h_create_from_handle",
-                             !(gf_uuid_is_null (loc.gfid)), out);
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        newinode = inode_find (subvol->itable, loc.gfid);
-        if (newinode) {
-                if (!stat) /* No need of lookup */
-                        goto found;
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
 
-                loc.inode = newinode;
-        } else {
-                loc.inode = inode_new (subvol->itable);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
-        }
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
 
-        ret = syncop_lookup (subvol, &loc, &iatt, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
-        if (ret) {
-                gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-                        "inode refresh of %s failed: %s",
-                        uuid_utoa (loc.gfid), strerror (errno));
-                goto out;
-        }
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
 
-        newinode = inode_link (loc.inode, 0, 0, &iatt);
-        if (newinode) {
-                if (newinode == loc.inode) {
-                        inode_ctx_set (newinode, THIS, &ctx_value);
-                }
-                inode_lookup (newinode);
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
         } else {
-                gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_LINK_FAILED,
-                        "inode linking of %s failed", uuid_utoa (loc.gfid));
-                goto out;
-        }
-
-        /* populate stat */
-        if (stat)
-                glfs_iatt_to_stat (fs, &iatt, stat);
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
+
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 
 found:
-        object = GF_CALLOC (1, sizeof(struct glfs_object),
-                            glfs_mt_glfs_object_t);
-        if (object == NULL) {
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-        /* populate the return object */
-        object->inode = newinode;
-        gf_uuid_copy (object->gfid, object->inode->gfid);
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
 out:
-        /* TODO: Check where the inode ref is being held? */
-        loc_wipe (&loc);
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
 int
-pub_glfs_h_close (struct glfs_object *object)
+pub_glfs_h_close(struct glfs_object *object)
 {
-        /* since glfs_h_* objects hold a reference to inode
-         * it is safe to keep lookup count to '0' */
-        inode_forget (object->inode, 0);
-        inode_unref (object->inode);
-        GF_FREE (object);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
 
-        return 0;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
 int
-pub_glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* validate in args */
-        if (object == NULL) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* fop/op */
-        ret = syncop_truncate (subvol, &loc, (off_t)offset, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* populate out args */
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
 struct glfs_object *
-pub_glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-                    const char *name, const char *data, struct stat *stat)
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((parent == NULL) || (name == NULL) ||
-                (data == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
-
-        /* fop/op */
-        ret = syncop_symlink (subvol, &loc, data, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
-
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        if (ret && object != NULL) {
-                pub_glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
 int
-pub_glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-                     size_t bufsiz)
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        char               *linkval = NULL;
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((object == NULL) || (buf == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* fop/op */
-        ret = syncop_readlink (subvol, &loc, &linkval, bufsiz, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* populate out args */
-        if (ret > 0)
-                memcpy (buf, linkval, ret);
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (linkval)
-                GF_FREE (linkval);
+    if (linkval)
+        GF_FREE(linkval);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
 int
-pub_glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
-             struct glfs_object *parent, const char *name)
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        inode_t            *pinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-        struct iatt         iatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((linksrc == NULL) || (parent == NULL) ||
-                (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, linksrc);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        if (inode->ia_type == IA_IFDIR) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
-
-        GLFS_LOC_FILL_INODE (inode, oldloc, out);
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        pinode = glfs_resolve_inode (fs, subvol, parent);
-        if (!pinode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* setup newloc based on parent */
-        newloc.parent = inode_ref (pinode);
-        newloc.name = name;
-        ret = glfs_loc_touchup (&newloc);
-        if (ret != 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        /* Filling the inode of the hard link to be same as that of the
-         * original file
-         */
-        newloc.inode = inode_ref (inode);
-
-        /* fop/op */
-        ret = syncop_link (subvol, &oldloc, &newloc, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        if (ret == 0)
-                ret = glfs_loc_link (&newloc, &iatt);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
 out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (pinode)
-                inode_unref (pinode);
+    if (pinode)
+        inode_unref(pinode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
 int
-pub_glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-                   const char *oldname, struct glfs_object *newdir,
-                   const char *newname)
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *oldpinode = NULL;
-        inode_t            *newpinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-        struct iatt         oldiatt = {0, };
-        struct iatt         newiatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((olddir == NULL) || (oldname == NULL) ||
-                (newdir == NULL) || (newname == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        oldpinode = glfs_resolve_inode (fs, subvol, olddir);
-        if (!oldpinode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        ret = glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
-                                    &oldiatt, 0 , 0);
-        if (ret != 0) {
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        newpinode = glfs_resolve_inode (fs, subvol, newdir);
-        if (!newpinode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        ret = glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
-                                    &newiatt, 0, 0);
-
-        if (ret && errno != ENOENT && newloc.parent)
-                goto out;
-
-        if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                        (newiatt.ia_type == IA_IFDIR)) {
-                        /* Either both old and new must be dirs,
-                         * or both must be non-dirs. Else, fail.
-                         */
-                        ret = -1;
-                        errno = EEXIST;
-                        goto out;
-                }
-        }
-
-        /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
-
-        ret = syncop_rename (subvol, &oldloc, &newloc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        if (ret == 0)
-                inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-                              newloc.parent, newloc.name, oldloc.inode,
-                              &oldiatt);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 
 out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-        if (oldpinode)
-                inode_unref (oldpinode);
+    if (oldpinode)
+        inode_unref(oldpinode);
 
-        if (newpinode)
-                inode_unref (newpinode);
+    if (newpinode)
+        inode_unref(newpinode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
-
 /*
  * Given a handle/gfid, find if the corresponding inode is present in
  * the inode table. If yes create and return the corresponding glfs_object.
  */
 struct glfs_object *
-glfs_h_find_handle (struct glfs *fs, unsigned char *handle, int len)
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
 {
-        inode_t            *newinode = NULL;
-        xlator_t           *subvol = NULL;
-        struct glfs_object *object = NULL;
-        uuid_t gfid;
-
-        /* validate in args */
-        if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        memcpy (gfid, handle, GFAPI_HANDLE_LENGTH);
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
 
-        /* make sure the gfid received is valid */
-        GF_VALIDATE_OR_GOTO ("glfs_h_find_handle",
-                             !(gf_uuid_is_null (gfid)), out);
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
 
-        newinode = inode_find (subvol->itable, gfid);
-        if (!newinode) {
-                goto out;
-        }
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
 
-        object = GF_CALLOC (1, sizeof(struct glfs_object),
-                            glfs_mt_glfs_object_t);
-        if (object == NULL) {
-                errno = ENOMEM;
-                goto out;
-        }
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
 
-        /* populate the return object. The ref taken here
-         * is un'refed when the application does glfs_h_close() */
-        object->inode = inode_ref(newinode);
-        gf_uuid_copy (object->gfid, object->inode->gfid);
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
 out:
-        /* inode_find takes a reference. Unref it. */
-        if (newinode)
-                inode_unref (newinode);
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
-
+    return object;
 }
 
 static void
-glfs_free_upcall_inode (void *to_free)
+glfs_free_upcall_inode(void *to_free)
 {
-        struct glfs_upcall_inode *arg = to_free;
+    struct glfs_upcall_inode *arg = to_free;
 
-        if (!arg)
-                return;
+    if (!arg)
+        return;
 
-        if (arg->object)
-                glfs_h_close (arg->object);
-        if (arg->p_object)
-                glfs_h_close (arg->p_object);
-        if (arg->oldp_object)
-                glfs_h_close (arg->oldp_object);
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
 
-        GF_FREE (arg);
+    GF_FREE(arg);
 }
 
 int
-glfs_h_poll_cache_invalidation (struct glfs *fs,
-                                struct glfs_upcall *up_arg,
-                                struct gf_upcall *upcall_data)
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
 {
-        int                                 ret           = -1;
-        struct glfs_object                  *p_object     = NULL;
-        struct glfs_object                  *oldp_object  = NULL;
-        struct glfs_object                  *object       = NULL;
-        struct gf_upcall_cache_invalidation *ca_data      = NULL;
-        struct glfs_upcall_inode            *up_inode_arg = NULL;
-
-        ca_data = upcall_data->data;
-        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                             ca_data, out);
-
-        object = glfs_h_find_handle (fs, upcall_data->gfid,
-                                     GFAPI_HANDLE_LENGTH);
-        if (!object) {
-                /* The reason handle creation will fail is because we
-                 * couldn't find the inode in the gfapi inode table.
-                 *
-                 * But since application would have taken inode_ref, the
-                 * only case when this can happen is when it has closed
-                 * the handle and hence will no more be interested in
-                 * the upcall for this particular gfid.
-                 */
-                gf_msg (THIS->name, GF_LOG_DEBUG, errno,
-                        API_MSG_CREATE_HANDLE_FAILED,
-                        "handle creation of %s failed",
-                         uuid_utoa (upcall_data->gfid));
-                errno = ESTALE;
-                goto out;
-        }
-
-        up_inode_arg = GF_CALLOC (1, sizeof (struct glfs_upcall_inode),
-                                  glfs_mt_upcall_inode_t);
-        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                             up_inode_arg, out);
-
-        up_inode_arg->object = object;
-        up_inode_arg->flags = ca_data->flags;
-        up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
-
-        /* XXX: Update stat as well incase of UP_*_TIMES.
-         * This will be addressed as part of INODE_UPDATE */
-        if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
-                glfs_iatt_to_stat (fs, &ca_data->stat, &up_inode_arg->buf);
-        }
-
-        if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
-                p_object = glfs_h_find_handle (fs,
-                                               ca_data->p_stat.ia_gfid,
-                                               GFAPI_HANDLE_LENGTH);
-                if (!p_object) {
-                        gf_msg (THIS->name, GF_LOG_DEBUG, errno,
-                                API_MSG_CREATE_HANDLE_FAILED,
-                                "handle creation of %s failed",
-                                 uuid_utoa (ca_data->p_stat.ia_gfid));
-                        errno = ESTALE;
-                        goto out;
-                }
-
-                glfs_iatt_to_stat (fs, &ca_data->p_stat, &up_inode_arg->p_buf);
-        }
-        up_inode_arg->p_object = p_object;
-
-        /* In case of RENAME, update old parent as well */
-        if (ca_data->flags & GFAPI_UP_RENAME) {
-                oldp_object = glfs_h_find_handle (fs,
-                                                  ca_data->oldp_stat.ia_gfid,
-                                                  GFAPI_HANDLE_LENGTH);
-                if (!oldp_object) {
-                        gf_msg (THIS->name, GF_LOG_DEBUG, errno,
-                                API_MSG_CREATE_HANDLE_FAILED,
-                                "handle creation of %s failed",
-                                 uuid_utoa (ca_data->oldp_stat.ia_gfid));
-                        errno = ESTALE;
-                        /* By the time we receive upcall old parent_dir may
-                         * have got removed. We still need to send upcall
-                         * for the file/dir and current parent handles. */
-                        up_inode_arg->oldp_object = NULL;
-                        ret = 0;
-                }
-
-                glfs_iatt_to_stat (fs, &ca_data->oldp_stat,
-                                   &up_inode_arg->oldp_buf);
-        }
-        up_inode_arg->oldp_object = oldp_object;
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
+
+    ret = 0;
 
-        up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
-        up_arg->event = up_inode_arg;
-        up_arg->free_event = glfs_free_upcall_inode;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
+}
 
-        ret = 0;
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
 
-out:
-        if (ret) {
-                /* Close p_object and oldp_object as well if being referenced.*/
-                if (object)
-                        glfs_h_close (object);
-
-                /* Set reason to prevent applications from using ->event */
-                up_arg->reason = GLFS_UPCALL_EVENT_NULL;
-                GF_FREE (up_inode_arg);
-        }
-        return ret;
+    if (to_free->event)
+        to_free->free_event(to_free->event);
 }
 
 /*
  * This API is used to poll for upcall events stored in the upcall list.
- * Current users of this API is NFS-Ganesha. Incase of any event received, it
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
  * will be mapped appropriately into 'glfs_upcall' along with the handle object
  * to be passed to NFS-Ganesha.
  *
@@ -2001,379 +2175,481 @@ out:
  * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
  * cannot be freed while in this routine.
  */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
 int
-pub_glfs_h_poll_upcall (struct glfs *fs, struct glfs_upcall **up_arg)
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
 {
-        upcall_entry       *u_list         = NULL;
-        upcall_entry       *tmp            = NULL;
-        xlator_t           *subvol         = NULL;
-        glusterfs_ctx_t    *ctx            = NULL;
-        int                 ret            = -1;
-        struct gf_upcall   *upcall_data    = NULL;
-
-        DECLARE_OLD_THIS;
-
-        if (!up_arg) {
-                errno = EINVAL;
-                goto err;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, err);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto restore;
-        }
-
-        /* Ideally applications should stop polling before calling
-         * 'glfs_fini'. Yet cross check if cleanup has started. */
-        pthread_mutex_lock (&fs->mutex);
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
         {
-                ctx = fs->ctx;
-
-                if (ctx->cleanup_started) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto out;
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
                 }
 
-                fs->pin_refcnt++;
-
-                /* once we call this function, the applications seems to be
-                 * interested in events, enable caching them */
-                fs->cache_upcalls = _gf_true;
-        }
-        pthread_mutex_unlock (&fs->mutex);
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
 
-        pthread_mutex_lock (&fs->upcall_list_mutex);
-        {
-                list_for_each_entry_safe (u_list, tmp,
-                                          &fs->upcall_list,
-                                          upcall_list) {
-                        list_del_init (&u_list->upcall_list);
-                        upcall_data = &u_list->upcall_data;
-                        break;
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
                 }
-        }
-        /* No other thread can delete this entry. So unlock it */
-        pthread_mutex_unlock (&fs->upcall_list_mutex);
-
-        if (upcall_data) {
-                switch (upcall_data->event_type) {
-                case GF_UPCALL_CACHE_INVALIDATION:
-                        *up_arg = GF_CALLOC (1, sizeof (struct gf_upcall),
-                                             glfs_mt_upcall_entry_t);
-                        if (!*up_arg) {
-                                errno = ENOMEM;
-                                break; /* goto free u_list */
-                        }
-
-                        /* XXX: Need to revisit this to support
-                         * GLFS_UPCALL_INODE_UPDATE if required. */
-                        ret = glfs_h_poll_cache_invalidation (fs, *up_arg,
-                                                              upcall_data);
-                        if (ret
-                            || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
-                                /* It could so happen that the file which got
-                                 * upcall notification may have got deleted by
-                                 * the same client. Irrespective of the error,
-                                 * return with an error or success+ENOENT. */
-                                if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
-                                        errno = ENOENT;
-
-                                GF_FREE (*up_arg);
-                                *up_arg = NULL;
-                        }
-                        break;
-                case GF_UPCALL_RECALL_LEASE:
-                        gf_log ("glfs_h_poll_upcall", GF_LOG_DEBUG,
-                                "UPCALL_RECALL_LEASE is not implemented yet");
-                case GF_UPCALL_EVENT_NULL:
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
                 /* no 'default:' label, to force handling all upcall events */
-                        errno = ENOENT;
-                        break;
-                }
-
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE (u_list);
-        } else {
-                /* fs->upcall_list was empty, no upcall events cached */
                 errno = ENOENT;
+                break;
         }
 
-        ret = 0;
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
+
+    ret = 0;
 
 out:
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs->pin_refcnt--;
-        }
-        pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
 restore:
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 err:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16);
-
 static gf_boolean_t log_upcall370 = _gf_true; /* log once */
 
-/* The old glfs_h_poll_upcall interface requires intimite knowledge of the
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
  * structures that are returned to the calling application. This is not
  * recommended, as the returned structures need to returned correctly (handles
  * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
  *
  * To the best of our knowledge, only NFS-Ganesha uses the upcall events
- * through gfapi. We keep this backwards compatability function around so that
+ * through gfapi. We keep this backwards compatibility function around so that
  * applications using the existing implementation do not break.
  *
  * WARNING: this function will be removed in the future.
  */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
 int
-pub_glfs_h_poll_upcall370 (struct glfs *fs, struct glfs_callback_arg *up_arg)
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
 {
-        struct glfs_upcall    *upcall        = NULL;
-        int                    ret           = -1;
-
-        if (log_upcall370) {
-                log_upcall370 = _gf_false;
-                gf_log (THIS->name, GF_LOG_WARNING, "this application is "
-                        "compiled against an old version of libgfapi, it "
-                        "should use glfs_free() to release the structure "
-                        "returned by glfs_h_poll_upcall() - for more details, "
-                        "see http://review.gluster.org/14701");
-        }
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
 
-        ret = pub_glfs_h_poll_upcall (fs, &upcall);
-        if (ret == 0) {
-                up_arg->fs = fs;
-                if (errno == ENOENT || upcall->event == NULL) {
-                        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
-                        goto out;
-                }
+            up_inode = upcall->event;
 
-                up_arg->reason = upcall->reason;
-
-                if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
-                        struct glfs_callback_inode_arg *cb_inode = NULL;
-                        struct glfs_upcall_inode       *up_inode = NULL;
-
-                        cb_inode = GF_CALLOC (1,
-                                              sizeof (struct glfs_callback_inode_arg),
-                                              glfs_mt_upcall_inode_t);
-                        if (!cb_inode) {
-                                errno = ENOMEM;
-                                ret = -1;
-                                goto out;
-                        }
-
-                        up_inode = upcall->event;
-
-                        /* copy attributes one by one, the memory layout might
-                         * be different between the old glfs_callback_inode_arg
-                         * and new glfs_upcall_inode */
-                        cb_inode->object = up_inode->object;
-                        cb_inode->flags = up_inode->flags;
-                        memcpy (&cb_inode->buf, &up_inode->buf,
-                                sizeof (struct stat));
-                        cb_inode->expire_time_attr = up_inode->expire_time_attr;
-                        cb_inode->p_object = up_inode->p_object;
-                        memcpy (&cb_inode->p_buf, &up_inode->p_buf,
-                                sizeof (struct stat));
-                        cb_inode->oldp_object = up_inode->oldp_object;
-                        memcpy (&cb_inode->oldp_buf, &up_inode->oldp_buf,
-                                sizeof (struct stat));
-
-                        up_arg->event_arg = cb_inode;
-                }
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
+
+            up_arg->event_arg = cb_inode;
         }
+    }
 
 out:
-        if (upcall) {
-                /* we can not use glfs_free() here, objects need to stay */
-                GF_FREE (upcall->event);
-                GF_FREE (upcall);
-        }
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0);
-
 #ifdef HAVE_ACL_LIBACL_H
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs-acl.h>
 #include <acl/libacl.h>
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type, const acl_t acl)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        int ret = -1;
-        char *acl_s = NULL;
-        const char *acl_key = NULL;
-        struct glfs_object *new_object = NULL;
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
-        DECLARE_OLD_THIS;
+    DECLARE_OLD_THIS;
 
-        if (!object || !acl) {
-                errno = EINVAL;
-                return ret;
-        }
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        acl_key = gf_posix_acl_get_key (type);
-        if (!acl_key)
-                goto out;
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        acl_s = acl_to_any_text (acl, NULL, ',',
-                                 TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
-        if (!acl_s)
-                goto out;
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
 
-        if (IA_ISLNK (object->inode->ia_type)) {
-                new_object = glfs_h_resolve_symlink (fs, object);
-                if (new_object == NULL)
-                        goto out;
-        } else
-                new_object = object;
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-        ret = pub_glfs_h_setxattrs (fs, new_object, acl_key, acl_s,
-                                    strlen (acl_s) + 1, 0);
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
 
-        acl_free (acl_s);
+    acl_free(acl_s);
 
 out:
-        if (IA_ISLNK (object->inode->ia_type) && new_object)
-                glfs_h_close (new_object);
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
 acl_t
-pub_glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type)
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
 {
-        int                 ret = 0;
-        acl_t acl = NULL;
-        char *acl_s = NULL;
-        dict_t *xattr = NULL;
-        const char *acl_key = NULL;
-        struct glfs_object *new_object = NULL;
-
-        DECLARE_OLD_THIS;
-
-        if (!object) {
-                errno = EINVAL;
-                return NULL;
-        }
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
 
-        acl_key = gf_posix_acl_get_key (type);
-        if (!acl_key)
-                goto out;
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        if (IA_ISLNK (object->inode->ia_type)) {
-                new_object = glfs_h_resolve_symlink (fs, object);
-                if (new_object == NULL)
-                        goto out;
-        } else
-                new_object = object;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = glfs_h_getxattrs_common (fs, new_object, &xattr, acl_key,
-                                       _gf_false);
-        if (ret)
-                goto out;
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        ret = dict_get_str (xattr, (char *)acl_key, &acl_s);
-        if (ret == -1)
-                goto out;
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
 
-        acl = acl_from_text (acl_s);
+    acl = acl_from_text(acl_s);
 
 out:
-        GF_FREE (acl_s);
-        if (IA_ISLNK (object->inode->ia_type) && new_object)
-                glfs_h_close (new_object);
+    if (xattr)
+        dict_unref(xattr);
 
-        __GLFS_EXIT_FS;
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return acl;
+    return acl;
 }
 #else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
 acl_t
-pub_glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type)
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
 {
-        errno = ENOTSUP;
-        return NULL;
+    errno = ENOTSUP;
+    return NULL;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type, const acl_t acl)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        errno = ENOTSUP;
-        return -1;
+    errno = ENOTSUP;
+    return -1;
 }
 #endif
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0);
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0);
 
 /* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
 ssize_t
-pub_glfs_h_anonymous_read (struct glfs *fs, struct glfs_object *object,
-                           const void *buf, size_t count, off_t offset)
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
 {
-        struct iovec    iov     = {0, };
-        ssize_t         ret     = 0;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        iov.iov_base = (void *) buf;
-        iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        ret = glfs_anonymous_preadv (fs, object, &iov, 1, offset, 0);
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0);
-
 /* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
 ssize_t
-pub_glfs_h_anonymous_write (struct glfs *fs, struct glfs_object *object,
-                            const void *buf, size_t count, off_t offset)
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
 {
-        struct iovec iov        = {0, };
-        ssize_t      ret        = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        iov.iov_base = (void *) buf;
-        iov.iov_len = count;
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
 
-        ret = glfs_anonymous_pwritev (fs, object, &iov, 1, offset, 0);
+    return ret;
+}
 
-        return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
+{
+    struct glfs_object *object = NULL;
+
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
+
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
+
+out:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
+
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
+
+    return xstat->object;
+
+out:
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
+int
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 740b759d0a0..4d039b9c76b 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -12,7 +12,6 @@
 #define _GLFS_HANDLES_H
 
 #include "glfs.h"
-#include <inttypes.h>
 
 /* GLFS OBJECT BASED OPERATIONS
  *
@@ -47,46 +46,39 @@
  *
  */
 
-/* Values for valid falgs to be used when using XXXsetattr, to set multiple
- attribute values passed via the related stat structure.
- */
-#define GFAPI_SET_ATTR_MODE  0x1
-#define GFAPI_SET_ATTR_UID   0x2
-#define GFAPI_SET_ATTR_GID   0x4
-#define GFAPI_SET_ATTR_SIZE  0x8
-#define GFAPI_SET_ATTR_ATIME 0x10
-#define GFAPI_SET_ATTR_MTIME 0x20
-
 /* Handle length for object handles returned from glfs_h_extract_handle or
  * glfs_h_create_from_handle */
 #define GFAPI_HANDLE_LENGTH 16
 
 /* These flags should be in sync to the ones defined in upcall.h */
-#define GFAPI_UP_NLINK   0x00000001   /* update nlink */
-#define GFAPI_UP_MODE    0x00000002   /* update mode and ctime */
-#define GFAPI_UP_OWN     0x00000004   /* update mode,uid,gid and ctime */
-#define GFAPI_UP_SIZE    0x00000008   /* update fsize */
-#define GFAPI_UP_TIMES   0x00000010   /* update all times */
-#define GFAPI_UP_ATIME   0x00000020   /* update atime only */
-#define GFAPI_UP_PERM    0x00000040   /* update fields needed for
-                                         permission checking */
-#define GFAPI_UP_RENAME  0x00000080   /* this is a rename op -
-                                         delete the cache entry */
-#define GFAPI_UP_FORGET  0x00000100   /* inode_forget on server side -
-                                         invalidate the cache entry */
-#define GFAPI_UP_PARENT_TIMES   0x00000200   /* update parent dir times */
-
-#define GFAPI_INODE_UPDATE_FLAGS (GFAPI_UP_NLINK | GFAPI_UP_MODE | \
-                                  GFAPI_UP_OWN | GFAPI_UP_SIZE | \
-                                  GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
 #endif
 
 __BEGIN_DECLS
@@ -102,49 +94,6 @@ __BEGIN_DECLS
 struct glfs_object;
 typedef struct glfs_object glfs_object_t;
 
-/*
- * Applications (currently NFS-Ganesha) can make use of this
- * structure to read upcall notifications sent by server.
- *
- * On success, applications need to check for 'reason' to decide
- * if any upcall event is received.
- *
- * Currently supported upcall_events -
- *      GFAPI_INODE_INVALIDATE -
- *              'event_arg' - glfs_upcall_inode
- *
- * After processing the event, applications need to free 'event_arg' with
- * glfs_free().
- *
- * Also similar to I/Os, the application should ideally stop polling
- * before calling glfs_fini(..). Hence making an assumption that
- * 'fs' & ctx structures cannot be freed while in this routine.
- */
-struct glfs_upcall;
-
-struct glfs*
-glfs_upcall_get_fs (struct glfs_upcall *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
-
-enum glfs_upcall_reason {
-        GLFS_UPCALL_EVENT_NULL = 0,
-        GLFS_UPCALL_INODE_INVALIDATE,    /* invalidate cache entry */
-};
-
-enum glfs_upcall_reason
-glfs_upcall_get_reason (struct glfs_upcall *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
-
-
-/*
- * After processing upcall event, glfs_free() should be called on the
- * glfs_upcall.
- */
-void*
-glfs_upcall_get_event (struct glfs_upcall *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
-
-
 /* Functions for getting details about the glfs_upcall_inode
  *
  * None of the pointers returned by the below functions should be free()'d,
@@ -154,153 +103,158 @@ glfs_upcall_get_event (struct glfs_upcall *arg) __THROW
  * to glfs_free().
  */
 struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
 
-struct glfs_object*
-glfs_upcall_inode_get_object (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
 
 uint64_t
-glfs_upcall_inode_get_flags (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
 
-struct stat*
-glfs_upcall_inode_get_stat (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
 
 uint64_t
-glfs_upcall_inode_get_expire (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
 
-struct glfs_object*
-glfs_upcall_inode_get_pobject (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
 
-struct stat*
-glfs_upcall_inode_get_pstat (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
 
-struct glfs_object*
-glfs_upcall_inode_get_oldpobject (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
-
-struct stat*
-glfs_upcall_inode_get_oldpstat (struct glfs_upcall_inode *arg) __THROW
-        GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
 
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
 
 /* Handle based operations */
 /* Operations that generate handles */
-struct glfs_object *glfs_h_lookupat (struct glfs *fs,
-                                     struct glfs_object *parent,
-                                     const char *path,
-                                     struct stat *stat, int follow) __THROW
-        GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
-
-struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, int flags, mode_t mode,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_create, 3.4.2);
-
-struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t flags,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
-
-struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t mode, dev_t dev,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
-
-struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-				    const char *name, const char *data,
-				    struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
 
 /* Operations on the actual objects */
-int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
-		   const char *path) __THROW
-        GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
 
-int glfs_h_close (struct glfs_object *object) __THROW
-        GFAPI_PUBLIC(glfs_h_close, 3.4.2);
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
 
-int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
-			       void *future) __THROW
-        GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
 
-int glfs_h_truncate (struct glfs *fs, struct glfs_object *object,
-                     off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
 
-int glfs_h_stat(struct glfs *fs, struct glfs_object *object,
-                struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
 
-int glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
-                struct statvfs *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
 
-int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
 
-int glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, void *value,
-		      size_t size) __THROW
-        GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
 
-int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *sb, int valid) __THROW
-        GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
 
-int glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, const void *value,
-		      size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
 
-int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-		     size_t bufsiz) __THROW
-        GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
 
-int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
-		 struct glfs_object *parent, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_h_link, 3.4.2);
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
 
-int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-		   const char *oldname, struct glfs_object *newdir,
-		   const char *newname) __THROW
-        GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
 
-int glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-			 const char *name) __THROW
-        GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
 
 /* Operations enabling opaque invariant handle to object transitions */
-ssize_t glfs_h_extract_handle (struct glfs_object *object,
-			       unsigned char *handle, int len) __THROW
-        GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
 
 /* Given a handle, looks up the inode and creates glfs_object.
  * In addition, if provided 'stat', copies the inode attributes
  */
-struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
-					       unsigned char *handle, int len,
-					       struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
 
 /* Operations enabling object handles to fd transitions */
-struct glfs_fd *glfs_h_opendir (struct glfs *fs,
-                                struct glfs_object *object) __THROW
-        GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
 
-struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
-			     int flags) __THROW
-        GFAPI_PUBLIC(glfs_h_open, 3.4.2);
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
 
 int
-glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
-        GFAPI_PUBLIC(glfs_h_access, 3.6.0);
-
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
 /*
   SYNOPSIS
 
@@ -310,7 +264,7 @@ glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
 
   This API is used to poll for upcall events stored in the
   upcall list. Current users of this API is NFS-Ganesha.
-  Incase of any event received, it will be mapped appropriately
+  In case of any event received, it will be mapped appropriately
   into 'glfs_upcall' along with the handle('glfs_object') to be
   passed to NFS-Ganesha.
 
@@ -332,28 +286,69 @@ glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
 */
 
 int
-glfs_h_poll_upcall (struct glfs *fs, struct glfs_upcall **cbk) __THROW
-        GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
 
 int
-glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                const acl_type_t type, const acl_t acl) __THROW
-        GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
 
 acl_t
-glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                const acl_type_t type) __THROW
-        GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
 
 size_t
-glfs_h_anonymous_write (struct glfs *fs, struct glfs_object *object,
-                        const void *buf, size_t count, off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
 
 ssize_t
-glfs_h_anonymous_read (struct glfs *fs, struct glfs_object *object,
-                      const void *buf, size_t count, off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
+
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
 
 __END_DECLS
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index fc62bbf027d..7cc3b18a104 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,25 +8,25 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_INTERNAL_H
 #define _GLFS_INTERNAL_H
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "upcall-utils.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
 #include "glfs-handles.h"
-#include "refcount.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
 #define DEFAULT_REVAL_COUNT 1
 
 /*
- * According to  pthread mutex and conditional variable ( cond, child_down_count,
- * upcall mutex and mutex) initialization of struct glfs members,
- * below GLFS_INIT_* flags are set in 'pthread_flags' member of struct glfs.
- * The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
  * as part of fs inititialization.
  *
  * These flag bits are validated in glfs_fini() to destroy all or partially
@@ -37,39 +37,42 @@
  *
  */
 
-#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label) do { \
-        int __ret = -1;                                          \
-        __ret = pthread_mutex_init (mutex, attr);                \
-        if (__ret == 0)                                          \
-                flags |= mask;                                   \
-        else                                                     \
-                goto label;                                      \
-} while (0)
-
-#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask) do {           \
-        if (flags & mask)                                        \
-                (void) pthread_mutex_destroy (mutex);            \
-} while (0)
-
-#define PTHREAD_COND_INIT(cond, attr, flags, mask, label) do {   \
-        int __ret = -1;                                          \
-        __ret = pthread_cond_init (cond, attr);                  \
-        if (__ret == 0)                                          \
-                flags |= mask;                                   \
-        else                                                     \
-                goto label;                                      \
-} while (0)
-
-#define PTHREAD_COND_DESTROY(cond, flags, mask) do {             \
-        if (flags & mask)                                        \
-                (void) pthread_cond_destroy (cond);              \
-} while (0)
-
-#define GLFS_INIT_MUTEX              0x00000001   /* pthread_mutex_flag */
-#define GLFS_INIT_COND               0x00000002   /* pthread_cond_flag */
-#define GLFS_INIT_COND_CHILD         0x00000004   /* pthread_cond_child_down_flag */
-#define GLFS_INIT_MUTEX_UPCALL       0x00000008   /* pthread_mutex_upcall_flag */
-
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
 
 #ifndef GF_DARWIN_HOST_OS
 #ifndef GFAPI_PUBLIC
@@ -78,225 +81,411 @@
 #ifndef GFAPI_PRIVATE
 #define GFAPI_PRIVATE(sym, ver) /**/
 #endif
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver) \
-        asm(".symver pub_"STR(fn)", "STR(fn)"@@GFAPI_"STR(ver))
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
 
-#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver) \
-        asm(".symver priv_"STR(fn)", "STR(fn)"@@GFAPI_PRIVATE_"STR(ver))
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
 
-#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver) \
-        asm(".symver pub_"STR(fn1)", "STR(fn2)"@GFAPI_"STR(ver))
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
 
-#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver) \
-        asm(".symver priv_"STR(fn1)", "STR(fn2)"@GFAPI_PRIVATE_"STR(ver))
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
 #define STR(str) #str
 #else
 #ifndef GFAPI_PUBLIC
-#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
 #endif
 #ifndef GFAPI_PRIVATE
-#define GFAPI_PRIVATE(sym, ver) __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
 #endif
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
 #define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
-#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver) /**/
-#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
 #endif
 
-#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
-	if (ret == -1 && errno == ESTALE) {	        \
-		if (reval < DEFAULT_REVAL_COUNT) {	\
-			reval++;			\
-			loc_wipe (loc);			\
-			goto label;			\
-		}					\
-	}						\
-	} while (0)
-
-#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
-	loc.inode = inode_ref (oinode);                \
-	gf_uuid_copy (loc.gfid, oinode->gfid);         \
-	ret = glfs_loc_touchup (&loc);                 \
-	if (ret != 0) {                                \
-		errno = EINVAL;                        \
-		goto label;                            \
-	}                                              \
-	} while (0)
-
-#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
-	loc.inode = inode_new (pinode->table);                            \
-	if (!loc.inode) {                                                 \
-		ret = -1;                                                 \
-		errno = ENOMEM;                                           \
-		goto label;                                               \
-	}                                                                 \
-	loc.parent = inode_ref (pinode);                                  \
-	loc.name = path;                                                  \
-	ret = glfs_loc_touchup (&loc);                                    \
-	if (ret != 0) {                                                   \
-		errno = EINVAL;                                           \
-		goto label;                                               \
-	}                                                                 \
-	} while (0)
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 struct glfs;
 
-struct _upcall_entry_t {
-        struct list_head  upcall_list;
-        struct gf_upcall  upcall_data;
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
 };
-typedef struct _upcall_entry_t upcall_entry;
+typedef struct _upcall_entry upcall_entry;
 
-typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
 
 struct glfs {
-	char               *volname;
-        uuid_t              vol_uuid;
-
-	glusterfs_ctx_t    *ctx;
-
-	pthread_t           poller;
-
-	glfs_init_cbk       init_cbk;
-	pthread_mutex_t     mutex;
-	pthread_cond_t      cond;
-        pthread_cond_t      child_down_cond; /* for broadcasting CHILD_DOWN */
-	int                 init;
-	int                 ret;
-	int                 err;
-
-	xlator_t           *active_subvol; /* active graph */
-        xlator_t           *mip_subvol;    /* graph for which migration is in
-                                            * progress */
-	xlator_t           *next_subvol;   /* Any new graph is put to
-                                            * next_subvol, the graph in
-                                            * next_subvol can either be move to
-                                            * mip_subvol (if any IO picks it up
-                                            * for migration), or be detroyed (if
-                                            * there is a new graph, and this was
-                                            * never picked for migration) */
-	xlator_t           *old_subvol;
-
-	char               *oldvolfile;
-	ssize_t             oldvollen;
-
-	inode_t            *cwd;
-
-	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
-
-	struct list_head    openfds;
-
-	gf_boolean_t        migration_in_progress;
-
-        gf_boolean_t        cache_upcalls; /* add upcalls to the upcall_list? */
-        struct list_head    upcall_list;
-        pthread_mutex_t     upcall_list_mutex; /* mutex for upcall entry list */
-
-        uint32_t            pin_refcnt;
-        uint32_t            pthread_flags; /* GLFS_INIT_* # defines set this flag */
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
 };
 
 /* This enum is used to maintain the state of glfd. In case of async fops
  * fd might be closed before the actual fop is complete. Therefore we need
  * to track whether the fd is closed or not, instead actually closing it.*/
-enum glfs_fd_state {
-        GLFD_INIT,
-        GLFD_OPEN,
-        GLFD_CLOSE
-};
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
 
 struct glfs_fd {
-	struct list_head   openfds;
-        GF_REF_DECL;
-	struct glfs       *fs;
-        enum glfs_fd_state state;
-	off_t              offset;
-	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
-	struct list_head   entries;
-	gf_dirent_t       *next;
-	struct dirent     *readdirbuf;
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
 };
 
 /* glfs object handle introduced for the alternate gfapi implementation based
    on glfs handles/gfid/inode
 */
 struct glfs_object {
-        inode_t         *inode;
-        uuid_t          gfid;
+    inode_t *inode;
+    uuid_t gfid;
 };
 
 struct glfs_upcall {
-        struct glfs             *fs;     /* glfs object */
-        enum glfs_upcall_reason  reason; /* Upcall event type */
-        void                    *event;  /* changes based in the event type */
-        void (*free_event)(void *);      /* free event after the usage */
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
 };
 
 struct glfs_upcall_inode {
-        struct glfs_object   *object;  /* Object which need to be acted upon */
-        int                   flags;   /* Cache UPDATE/INVALIDATE flags */
-        struct stat           buf;     /* Latest stat of this entry */
-        unsigned int          expire_time_attr; /* the amount of time for which
-                                                 * the application need to cache
-                                                 * this entry */
-        struct glfs_object   *p_object; /* parent Object to be updated */
-        struct stat           p_buf;    /* Latest stat of parent dir handle */
-        struct glfs_object   *oldp_object; /* Old parent Object to be updated */
-        struct stat           oldp_buf; /* Latest stat of old parent dir handle */
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
+
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
+
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
+};
+
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
+};
+
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
+
+typedef void(glfs_mem_release_t)(void *ptr);
+
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
 };
 
-#define DEFAULT_EVENT_POOL_SIZE           16384
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-
-#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T     256
-
-int glfs_mgmt_init (struct glfs *fs);
-void glfs_init_done (struct glfs *fs, int ret)
-        GFAPI_PRIVATE(glfs_init_done, 3.4.0);
-int glfs_process_volfp (struct glfs *fs, FILE *fp);
-int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path,
-                       loc_t *loc, struct iatt *iatt, int reval)
-        GFAPI_PRIVATE(glfs_resolve, 3.7.0);
-int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		   struct iatt *iatt, int reval);
-fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
-
-fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
-
-int glfs_first_lookup (xlator_t *subvol);
-
-void glfs_process_upcall_event (struct glfs *fs, void *data)
-        GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
-
-
-#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                         \
-do {                                                                \
-        if (!fs) {                                                  \
-                errno = EINVAL;                                     \
-                goto label;                                         \
-        }                                                           \
-        old_THIS = THIS;                                            \
-        THIS = fs->ctx->master;                                     \
-} while (0)
-
-#define __GLFS_EXIT_FS                                              \
-do {                                                                \
-        THIS = old_THIS;                                            \
-} while (0)
-
-#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                       \
-do {                                                                \
-        if (!glfd || !glfd->fd || !glfd->fd->inode ||               \
-             glfd->state != GLFD_OPEN) {                           \
-                errno = EBADF;                                      \
-                goto label;                                         \
-        }                                                           \
-        old_THIS = THIS;                                            \
-        THIS = glfd->fd->inode->table->xl->ctx->master;             \
-} while (0)
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
+
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
 
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
+
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
+
+static inline void
+__glfs_free(void *free_ptr)
+{
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
+}
+
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
+
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
+
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
 
 /*
   By default all lock attempts from user context must
@@ -305,64 +494,93 @@ do {                                                                \
   we can give up the mutex during syncop calls so
   that bottom up calls (particularly CHILD_UP notify)
   can do a mutex_lock() on @glfs without deadlocking
-  the filesystem
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
 */
 static inline int
-glfs_lock (struct glfs *fs)
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
 {
-	pthread_mutex_lock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
 
-	while (!fs->init)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
 
-	while (fs->migration_in_progress)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
 
-	return 0;
+    return 0;
 }
 
-
 static inline void
-glfs_unlock (struct glfs *fs)
+glfs_unlock(struct glfs *fs)
 {
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 }
 
-struct glfs_fd *glfs_fd_new (struct glfs *fs);
-void glfs_fd_bind (struct glfs_fd *glfd);
-
-xlator_t *glfs_active_subvol (struct glfs *fs)
-        GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
-xlator_t *__glfs_active_subvol (struct glfs *fs);
-void glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
-        GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
-
-inode_t *glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
-
-inode_t *glfs_cwd_get (struct glfs *fs);
-int glfs_cwd_set (struct glfs *fs, inode_t *inode);
-inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-			     struct glfs_object *object);
-int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
-int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
-
-int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		       struct iatt *iatt);
-int glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-                          const char *origpath, loc_t *loc, struct iatt *iatt,
-                          int follow, int reval)
-        GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
-int glfs_loc_touchup (loc_t *loc)
-	GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
-void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
-int glfs_loc_link (loc_t *loc, struct iatt *iatt);
-int glfs_loc_unlink (loc_t *loc);
-dict_t *dict_for_key_value (const char *name, const char *value, size_t size);
-int glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-			   const char *name);
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
+
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
 
 /* Sends RPC call to glusterd to fetch required volume info */
-int glfs_get_volume_info (struct glfs *fs);
+int
+glfs_get_volume_info(struct glfs *fs);
 
 /*
   SYNOPSIS
@@ -388,8 +606,8 @@ int glfs_get_volume_info (struct glfs *fs);
        NULL   : Otherwise.
 */
 
-struct glfs *glfs_new_from_ctx (glusterfs_ctx_t *ctx)
-        GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
 
 /*
   SYNOPSIS
@@ -414,28 +632,30 @@ struct glfs *glfs_new_from_ctx (glusterfs_ctx_t *ctx)
        void
 */
 
-void glfs_free_from_ctx (struct glfs *fs)
-         GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
 
-int glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
-                                        struct gf_upcall *from_up_data);
 int
-glfs_h_poll_cache_invalidation (struct glfs *fs,
-                                struct glfs_upcall *up_arg,
-                                struct gf_upcall *upcall_data);
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
 
 ssize_t
-glfs_anonymous_preadv (struct glfs *fs, struct glfs_object *object,
-                       const struct iovec *iovec, int iovcnt,
-                       off_t offset, int flags);
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
 ssize_t
-glfs_anonymous_pwritev (struct glfs *fs, struct glfs_object *object,
-                        const struct iovec *iovec, int iovcnt,
-                        off_t offset, int flags);
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
 
 struct glfs_object *
-glfs_h_resolve_symlink (struct glfs *fs, struct glfs_object *object);
-
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
 
 /* Deprecated structures that were passed to client applications, replaced by
  * accessor functions. Do not use these in new applications, and update older
@@ -446,25 +666,91 @@ glfs_h_resolve_symlink (struct glfs *fs, struct glfs_object *object);
  * WARNING: These structures will be removed in the future.
  */
 struct glfs_callback_arg {
-        struct glfs             *fs;
-        enum glfs_upcall_reason  reason;
-        void                    *event_arg;
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
 };
 
 struct glfs_callback_inode_arg {
-        struct glfs_object      *object; /* Object which need to be acted upon */
-        int                     flags; /* Cache UPDATE/INVALIDATE flags */
-        struct stat             buf; /* Latest stat of this entry */
-        unsigned int            expire_time_attr; /* the amount of time for which
-                                                   * the application need to cache
-                                                   * this entry
-                                                   */
-        struct glfs_object      *p_object; /* parent Object to be updated */
-        struct stat             p_buf; /* Latest stat of parent dir handle */
-        struct glfs_object      *oldp_object; /* Old parent Object
-                                               * to be updated */
-        struct stat             oldp_buf; /* Latest stat of old parent
-                                           * dir handle */
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
 };
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
 
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
+
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
+
+void
+unset_fop_attr(dict_t **fop_attr);
+
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 00a9c929a04..100dcc16cc0 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,180 +8,176 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
-#include <inttypes.h>
-#include <limits.h>
 
-#include "xlator.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
 #include "gfapi-messages.h"
 
-
 int
-graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
 {
-	xlator_t      *new_subvol = NULL;
-	xlator_t      *old_subvol = NULL;
-	inode_table_t *itable = NULL;
-	int            ret = -1;
-
-	new_subvol = graph->top;
-
-	/* This is called in a bottom-up context, it should specifically
-	   NOT be glfs_lock()
-	*/
-	pthread_mutex_lock (&fs->mutex);
-	{
-		if (new_subvol->switched ||
-		    new_subvol == fs->active_subvol ||
-		    new_subvol == fs->next_subvol ||
-                    new_subvol == fs->mip_subvol) {
-			/* Spurious CHILD_UP event on old graph */
-			ret = 0;
-			goto unlock;
-		}
-
-		if (!new_subvol->itable) {
-			itable = inode_table_new (131072, new_subvol);
-			if (!itable) {
-				errno = ENOMEM;
-				ret = -1;
-				goto unlock;
-			}
-
-			new_subvol->itable = itable;
-		}
-
-		old_subvol = fs->next_subvol;
-		fs->next_subvol = new_subvol;
-		fs->next_subvol->winds++; /* first ref */
-		ret = 0;
-	}
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
 unlock:
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (old_subvol)
-		/* wasn't picked up so far, skip */
-		glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
 
-	return ret;
+    return ret;
 }
 
-
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-	glusterfs_graph_t   *graph = NULL;
-	struct glfs	    *fs = NULL;
-
-	graph = data;
-	fs = this->private;
-
-	switch (event) {
-	case GF_EVENT_GRAPH_NEW:
-		gf_msg (this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
-                        "New graph %s (%d) coming up",
-			uuid_utoa ((unsigned char *)graph->graph_uuid),
-			graph->id);
-		break;
-	case GF_EVENT_CHILD_UP:
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        graph->used = 1;
-                }
-                pthread_mutex_unlock (&fs->mutex);
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 0);
-		break;
-	case GF_EVENT_CHILD_DOWN:
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        graph->used = 0;
-                        pthread_cond_broadcast (&fs->child_down_cond);
-                }
-                pthread_mutex_unlock (&fs->mutex);
-		glfs_init_done (fs, 1);
-		break;
-	case GF_EVENT_CHILD_CONNECTING:
-		break;
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
         case GF_EVENT_UPCALL:
-                glfs_process_upcall_event (fs, data);
-                break;
-	default:
-		gf_msg_debug (this->name, 0, "got notify event %d", event);
-		break;
-	}
-
-	return 0;
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!this)
-		return ret;
+    if (!this)
+        return ret;
 
-	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
-	if (ret) {
-		gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        API_MSG_MEM_ACCT_INIT_FAILED, "Failed to initialise "
-                        "memory accounting");
-		return ret;
-	}
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-
 }
 
 /* place-holder fops */
 int
-glfs_forget (xlator_t *this, inode_t *inode)
+glfs_forget(xlator_t *this, inode_t *inode)
 {
-	return 0;
+    return 0;
 }
 
 int
-glfs_release (xlator_t *this, fd_t *fd)
+glfs_release(xlator_t *this, fd_t *fd)
 {
-	return 0;
+    return 0;
 }
 
 int
-glfs_releasedir (xlator_t *this, fd_t *fd)
+glfs_releasedir(xlator_t *this, fd_t *fd)
 {
-	return 0;
+    return 0;
 }
 
 struct xlator_dumpops dumpops;
 
-
 struct xlator_fops fops;
 
-
 struct xlator_cbks cbks = {
-	.forget	    = glfs_forget,
-	.release    = glfs_release,
-	.releasedir = glfs_releasedir
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
 };
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index 52033360853..bfa325a3ad9 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,24 +11,25 @@
 #ifndef _GLFS_MEM_TYPES_H
 #define _GLFS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_call_pool_t = GF_MEM_TYPE_START,
-        glfs_mt_xlator_t,
-	glfs_mt_glfs_fd_t,
-	glfs_mt_glfs_io_t,
-	glfs_mt_volfile_t,
-	glfs_mt_xlator_cmdline_option_t,
-        glfs_mt_server_cmdline_t,
-	glfs_mt_glfs_object_t,
-	glfs_mt_readdirbuf_t,
-        glfs_mt_upcall_entry_t,
-	glfs_mt_acl_t,
-        glfs_mt_upcall_inode_t,
-        glfs_mt_realpath_t,
-	glfs_mt_end
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
 };
 #endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index f476ce14d54..7c82b8cd162 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -16,943 +15,1035 @@
 #include <signal.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
-#include "glusterfs3.h"
-#include "portmap-xdr.h"
-#include "xdr-common.h"
 #include "xdr-generic.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
-#include "xlator.h"
+#include <glusterfs/syncop.h>
 
 #include "glfs-internal.h"
-#include "glfs-mem-types.h"
 #include "gfapi-messages.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
-int glfs_volfile_fetch (struct glfs *fs);
-int32_t glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                                  struct glfs *fs);
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
 
 int
-glfs_process_volfp (struct glfs *fs, FILE *fp)
+glfs_process_volfp(struct glfs *fs, FILE *fp)
 {
-	glusterfs_graph_t  *graph = NULL;
-	int		    ret = -1;
-	xlator_t	   *trav = NULL;
-	glusterfs_ctx_t	   *ctx = NULL;
-
-	ctx = fs->ctx;
-	graph = glusterfs_graph_construct (fp);
-	if (!graph) {
-		gf_msg ("glfs", GF_LOG_ERROR, errno,
-                        API_MSG_GRAPH_CONSTRUCT_FAILED,
-                        "failed to construct the graph");
-		goto out;
-	}
-
-	for (trav = graph->first; trav; trav = trav->next) {
-		if (strcmp (trav->type, "mount/fuse") == 0) {
-			gf_msg ("glfs", GF_LOG_ERROR, EINVAL,
-                                API_MSG_FUSE_XLATOR_ERROR,
-				"fuse xlator cannot be specified "
-				"in volume file");
-			goto out;
-		}
-	}
-
-	ret = glusterfs_graph_prepare (graph, ctx);
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = glusterfs_graph_activate (graph, ctx);
-
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = 0;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-	if (fp)
-		fclose (fp);
+    if (fp)
+        fclose(fp);
 
-	if (!ctx->active) {
-		ret = -1;
-	}
+    if (!ctx->active) {
+        ret = -1;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	struct glfs *fs = NULL;
-	xlator_t    *this = NULL;
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
 
-	this = mydata;
-	fs = this->private;
+    this = mydata;
+    fs = this->private;
 
-	glfs_volfile_fetch (fs);
+    glfs_volfile_fetch(fs);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	return 0;
+    return 0;
 }
 
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
-	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-				 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-	.progname  = "GlusterFS Callback",
-	.prognum   = GLUSTER_CBK_PROGRAM,
-	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = mgmt_cbk_actors,
-	.numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-	[GF_HNDSK_NULL]		= "NULL",
-	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
-	[GF_HNDSK_GETSPEC]	= "GETSPEC",
-	[GF_HNDSK_PING]		= "PING",
-	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
-        [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-	.progname  = "GlusterFS Handshake",
-	.prognum   = GLUSTER_HNDSK_PROGRAM,
-	.progver   = GLUSTER_HNDSK_VERSION,
-	.procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-		     glusterfs_ctx_t *ctx,
-		     rpc_clnt_prog_t *prog, int procnum,
-		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-	int			ret	    = -1;
-	int			count	   = 0;
-	struct iovec		iov	    = {0, };
-	struct iobuf		*iobuf = NULL;
-	struct iobref		*iobref = NULL;
-	ssize_t			xdr_size = 0;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	if (req) {
-		xdr_size = xdr_sizeof (xdrproc, req);
-
-		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-		if (!iobuf) {
-			goto out;
-		};
-
-		iobref_add (iobref, iobuf);
-
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len  = iobuf_pagesize (iobuf);
-
-		/* Create the xdr payload */
-		ret = xdr_serialize_generic (iov, req, xdrproc);
-		if (ret == -1) {
-			gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                API_MSG_XDR_PAYLOAD_FAILED,
-				"failed to create XDR payload");
-			goto out;
-		}
-		iov.iov_len = ret;
-		count = 1;
-	}
-
-	/* Send the msg */
-	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-			       &iov, count,
-			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-	if (iobref)
-		iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (iobuf)
-		iobuf_unref (iobuf);
-	return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /*
  * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
  */
 int
-mgmt_get_volinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                        ret                  = 0;
-        char                       *volume_id_str       = NULL;
-        dict_t                     *dict                = NULL;
-        char                       key[1024]            = {0};
-        gf_get_volume_info_rsp     rsp                  = {0,};
-        call_frame_t               *frame               = NULL;
-        glusterfs_ctx_t            *ctx                 = NULL;
-        struct glfs                *fs                  = NULL;
-        struct syncargs            *args;
-
-        frame = myframe;
-        ctx = frame->this->ctx;
-        args = frame->local;
-
-        if (!ctx) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "NULL context");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        fs = ((xlator_t *)ctx->master)->private;
-
-        if (-1 == req->rpc_status) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY,
-                        "GET_VOLUME_INFO RPC call is not successfull");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
-
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        API_MSG_XDR_RESPONSE_DECODE_FAILED,
-                        "Failed to decode xdr response for GET_VOLUME_INFO");
-                goto out;
-        }
-
-        gf_msg_debug (frame->this->name, 0, "Received resp to GET_VOLUME_INFO "
-                      "RPC: %d", rsp.op_ret);
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
-        }
-
-        if (!rsp.dict.dict_len) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "Response received for "
-                        "GET_VOLUME_INFO RPC call is not valid");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "volume_id");
-        ret = dict_get_str (dict, key, &volume_id_str);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (volume_id_str) {
-                gf_msg_debug (frame->this->name, 0,
-                              "Volume Id: %s", volume_id_str);
-                pthread_mutex_lock (&fs->mutex);
-                gf_uuid_parse (volume_id_str, fs->vol_uuid);
-                pthread_mutex_unlock (&fs->mutex);
-        }
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
 
-        if (ret) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, errno,
-                        API_MSG_GET_VOLINFO_CBK_FAILED, "In GET_VOLUME_INFO "
-                        "cbk, received error: %s", strerror(errno));
-        }
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
 
-        if (rsp.op_errstr && *rsp.op_errstr)
-                free (rsp.op_errstr);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        gf_msg_debug (frame->this->name, 0, "Returning: %d", ret);
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
 
-        __wake (args);
+    __wake(args);
 
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
 int
-pub_glfs_get_volumeid (struct glfs *fs, char *volid, size_t size)
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
 {
-        /* TODO: Define a global macro to store UUID size */
-        size_t uuid_size = 16;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        pthread_mutex_lock (&fs->mutex);
-        {
-                /* check if the volume uuid is initialized */
-                if (!gf_uuid_is_null (fs->vol_uuid)) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto done;
-                }
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
         }
-        pthread_mutex_unlock (&fs->mutex);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        /* Need to fetch volume_uuid */
-        glfs_get_volume_info (fs);
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
 
-        if (gf_uuid_is_null (fs->vol_uuid)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_FETCH_VOLUUID_FAILED, "Unable to fetch "
-                        "volume UUID");
-                goto out;
-        }
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
 
 done:
-        if (!volid || !size) {
-                gf_msg_debug (THIS->name, 0, "volumeid/size is null");
-                __GLFS_EXIT_FS;
-                return uuid_size;
-        }
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
 
-        if (size < uuid_size) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE,
-                        "Insufficient size passed");
-                errno = ERANGE;
-                goto out;
-        }
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (volid, fs->vol_uuid, uuid_size);
+    memcpy(volid, fs->vol_uuid, uuid_size);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        return uuid_size;
+    return uuid_size;
 
 out:
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0);
-
 int
-glfs_get_volume_info (struct glfs *fs)
+glfs_get_volume_info(struct glfs *fs)
 {
-        call_frame_t     *frame = NULL;
-        glusterfs_ctx_t  *ctx   = NULL;
-        struct syncargs  args   = {0, };
-        int              ret    = 0;
-
-        ctx = fs->ctx;
-        frame = create_frame (THIS, ctx->pool);
-        if (!frame) {
-                gf_msg ("glfs", GF_LOG_ERROR, ENOMEM,
-                        API_MSG_FRAME_CREAT_FAILED,
-                        "failed to create the frame");
-                ret = -1;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
 
-        frame->local = &args;
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        __yawn ((&args));
+    frame->local = &args;
 
-        ret = glfs_get_volume_info_rpc (frame, THIS, fs);
-        if (ret)
-                goto out;
+    __yawn((&args));
 
-        __yield ((&args));
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    __yield((&args));
+
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                          struct glfs *fs)
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
 {
-        gf_get_volume_info_req  req       = {{0,}};
-        int                     ret       = 0;
-        glusterfs_ctx_t         *ctx      = NULL;
-        dict_t                  *dict     = NULL;
-        int32_t                 flags     = 0;
-
-        if (!frame || !this || !fs) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = fs->ctx;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        if (fs->volname) {
-                ret = dict_set_str (dict, "volname", fs->volname);
-                if (ret)
-                        goto out;
-        }
-
-        // Set the flags for the fields which we are interested in
-        flags = (int32_t)GF_GET_VOLUME_UUID; //ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GET_VOLUME_INFO,
-                                   mgmt_get_volinfo_cbk,
-                                   (xdrproc_t)xdr_gf_get_volume_info_req);
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
 {
-	int ret = -1;
-
-        pthread_mutex_lock (&fs->mutex);
-
-	fs->oldvollen = size;
-	if (!fs->oldvolfile) {
-		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
-	} else {
-		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
-	}
-
-	if (!fs->oldvolfile) {
-		fs->oldvollen = 0;
-	} else {
-		memcpy (fs->oldvolfile, volfile, size);
-		fs->oldvollen = size;
-		ret = 0;
-	}
+    int ret = -1;
+
+    pthread_mutex_lock(&fs->mutex);
+
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
+
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
 
-        pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-	gf_getspec_rsp		 rsp   = {0,};
-	call_frame_t		*frame = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-	int			 ret   = 0;
-	ssize_t			 size = 0;
-	FILE			*tmpfp = NULL;
-	int			 need_retry = 0;
-	struct glfs		*fs = NULL;
-
-	frame = myframe;
-	ctx = frame->this->ctx;
-
-	if (!ctx) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "NULL context");
-		errno = EINVAL;
-		ret = -1;
-		goto out;
-	}
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	if (-1 == req->rpc_status) {
-		ret = -1;
-		need_retry = 1;
-		goto out;
-	}
-
-	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-	if (ret < 0) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        API_MSG_XDR_DECODE_FAILED, "XDR decoding error");
-		ret   = -1;
-		goto out;
-	}
-
-	if (-1 == rsp.op_ret) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, rsp.op_errno,
-                        API_MSG_GET_VOLFILE_FAILED,
-			"failed to get the 'volume file' from server");
-		ret = -1;
-		errno = rsp.op_errno;
-		goto out;
-	}
-
-	ret = 0;
-	size = rsp.op_ret;
-
-	if ((size == fs->oldvollen) &&
-	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
-		gf_msg (frame->this->name, GF_LOG_INFO, 0,
-                        API_MSG_VOLFILE_INFO,
-			"No change in volfile, continuing");
-		goto out;
-	}
-
-	tmpfp = tmpfile ();
-	if (!tmpfp) {
-		ret = -1;
-		goto out;
-	}
-
-	fwrite (rsp.spec, size, 1, tmpfp);
-	fflush (tmpfp);
-	if (ferror (tmpfp)) {
-		ret = -1;
-		goto out;
-	}
-
-	/*  Check if only options have changed. No need to reload the
-	*  volfile if topology hasn't changed.
-	*  glusterfs_volfile_reconfigure returns 3 possible return states
-	*  return 0	     =======> reconfiguration of options has succeeded
-	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
-	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
-	*/
-
-	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
-					     fs->oldvolfile);
-	if (ret == 0) {
-		gf_msg_debug ("glusterfsd-mgmt", 0, "No need to re-load "
-                              "volfile, reconfigure done");
-		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-		goto out;
-	}
-
-	if (ret < 0) {
-		gf_msg_debug ("glusterfsd-mgmt", 0, "Reconfigure failed !!");
-		goto out;
-	}
-
-	ret = glfs_process_volfp (fs, tmpfp);
-	/* tmpfp closed */
-	tmpfp = NULL;
-	if (ret)
-		goto out;
-
-	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
 out:
-	STACK_DESTROY (frame->root);
-
-	if (rsp.spec)
-		free (rsp.spec);
-
-	// Stop if server is running at an unsupported op-version
-	if (ENOTSUP == ret) {
-		gf_msg ("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION,
-                        "Server is operating at an op-version which is not "
-                        "supported");
-		errno = ENOTSUP;
-		glfs_init_done (fs, -1);
-	}
-
-	if (ret && ctx && !ctx->active) {
-		/* Do it only for the first time */
-		/* Failed to get the volume file, something wrong,
-		   restart the process */
-		gf_msg ("glfs-mgmt", GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY,
-			"failed to fetch volume file (key:%s)",
-			ctx->cmd_args.volfile_id);
-		if (!need_retry) {
-			if (!errno)
-				errno = EINVAL;
-			glfs_init_done (fs, -1);
-		}
-	}
-
-	if (tmpfp)
-		fclose (tmpfp);
-
-	return 0;
-}
-
-
-int
-glfs_volfile_fetch (struct glfs *fs)
-{
-	cmd_args_t	 *cmd_args = NULL;
-	gf_getspec_req	  req = {0, };
-	int		  ret = 0;
-	call_frame_t	 *frame = NULL;
-	glusterfs_ctx_t	 *ctx = NULL;
-        dict_t           *dict = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	frame = create_frame (THIS, ctx->pool);
-
-	req.key = cmd_args->volfile_id;
-	req.flags = 0;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED,
-                        "Failed to set min-op-version in request dict");
-                goto out;
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
         }
+    }
 
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED,
-                        "Failed to set max-op-version in request dict");
-                goto out;
-        }
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
 
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        API_MSG_DICT_SERIALIZE_FAILED,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+    return 0;
+}
 
-	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-				   GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
-				   (xdrproc_t)xdr_gf_getspec_req);
+int
+glfs_volfile_fetch(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 out:
-        return ret;
-}
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
 
+    return ret;
+}
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-		 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-	xlator_t	*this = NULL;
-	glusterfs_ctx_t *ctx = NULL;
-        server_cmdline_t *server = NULL;
-        rpc_transport_t  *rpc_trans = NULL;
-	struct glfs	 *fs = NULL;
-	int		 ret = 0;
-	struct dnscache6 *dnscache = NULL;
-
-	this = mydata;
-        rpc_trans = rpc->conn.trans;
-
-	ctx = this->ctx;
-	if (!ctx)
-		goto out;
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	switch (event) {
-	case RPC_CLNT_DISCONNECT:
-		if (!ctx->active) {
-                        gf_msg ("glfs-mgmt", GF_LOG_ERROR, errno,
-                                API_MSG_REMOTE_HOST_CONN_FAILED,
-                                "failed to connect with remote-host: %s (%s)",
-                                ctx->cmd_args.volfile_server,
-                                strerror (errno));
-
-                        if (!rpc->disabled) {
-                                /*
-                                 * Check if dnscache is exhausted for current server
-                                 * and continue until cache is exhausted
-                                 */
-                                dnscache = rpc_trans->dnscache;
-                                if (dnscache && dnscache->next) {
-                                        break;
-                                }
-                        }
-                        server = ctx->cmd_args.curr_server;
-                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
-                                errno = ENOTCONN;
-                                gf_msg ("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
-                                        API_MSG_VOLFILE_SERVER_EXHAUST,
-                                        "Exhausted all volfile servers");
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-                        server = list_entry (server->list.next, typeof(*server),
-                                             list);
-                        ctx->cmd_args.curr_server = server;
-                        ctx->cmd_args.volfile_server_port = server->port;
-                        ctx->cmd_args.volfile_server = server->volfile_server;
-                        ctx->cmd_args.volfile_server_transport = server->transport;
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "transport-type",
-                                            server->transport);
-                        if (ret != 0) {
-                                gf_msg ("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
-                                        API_MSG_DICT_SET_FAILED,
-                                        "failed to set transport-type: %s",
-                                        server->transport);
-                                errno = ENOTCONN;
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        if (strcmp(server->transport, "unix") == 0) {
-                                ret = dict_set_str (rpc_trans->options,
-                                                    "transport.socket.connect-path",
-                                                    server->volfile_server);
-                                if (ret != 0) {
-                                        gf_msg ("glfs-mgmt", GF_LOG_ERROR,
-                                                ENOTCONN,
-                                                API_MSG_DICT_SET_FAILED,
-                                                "failed to set socket.connect-path: %s",
-                                                server->volfile_server);
-                                        errno = ENOTCONN;
-                                        glfs_init_done (fs, -1);
-                                        break;
-                                }
-                                /* delete the remote-host and remote-port keys
-                                 * in case they were set while looping through
-                                 * list of volfile servers previously
-                                 */
-                                dict_del (rpc_trans->options, "remote-host");
-                                dict_del (rpc_trans->options, "remote-port");
-                        } else {
-                                ret = dict_set_int32 (rpc_trans->options,
-                                                      "remote-port",
-                                                      server->port);
-                                if (ret != 0) {
-                                        gf_msg ("glfs-mgmt", GF_LOG_ERROR,
-                                                ENOTCONN,
-                                                API_MSG_DICT_SET_FAILED,
-                                                "failed to set remote-port: %d",
-                                                server->port);
-                                        errno = ENOTCONN;
-                                        glfs_init_done (fs, -1);
-                                        break;
-                                }
-
-                                ret = dict_set_str (rpc_trans->options,
-                                                    "remote-host",
-                                                    server->volfile_server);
-                                if (ret != 0) {
-                                        gf_msg ("glfs-mgmt", GF_LOG_ERROR,
-                                                ENOTCONN,
-                                                API_MSG_DICT_SET_FAILED,
-                                                "failed to set remote-host: %s",
-                                                server->volfile_server);
-                                        errno = ENOTCONN;
-                                        glfs_init_done (fs, -1);
-                                        break;
-                                }
-                                /* delete the "transport.socket.connect-path"
-                                 * key in case if it was set while looping
-                                 * through list of volfile servers previously
-                                 */
-                                dict_del (rpc_trans->options,
-                                          "transport.socket.connect-path");
-                        }
-
-                        gf_msg ("glfs-mgmt", GF_LOG_INFO, 0,
-                                API_MSG_VOLFILE_CONNECTING,
-                                "connecting to next volfile server %s"
-                                " at port %d with transport: %s",
-                                server->volfile_server, server->port,
-                                server->transport);
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
                 }
-                break;
-	case RPC_CLNT_CONNECT:
-		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-		ret = glfs_volfile_fetch (fs);
-		if (ret && (ctx->active == NULL)) {
-			/* Do it only for the first time */
-			/* Exit the process.. there are some wrong options */
-			gf_msg ("glfs-mgmt", GF_LOG_ERROR, EINVAL,
-                                API_MSG_INVALID_ENTRY,
-				"failed to fetch volume file (key:%s)",
-                                ctx->cmd_args.volfile_id);
-                        errno = EINVAL;
-                        glfs_init_done (fs, -1);
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
                 }
 
-                break;
-	default:
-		break;
-	}
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
 out:
-	return 0;
+    return 0;
 }
 
-
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-	int ret = 0;
+    int ret = 0;
 
-	switch (op)
-	{
-		case GF_EN_DEFRAG_STATUS:
-			break;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
 
-		default:
-			break;
-	}
+        default:
+            break;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_init (struct glfs *fs)
+glfs_mgmt_init(struct glfs *fs)
 {
-	cmd_args_t		*cmd_args = NULL;
-	struct rpc_clnt		*rpc = NULL;
-	dict_t			*options = NULL;
-	int			ret = -1;
-	int			port = GF_DEFAULT_BASE_PORT;
-	char			*host = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	if (ctx->mgmt)
-		return 0;
-
-	if (cmd_args->volfile_server_port)
-		port = cmd_args->volfile_server_port;
-
-	if (cmd_args->volfile_server) {
-		host = cmd_args->volfile_server;
-        } else if (cmd_args->volfile_server_transport &&
-                   !strcmp (cmd_args->volfile_server_transport, "unix")) {
-                host = DEFAULT_GLUSTERD_SOCKFILE;
-        } else {
-                host = "localhost";
-        }
-
-        if (!strcmp (cmd_args->volfile_server_transport, "unix")) {
-                ret = rpc_transport_unix_options_build (&options, host, 0);
-        } else {
-                ret = rpc_transport_inet_options_build (&options, host, port);
-        }
-
-	if (ret)
-		goto out;
-
-        if (sys_access (SECURE_ACCESS_FILE, F_OK) == 0) {
-                ctx->secure_mgmt = 1;
-        }
-
-	rpc = rpc_clnt_new (options, THIS, THIS->name, 8);
-	if (!rpc) {
-		ret = -1;
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_CREATE_RPC_CLIENT_FAILED,
-			"failed to create rpc clnt");
-		goto out;
-	}
-
-	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-	if (ret) {
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_REG_NOTIFY_FUNC_FAILED,
-			"failed to register notify function");
-		goto out;
-	}
-
-	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-	if (ret) {
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_REG_CBK_FUNC_FAILED,
-			"failed to register callback function");
-		goto out;
-	}
-
-	ctx->notify = glusterfs_mgmt_notify;
-
-	/* This value should be set before doing the 'rpc_clnt_start()' as
-	   the notify function uses this variable */
-	ctx->mgmt = rpc;
-
-	ret = rpc_clnt_start (rpc);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-	return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index b84e5d8f58c..8a393ecb464 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
@@ -16,59 +15,61 @@
 #include <inttypes.h>
 #include <limits.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
 #include "gfapi-messages.h"
-
+#include <glusterfs/inode.h>
 #include "glfs-internal.h"
 
-#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
-
-
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 int
-glfs_first_lookup_safe (xlator_t *subvol)
+glfs_first_lookup_safe(xlator_t *subvol)
 {
-	loc_t  loc = {0, };
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
 
-	loc.inode = subvol->itable->root;
-	memset (loc.gfid, 0, 16);
-	loc.gfid[15] = 1;
-	loc.path = "/";
-	loc.name = "";
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
 
-	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
 
-	gf_msg_debug (subvol->name, 0, "first lookup complete %d", ret);
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
 
-	return ret;
+    return ret;
 }
 
-
 int
-__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
 {
-	int ret = -1;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		ret = glfs_first_lookup_safe (subvol);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return ret;
-}
+    int ret = -1;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return ret;
+}
 
 /**
  * We have to check if need_lookup flag is set in both old and the new inodes.
@@ -78,747 +79,837 @@ __glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
  * below xlators can set their respective contexts.
  */
 inode_t *
-glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode,
-                         gf_boolean_t need_lookup)
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
 {
-	loc_t        loc = {0, };
-	int          ret = -1;
-	struct iatt  iatt = {0, };
-	inode_t     *newinode = NULL;
-        gf_boolean_t lookup_needed = _gf_false;
-        uint64_t     ctx_value = LOOKUP_NOT_NEEDED;
-
-
-	if (!oldinode)
-		return NULL;
-
-	if (!need_lookup && oldinode->table->xl == subvol)
-		return inode_ref (oldinode);
-
-	newinode = inode_find (subvol->itable, oldinode->gfid);
-	if (!need_lookup && newinode) {
-
-                lookup_needed = inode_needs_lookup (newinode, THIS);
-                if (!lookup_needed)
-                        return newinode;
-        }
-
-	gf_uuid_copy (loc.gfid, oldinode->gfid);
-        if (!newinode)
-                loc.inode = inode_new (subvol->itable);
-        else
-                loc.inode = newinode;
-
-	if (!loc.inode)
-		return NULL;
-
-	ret = syncop_lookup (subvol, &loc, &iatt, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret) {
-		gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (oldinode->gfid), strerror (errno));
-		loc_wipe (&loc);
-		return NULL;
-	}
-
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-        if (newinode) {
-                if (newinode == loc.inode)
-                        inode_ctx_set (newinode, THIS, &ctx_value);
-                inode_lookup (newinode);
-        } else {
-                gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_LINK_FAILED,
-                        "inode linking of %s failed",
-                        uuid_utoa ((unsigned char *)&iatt.ia_gfid));
-        }
-
-	loc_wipe (&loc);
-
-	return newinode;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
 }
 
-
 inode_t *
-__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-                      gf_boolean_t need_lookup)
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
 {
-	inode_t *newinode = NULL;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newinode = glfs_refresh_inode_safe (subvol, inode, need_lookup);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return newinode;
+    inode_t *newinode = NULL;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
+
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return newinode;
 }
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
 int
-priv_glfs_loc_touchup (loc_t *loc)
+priv_glfs_loc_touchup(loc_t *loc)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        ret = loc_touchup (loc, loc->name);
-        if (ret < 0) {
-                errno = -ret;
-                ret = -1;
-        }
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0);
-
 int
-glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		      char **lpath)
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
 {
-	loc_t  loc = {0, };
-	char  *path = NULL;
-	char  *rpath = NULL;
-	int    ret = -1;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-	ret = inode_path (inode, NULL, &rpath);
-	if (ret < 0)
-		goto out;
-	loc.path = rpath;
-
-	ret = syncop_readlink (subvol, &loc, &path, 4096, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret < 0)
-		goto out;
-
-	if (lpath)
-		*lpath = path;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-
 int
-glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		   struct iatt *iatt)
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
 {
-	loc_t       loc = {0, };
-	int         ret = -1;
-	char       *path = NULL;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = inode_path (loc.inode, NULL, &path);
-	loc.path = path;
-	if (ret < 0)
-		goto out;
-
-	ret = syncop_lookup (subvol, &loc, iatt, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return ret;
+    return ret;
 }
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
+    return ret;
+}
 
 inode_t *
-glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt,
-			int force_lookup)
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
 {
-	loc_t        loc = {0, };
-	inode_t     *inode = NULL;
-	int          reval = 0;
-	int          ret = -1;
-	int          glret = -1;
-	struct iatt  ciatt = {0, };
-	uuid_t       gfid;
-	dict_t      *xattr_req = NULL;
-        uint64_t     ctx_value = LOOKUP_NOT_NEEDED;
-
-	loc.name = component;
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, parent->gfid);
-
-        /* /.. and /. should point back to /
-           we lookup using inode and gfid of root
-           Fill loc.name so that we make use md-cache.
-           md-cache is not valid for nameless lookups.
-        */
-        if (__is_root_gfid (parent->gfid) &&
-            (strcmp (component, "..") == 0)) {
-                loc.inode = inode_ref (parent);
-                loc.name = ".";
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
         } else {
-                if (strcmp (component, ".") == 0)
-                        loc.inode = inode_ref (parent);
-                else if (strcmp (component, "..") == 0)
-                        loc.inode = inode_parent (parent, 0, 0);
-                else
-                        loc.inode = inode_grep (parent->table, parent,
-                                                component);
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
         }
 
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
 
-	if (loc.inode) {
-		gf_uuid_copy (loc.gfid, loc.inode->gfid);
-		reval = 1;
-
-                if (!(force_lookup || inode_needs_lookup (loc.inode, THIS))) {
-			inode = inode_ref (loc.inode);
-			ciatt.ia_type = inode->ia_type;
-			goto found;
-		}
-	} else {
-		gf_uuid_generate (gfid);
-		loc.inode = inode_new (parent->table);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+    if (!loc.name)
+        loc.name = component;
 
-                xattr_req = dict_new ();
-                if (!xattr_req) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
 
-                ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-                if (ret) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	}
-
-	glret = priv_glfs_loc_touchup (&loc);
-	if (glret < 0) {
-		ret = -1;
-		goto out;
-	}
-
-        ret = syncop_lookup (subvol, &loc, &ciatt, NULL, xattr_req, NULL);
-        if (ret && reval) {
-                /*
-                 * A stale mapping might exist for a dentry/inode that has been
-                 * removed from another client.
-                 */
-                if (-ret == ENOENT)
-                        inode_unlink(loc.inode, loc.parent,
-                                     loc.name);
-		inode_unref (loc.inode);
-	        gf_uuid_clear (loc.gfid);
-		loc.inode = inode_new (parent->table);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		xattr_req = dict_new ();
-		if (!xattr_req) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		gf_uuid_generate (gfid);
-
-		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-		if (ret) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		ret = syncop_lookup (subvol, &loc, &ciatt, NULL,
-				     xattr_req, NULL);
-	}
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-        if (!inode) {
-                gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_LINK_FAILED,
-                        "inode linking of %s failed",
-                        uuid_utoa ((unsigned char *)&ciatt.ia_gfid));
-                goto out;
-        } else if (inode == loc.inode)
-                inode_ctx_set (inode, THIS, &ctx_value);
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
+        }
+
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        gf_uuid_generate(gfid);
+
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
+
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
 found:
-	if (inode)
-		inode_lookup (inode);
-	if (iatt)
-		*iatt = ciatt;
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
 out:
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	loc_wipe (&loc);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
 
-	return inode;
+    return inode;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
 int
-priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-		 const char *origpath, loc_t *loc, struct iatt *iatt,
-		 int follow, int reval)
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
 {
-	inode_t    *inode = NULL;
-	inode_t    *parent = NULL;
-	char       *saveptr = NULL;
-	char       *path = NULL;
-	char       *component = NULL;
-	char       *next_component = NULL;
-	int         ret = -1;
-	struct iatt ciatt = {0, };
-
-	path = gf_strdup (origpath);
-	if (!path) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	parent = NULL;
-	if (at && path[0] != '/') {
-		/* A relative resolution of a path which starts with '/'
-		   is equal to an absolute path resolution.
-		*/
-		inode = inode_ref (at);
-	} else {
-		inode = inode_ref (subvol->itable->root);
-
-		if (strcmp (path, "/") == 0)
-			glfs_resolve_base (fs, subvol, inode, &ciatt);
-	}
-
-	for (component = strtok_r (path, "/", &saveptr);
-	     component; component = next_component) {
-
-		next_component = strtok_r (NULL, "/", &saveptr);
-
-		if (parent)
-			inode_unref (parent);
-
-		parent = inode;
-
-		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt,
-						/* force hard lookup on the last
-						   component, as the caller
-						   wants proper iatt filled
-						*/
-						(reval || (!next_component &&
-						iatt)));
-		if (!inode) {
-                        ret = -1;
-			break;
-                }
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
 
-		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
-			/* If the component is not the last piece,
-			   then following it is necessary even if
-			   not requested by the caller
-			*/
-			char *lpath = NULL;
-			loc_t sym_loc = {0,};
-
-			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
-				errno = ELOOP;
-				ret = -1;
-				if (inode) {
-					inode_unref (inode);
-					inode = NULL;
-				}
-				break;
-			}
-
-			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
-			inode_unref (inode);
-			inode = NULL;
-			if (ret < 0)
-				break;
-
-			ret = priv_glfs_resolve_at (fs, subvol, parent, lpath,
-					       &sym_loc,
-					       /* followed iatt becomes the
-						  component iatt
-					       */
-					       &ciatt,
-					       /* always recurisvely follow while
-						  following symlink
-					       */
-					       follow + 1, reval);
-			if (ret == 0)
-				inode = inode_ref (sym_loc.inode);
-			loc_wipe (&sym_loc);
-			GF_FREE (lpath);
-		}
-
-		if (!next_component)
-			break;
-
-		if (!IA_ISDIR (ciatt.ia_type)) {
-			/* next_component exists and this component is
-			   not a directory
-			*/
-			inode_unref (inode);
-			inode = NULL;
-			ret = -1;
-			errno = ENOTDIR;
-			break;
-		}
-	}
-
-	if (parent && next_component)
-		/* resolution failed mid-way */
-		goto out;
-
-	/* At this point, all components up to the last parent directory
-	   have been resolved successfully (@parent). Resolution of basename
-	   might have failed (@inode) if at all.
-	*/
-
-	loc->parent = parent;
-	if (parent) {
-		gf_uuid_copy (loc->pargfid, parent->gfid);
-		loc->name = component;
-	}
-
-	loc->inode = inode;
-	if (inode) {
-		gf_uuid_copy (loc->gfid, inode->gfid);
-		if (iatt)
-			*iatt = ciatt;
-		ret = 0;
-	}
-
-        if (priv_glfs_loc_touchup (loc) < 0) {
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
                 ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
         }
-out:
-	GF_FREE (path);
 
-	/* do NOT loc_wipe here as only last component might be missing */
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
 
-	return ret;
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0);
-
-
 int
-glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
-		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
 {
-	int ret = -1;
-	inode_t *cwd = NULL;
-
-	if (origpath[0] == '/')
-		return priv_glfs_resolve_at (fs, subvol, NULL, origpath, loc,
-                                             iatt, follow, reval);
-
-	cwd = glfs_cwd_get (fs);
-        if (NULL == cwd) {
-                gf_msg (subvol->name, GF_LOG_WARNING, EIO,
-                        API_MSG_GET_CWD_FAILED, "Failed to get cwd");
-                errno = EIO;
-                goto out;
-        }
+    int ret = -1;
+    inode_t *cwd = NULL;
 
-	ret = priv_glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
                                     follow, reval);
-	if (cwd)
-		inode_unref (cwd);
+
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
+
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
 
 out:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
 int
-priv_glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt, int reval)
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
 
-	return ret;
+    return ret;
 }
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0);
 
 int
-glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	       loc_t *loc, struct iatt *iatt, int reval)
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
-			    xlator_t *newsubvol, fd_t *newfd)
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
 {
-	dict_t *lockinfo = NULL;
-	int ret = 0;
-	char uuid1[64];
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
-
-	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
-
-	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
-				GF_XATTR_LOCKINFO_KEY, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_FGETXATTR_FAILED,
-			"fgetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
-		gf_msg (fs->volname, GF_LOG_WARNING, 0,
-                        API_MSG_LOCKINFO_KEY_MISSING,
-			"missing lockinfo key (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_msg (fs->volname, GF_LOG_WARNING, 0,
-                        API_MSG_FSETXATTR_FAILED,
-			"fsetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
 out:
-	if (lockinfo)
-		dict_unref (lockinfo);
-	return ret;
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
 }
 
-
 fd_t *
-glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
 {
-	fd_t *newfd = NULL;
-	inode_t *oldinode = NULL;
-	inode_t *newinode = NULL;
-	xlator_t *oldsubvol = NULL;
-	int ret = -1;
-	loc_t loc = {0, };
-	char uuid1[64];
-
-
-	oldinode = oldfd->inode;
-	oldsubvol = oldinode->table->xl;
-
-	if (oldsubvol == newsubvol)
-		return fd_ref (oldfd);
-
-	if (!oldsubvol->switched) {
-		ret = syncop_fsync (oldsubvol, oldfd, 0, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-		if (ret) {
-			gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                                API_MSG_FSYNC_FAILED, "fsync() failed "
-                                "(%s) on %s graph %s (%d)", strerror (errno),
-				uuid_utoa_r (oldfd->inode->gfid, uuid1),
-				graphid_str (oldsubvol), oldsubvol->graph->id);
-		}
-	}
-
-	newinode = glfs_refresh_inode_safe (newsubvol, oldinode, _gf_false);
-	if (!newinode) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-			"inode (%s) refresh failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	newfd = fd_create (newinode, getpid());
-	if (!newfd) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_FDCREATE_FAILED,
-			"fd_create (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	loc.inode = inode_ref (newinode);
-
-        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_msg (fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED,
-                        "inode_path failed");
-                goto out;
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
         }
 
-        gf_uuid_copy (loc.gfid, oldinode->gfid);
-
-
-	if (IA_ISDIR (oldinode->ia_type))
-		ret = syncop_opendir (newsubvol, &loc, newfd, NULL, NULL);
-	else
-		ret = syncop_open (newsubvol, &loc,
-				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
-				   newfd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	loc_wipe (&loc);
-
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_SYNCOP_OPEN_FAILED,
-			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
-			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
-					  newfd);
-
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_LOCK_MIGRATE_FAILED,
-			"lock migration (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-        newfd->flags = oldfd->flags;
-	fd_bind (newfd);
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
 out:
-	if (newinode)
-		inode_unref (newinode);
+    if (newinode)
+        inode_unref(newinode);
 
-	if (ret) {
-		fd_unref (newfd);
-		newfd = NULL;
-	}
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
 
-	return newfd;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
 {
-	fd_t *oldfd = NULL;
-	fd_t *newfd = NULL;
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
 
-	oldfd = glfd->fd;
+    oldfd = glfd->fd;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return newfd;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	if (glfd->fd->inode->table->xl == subvol)
-		return fd_ref (glfd->fd);
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
 
-	fd = __glfs_migrate_fd (fs, subvol, glfd);
-	if (!fd)
-		return NULL;
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		fd_unref (glfd->fd);
-		glfd->fd = fd_ref (fd);
-	}
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
 
-	return fd;
+    return fd;
 }
 
-
 fd_t *
-glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	glfs_lock (fs);
-	{
-		fd = __glfs_resolve_fd (fs, subvol, glfd);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
 
-	return fd;
+    return fd;
 }
 
-
 void
-__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
 {
-	struct glfs_fd *glfd = NULL;
-	fd_t *fd = NULL;
-
-	list_for_each_entry (glfd, &fs->openfds, openfds) {
-		if (gf_uuid_is_null (glfd->fd->inode->gfid)) {
-			gf_msg (fs->volname, GF_LOG_INFO, 0,
-                                API_MSG_OPENFD_SKIPPED,
-				"skipping openfd %p/%p in graph %s (%d)",
-				glfd, glfd->fd,	graphid_str(subvol),
-				subvol->graph->id);
-			/* create in progress, defer */
-			continue;
-		}
-
-		fd = __glfs_migrate_fd (fs, subvol, glfd);
-		if (fd) {
-			fd_unref (glfd->fd);
-			glfd->fd = fd;
-		}
-	}
-}
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
 
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
+}
 
 /* Note that though it appears that this function executes under fs->mutex,
  * it is not fully executed under fs->mutex. i.e. there are functions like
@@ -828,284 +919,281 @@ __glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
  * function need not have the same value by the end of the function.
  */
 xlator_t *
-__glfs_active_subvol (struct glfs *fs)
+__glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *new_subvol = NULL;
-	int            ret = -1;
-	inode_t       *new_cwd = NULL;
-
-	if (!fs->next_subvol)
-		return fs->active_subvol;
-
-	new_subvol = fs->mip_subvol = fs->next_subvol;
-        fs->next_subvol = NULL;
-
-	ret = __glfs_first_lookup (fs, new_subvol);
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_INFO, errno,
-                        API_MSG_FIRST_LOOKUP_GRAPH_FAILED,
-			"first lookup on graph %s (%d) failed (%s)",
-			graphid_str (new_subvol), new_subvol->graph->id,
-			strerror (errno));
-		return NULL;
-	}
-
-	if (fs->cwd) {
-		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd,
-                                                _gf_false);
-
-		if (!new_cwd) {
-			char buf1[64];
-			gf_msg (fs->volname, GF_LOG_INFO, errno,
-                                API_MSG_CWD_GRAPH_REF_FAILED,
-				"cwd refresh of %s graph %s (%d) failed (%s)",
-				uuid_utoa_r (fs->cwd->gfid, buf1),
-				graphid_str (new_subvol),
-				new_subvol->graph->id, strerror (errno));
-			return NULL;
-		}
-	}
-
-	__glfs_migrate_openfds (fs, new_subvol);
-
-	/* switching @active_subvol and @cwd
-	   should be atomic
-	*/
-	fs->old_subvol = fs->active_subvol;
-	fs->active_subvol = fs->mip_subvol;
-	fs->mip_subvol = NULL;
-
-	if (new_cwd) {
-		__glfs_cwd_set (fs, new_cwd);
-		inode_unref (new_cwd);
-	}
-
-	gf_msg (fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH,
-                "switched to graph %s (%d)",
-		graphid_str (new_subvol), new_subvol->graph->id);
-
-	return new_subvol;
-}
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
+
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
 
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
 
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
+
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
+
+    return new_subvol;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
 void
-priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
 {
-	int ref = 0;
-	xlator_t *active_subvol = NULL;
-
-	if (!subvol)
-		return;
-
-	glfs_lock (fs);
-	{
-		ref = (--subvol->winds);
-		active_subvol = fs->active_subvol;
-	}
-	glfs_unlock (fs);
-
-	if (ref == 0) {
-		assert (subvol != active_subvol);
-		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
-	}
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0);
-
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
 xlator_t *
-priv_glfs_active_subvol (struct glfs *fs)
+priv_glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *subvol = NULL;
-	xlator_t      *old_subvol = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
 
-	glfs_lock (fs);
-	{
-		subvol = __glfs_active_subvol (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
 
-		if (subvol)
-			subvol->winds++;
+        if (subvol)
+            subvol->winds++;
 
-		if (fs->old_subvol) {
-			old_subvol = fs->old_subvol;
-			fs->old_subvol = NULL;
-			old_subvol->switched = 1;
-		}
-	}
-	glfs_unlock (fs);
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
 
-	if (old_subvol)
-		priv_glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
 
-	return subvol;
+    return subvol;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0);
-
 int
-__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	if (inode->table->xl != fs->active_subvol) {
-		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode,
-                                              _gf_false);
-		if (!inode)
-			return -1;
-	} else {
-		inode_ref (inode);
-	}
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
 
-	if (fs->cwd)
-		inode_unref (fs->cwd);
+    if (fs->cwd)
+        inode_unref(fs->cwd);
 
-	fs->cwd = inode;
+    fs->cwd = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_cwd_set (struct glfs *fs, inode_t *inode)
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	int ret = 0;
+    int ret = 0;
 
-	glfs_lock (fs);
-	{
-		ret = __glfs_cwd_set (fs, inode);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-__glfs_cwd_get (struct glfs *fs)
+__glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	if (!fs->cwd)
-		return NULL;
+    if (!fs->cwd)
+        return NULL;
 
-	if (fs->cwd->table->xl == fs->active_subvol) {
-		cwd = inode_ref (fs->cwd);
-		return cwd;
-	}
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
 
-	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd, _gf_false);
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-glfs_cwd_get (struct glfs *fs)
+glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	glfs_lock (fs);
-	{
-		cwd = __glfs_cwd_get (fs);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
 {
-	inode_t *inode = NULL;
-        gf_boolean_t lookup_needed = _gf_false;
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
 
-        lookup_needed = inode_needs_lookup (object->inode, THIS);
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
 
-	if (!lookup_needed && object->inode->table->xl == subvol)
-		return inode_ref (object->inode);
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
 
-	inode = __glfs_refresh_inode (fs, fs->active_subvol,
-                                      object->inode, lookup_needed);
-	if (!inode)
-		return NULL;
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		inode_unref (object->inode);
-		object->inode = inode_ref (inode);
-	}
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
 
-	return inode;
+    return inode;
 }
 
 inode_t *
-glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	glfs_lock (fs);
-	{
-		inode = __glfs_resolve_inode(fs, subvol, object);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
 
-	return inode;
+    return inode;
 }
 
 int
-glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
 {
-	struct glfs_object *object = NULL;
+    struct glfs_object *object = NULL;
 
-	object = GF_CALLOC (1, sizeof(struct glfs_object),
-			    glfs_mt_glfs_object_t);
-	if (object == NULL) {
-		errno = ENOMEM;
-		return -1;
-	}
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
 
-	object->inode = loc->inode;
-	gf_uuid_copy (object->gfid, object->inode->gfid);
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-	/* we hold the reference */
-	loc->inode = NULL;
+    /* we hold the reference */
+    loc->inode = NULL;
 
-	*retobject = object;
+    *retobject = object;
 
-	return 0;
+    return 0;
 }
 
 struct glfs_object *
-glfs_h_resolve_symlink (struct glfs *fs, struct glfs_object *object)
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
 {
-
-        xlator_t                *subvol         = NULL;
-        loc_t                   sym_loc         = {0,};
-        struct iatt             iatt            = {0,};
-        char                    *lpath          = NULL;
-        int                     ret             = 0;
-        struct glfs_object      *target_object  = NULL;
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        ret = glfs_resolve_symlink (fs, subvol, object->inode, &lpath);
-        if (ret < 0)
-                goto out;
-
-        ret = glfs_resolve_at (fs, subvol, NULL, lpath,
-                               &sym_loc, &iatt,
-                              /* always recurisvely follow while
-                                following symlink
-                              */
-                               1, 0);
-        if (ret == 0)
-                ret = glfs_create_object (&sym_loc, &target_object);
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
 
 out:
-        loc_wipe (&sym_loc);
-        GF_FREE (lpath);
-        return target_object;
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
 }
diff --git a/api/src/glfs.c b/api/src/glfs.c
index a9d56071743..b4bf1423f6d 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /*
   TODO:
   - set proper pid/lk_owner to call frames (currently buried in syncop)
@@ -31,667 +30,743 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <limits.h>
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
+#endif
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
-#include "gfapi-messages.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 
+#include "gfapi-messages.h"
 #include "glfs.h"
 #include "glfs-internal.h"
-#include "hashfn.h"
-#include "rpc-clnt.h"
-
 
 static gf_boolean_t
-vol_assigned (cmd_args_t *args)
+vol_assigned(cmd_args_t *args)
 {
-	return args->volfile || args->volfile_server;
+    return args->volfile || args->volfile_server;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-	call_pool_t   *pool = NULL;
-	int	       ret = -1;
+    call_pool_t *pool = NULL;
+    int ret = -1;
 
-	if (!ctx) {
-		goto err;
-        }
-
-        ret = xlator_mem_acct_init (THIS, glfs_mt_end + 1);
-        if (ret != 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                        API_MSG_MEM_ACCT_INIT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    if (!ctx) {
+        goto err;
+    }
 
-        /* reset ret to -1 so that we don't need to explicitly
-         * set it in all error paths before "goto err"
-         */
-
-        ret = -1;
-
-	ctx->process_uuid = generate_glusterfs_ctx_id ();
-	if (!ctx->process_uuid) {
-		goto err;
-	}
-
-	ctx->page_size	= 128 * GF_UNIT_KB;
-
-	ctx->iobuf_pool = iobuf_pool_new ();
-	if (!ctx->iobuf_pool) {
-		goto err;
-	}
-
-	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE,
-                                          STARTING_EVENT_THREADS);
-	if (!ctx->event_pool) {
-		goto err;
-	}
-
-	ctx->env = syncenv_new (0, 0, 0);
-	if (!ctx->env) {
-		goto err;
-	}
-
-	pool = GF_CALLOC (1, sizeof (call_pool_t),
-			  glfs_mt_call_pool_t);
-	if (!pool) {
-		goto err;
-	}
-
-	/* frame_mem_pool size 112 * 4k */
-	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-	if (!pool->frame_mem_pool) {
-		goto err;
-	}
-	/* stack_mem_pool size 256 * 1024 */
-	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-	if (!pool->stack_mem_pool) {
-		goto err;
-	}
-
-	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-	if (!ctx->stub_mem_pool) {
-		goto err;
-	}
-
-	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-	if (!ctx->dict_pool)
-		goto err;
-
-	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-	if (!ctx->dict_pair_pool)
-		goto err;
-
-	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-	if (!ctx->dict_data_pool)
-		goto err;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t,
-                                         GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
-        if (!ctx->logbuf_pool)
-                goto err;
-
-	INIT_LIST_HEAD (&pool->all_frames);
-	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
-        INIT_LIST_HEAD (&ctx->cmd_args.volfile_servers);
-
-	LOCK_INIT (&pool->lock);
-	ctx->pool = pool;
-
-	LOCK_INIT (&ctx->lock);
-
-	ret = 0;
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
 err:
-	if (ret && pool) {
-		if (pool->frame_mem_pool)
-			mem_pool_destroy (pool->frame_mem_pool);
-		if (pool->stack_mem_pool)
-			mem_pool_destroy (pool->stack_mem_pool);
-		GF_FREE (pool);
-	}
-
-	if (ret && ctx) {
-		if (ctx->stub_mem_pool)
-			mem_pool_destroy (ctx->stub_mem_pool);
-		if (ctx->dict_pool)
-			mem_pool_destroy (ctx->dict_pool);
-		if (ctx->dict_data_pool)
-			mem_pool_destroy (ctx->dict_data_pool);
-		if (ctx->dict_pair_pool)
-			mem_pool_destroy (ctx->dict_pair_pool);
-                if (ctx->logbuf_pool)
-                        mem_pool_destroy (ctx->logbuf_pool);
-	}
-
-	return ret;
-}
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
 
+    return ret;
+}
 
 static int
-create_master (struct glfs *fs)
+create_master(struct glfs *fs)
 {
-	int		 ret = 0;
-	xlator_t	*master = NULL;
-
-	master = GF_CALLOC (1, sizeof (*master),
-			    glfs_mt_xlator_t);
-	if (!master)
-		goto err;
+    int ret = 0;
+    xlator_t *master = NULL;
 
-	master->name = gf_strdup ("gfapi");
-	if (!master->name)
-		goto err;
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
 
-	if (xlator_set_type (master, "mount/api") == -1) {
-		gf_msg ("glfs", GF_LOG_ERROR, 0,
-                        API_MSG_MASTER_XLATOR_INIT_FAILED, "master xlator "
-                        "for %s initialization failed", fs->volname);
-		goto err;
-	}
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
 
-	master->ctx	 = fs->ctx;
-	master->private	 = fs;
-	master->options	 = get_new_dict ();
-	if (!master->options)
-		goto err;
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
 
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
 
-	ret = xlator_init (master);
-	if (ret) {
-		gf_msg ("glfs", GF_LOG_ERROR, 0,
-                        API_MSG_GFAPI_XLATOR_INIT_FAILED,
-			"failed to initialize gfapi translator");
-		goto err;
-	}
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
 
-	fs->ctx->master = master;
-	THIS = master;
+    fs->ctx->master = master;
+    THIS = master;
 
-	return 0;
+    return 0;
 
 err:
-	if (master) {
-		xlator_destroy (master);
-	}
+    if (master) {
+        xlator_destroy(master);
+    }
 
-	return -1;
+    return -1;
 }
 
-
 static FILE *
-get_volfp (struct glfs *fs)
+get_volfp(struct glfs *fs)
 {
-	cmd_args_t  *cmd_args = NULL;
-	FILE	    *specfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-		gf_msg ("glfs", GF_LOG_ERROR, errno,
-                        API_MSG_VOLFILE_OPEN_FAILED,
-			"volume file %s open failed: %s",
-			cmd_args->volfile,
-			strerror (errno));
-		return NULL;
-	}
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
 
-	gf_msg_debug ("glfs", 0, "loading volume file %s", cmd_args->volfile);
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
 
-	return specfp;
+    return specfp;
 }
 
-
 int
-glfs_volumes_init (struct glfs *fs)
+glfs_volumes_init(struct glfs *fs)
 {
-	FILE		   *fp = NULL;
-	cmd_args_t	   *cmd_args = NULL;
-	int		    ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    cmd_args = &fs->ctx->cmd_args;
 
-	cmd_args = &fs->ctx->cmd_args;
+    if (!vol_assigned(cmd_args))
+        return -1;
 
-	if (!vol_assigned (cmd_args))
-		return -1;
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-	if (cmd_args->volfile_server) {
-		ret = glfs_mgmt_init (fs);
-		goto out;
-	}
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
 
-	fp = get_volfp (fs);
+    fp = get_volfp(fs);
 
-	if (!fp) {
-		gf_msg ("glfs", GF_LOG_ERROR, ENOENT,
-                        API_MSG_VOL_SPEC_FILE_ERROR,
-			"Cannot reach volume specification file");
-		ret = -1;
-		goto out;
-	}
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
 
-	ret = glfs_process_volfp (fs, fp);
-	if (ret)
-		goto out;
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
 int
-pub_glfs_set_xlator_option (struct glfs *fs, const char *xlator,
-                            const char *key, const char *value)
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
 {
-	xlator_cmdline_option_t *option = NULL;
+    xlator_cmdline_option_t *option = NULL;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	option = GF_CALLOC (1, sizeof (*option),
-			    glfs_mt_xlator_cmdline_option_t);
-	if (!option)
-		goto enomem;
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
 
-	INIT_LIST_HEAD (&option->cmd_args);
+    INIT_LIST_HEAD(&option->cmd_args);
 
-	option->volume = gf_strdup (xlator);
-	if (!option->volume)
-		goto enomem;
-	option->key = gf_strdup (key);
-	if (!option->key)
-		goto enomem;
-	option->value = gf_strdup (value);
-	if (!option->value)
-		goto enomem;
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
 
-	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-	return 0;
+    return 0;
 enomem:
-	errno = ENOMEM;
+    errno = ENOMEM;
 
-	if (!option) {
-                __GLFS_EXIT_FS;
-		return -1;
-        }
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
 
-	GF_FREE (option->volume);
-	GF_FREE (option->key);
-	GF_FREE (option->value);
-	GF_FREE (option);
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
 int
-pub_glfs_unset_volfile_server (struct glfs *fs, const char *transport,
-                               const char *host, const int port)
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
 {
-        cmd_args_t       *cmd_args = NULL;
-        server_cmdline_t *server = NULL;
-        server_cmdline_t *tmp = NULL;
-        char             *transport_val = NULL;
-        int               port_val = 0;
-        int               ret = -1;
-
-        if (!fs || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        cmd_args = &fs->ctx->cmd_args;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        if (transport) {
-                transport_val = gf_strdup (transport);
-        } else {
-                transport_val = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
-        }
+    cmd_args = &fs->ctx->cmd_args;
 
-        if (!transport_val) {
-                errno = ENOMEM;
-                goto out;
-        }
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
 
-        if (port) {
-                port_val = port;
-        } else {
-                port_val = GF_DEFAULT_BASE_PORT;
-        }
-
-        list_for_each_entry_safe (server, tmp,
-                                  &cmd_args->curr_server->list,
-                                  list) {
-                if ((!strcmp(server->volfile_server, host) &&
-                     !strcmp(server->transport, transport_val) &&
-                     (server->port == port_val))) {
-                        list_del (&server->list);
-                        ret = 0;
-                        goto out;
-                }
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        GF_FREE (transport_val);
-        __GLFS_EXIT_FS;
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
 int
-pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
-                             const char *host, int port)
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
 {
-        cmd_args_t            *cmd_args = NULL;
-        server_cmdline_t      *server = NULL;
-        server_cmdline_t      *tmp = NULL;
-        int                    ret = -1;
-
-        if (!fs || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
 
-        cmd_args = &fs->ctx->cmd_args;
-
-        cmd_args->max_connect_attempts = 1;
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            glfs_mt_server_cmdline_t);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        if (!server) {
-                errno = ENOMEM;
-                goto out;
-        }
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
 
-        INIT_LIST_HEAD (&server->list);
-
-        server->volfile_server = gf_strdup (host);
-        if (!server->volfile_server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        if (transport) {
-                /* volfile fetch support over tcp|unix only */
-                if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
-                        server->transport = gf_strdup (transport);
-                } else if (!strcmp(transport, "rdma")) {
-                        server->transport = gf_strdup ("tcp");
-                        gf_msg ("glfs", GF_LOG_WARNING, EINVAL,
-                                API_MSG_INVALID_ENTRY,
-                                "transport RDMA is deprecated, "
-                                "falling back to tcp");
-                } else {
-                        gf_msg ("glfs", GF_LOG_TRACE, EINVAL,
-                                API_MSG_INVALID_ENTRY,
-                                "transport %s is not supported, "
-                                "possible values tcp|unix",
-                                transport);
-                        ret = -1;
-                        goto out;
-                }
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
         } else {
-                server->transport = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
-        }
-
-        if (!server->transport) {
-                errno = ENOMEM;
-                goto out;
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
         }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
 
-        if (strcmp(server->transport, "unix")) {
-                if (port) {
-                        server->port = port;
-                } else {
-                        server->port = GF_DEFAULT_BASE_PORT;
-                }
-        } else {
-                server->port = 0;
-        }
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
 
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->volfile_server_transport = server->transport;
-                cmd_args->volfile_server_port = server->port;
-                cmd_args->curr_server = server;
-        }
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
 
-        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
-                if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
-                     !strcmp(tmp->transport, server->transport) &&
-                     (tmp->port == server->port))) {
-                        errno = EEXIST;
-                        ret = -1;
-                        goto out;
-                }
-        }
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
 
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
 
-        ret = 0;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server->transport);
-                        GF_FREE (server);
-                }
-        }
+    if (server_host) {
+        GF_FREE(server_host);
+    }
 
-        __GLFS_EXIT_FS;
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0);
-
 /* *
  * Used to free the arguments allocated by glfs_set_volfile_server()
  */
-void
-glfs_free_volfile_servers (cmd_args_t *cmd_args)
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
 {
-        server_cmdline_t *server = NULL;
-        server_cmdline_t *tmp = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
+out:
+    return;
+}
 
-        GF_VALIDATE_OR_GOTO (THIS->name, cmd_args, out);
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
+{
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
 
-        list_for_each_entry_safe (server, tmp, &cmd_args->volfile_servers,
-                                  list) {
-                list_del_init (&server->list);
-                GF_FREE (server->volfile_server);
-                GF_FREE (server->transport);
-                GF_FREE (server);
-        }
-        cmd_args->curr_server = NULL;
-out:
+    if (!&(cmd_args->xlator_options))
         return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
 int
-pub_glfs_setfsuid (uid_t fsuid)
+pub_glfs_setfsuid(uid_t fsuid)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsuid (&fsuid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
 int
-pub_glfs_setfsgid (gid_t fsgid)
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsgid (&fsgid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
 
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
+}
 
 int
-pub_glfs_setfsgroups (size_t size, const gid_t *list)
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsgroups(size, list);
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2);
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
+{
+    char *lease_id = NULL;
+    int ret = -1;
 
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
 
-struct glfs *
-pub_glfs_from_glfd (struct glfs_fd *glfd)
+int
+get_fop_attr_thrd_key(dict_t **fop_attr)
 {
-	return glfd->fs;
-}
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
 
 void
-glfs_fd_destroy (void *data)
+unset_fop_attr(dict_t **fop_attr)
 {
-        struct glfs_fd  *glfd = NULL;
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
 
-        if (!data)
-                return;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
+struct glfs *
+pub_glfs_from_glfd(struct glfs_fd *glfd)
+{
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
 
-        glfd = (struct glfs_fd *)data;
+    return glfd->fs;
+}
 
-        glfs_lock (glfd->fs);
-        {
-                list_del_init (&glfd->openfds);
-        }
-        glfs_unlock (glfd->fs);
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
+{
+    if (!glfd)
+        return;
 
-        if (glfd->fd) {
-                fd_unref (glfd->fd);
-                glfd->fd = NULL;
-        }
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
 
-        GF_FREE (glfd->readdirbuf);
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-        GF_FREE (glfd);
-}
+    GF_FREE(glfd->readdirbuf);
 
+    GF_FREE(glfd);
+}
 
 struct glfs_fd *
-glfs_fd_new (struct glfs *fs)
+glfs_fd_new(struct glfs *fs)
 {
-	struct glfs_fd  *glfd = NULL;
+    struct glfs_fd *glfd = NULL;
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
-	if (!glfd)
-		return NULL;
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
 
-	glfd->fs = fs;
+    glfd->fs = fs;
 
-	INIT_LIST_HEAD (&glfd->openfds);
+    INIT_LIST_HEAD(&glfd->openfds);
 
-        GF_REF_INIT (glfd, glfs_fd_destroy);
+    GF_REF_INIT(glfd, glfs_fd_destroy);
 
-	return glfd;
+    return glfd;
 }
 
-
 void
-glfs_fd_bind (struct glfs_fd *glfd)
+glfs_fd_bind(struct glfs_fd *glfd)
 {
-	struct glfs *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = glfd->fs;
+    fs = glfd->fs;
 
-	glfs_lock (fs);
-	{
-		list_add_tail (&glfd->openfds, &fs->openfds);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
 }
 
-
 static void *
-glfs_poller (void *data)
+glfs_poller(void *data)
 {
-	struct glfs  *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = data;
+    fs = data;
 
-	event_dispatch (fs->ctx->event_pool);
+    gf_event_dispatch(fs->ctx->event_pool);
 
-	return NULL;
+    return NULL;
 }
 
 static struct glfs *
-glfs_new_fs (const char *volname)
+glfs_new_fs(const char *volname)
 {
-        struct glfs     *fs             = NULL;
+    struct glfs *fs = NULL;
 
-        fs = CALLOC (1, sizeof (*fs));
-        if (!fs)
-                return NULL;
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
 
-        INIT_LIST_HEAD (&fs->openfds);
-        INIT_LIST_HEAD (&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
 
-        PTHREAD_MUTEX_INIT (&fs->mutex, NULL, fs->pthread_flags,
-                            GLFS_INIT_MUTEX, err);
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
 
-        PTHREAD_COND_INIT (&fs->cond, NULL, fs->pthread_flags,
-                           GLFS_INIT_COND, err);
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
 
-        PTHREAD_COND_INIT (&fs->child_down_cond, NULL, fs->pthread_flags,
-                           GLFS_INIT_COND_CHILD, err);
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
 
-        PTHREAD_MUTEX_INIT (&fs->upcall_list_mutex, NULL, fs->pthread_flags,
-                            GLFS_INIT_MUTEX_UPCALL, err);
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
 
-        fs->volname = strdup (volname);
-        if (!fs->volname)
-                goto err;
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
 
-        fs->pin_refcnt = 0;
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
 
-        return fs;
+    return fs;
 
 err:
-        glfs_free_from_ctx (fs);
-        return NULL;
+    glfs_free_from_ctx(fs);
+    return NULL;
 }
 
 extern xlator_t global_xlator;
@@ -699,768 +774,1033 @@ extern glusterfs_ctx_t *global_ctx;
 extern pthread_mutex_t global_ctx_mutex;
 
 static int
-glfs_init_global_ctx ()
+glfs_init_global_ctx()
 {
-        int              ret = 0;
-        glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
 
-        pthread_mutex_lock (&global_ctx_mutex);
-        {
-                if (global_xlator.ctx)
-                        goto unlock;
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
 
-                ctx = glusterfs_ctx_new ();
-                if (!ctx) {
-                        ret = -1;
-                        goto unlock;
-                }
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-                gf_log_globals_init (ctx, GF_LOG_NONE);
+        gf_log_globals_init(ctx, GF_LOG_NONE);
 
-                global_ctx = ctx;
-                global_xlator.ctx = global_ctx;
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
 
-                ret = glusterfs_ctx_defaults_init (ctx);
-                if (ret) {
-                        global_ctx = NULL;
-                        global_xlator.ctx = NULL;
-                        goto unlock;
-                }
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
         }
+    }
 unlock:
-        pthread_mutex_unlock (&global_ctx_mutex);
+    pthread_mutex_unlock(&global_ctx_mutex);
 
-        if (ret)
-                FREE (ctx);
+    if (ret)
+        FREE(ctx);
 
-        return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
 struct glfs *
-pub_glfs_new (const char *volname)
+pub_glfs_new(const char *volname)
 {
-	struct glfs     *fs             = NULL;
-	int              ret            = -1;
-	glusterfs_ctx_t *ctx            = NULL;
-        xlator_t        *old_THIS       = NULL;
-
-        if (!volname) {
-                errno = EINVAL;
-                return NULL;
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
         }
+        errno = EINVAL;
+        return NULL;
+    }
 
-        fs = glfs_new_fs (volname);
-        if (!fs)
-                return NULL;
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                goto fini;
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
 
-        /* first globals init, for gf_mem_acct_enable_set () */
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
 
-        ret = glusterfs_globals_init (ctx);
-        if (ret)
-                goto fini;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
 
-        old_THIS = THIS;
-        ret = glfs_init_global_ctx ();
-        if (ret)
-                goto fini;
+    /* first globals init, for gf_mem_acct_enable_set () */
 
-        /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto fini;
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
 
-        fs->ctx = ctx;
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
 
-        ret = glfs_set_logging (fs, "/dev/null", 0);
-        if (ret)
-                goto fini;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
-        if (!(fs->ctx->cmd_args.volfile_id))
-                goto fini;
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
 
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
         goto out;
 
-fini:
-         glfs_fini (fs);
-         fs = NULL;
-out:
-        if (old_THIS)
-                THIS = old_THIS;
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
+#endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
 
-        return fs;
-}
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0);
+    if (old_THIS)
+        THIS = old_THIS;
 
+    return fs;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
 struct glfs *
-priv_glfs_new_from_ctx (glusterfs_ctx_t *ctx)
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
 {
-        struct glfs    *fs = NULL;
+    struct glfs *fs = NULL;
 
-        if (!ctx)
-                goto out;
+    if (!ctx)
+        goto out;
 
-        fs = glfs_new_fs ("");
-        if (!fs)
-                goto out;
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
 
-        fs->ctx = ctx;
+    fs->ctx = ctx;
 
 out:
-        return fs;
+    return fs;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0);
-
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
 void
-priv_glfs_free_from_ctx (struct glfs *fs)
+priv_glfs_free_from_ctx(struct glfs *fs)
 {
-        upcall_entry       *u_list   = NULL;
-        upcall_entry       *tmp      = NULL;
-
-        if (!fs)
-                return;
-
-        /* cleanup upcall structures */
-        list_for_each_entry_safe (u_list, tmp,
-                                  &fs->upcall_list,
-                                  upcall_list) {
-                list_del_init (&u_list->upcall_list);
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE (u_list);
-        }
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
 
-        PTHREAD_MUTEX_DESTROY (&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
+    if (!fs)
+        return;
 
-        PTHREAD_COND_DESTROY (&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
 
-        PTHREAD_COND_DESTROY (&fs->child_down_cond, fs->pthread_flags,
-                              GLFS_INIT_COND_CHILD);
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
 
-        PTHREAD_MUTEX_DESTROY (&fs->upcall_list_mutex, fs->pthread_flags,
-                               GLFS_INIT_MUTEX_UPCALL);
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
 
-        FREE (fs->volname);
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
 
-        FREE (fs);
-}
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
+
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0);
+    FREE(fs->volname);
 
+    FREE(fs);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
 int
-pub_glfs_set_volfile (struct glfs *fs, const char *volfile)
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
 {
-	cmd_args_t  *cmd_args = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    if (vol_assigned(cmd_args))
+        return -1;
 
-	cmd_args->volfile = gf_strdup (volfile);
-        if (!cmd_args->volfile)
-                return -1;
-	return 0;
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
 int
-pub_glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
 {
-        int              ret     = -1;
-        char            *tmplog  = NULL;
-        glusterfs_ctx_t *old_ctx = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    int ret = -1;
+    char *tmplog = NULL;
 
-        old_ctx = THIS->ctx;
-        THIS->ctx = fs->ctx;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        if (!logfile) {
-                ret = gf_set_log_file_path (&fs->ctx->cmd_args, fs->ctx);
-                if (ret)
-                        goto out;
-                tmplog = fs->ctx->cmd_args.log_file;
-        } else {
-                tmplog = (char *)logfile;
-        }
-
-        /* finish log set parameters before init */
-        if (loglevel >= 0)
-                gf_log_set_loglevel (loglevel);
-
-        ret = gf_log_init (fs->ctx, tmplog, NULL);
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
         if (ret)
-                goto out;
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
+
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
 
-        ret = gf_log_inject_timer_event (fs->ctx);
-        if (ret)
-                goto out;
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
 
 out:
-        THIS->ctx = old_ctx;
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0);
-
-
 int
-glfs_init_wait (struct glfs *fs)
+glfs_init_wait(struct glfs *fs)
 {
-	int   ret = -1;
-
-	/* Always a top-down call, use glfs_lock() */
-	glfs_lock (fs);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond,
-					   &fs->mutex);
-		ret = fs->ret;
-		errno = fs->err;
-	}
-	glfs_unlock (fs);
-
-	return ret;
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
 void
-priv_glfs_init_done (struct glfs *fs, int ret)
+priv_glfs_init_done(struct glfs *fs, int ret)
 {
-	glfs_init_cbk init_cbk;
-
-	if (!fs) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL,
-			"fs is NULL");
-		goto out;
-	}
-
-	init_cbk = fs->init_cbk;
-
-	/* Always a bottom-up call, use mutex_lock() */
-	pthread_mutex_lock (&fs->mutex);
-	{
-		fs->init = 1;
-		fs->ret = ret;
-		fs->err = errno;
-
-		if (!init_cbk)
-			pthread_cond_broadcast (&fs->cond);
-	}
-	pthread_mutex_unlock (&fs->mutex);
-
-	if (init_cbk)
-		init_cbk (fs, ret);
-out:
-	return;
-}
+    glfs_init_cbk init_cbk;
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0);
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
 
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (init_cbk)
+        init_cbk(fs, ret);
+out:
+    return;
+}
 
 int
-glfs_init_common (struct glfs *fs)
+glfs_init_common(struct glfs *fs)
 {
-	int  ret = -1;
+    int ret = -1;
 
-	ret = create_master (fs);
-	if (ret)
-		return ret;
+    ret = create_master(fs);
+    if (ret)
+        return ret;
 
-	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
-	if (ret)
-		return ret;
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
 
-	ret = glfs_volumes_init (fs);
-	if (ret)
-		return ret;
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
 
-	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
-	return ret;
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
 }
 
-
 int
-glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
 {
-	int  ret = -1;
+    int ret = -1;
 
-	if (!fs || !fs->ctx) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_INVALID_ENTRY,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-	fs->init_cbk = cbk;
+    fs->init_cbk = cbk;
 
-	ret = glfs_init_common (fs);
+    ret = glfs_init_common(fs);
 
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
 int
-pub_glfs_init (struct glfs *fs)
+pub_glfs_init(struct glfs *fs)
 {
-	int  ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
+    DECLARE_OLD_THIS;
 
-	if (!fs || !fs->ctx) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_INVALID_ENTRY,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	ret = glfs_init_common (fs);
-	if (ret)
-		goto out;
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
 
-	ret = glfs_init_wait (fs);
+    ret = glfs_init_wait(fs);
 out:
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        /* Set the initial current working directory to "/" */
-        if (ret >= 0) {
-                ret = glfs_chdir (fs, "/");
-        }
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0);
-
 static int
-glusterfs_ctx_destroy (glusterfs_ctx_t *ctx)
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
 {
-        call_pool_t       *pool            = NULL;
-        int               ret              = 0;
-        glusterfs_graph_t *trav_graph      = NULL;
-        glusterfs_graph_t *tmp             = NULL;
-
-        if (ctx == NULL)
-                return 0;
-
-        if (ctx->cmd_args.curr_server)
-                glfs_free_volfile_servers (&ctx->cmd_args);
-
-        /* For all the graphs, crawl through the xlator_t structs and free
-         * all its members except for the mem_acct member,
-         * as GF_FREE will be referencing it.
-         */
-        list_for_each_entry_safe (trav_graph, tmp, &ctx->graphs, list) {
-                xlator_tree_free_members (trav_graph->first);
-        }
-
-        /* Free the memory pool */
-        if (ctx->stub_mem_pool)
-                mem_pool_destroy (ctx->stub_mem_pool);
-        if (ctx->dict_pool)
-                mem_pool_destroy (ctx->dict_pool);
-        if (ctx->dict_data_pool)
-                mem_pool_destroy (ctx->dict_data_pool);
-        if (ctx->dict_pair_pool)
-                mem_pool_destroy (ctx->dict_pair_pool);
-        if (ctx->logbuf_pool)
-                mem_pool_destroy (ctx->logbuf_pool);
-
-        pool = ctx->pool;
-        if (pool) {
-                if (pool->frame_mem_pool)
-                        mem_pool_destroy (pool->frame_mem_pool);
-                if (pool->stack_mem_pool)
-                        mem_pool_destroy (pool->stack_mem_pool);
-                LOCK_DESTROY (&pool->lock);
-                GF_FREE (pool);
-        }
-
-        /* Free the event pool */
-        ret = event_pool_destroy (ctx->event_pool);
-
-        /* Free the iobuf pool */
-        iobuf_pool_destroy (ctx->iobuf_pool);
-
-        GF_FREE (ctx->process_uuid);
-        GF_FREE (ctx->cmd_args.volfile_id);
-
-        LOCK_DESTROY (&ctx->lock);
-        pthread_mutex_destroy (&ctx->notify_lock);
-        pthread_cond_destroy (&ctx->notify_cond);
-
-        /* Free all the graph structs and its containing xlator_t structs
-         * from this point there should be no reference to GF_FREE/GF_CALLOC
-         * as it will try to access mem_acct and the below funtion would
-         * have freed the same.
-         */
-        list_for_each_entry_safe (trav_graph, tmp, &ctx->graphs, list) {
-                glusterfs_graph_destroy_residual (trav_graph);
-        }
-
-        FREE (ctx);
-
-        return ret;
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
 int
-pub_glfs_fini (struct glfs *fs)
+pub_glfs_fini(struct glfs *fs)
 {
-        int                ret = -1;
-        int                countdown = 100;
-        xlator_t           *subvol = NULL;
-        glusterfs_ctx_t    *ctx = NULL;
-        glusterfs_graph_t  *graph = NULL;
-        call_pool_t        *call_pool = NULL;
-        int                fs_init = 0;
-        int                err = -1;
-
-        DECLARE_OLD_THIS;
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        ctx = fs->ctx;
-        if (!ctx) {
-                goto free_fs;
-        }
-
-        if (ctx->mgmt) {
-                rpc_clnt_disable (ctx->mgmt);
-                ctx->mgmt = NULL;
-        }
-
-        call_pool = fs->ctx->pool;
-
-        while (countdown--) {
-                /* give some time for background frames to finish */
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        /* Do we need to increase countdown? */
-                        if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
-                                gf_msg_trace ("glfs", 0,
-                                        "call_pool_cnt - %"PRId64","
-                                        "pin_refcnt - %d",
-                                        call_pool->cnt, fs->pin_refcnt);
-
-                                ctx->cleanup_started = 1;
-                                pthread_mutex_unlock (&fs->mutex);
-                                break;
-                        }
-                }
-                pthread_mutex_unlock (&fs->mutex);
-                usleep (100000);
-        }
-
-        /* leaked frames may exist, we ignore */
-
-        /*We deem glfs_fini as successful if there are no pending frames in the call
-         *pool*/
-        ret = (call_pool->cnt == 0)? 0: -1;
-
-        pthread_mutex_lock (&fs->mutex);
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
         {
-                fs_init = fs->init;
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
         }
-        pthread_mutex_unlock (&fs->mutex);
-
-        if (fs_init != 0) {
-                subvol = glfs_active_subvol (fs);
-                if (subvol) {
-                        /* PARENT_DOWN within glfs_subvol_done() is issued
-                           only on graph switch (new graph should activiate
-                           and decrement the extra @winds count taken in
-                           glfs_graph_setup()
-
-                           Since we are explicitly destroying,
-                           PARENT_DOWN is necessary
-                        */
-                        xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                        /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
-                           in case of asynchrnous cleanup
-                        */
-                        graph = subvol->graph;
-                        err = pthread_mutex_lock (&fs->mutex);
-                        if (err != 0) {
-                                gf_msg ("glfs", GF_LOG_ERROR, err,
-                                        API_MSG_FSMUTEX_LOCK_FAILED,
-                                        "pthread lock on glfs mutex, "
-                                        "returned error: (%s)", strerror (err));
-                                goto fail;
-                        }
-                        /* check and wait for CHILD_DOWN for active subvol*/
-                        {
-                                while (graph->used) {
-                                        err = pthread_cond_wait (&fs->child_down_cond,
-                                                                 &fs->mutex);
-                                        if (err != 0)
-                                                gf_msg ("glfs", GF_LOG_INFO, err,
-                                                        API_MSG_COND_WAIT_FAILED,
-                                                        "%s cond wait failed %s",
-                                                        subvol->name,
-                                                        strerror (err));
-                                }
-                        }
-
-                        err = pthread_mutex_unlock (&fs->mutex);
-                        if (err != 0) {
-                                gf_msg ("glfs", GF_LOG_ERROR, err,
-                                        API_MSG_FSMUTEX_UNLOCK_FAILED,
-                                        "pthread unlock on glfs mutex, "
-                                        "returned error: (%s)", strerror (err));
-                                goto fail;
-                        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
+
+    /* leaked frames may exist, we ignore */
+
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
                 }
-                glfs_subvol_done (fs, subvol);
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
         }
-
-        ctx->cleanup_started = 1;
-
-        if (fs_init != 0) {
-                /* Destroy all the inode tables of all the graphs.
-                 * NOTE:
-                 * - inode objects should be destroyed before calling fini()
-                 *   of each xlator, as fini() and forget() of the xlators
-                 *   can share few common locks or data structures, calling
-                 *   fini first might destroy those required by forget
-                 *   ( eg: in quick-read)
-                 * - The call to inode_table_destroy_all is not required when
-                 *   the cleanup during graph switch is implemented to perform
-                 *   inode table destroy.
-                 */
-                inode_table_destroy_all (ctx);
-
-                /* Call fini() of all the xlators in the active graph
-                 * NOTE:
-                 * - xlator fini() should be called before destroying any of
-                 *   the threads. (eg: fini() in protocol-client uses timer
-                 *   thread) */
-                glusterfs_graph_deactivate (ctx->active);
-
-                /* Join the syncenv_processor threads and cleanup
-                 * syncenv resources*/
-                syncenv_destroy (ctx->env);
-
-                /* Join the poller thread */
-                if (event_dispatch_destroy (ctx->event_pool) < 0)
-                        ret = -1;
-        }
-
-        /* log infra has to be brought down before destroying
-         * timer registry, as logging uses timer infra
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
          */
-        if (gf_log_fini (ctx) != 0)
-                ret = -1;
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
 
-        /* Join the timer thread */
-        if (fs_init != 0) {
-                gf_timer_registry_destroy (ctx);
-        }
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
 
-        /* Destroy the context and the global pools */
-        if (glusterfs_ctx_destroy (ctx) != 0)
-                ret = -1;
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
 
 free_fs:
-        glfs_free_from_ctx (fs);
+    glfs_free_from_ctx(fs);
+
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
 
 fail:
-        if (!ret)
-                ret = err;
+    if (!ret)
+        ret = err;
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
 ssize_t
-pub_glfs_get_volfile (struct glfs *fs, void *buf, size_t len)
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
 {
-        ssize_t         res = -1;
+    ssize_t res = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        glfs_lock(fs);
-        if (len >= fs->oldvollen) {
-                gf_msg_trace ("glfs", 0, "copying %zu to %p", len, buf);
-                memcpy(buf,fs->oldvolfile,len);
-                res = len;
-        }
-        else {
-                res = len - fs->oldvollen;
-                gf_msg_trace ("glfs", 0, "buffer is %zd too short", -res);
-        }
-        glfs_unlock(fs);
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return res;
+    return res;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0);
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
 int
-pub_glfs_ipc (struct glfs *fs, int opcode, void *xd_in, void **xd_out)
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
 {
-	xlator_t        *subvol = NULL;
-        int             ret = -1;
+    xlator_t *subvol = NULL;
+    int ret = -1;
 
-	DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-	ret = syncop_ipc (subvol, opcode, (dict_t *) xd_in, (dict_t **) xd_out);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        glfs_subvol_done (fs, subvol);
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ipc, 4.0.0);
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
 int
-pub_glfs_ipc37 (struct glfs *fs, int opcode)
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
 {
-        return pub_glfs_ipc (fs, opcode, NULL, NULL);
-}
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-GFAPI_SYMVER_PUBLIC(glfs_ipc37, glfs_ipc, 3.7.0);
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
 void
-pub_glfs_free (void *ptr)
+pub_glfs_free(void *ptr)
 {
-        int mem_type = 0;
+    GLFS_FREE(ptr);
+}
 
-        mem_type = gf_get_mem_type (ptr);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
 
-        switch (mem_type) {
-        case glfs_mt_upcall_entry_t:
-        {
-                struct glfs_upcall *to_free = ptr;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
+{
+    return arg->reason;
+}
 
-                if (to_free->event)
-                        to_free->free_event (to_free->event);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
 
-                GF_FREE (ptr);
-                break;
-        }
-        default:
-                GF_FREE (ptr);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
+{
+    return arg->object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
+}
 
-struct glfs*
-pub_glfs_upcall_get_fs (struct glfs_upcall *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
 {
-        return arg->fs;
+    return arg->expire_time_attr;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16);
 
-enum glfs_upcall_reason
-pub_glfs_upcall_get_reason (struct glfs_upcall *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
 {
-        return arg->reason;
+    return arg->p_object;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16);
 
-void*
-pub_glfs_upcall_get_event (struct glfs_upcall *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
 {
-        return arg->event;
+    return &arg->p_buf;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16);
 
-struct glfs_object*
-pub_glfs_upcall_inode_get_object (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
 {
-        return arg->object;
+    return arg->oldp_object;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16);
 
-uint64_t
-pub_glfs_upcall_inode_get_flags (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
 {
-        return arg->flags;
+    return &arg->oldp_buf;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16);
 
-struct stat*
-pub_glfs_upcall_inode_get_stat (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
 {
-        return &arg->buf;
+    return arg->object;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16);
 
-uint64_t
-pub_glfs_upcall_inode_get_expire (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
 {
-        return arg->expire_time_attr;
+    return arg->lease_type;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16);
 
-struct glfs_object*
-pub_glfs_upcall_inode_get_pobject (struct glfs_upcall_inode *arg)
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
+int
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
 {
-        return arg->p_object;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
+        }
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16);
 
-struct stat*
-pub_glfs_upcall_inode_get_pstat (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
 {
-        return &arg->p_buf;
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
+        }
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
+        }
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_pstat, 3.7.16);
 
-struct glfs_object*
-pub_glfs_upcall_inode_get_oldpobject (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
 {
-        return arg->oldp_object;
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_oldpobject, 3.7.16);
 
-struct stat*
-pub_glfs_upcall_inode_get_oldpstat (struct glfs_upcall_inode *arg)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
+int
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
 {
-        return &arg->oldp_buf;
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
+        }
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
+        }
+    }
+
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
+
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
+
+    __GLFS_EXIT_FS;
+
+    return 0;
+err:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
 }
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_oldpstat, 3.7.16);
diff --git a/api/src/glfs.h b/api/src/glfs.h
index c81b7cc5d0b..279d11d58ee 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_H
 #define _GLFS_H
 
@@ -21,6 +20,17 @@
   both the library and the application.
 */
 
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
 #ifndef _FILE_OFFSET_BITS
 #define _FILE_OFFSET_BITS 64
 #endif
@@ -41,6 +51,40 @@
 #include <sys/cdefs.h>
 #include <dirent.h>
 #include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
 
 #if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
 #include <sys/acl.h>
@@ -51,19 +95,20 @@ typedef int acl_type_t;
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
 #endif
 
 #ifndef GF_DARWIN_HOST_OS
-#define GFAPI_PUBLIC(sym, ver) /**/
+#define GFAPI_PUBLIC(sym, ver)  /**/
 #define GFAPI_PRIVATE(sym, ver) /**/
 #else
 #define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
-#define GFAPI_PRIVATE(sym, ver) __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
 #endif
 
 __BEGIN_DECLS
@@ -92,8 +137,8 @@ typedef struct glfs glfs_t;
 
   @volname: Name of the volume. This identifies the server-side volume and
             the fetched volfile (equivalent of --volfile-id command line
-	    parameter to glusterfsd). When used with glfs_set_volfile() the
-	    @volname has no effect (except for appearing in log messages).
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
 
   RETURN VALUES
 
@@ -102,9 +147,8 @@ typedef struct glfs glfs_t;
 
 */
 
-glfs_t *glfs_new (const char *volname) __THROW
-        GFAPI_PUBLIC(glfs_new, 3.4.0);
-
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
 
 /*
   SYNOPSIS
@@ -133,9 +177,9 @@ glfs_t *glfs_new (const char *volname) __THROW
 
 */
 
-int glfs_set_volfile (glfs_t *fs, const char *volfile) __THROW
-        GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
-
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
 
 /*
   SYNOPSIS
@@ -186,12 +230,15 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile) __THROW
 
 */
 
-int glfs_set_volfile_server (glfs_t *fs, const char *transport,
-                             const char *host, int port) __THROW
-        GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
-int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
-                               const char *host, int port) __THROW
-        GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
+
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
 /*
   SYNOPSIS
 
@@ -221,9 +268,9 @@ int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
 
 */
 
-int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW
-        GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
-
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
 
 /*
   SYNOPSIS
@@ -249,9 +296,8 @@ int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW
 
 */
 
-int glfs_init (glfs_t *fs) __THROW
-        GFAPI_PUBLIC(glfs_init, 3.4.0);
-
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
 
 /*
   SYNOPSIS
@@ -283,8 +329,8 @@ int glfs_init (glfs_t *fs) __THROW
    0 : Success.
 */
 
-int glfs_fini (glfs_t *fs) __THROW
-        GFAPI_PUBLIC(glfs_fini, 3.4.0);
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
 
 /*
   SYNOPSIS
@@ -314,9 +360,9 @@ int glfs_fini (glfs_t *fs) __THROW
       <0: volfile length exceeds @len by N bytes (@buf unchanged)
 */
 
-ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
-
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
 
 /*
   SYNOPSIS
@@ -329,9 +375,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
        the management server (glusterd) to fetch volume uuid and stores it
        in the glusterfs_context linked to the glfs object fs which can be used
        in the subsequent calls. Later it parses that UUID to convert it from
-       cannonical string format into an opaque byte array and copy it into
-       the volid array. Incase if either of the input parameters, volid or size,
-       is NULL, number of bytes required to copy the volume UUID is returned.
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
 
   PARAMETERS
 
@@ -346,9 +392,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
         Others : length of the volume UUID stored.
 */
 
-int glfs_get_volumeid (struct glfs *fs, char *volid, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
-
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
 
 /*
  * FILE OPERATION
@@ -370,6 +416,119 @@ struct glfs_fd;
 typedef struct glfs_fd glfs_fd_t;
 
 /*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
  * PER THREAD IDENTITY MODIFIERS
  *
  * The following operations enable to set a per thread identity context
@@ -386,18 +545,27 @@ typedef struct glfs_fd glfs_fd_t;
  *    caller
  *  - The groups once set, need to be unset by setting the size to 0 (in which
  *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
  *  - Once a process for a thread of operation choses to set the IDs, all glfs
  *    calls made from that thread would default to the IDs set for the thread.
  *    As a result use these APIs with care and ensure that the set IDs are
  *    reverted to global process defaults as required.
  *
  */
-int glfs_setfsuid (uid_t fsuid) __THROW
-        GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
-int glfs_setfsgid (gid_t fsgid) __THROW
-        GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
-int glfs_setfsgroups (size_t size, const gid_t *list) __THROW
-        GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
 
 /*
   SYNOPSIS
@@ -424,9 +592,9 @@ int glfs_setfsgroups (size_t size, const gid_t *list) __THROW
 
  */
 
-glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW
-        GFAPI_PUBLIC(glfs_open, 3.4.0);
-
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
 
 /*
   SYNOPSIS
@@ -454,19 +622,20 @@ glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW
 
  */
 
-glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
-		       mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_creat, 3.4.0);
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
 
-int glfs_close (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_close, 3.4.0);
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
 
-glfs_t *glfs_from_glfd (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
 
-int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
-			    const char *value) __THROW
-        GFAPI_PUBLIC(glfs_set_xlator_options, 3.4.0);
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
 
 /*
 
@@ -489,292 +658,828 @@ int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
   time of issuing the async IO call. This can be used by the
   caller to differentiate different instances of the async requests
   in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
 */
 
-typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
 
 // glfs_{read,write}[_async]
 
-ssize_t glfs_read (glfs_fd_t *fd, void *buf,
-                   size_t count, int flags) __THROW
-        GFAPI_PUBLIC(glfs_read, 3.4.0);
-ssize_t glfs_write (glfs_fd_t *fd, const void *buf,
-                    size_t count, int flags) __THROW
-        GFAPI_PUBLIC(glfs_write, 3.4.0);
-int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
-		     glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_read_async, 3.4.0);
-int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
-		      glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_write_async, 3.4.0);
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
 
 // glfs_{read,write}v[_async]
 
-ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		    int flags) __THROW
-        GFAPI_PUBLIC(glfs_readv, 3.4.0);
-ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     int flags) __THROW
-        GFAPI_PUBLIC(glfs_writev, 3.4.0);
-int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		      int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_readv_async, 3.4.0);
-int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_writev_async, 3.4.0);
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
 
 // glfs_p{read,write}[_async]
 
-ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		    int flags) __THROW
-        GFAPI_PUBLIC(glfs_pread, 3.4.0);
-ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
-		     off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_pwrite, 3.4.0);
-int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		      int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pread_async, 3.4.0);
-int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
-		       int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pwrite_async, 3.4.0);
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
 
 // glfs_p{read,write}v[_async]
 
-ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_preadv, 3.4.0);
-ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		      off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
-int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov,
-                       int count, off_t offset, int flags,
-                       glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_preadv_async, 3.4.0);
-int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov,
-                        int count, off_t offset, int flags,
-                        glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pwritev_async, 3.4.0);
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
+
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
+
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
 
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
 
-off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence) __THROW
-        GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
 
-int glfs_truncate (glfs_t *fs, const char *path, off_t length) __THROW
-        GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
 
-int glfs_ftruncate (glfs_fd_t *fd, off_t length) __THROW
-        GFAPI_PUBLIC(glfs_ftruncate, 3.4.0);
-int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
-			  void *data) __THROW
-        GFAPI_PUBLIC(glfs_ftruncate_async, 3.4.0);
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
 
-int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_lstat, 3.4.0);
-int glfs_stat (glfs_t *fs, const char *path, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_stat, 3.4.0);
-int glfs_fstat (glfs_fd_t *fd, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
 
-int glfs_fsync (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fsync, 3.4.0);
-int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_fsync_async, 3.4.0);
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
 
-int glfs_fdatasync (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fdatasync, 3.4.0);
-int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_fdatasync_async, 3.4.0);
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
 
-int glfs_access (glfs_t *fs, const char *path, int mode) __THROW
-        GFAPI_PUBLIC(glfs_access, 3.4.0);
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
 
-int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
 
-int glfs_readlink (glfs_t *fs, const char *path,
-                   char *buf, size_t bufsiz) __THROW
-        GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
 
-int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
-        GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
 
-int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
 
-int glfs_unlink (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
 
-int glfs_rmdir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
 
-int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_rename, 3.4.0);
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
 
-int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_link, 3.4.0);
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
 
-glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
+
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
+
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
+
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
 
 /*
- * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
- * but the interface has ambiguity about the size of @dirent to be allocated
- * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
- * all known systems which are tested againt glusterfs/gfapi, but may be
- * insufficient in the future.
+ * @cmd and @flock are as specified in man fcntl(2).
  */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_file_lock: Request extended byte range lock on a file
+
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
 
-int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
-		    struct dirent **result) __THROW
-        GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+  PARAMETERS
+
+  @fd: File descriptor
+
+  @cmd: As specified in man fcntl(2).
+
+  @flock: As specified in man fcntl(2).
+
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
+
+  RETURN VALUES
 
-int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
-			struct dirent **result) __THROW
-        GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
+
+ */
+
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
+
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
+
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
 
 /*
- * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
- * when called on the same directory handle. However they ARE thread safe
- * AND re-entrant when called on different directory handles (which may be
- * referring to the same directory too.)
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
  */
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
 
-struct dirent *glfs_readdir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
+
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
 
-struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
+
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
+ */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
 
-long glfs_telldir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
 
-void glfs_seekdir (glfs_fd_t *fd, long offset) __THROW
-        GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
+
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
+
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
+
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
+
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
+
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
 
-int glfs_closedir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
 
-int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf) __THROW
-        GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
 
-int glfs_chmod (glfs_t *fs, const char *path, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
 
-int glfs_fchmod (glfs_fd_t *fd, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+/*
+  SYNOPSIS
 
-int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_chown, 3.4.0);
+  glfs_lease: Takes a lease on a file.
 
-int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_lchown, 3.4.0);
+  DESCRIPTION
 
-int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_fchown, 3.4.0);
+  This function takes lease on an open file.
 
-int glfs_utimens (glfs_t *fs, const char *path,
-                  const struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_utimens, 3.4.0);
+  PARAMETERS
 
-int glfs_lutimens (glfs_t *fs, const char *path,
-                   const struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
 
-int glfs_futimens (glfs_fd_t *fd, const struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_futimens, 3.4.0);
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
 
-ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
-		       void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
 
-ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
-			void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
+      @lease.lease_id - A unique identification of lease, 128bits.
 
-ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
-			void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
 
-ssize_t glfs_listxattr (glfs_t *fs, const char *path,
-                        void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
 
-ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
-			 size_t size) __THROW
-        GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
 
-ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
 
-int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
-		   const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
+/*
+  SYNOPSIS
 
-int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
-		    const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
 
-int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
-		    const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
+  DESCRIPTION
 
-int glfs_removexattr (glfs_t *fs, const char *path, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
+  The functions are used to set attributes on the file.
 
-int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
+  PARAMETERS
 
-int glfs_fremovexattr (glfs_fd_t *fd, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
+  @glfs_fsetattr
 
-int glfs_fallocate(glfs_fd_t *fd, int keep_size,
-                   off_t offset, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
 
-int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_discard, 3.5.0);
+  @glfs_setattr
 
+         @fs: File object.
 
-int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
-			glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_discard_async, 3.5.0);
+         @path: The path of the file that is being operated on.
 
-int glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
-        GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+         @follow: Flag used to resolve symlink.
 
-int glfs_zerofill_async (glfs_fd_t *fd, off_t length, off_t len,
-                        glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_zerofill_async, 3.5.0);
 
-char *glfs_getcwd (glfs_t *fs, char *buf, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+  @stat: Struct that has information about the file.
 
-int glfs_chdir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
 
-int glfs_fchdir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
 
-char *glfs_realpath34 (glfs_t *fs, const char *path, char *resolved_path) __THROW
-        GFAPI_PUBLIC(glfs_realpath, 3.4.0);
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
 
-char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path) __THROW
-        GFAPI_PUBLIC(glfs_realpath, 3.7.17);
-/*
- * @cmd and @flock are as specified in man fcntl(2).
  */
-int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
-        GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
 
-glfs_fd_t *glfs_dup (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_dup, 3.4.0);
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
+
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
 
 /*
- * Nobody needs this call at all yet except for the test script.
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
  */
-int glfs_ipc (glfs_fd_t *fd, int cmd,  void *xd_in, void **xd_out) __THROW
-        GFAPI_PUBLIC(glfs_ipc, 4.0.0);
 
-__END_DECLS
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
 
+__END_DECLS
 #endif /* !_GLFS_H */
diff --git a/autogen.sh b/autogen.sh
index 73019f29e4c..c8cdc3f89fa 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -85,11 +85,7 @@ $TOOL --automake --copy --force
 echo Running ${AUTOCONF}...
 $AUTOCONF
 echo Running ${AUTOMAKE}...
-$AUTOMAKE --add-missing --force-missing --copy --foreign
-
-# Run autogen in the argp-standalone sub-directory
-echo "Running autogen.sh in argp-standalone ..."
-( cd contrib/argp-standalone;./autogen.sh )
+$AUTOMAKE --add-missing --force-missing --copy
 
 # Instruct user on next steps
 echo
diff --git a/build-aux/checkpatch.pl b/build-aux/checkpatch.pl
index 205567307b1..17ae4e4d579 100755
--- a/build-aux/checkpatch.pl
+++ b/build-aux/checkpatch.pl
@@ -3099,7 +3099,7 @@ sub process {
             if (ERROR("SPACING",
                       "space required before the open brace '{'\n" . $herecurr) &&
                 $fix) {
-                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\))){/$1 {/;
+                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\)))\{/$1 {/;
             }
         }
 
diff --git a/build-aux/pkg-version b/build-aux/pkg-version
index 83d4a5f9136..17ceab70c03 100755
--- a/build-aux/pkg-version
+++ b/build-aux/pkg-version
@@ -1,9 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 
 # To override version/release from git,
 # create VERSION file containing text with version/release
 # eg. v3.4.0-1
-PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*"`
+
+# One thing to note, If one does 'git clone --depth N glusterfs.git',
+# the git describe command doesn't work. Hence you notice below that
+# we have added timestamp as version (YYYY.MM.DD) and release (HH.mmss)
+PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*" 2>/dev/null`
 
 get_version()
 {
@@ -18,7 +22,11 @@ get_version()
       sub(/^v/,"") ; print $1
     }'
 
-    echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].'
+    version=$(echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].')
+    if [ "x${version}" == "x" ] ; then
+       version=$(date +%Y.%m.%d | tr -d '\n')
+    fi
+    echo $version | tr -d '\n'
 }
 
 get_release()
@@ -37,7 +45,11 @@ get_release()
       else if (NF == 4) print $2, $3, "git" substr($4, 2)
     }'
 
-    echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].'
+    release=$(echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].')
+    if [ "x${release}" == "x" ] ; then
+       release=$(date +%H.%M%S | tr -d '\n')
+    fi
+    echo $release | tr -d '\n'
 }
 
 if test "x$1" = "x--full"; then
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index f5b8d00e673..16063f27c7f 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -5,6 +5,7 @@ gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c cli-cmd-global.c \
 	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c cli-cmd-snapshot.c
 
 gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) \
+		$(top_builddir)/libglusterd/src/libglusterd.la \
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
 		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 		$(XML_LIBS)
@@ -13,21 +14,24 @@ gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h cli-quotad-client.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
-	-I$(top_srcdir)/rpc/xdr/src\
-	-I$(top_builddir)/rpc/xdr/src\
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/libglusterd/src \
+	-I$(top_builddir)/rpc/xdr/src \
 	-DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
-	$(XML_CPPFLAGS)
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DGLFSHEAL_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE)
 
-AM_CFLAGS = -Wall $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
 
+$(top_builddir)/libglusterd/src/libglusterd.la:
+	$(MAKE) -C $(top_builddir)/libglusterd/src/ all
+
 install-data-hook:
 	$(mkdir_p) $(DESTDIR)$(localstatedir)/run/gluster
diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
index f4544da367c..2c9a5f01bb1 100644
--- a/cli/src/cli-cmd-global.c
+++ b/cli/src/cli-cmd-global.c
@@ -23,168 +23,173 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
-#include "common-utils.h"
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
 
 int
-cli_cmd_global_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount);
-int cli_cmd_ganesha_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                                         const char **words, int wordcount);
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
+
 int
-cli_cmd_get_state_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount);
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount);
 
 struct cli_cmd global_cmds[] = {
-        { "global help",
-           cli_cmd_global_help_cbk,
-           "list global commands",
-        },
-        { "nfs-ganesha {enable| disable} ",
-           cli_cmd_ganesha_cbk,
-          "Enable/disable NFS-Ganesha support",
-        },
-        { "get-state [<daemon>] [odir </path/to/output/dir/>] [file <filename>]",
-          cli_cmd_get_state_cbk,
-          "Get local state representation of mentioned daemon",
-        },
-        {NULL,  NULL,  NULL}
-};
+    {
+        "global help",
+        cli_cmd_global_help_cbk,
+        "list global commands",
+    },
+    {
+        "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+        "[file <filename>]] [detail|volumeoptions]",
+        cli_cmd_get_state_cbk,
+        "Get local state representation of mentioned daemon",
+    },
+    {
+        "nfs-ganesha {enable| disable} ",
+        cli_cmd_ganesha_cbk,
+        "Enable/disable NFS-Ganesha support",
+    },
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_global_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *global_cmd = NULL;
-        int                   count     = 0;
-
-        cmd = GF_CALLOC (1, sizeof (global_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, global_cmds, sizeof (global_cmds));
-        count = (sizeof (global_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
-
-        for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
-                if (_gf_false == global_cmd->disable)
-                        cli_out ("%s - %s", global_cmd->pattern,
-                                 global_cmd->desc);
-
-        GF_FREE (cmd);
-        return 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *global_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(global_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, global_cmds, sizeof(global_cmds));
+    count = (sizeof(global_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster global commands");
+    cli_out("========================\n");
+    for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
+        if (_gf_false == global_cmd->disable)
+            cli_out("%s - %s", global_cmd->pattern, global_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_global_register (struct cli_state *state)
+cli_cmd_global_register(struct cli_state *state)
 {
-        int ret = 0;
-        struct cli_cmd *cmd =  NULL;
-        for (cmd = global_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd);
-                        if (ret)
-                                goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+    for (cmd = global_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
-
+    return ret;
 }
 
-int cli_cmd_ganesha_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 
 {
-         int                     sent        =   0;
-         int                     parse_error =   0;
-         int                     ret         =  -1;
-         rpc_clnt_procedure_t    *proc       =  NULL;
-         call_frame_t            *frame      =  NULL;
-         dict_t                  *options    =  NULL;
-         cli_local_t             *local      =  NULL;
-         char                    *op_errstr  =  NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_ganesha_parse (state, words, wordcount,
-                                     &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Setting global option failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Setting global option failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_get_state_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     sent        =   0;
-        int                     parse_error =   0;
-        int                     ret         =  -1;
-        rpc_clnt_procedure_t    *proc       =  NULL;
-        call_frame_t            *frame      =  NULL;
-        dict_t                  *options    =  NULL;
-        cli_local_t             *local      =  NULL;
-        char                    *op_errstr  =  NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_get_state_parse (state, words, wordcount, &options,
-                                       &op_errstr);
-
-        if (ret) {
-                if (op_errstr) {
-                        cli_err ("%s", op_errstr);
-                        cli_usage_out (word->pattern);
-                        GF_FREE (op_errstr);
-                } else
-                        cli_usage_out (word->pattern);
-
-                parse_error = 1;
-                goto out;
-        }
-
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    ret = cli_cmd_get_state_parse(state, words, wordcount, &options,
+                                  &op_errstr);
+
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            cli_usage_out(word->pattern);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting daemon state failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting daemon state failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
-
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 9f8c159f073..e961d88da86 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -18,11 +18,9 @@
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 extern struct cli_cmd volume_cmds[];
+extern struct cli_cmd bitrot_cmds[];
+extern struct cli_cmd quota_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
@@ -32,63 +30,88 @@ extern struct cli_cmd global_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
-cli_cmd_quit_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quit_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                 const char **words, int wordcount)
 {
-        exit (0);
+    exit(0);
 }
 
-int
-cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+static gf_boolean_t
+cli_is_help_command(const char *pattern)
 {
-        struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds, snapshot_cmds,
-                                       global_cmds, NULL};
-        struct cli_cmd        *cmd_ind = NULL;
-        int                   i = 0;
-
-         /* cli_system_cmds commands for internal usage
-           they are not exposed
-         */
-        for (i=0; cmd[i]!=NULL; i++)
-                for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++)
-                        if (_gf_false == cmd_ind->disable)
-                                cli_out ("%s - %s", cmd_ind->pattern,
-                                         cmd_ind->desc);
-
-        return 0;
+    /* FixFixFix
+     * This is not the best way to determine whether
+     * this is a help command
+     */
+    if (strstr(pattern, "help"))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-struct cli_cmd cli_misc_cmds[] = {
-        { "quit",
-          cli_cmd_quit_cbk,
-          "quit"},
+int
+cli_cmd_display_help(struct cli_state *state, struct cli_cmd_word *in_word,
+                     const char **words, int wordcount)
+{
+    static struct cli_cmd *cmd[] = {
+        cli_misc_cmds, cli_probe_cmds, volume_cmds, bitrot_cmds,
+        quota_cmds,    snapshot_cmds,  global_cmds, NULL};
+    struct cli_cmd *cmd_ind = NULL;
+    int i = 0;
+    gf_boolean_t list_all = _gf_false;
+
+    /* cli_system_cmds commands for internal usage
+      they are not exposed
+    */
+
+    /* If "help all" */
+    if (wordcount == 2)
+        list_all = _gf_true;
+
+    for (i = 0; cmd[i] != NULL; i++) {
+        for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++) {
+            if ((_gf_false == cmd_ind->disable) &&
+                cli_is_help_command(cmd_ind->pattern)) {
+                if (list_all && (cmd_ind->cbk)) {
+                    cmd_ind->cbk(state, in_word, words, wordcount);
+                } else {
+                    cli_out(" %-25s- %s", cmd_ind->pattern, cmd_ind->desc);
+                }
+            }
+        }
+    }
 
-        { "help",
-           cli_cmd_display_help,
-           "display command options"},
+    cli_out("\n");
+    return 0;
+}
+
+struct cli_cmd cli_help_cmds[] = {
+    {"help [all]", cli_cmd_display_help, "display help for command classes"},
 
-        { "exit",
-          cli_cmd_quit_cbk,
-          "exit"},
+    {NULL, NULL, NULL}};
 
-        { NULL, NULL, NULL }
-};
+struct cli_cmd cli_misc_cmds[] = {{"quit", cli_cmd_quit_cbk, "quit"},
+                                  {"exit", cli_cmd_quit_cbk, "exit"},
 
+                                  {NULL, NULL, NULL}};
 
 int
-cli_cmd_misc_register (struct cli_state *state)
+cli_cmd_misc_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = cli_help_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index d6ac6fd7077..34620b4a31b 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -13,2642 +13,2756 @@
 #include <stdint.h>
 #include <pthread.h>
 #include <fnmatch.h>
+#include <time.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
 
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
 #define MAX_SNAP_DESCRIPTION_LEN 1024
 
-struct snap_config_opt_vals_ snap_confopt_vals[] = {
-        {.op_name        = "snap-max-hard-limit",
-         .question       = "Changing snapshot-max-hard-limit "
-                           "will limit the creation of new snapshots "
-                           "if they exceed the new limit.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = "snap-max-soft-limit",
-         .question       = "If Auto-delete is enabled, snap-max-soft-limit will"
-                           " trigger deletion of oldest snapshot, on the "
-                           "creation of new snapshot, when the "
-                           "snap-max-soft-limit is reached.\n"
-                           "Do you want to change the snap-max-soft-limit?"
-        },
-        {.op_name        = "both",
-        .question        = "Changing snapshot-max-hard-limit "
-                           "will limit the creation of new snapshots "
-                           "if they exceed the new snapshot-max-hard-limit.\n"
-                           "If Auto-delete is enabled, snap-max-soft-limit will"
-                           " trigger deletion of oldest snapshot, on the "
-                           "creation of new snapshot, when the "
-                           "snap-max-soft-limit is reached.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = NULL,
-        }
-};
+static struct snap_config_opt_vals_ snap_confopt_vals[] = {
+    {.op_name = "snap-max-hard-limit",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new limit.\n"
+                 "Do you want to continue?"},
+    {.op_name = "snap-max-soft-limit",
+     .question = "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to change the snap-max-soft-limit?"},
+    {.op_name = "both",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new snapshot-max-hard-limit.\n"
+                 "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to continue?"},
+    {
+        .op_name = NULL,
+    }};
 
 enum cli_snap_config_set_types {
-        GF_SNAP_CONFIG_SET_HARD = 0,
-        GF_SNAP_CONFIG_SET_SOFT = 1,
-        GF_SNAP_CONFIG_SET_BOTH = 2,
+    GF_SNAP_CONFIG_SET_HARD = 0,
+    GF_SNAP_CONFIG_SET_SOFT = 1,
+    GF_SNAP_CONFIG_SET_BOTH = 2,
 };
 typedef enum cli_snap_config_set_types cli_snap_config_set_types;
 
+typedef struct _cli_brick {
+    struct list_head list;
+    const char *name;
+    int32_t len;
+} cli_brick_t;
+
 int
-cli_cmd_validate_volume (char *volname);
+cli_cmd_validate_volume(char *volname);
 
 static const char *
-id_sel (void *wcon)
+id_sel(void *wcon)
 {
-        return (const char *)wcon;
+    return (const char *)wcon;
 }
 
 static char *
-str_getunamb (const char *tok, char **opwords)
+str_getunamb(const char *tok, char **opwords)
 {
-        return (char *)cli_getunamb (tok, (void **)opwords, id_sel);
+    return (char *)cli_getunamb(tok, (void **)opwords, id_sel);
 }
 
 int32_t
-cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
-                      char **bricks, int *brick_count)
+cli_cmd_ta_brick_parse(const char **words, int wordcount, char **ta_brick)
 {
-        int                    ret = 0;
-        char             *tmp_list = NULL;
-        char    brick_list[120000] = {0,};
-        char                *space = " ";
-        char            *delimiter = NULL;
-        char            *host_name = NULL;
-        char        *free_list_ptr = NULL;
-        char               *tmpptr = NULL;
-        int                      j = 0;
-        int         brick_list_len = 0;
-        char             *tmp_host = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (wordcount);
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_index > 0);
-        GF_ASSERT (brick_index < wordcount);
-
-        strncpy (brick_list, space, strlen (space));
-        brick_list_len++;
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr (words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        cli_err ("Total brick list is larger than a request. "
-                                 "Can take (brick_count %d)", *brick_count);
-                        ret = -1;
-                        goto out;
-                }
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *delimiter = NULL;
+    cli_brick_t *brick = NULL;
+    int ret = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+
+    if (validate_brick_name((char *)words[wordcount - 1])) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words[wordcount - 1]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words[wordcount - 1], ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret)
+            goto out;
+    }
 
-                tmp_host = gf_strdup ((char *)words[brick_index]);
-                if (!tmp_host) {
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                        ret = -1;
-                        goto out;
-                }
-                get_host_name (tmp_host, &host_name);
-                if (!host_name) {
-                        ret = -1;
-                        gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                               "memory");
-                        goto out;
-                }
+    tmp_host = gf_strdup((char *)words[wordcount - 1]);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to retrieve "
+               "hostname");
+        goto out;
+    }
+
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+    }
+
+    brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+    if (brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-                if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1") &&
-                      strncmp (host_name, "0.", 2))) {
-                        cli_err ("Please provide a valid hostname/ip other "
-                                 "than localhost, 127.0.0.1 or loopback "
-                                 "address (0.0.0.0 to 0.255.255.255).");
-                        ret = -1;
-                        GF_FREE (tmp_host);
-                        goto out;
-                }
-                if (!valid_internet_address (host_name, _gf_false)) {
-                        cli_err ("internet address '%s' does not conform to "
-                                 "standards", host_name);
-                }
-                GF_FREE (tmp_host);
-                tmp_list = gf_strdup (brick_list + 1);
-                if (free_list_ptr) {
-                        GF_FREE (free_list_ptr);
-                        free_list_ptr = NULL;
-                }
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(j < *brick_count) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_err ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_len += (strlen (words[brick_index]) + 1);
-                ++(*brick_count);
-                ++brick_index;
-        }
+    brick->name = words[wordcount - 1];
+    brick->len = strlen(words[wordcount - 1]);
+    *ta_brick = GF_MALLOC(brick->len + 3, gf_common_mt_char);
+    if (*ta_brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-        *bricks = gf_strdup (brick_list);
-        if (!*bricks)
-                ret = -1;
+    strcat(*ta_brick, " ");
+    strcat(*ta_brick, brick->name);
+    strcat(*ta_brick, " ");
 out:
-        GF_FREE (free_list_ptr);
-        return ret;
+    if (tmp_host) {
+        GF_FREE(tmp_host);
+        tmp_host = NULL;
+    }
+    if (brick) {
+        GF_FREE(brick);
+        brick = NULL;
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_create_disperse_check (struct cli_state *state, int *disperse,
-                               int *redundancy, int *data, int count)
+cli_cmd_bricks_parse(const char **words, int wordcount, int brick_index,
+                     char **bricks, int *brick_count)
 {
-        int i = 0;
-        int tmp = 0;
-        gf_answer_t answer = GF_ANSWER_NO;
-        char question[128];
-
-        const char *question1 = "There isn't an optimal redundancy value "
-                                "for this configuration. Do you want to "
-                                "create the volume with redundancy 1 ?";
-
-        const char *question2 = "The optimal redundancy for this "
-                                "configuration is %d. Do you want to create "
-                                "the volume with this value ?";
-
-        const char *question3 = "This configuration is not optimal on most "
-                                "workloads. Do you want to use it ?";
-
-        const char *question4 = "Redundancy for this configuration is %d. "
-                                "Do you want to create "
-                                "the volume with this value ?";
-
-        if (*data > 0) {
-                if (*disperse > 0 && *redundancy > 0) {
-                        if (*disperse != (*data + *redundancy)) {
-                                cli_err ("Disperse count(%d) should be equal "
-                                         "to sum of disperse-data count(%d) and "
-                                         "redundancy count(%d)", *disperse,
-                                         *data, *redundancy);
-                                return -1;
-                        }
-                } else if (*redundancy > 0) {
-                        *disperse = *data + *redundancy;
-                } else if (*disperse > 0) {
-                        *redundancy = *disperse - *data;
-                } else {
-                        if ((count - *data) >= *data) {
-                                cli_err ("Please provide redundancy count "
-                                         "along with disperse-data count");
-                                return -1;
-                        } else {
-                                sprintf (question, question4, count - *data);
-                                answer = cli_cmd_get_confirmation (state,
-                                                                   question);
-                                if (answer == GF_ANSWER_NO)
-                                        return -1;
-                                *redundancy = count - *data;
-                                *disperse = count;
-                        }
-                }
+    int ret = 0;
+    char *delimiter = NULL;
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *bricks_str = NULL;
+    int len = 0;
+    int brick_list_len = 1; /* For initial space */
+    struct list_head brick_list = {
+        0,
+    };
+    cli_brick_t *brick = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+    GF_ASSERT(bricks);
+    GF_ASSERT(brick_index > 0);
+    GF_ASSERT(brick_index < wordcount);
+
+    INIT_LIST_HEAD(&brick_list);
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "Wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
 
-        if (*disperse <= 0) {
-                if (count < 3) {
-                        cli_err ("number of bricks must be greater "
-                                 "than 2");
+        tmp_host = gf_strdup((char *)words[brick_index]);
+        if (!tmp_host) {
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            ret = -1;
+            goto out;
+        }
+        get_host_name(tmp_host, &host_name);
+        if (!host_name) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to allocate "
+                   "memory");
+            GF_FREE(tmp_host);
+            goto out;
+        }
+
+        if (!(strcmp(host_name, "localhost") &&
+              strcmp(host_name, "127.0.0.1") && strncmp(host_name, "0.", 2))) {
+            cli_err(
+                "Please provide a valid hostname/ip other "
+                "than localhost, 127.0.0.1 or loopback "
+                "address (0.0.0.0 to 0.255.255.255).");
+            ret = -1;
+            GF_FREE(tmp_host);
+            goto out;
+        }
+        if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+            cli_err(
+                "internet address '%s' does not conform to "
+                "standards",
+                host_name);
+        }
+        GF_FREE(tmp_host);
+        list_for_each_entry(brick, &brick_list, list)
+        {
+            if (strcmp(brick->name, words[brick_index]) == 0) {
+                ret = -1;
+                cli_err("Found duplicate exports %s", words[brick_index]);
+                goto out;
+            }
+        }
 
-                        return -1;
-                }
-                *disperse = count;
+        brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+        if (brick == NULL) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            goto out;
         }
+        len = strlen(words[brick_index]);
+        brick->name = words[brick_index];
+        brick->len = len;
+        list_add_tail(&brick->list, &brick_list);
 
-        if (*redundancy == -1) {
-                tmp = *disperse - 1;
-                for (i = tmp / 2;
-                     (i > 0) && ((tmp & -tmp) != tmp);
-                     i--, tmp--);
+        brick_list_len += len + 1; /* Brick name + space */
+        ++(*brick_count);
+        ++brick_index;
+    }
 
-                if (i == 0) {
-                        answer = cli_cmd_get_confirmation(state, question1);
-                        if (answer == GF_ANSWER_NO)
-                                return -1;
+    /* If brick count is not valid exit here */
+    if (!*brick_count) {
+        cli_err("No bricks specified");
+        ret = -1;
+        goto out;
+    }
 
-                        *redundancy = 1;
-                }
-                else
-                {
-                        *redundancy = *disperse - tmp;
-                        if (*redundancy > 1) {
-                                sprintf(question, question2, *redundancy);
-                                answer = cli_cmd_get_confirmation(state,
-                                                                  question);
-                                if (answer == GF_ANSWER_NO)
-                                        return -1;
-                        }
-                }
+    brick_list_len++; /* For terminating null char */
 
-                tmp = 0;
+    bricks_str = GF_MALLOC(brick_list_len, gf_common_mt_char);
+    if (bricks_str == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+    *bricks = bricks_str;
+    *bricks_str = ' ';
+    bricks_str++;
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        memcpy(bricks_str, brick->name, brick->len);
+        bricks_str[brick->len] = ' ';
+        bricks_str += brick->len + 1;
+        GF_FREE(brick);
+    }
+    *bricks_str = 0;
+
+out:
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        GF_FREE(brick);
+    }
+
+    return ret;
+}
+
+int32_t
+cli_cmd_create_disperse_check(struct cli_state *state, int *disperse,
+                              int *redundancy, int *data, int count)
+{
+    int i = 0;
+    int tmp = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char question[128];
+
+    const char *question1 =
+        "There isn't an optimal redundancy value "
+        "for this configuration. Do you want to "
+        "create the volume with redundancy 1 ?";
+
+    const char *question2 =
+        "The optimal redundancy for this "
+        "configuration is %d. Do you want to create "
+        "the volume with this value ?";
+
+    const char *question3 =
+        "This configuration is not optimal on most "
+        "workloads. Do you want to use it ?";
+
+    const char *question4 =
+        "Redundancy for this configuration is %d. "
+        "Do you want to create "
+        "the volume with this value ?";
+
+    if (*data > 0) {
+        if (*disperse > 0 && *redundancy > 0) {
+            if (*disperse != (*data + *redundancy)) {
+                cli_err(
+                    "Disperse count(%d) should be equal "
+                    "to sum of disperse-data count(%d) and "
+                    "redundancy count(%d)",
+                    *disperse, *data, *redundancy);
+                return -1;
+            }
+        } else if (*redundancy > 0) {
+            *disperse = *data + *redundancy;
+        } else if (*disperse > 0) {
+            *redundancy = *disperse - *data;
         } else {
-                tmp = *disperse - *redundancy;
+            if ((count - *data) >= *data) {
+                cli_err(
+                    "Please provide redundancy count "
+                    "along with disperse-data count");
+                return -1;
+            } else {
+                sprintf(question, question4, count - *data);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+                *redundancy = count - *data;
+                *disperse = count;
+            }
         }
+    }
 
-        if (*redundancy > (*disperse - 1) / 2) {
-                cli_err ("redundancy must be less than %d for a "
-                         "disperse %d volume",
-                         (*disperse + 1) / 2, *disperse);
+    if (*disperse <= 0) {
+        if (count < 3) {
+            cli_err(
+                "number of bricks must be greater "
+                "than 2");
 
-                return -1;
+            return -1;
         }
+        *disperse = count;
+    }
+
+    if (*redundancy == -1) {
+        tmp = *disperse - 1;
+        for (i = tmp / 2; (i > 0) && ((tmp & -tmp) != tmp); i--, tmp--)
+            ;
+
+        if (i == 0) {
+            answer = cli_cmd_get_confirmation(state, question1);
+            if (answer == GF_ANSWER_NO)
+                return -1;
 
-        if ((tmp & -tmp) != tmp) {
-                answer = cli_cmd_get_confirmation(state, question3);
+            *redundancy = 1;
+        } else {
+            *redundancy = *disperse - tmp;
+            if (*redundancy > 1) {
+                sprintf(question, question2, *redundancy);
+                answer = cli_cmd_get_confirmation(state, question);
                 if (answer == GF_ANSWER_NO)
-                        return -1;
+                    return -1;
+            }
         }
 
-        return 0;
+        tmp = 0;
+    } else {
+        tmp = *disperse - *redundancy;
+    }
+
+    if ((*redundancy < 1) || (*redundancy > (*disperse - 1) / 2)) {
+        cli_err(
+            "redundancy must be greater than or equal to 1 and "
+            "less than %d for a disperse %d volume",
+            (*disperse + 1) / 2, *disperse);
+
+        return -1;
+    }
+
+    if ((tmp & -tmp) != tmp) {
+        answer = cli_cmd_get_confirmation(state, question3);
+        if (answer == GF_ANSWER_NO)
+            return -1;
+    }
+
+    return 0;
 }
 
 static int32_t
-cli_validate_disperse_volume (char *word, gf1_cluster_type type,
-                              const char **words, int32_t wordcount,
-                              int32_t index, int32_t *disperse_count,
-                              int32_t *redundancy_count,
-                              int32_t *data_count)
+cli_validate_disperse_volume(char *word, gf1_cluster_type type,
+                             const char **words, int32_t wordcount,
+                             int32_t index, int32_t *disperse_count,
+                             int32_t *redundancy_count, int32_t *data_count)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        switch (type) {
+    switch (type) {
         case GF_CLUSTER_TYPE_NONE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                if (strcmp (word, "disperse") == 0) {
-                        if (*disperse_count >= 0) {
-                                cli_err ("disperse option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index + 1], disperse_count);
-                        if (ret == -1 && errno == EINVAL) {
-                                *disperse_count = 0;
-                                ret = 1;
-                        } else if (ret == -1) {
-                                goto out;
-                        } else {
-                                if (*disperse_count < 3) {
-                                        cli_err ("disperse count must "
-                                                 "be greater than 2");
-                                        goto out;
-                                }
-                                ret = 2;
-                        }
-                } else if (strcmp (word, "disperse-data") == 0) {
-                        if (*data_count >= 0) {
-                                cli_err ("disperse-data option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index+1], data_count);
-                        if (ret == -1 || *data_count < 2) {
-                                cli_err ("disperse-data must be greater than 1");
-                                goto out;
-                        }
-                        ret = 2;
-                } else if (strcmp (word, "redundancy") == 0) {
-                        if (*redundancy_count >= 0) {
-                                cli_err ("redundancy option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index+1], redundancy_count);
-                        if (ret == -1 || *redundancy_count < 1) {
-                                cli_err ("redundancy must be greater than 0");
-                                goto out;
-                        }
-                        ret = 2;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                cli_err ("striped-replicated-dispersed volume "
-                         "is not supported");
-                goto out;
-        case GF_CLUSTER_TYPE_TIER:
-                cli_err ("tier-dispersed volume is not "
-                         "supported");
-                goto out;
-        case GF_CLUSTER_TYPE_STRIPE:
-                cli_err ("striped-dispersed volume is not "
-                         "supported");
-                goto out;
+            if (strcmp(word, "disperse") == 0) {
+                if (*disperse_count >= 0) {
+                    cli_err("disperse option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], disperse_count);
+                if (ret == -1 && errno == EINVAL) {
+                    *disperse_count = 0;
+                    ret = 1;
+                } else if (ret == -1) {
+                    goto out;
+                } else {
+                    if (*disperse_count < 3) {
+                        cli_err(
+                            "disperse count must "
+                            "be greater than 2");
+                        goto out;
+                    }
+                    ret = 2;
+                }
+            } else if (strcmp(word, "disperse-data") == 0) {
+                if (*data_count >= 0) {
+                    cli_err("disperse-data option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], data_count);
+                if (ret == -1 || *data_count < 2) {
+                    cli_err("disperse-data must be greater than 1");
+                    goto out;
+                }
+                ret = 2;
+            } else if (strcmp(word, "redundancy") == 0) {
+                if (*redundancy_count >= 0) {
+                    cli_err("redundancy option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], redundancy_count);
+                if (ret == -1 || *redundancy_count < 1) {
+                    cli_err("redundancy must be greater than 0");
+                    goto out;
+                }
+                ret = 2;
+            }
+            break;
         case GF_CLUSTER_TYPE_REPLICATE:
-                cli_err ("replicated-dispersed volume is not "
-                         "supported");
-                goto out;
+            cli_err(
+                "replicated-dispersed volume is not "
+                "supported");
+            goto out;
         default:
-                cli_err ("Invalid type given");
-                break;
-        }
+            cli_err("Invalid type given");
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_validate_volname (const char *volname)
+cli_validate_volname(const char *volname)
 {
-        int32_t            ret                       = -1;
-        int32_t            i                         = -1;
-        static const char * const invalid_volnames[] = {
-                                      "volume", "type", "subvolumes", "option",
-                                      "end-volume", "all", "volume_not_in_ring",
-                                      "description", "force",
-                                      "snap-max-hard-limit",
-                                      "snap-max-soft-limit", "auto-delete",
-                                      "activate-on-create", NULL};
-
-        if (volname[0] == '-')
-                goto out;
-
-        for (i = 0; invalid_volnames[i]; i++) {
-                if (!strcmp (volname, invalid_volnames[i])) {
-                        cli_err ("\"%s\" cannot be the name of a volume.",
-                                 volname);
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    int32_t i = -1;
+    int volname_len;
+    static const char *const invalid_volnames[] = {"volume",
+                                                   "type",
+                                                   "subvolumes",
+                                                   "option",
+                                                   "end-volume",
+                                                   "all",
+                                                   "volume_not_in_ring",
+                                                   "description",
+                                                   "force",
+                                                   "snap-max-hard-limit",
+                                                   "snap-max-soft-limit",
+                                                   "auto-delete",
+                                                   "activate-on-create",
+                                                   NULL};
 
-        if (strchr (volname, '/'))
-                goto out;
+    if (volname[0] == '-')
+        goto out;
 
-        if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                cli_err("Volume name exceeds %d characters.",
-                        GD_VOLUME_NAME_MAX);
-                goto out;
+    for (i = 0; invalid_volnames[i]; i++) {
+        if (!strcmp(volname, invalid_volnames[i])) {
+            cli_err("\"%s\" cannot be the name of a volume.", volname);
+            goto out;
         }
+    }
 
-        for (i = 0; i < strlen (volname); i++) {
-                if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                   (volname[i] != '-')) {
-                        cli_err ("Volume name should not contain \"%c\""
-                                 " character.\nVolume names can only"
-                                 "contain alphanumeric, '-' and '_' "
-                                 "characters.", volname[i]);
-                        goto out;
-                }
-        }
+    if (strchr(volname, '/'))
+        goto out;
 
-        ret = 0;
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volume name exceeds %d characters.", GD_VOLUME_NAME_MAX);
+        goto out;
+    }
+
+    for (i = 0; i < volname_len; i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            goto out;
+        }
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
-                             int wordcount, dict_t **options)
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **brick_list)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     sub_count = 1;
-        int     brick_index = 0;
-        char    *trans_type = NULL;
-        int32_t index = 0;
-        char    *bricks = NULL;
-        int32_t brick_count = 0;
-        char    *opwords[] = { "replica", "stripe", "transport", "disperse",
-                               "redundancy", "disperse-data", "arbiter", NULL };
-
-        char    *w = NULL;
-        int      op_count = 0;
-        int32_t  replica_count = 1;
-        int32_t  arbiter_count = 0;
-        int32_t  stripe_count = 1;
-        int32_t  disperse_count = -1;
-        int32_t  redundancy_count = -1;
-        int32_t  disperse_data_count = -1;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int sub_count = 1;
+    int brick_index = 0;
+    char *trans_type = NULL;
+    int32_t index = 0;
+    char *bricks = NULL;
+    char *ta_brick = NULL;
+    int32_t brick_count = 0;
+    static char *opwords[] = {"replica",  "stripe",       "transport",
+                              "disperse", "redundancy",   "disperse-data",
+                              "arbiter",  "thin-arbiter", NULL};
+
+    char *w = NULL;
+    int op_count = 0;
+    int32_t replica_count = 1;
+    int32_t arbiter_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t stripe_count = 1;
+    int32_t disperse_count = -1;
+    int32_t redundancy_count = -1;
+    int32_t disperse_data_count = -1;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
-        }
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
 
-        type = GF_CLUSTER_TYPE_NONE;
-        index = 3;
+    type = GF_CLUSTER_TYPE_NONE;
+    index = 3;
 
-        while (op_count < 3) {
+    while (op_count < 3) {
+        ret = -1;
+        w = str_getunamb(words[index], opwords);
+        if (!w) {
+            break;
+        } else if ((strcmp(w, "replica")) == 0) {
+            switch (type) {
+                case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                case GF_CLUSTER_TYPE_REPLICATE:
+                    cli_err("replica option given twice");
+                    goto out;
+                case GF_CLUSTER_TYPE_NONE:
+                    type = GF_CLUSTER_TYPE_REPLICATE;
+                    break;
+                case GF_CLUSTER_TYPE_STRIPE:
+                    cli_err("stripe option not supported");
+                    goto out;
+                case GF_CLUSTER_TYPE_DISPERSE:
+                    cli_err(
+                        "replicated-dispersed volume is not "
+                        "supported");
+                    goto out;
+                default:
+                    cli_err("Invalid type given");
+                    goto out;
+            }
+
+            if (wordcount < (index + 2)) {
                 ret = -1;
-                w = str_getunamb (words[index], opwords);
-                if (!w) {
-                        break;
-                } else if ((strcmp (w, "replica")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replica option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_TIER:
-                                cli_err ("replicated-tiered volume is not "
-                                         "supported");
-                                goto out;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("replicated-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        default:
-                                cli_err ("Invalid type given");
-                                goto out;
-                        }
-
-                        if (wordcount < (index+2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        replica_count = strtol (words[index+1], NULL, 0);
-                        if (replica_count < 2) {
-                                cli_err ("replica count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
-                        if (words[index]) {
-                                if (!strcmp (words[index], "arbiter")) {
-                                        ret = gf_string2int (words[index+1],
-                                                             &arbiter_count);
-                                        if (ret == -1 || arbiter_count != 1 ||
-                                            replica_count != 3) {
-                                                cli_err ("For arbiter "
-                                                         "configuration, "
-                                                         "replica count must be"
-                                                         " 3 and arbiter count "
-                                                         "must be 1. The 3rd "
-                                                         "brick of the replica "
-                                                         "will be the arbiter");
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        ret = dict_set_int32 (dict, "arbiter-count",
-                                                              arbiter_count);
-                                        if (ret)
-                                                goto out;
-                                        index += 2;
-                                }
-                        }
-
-                } else if ((strcmp (w, "stripe")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("stripe option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_STRIPE;
-                                break;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("striped-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_TIER:
-                                cli_err ("striped-tier volume is not "
-                                         "supported");
-                                goto out;
-                        default:
-                                cli_err ("Invalid type given");
-                                goto out;
-                        }
-                        if (wordcount < (index + 2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        stripe_count = strtol (words[index+1], NULL, 0);
-                        if (stripe_count < 2) {
-                                cli_err ("stripe count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
+                goto out;
+            }
 
-                } else if ((strcmp (w, "transport")) == 0) {
-                        if (trans_type) {
-                                cli_err ("'transport' option given more"
-                                         " than one time");
-                                goto out;
-                        }
-                        if ((strcasecmp (words[index+1], "tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp");
-                        } else if ((strcasecmp (words[index+1], "rdma") == 0)) {
-                                trans_type = gf_strdup ("rdma");
-                        } else if ((strcasecmp (words[index+1], "tcp,rdma") == 0) ||
-                                   (strcasecmp (words[index+1], "rdma,tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp,rdma");
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "incorrect transport"
-                                        " protocol specified");
-                                ret = -1;
-                                goto out;
-                        }
-                        index += 2;
-
-                } else if ((strcmp (w, "disperse") == 0) ||
-                           (strcmp (w, "redundancy") == 0) ||
-                           (strcmp (w, "disperse-data") == 0)) {
-                        ret = cli_validate_disperse_volume (w, type, words,
-                                      wordcount, index, &disperse_count,
-                                      &redundancy_count, &disperse_data_count);
-                        if (ret < 0)
-                                goto out;
-                        index += ret;
-                        type = GF_CLUSTER_TYPE_DISPERSE;
-                } else if ((strcmp (w, "arbiter") == 0)) {
-                        cli_err ("arbiter option must be preceded by replica "
-                                 "option.");
+            replica_count = strtol(words[index + 1], NULL, 0);
+            if (replica_count < 2) {
+                cli_err(
+                    "replica count should be greater"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
+
+            index += 2;
+            if (words[index]) {
+                if (!strcmp(words[index], "arbiter")) {
+                    ret = gf_string2int(words[index + 1], &arbiter_count);
+                    if ((ret == -1) || (arbiter_count != 1)) {
+                        cli_err(
+                            "For arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the arbiter");
                         ret = -1;
                         goto out;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
+                    }
+                    ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                } else if (!strcmp(words[index], "thin-arbiter")) {
+                    ret = gf_string2int(words[index + 1], &thin_arbiter_count);
+                    if ((ret == -1) || (thin_arbiter_count != 1) ||
+                        (replica_count != 2)) {
+                        cli_err(
+                            "For thin-arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and thin-arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the thin-arbiter brick");
                         ret = -1;
                         goto out;
-                }
-                op_count++;
-        }
-
-        if (!trans_type)
-                trans_type = gf_strdup ("tcp");
-
-        if (index >= wordcount) {
-                ret = -1;
-                goto out;
-        }
-
-        brick_index = index;
-
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
-
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
-
-        /* If brick-count is not valid when replica or stripe is
-           given, exit here */
-        if (!brick_count) {
-                cli_err ("No bricks specified");
-                ret = -1;
-                goto out;
-        }
-
-        if (type == GF_CLUSTER_TYPE_DISPERSE) {
-                ret = cli_cmd_create_disperse_check (state, &disperse_count,
-                                                     &redundancy_count,
-                                                     &disperse_data_count,
-                                                     brick_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "disperse-count",
-                                              disperse_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "redundancy-count",
-                                              redundancy_count);
-                if (ret)
+                    }
+                    ret = dict_set_int32(dict, "thin-arbiter-count",
+                                         thin_arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                }
+            }
+
+            /* Do this to keep glusterd happy with sending
+               "replica 3 arbiter 1" options to server */
+            if ((arbiter_count == 1) && (replica_count == 2))
+                replica_count += arbiter_count;
+
+            if (replica_count == 2 && thin_arbiter_count == 0) {
+                if (strcmp(words[wordcount - 1], "force")) {
+                    question =
+                        "Replica 2 volumes are prone"
+                        " to split-brain. Use "
+                        "Arbiter or Replica 3 to "
+                        "avoid this. See: "
+                        "http://docs.gluster.org/en/latest/"
+                        "Administrator%20Guide/"
+                        "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                        "\nDo you still want to "
+                        "continue?\n";
+                    answer = cli_cmd_get_confirmation(state, question);
+                    if (GF_ANSWER_NO == answer) {
+                        gf_log("cli", GF_LOG_ERROR,
+                               "Volume create "
+                               "cancelled, exiting");
+                        ret = -1;
                         goto out;
-
-                sub_count = disperse_count;
-        } else
-                sub_count = stripe_count * replica_count;
-
-        if (brick_count % sub_count) {
-                if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "stripe count");
-                else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "replica count");
-                else if (type == GF_CLUSTER_TYPE_DISPERSE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "disperse count");
-                else
-                        cli_err ("number of bricks given doesn't match "
-                                 "required count");
-
+                    }
+                }
+            }
+            ret = dict_set_int32(dict, "replica-count", replica_count);
+            if (ret)
+                goto out;
+
+        } else if ((strcmp(w, "stripe")) == 0) {
+            cli_err("stripe option not supported");
+            goto out;
+        } else if ((strcmp(w, "transport")) == 0) {
+            if (trans_type) {
+                cli_err(
+                    "'transport' option given more"
+                    " than one time");
+                goto out;
+            }
+            if ((strcasecmp(words[index + 1], "tcp") == 0)) {
+                trans_type = gf_strdup("tcp");
+            } else if ((strcasecmp(words[index + 1], "rdma") == 0)) {
+                trans_type = gf_strdup("rdma");
+            } else if ((strcasecmp(words[index + 1], "tcp,rdma") == 0) ||
+                       (strcasecmp(words[index + 1], "rdma,tcp") == 0)) {
+                trans_type = gf_strdup("tcp,rdma");
+            } else {
+                gf_log("", GF_LOG_ERROR,
+                       "incorrect transport"
+                       " protocol specified");
                 ret = -1;
                 goto out;
+            }
+            index += 2;
+
+        } else if ((strcmp(w, "disperse") == 0) ||
+                   (strcmp(w, "redundancy") == 0) ||
+                   (strcmp(w, "disperse-data") == 0)) {
+            ret = cli_validate_disperse_volume(
+                w, type, words, wordcount, index, &disperse_count,
+                &redundancy_count, &disperse_data_count);
+            if (ret < 0)
+                goto out;
+            index += ret;
+            type = GF_CLUSTER_TYPE_DISPERSE;
+        } else if ((strcmp(w, "arbiter") == 0)) {
+            cli_err(
+                "arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else if ((strcmp(w, "thin-arbiter") == 0)) {
+            cli_err(
+                "thin-arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+        op_count++;
+    }
 
-        /* Everything is parsed fine. start setting info in dict */
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    if (!trans_type)
+        trans_type = gf_strdup("tcp");
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret)
-                goto out;
+    if (index >= wordcount) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "transport", trans_type);
-        if (ret)
-                goto out;
-        trans_type = NULL;
+    brick_index = index;
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    // Exclude the thin-arbiter-brick i.e. last brick in the bricks list
+    if (thin_arbiter_count == 1) {
+        ret = cli_cmd_bricks_parse(words, wc - 1, brick_index, &bricks,
+                                   &brick_count);
         if (ret)
-                goto out;
-
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
-
-        *options = dict;
-
-out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+            goto out;
 
-        GF_FREE (trans_type);
-
-        return ret;
-}
+        ret = cli_cmd_ta_brick_parse(words, wc, &ta_brick);
 
-int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **options)
-{
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
+    } else {
+        ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks,
+                                   &brick_count);
+    }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (ret)
+        goto out;
 
-        dict = dict_new ();
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = cli_cmd_create_disperse_check(state, &disperse_count,
+                                            &redundancy_count,
+                                            &disperse_data_count, brick_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "disperse-count", disperse_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "redundancy-count", redundancy_count);
+        if (ret)
+            goto out;
+
+        sub_count = disperse_count;
+    } else
+        sub_count = stripe_count * replica_count;
+
+    if (brick_count % sub_count) {
+        if (type == GF_CLUSTER_TYPE_STRIPE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "stripe count");
+        else if (type == GF_CLUSTER_TYPE_REPLICATE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "replica count");
+        else if (type == GF_CLUSTER_TYPE_DISPERSE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "disperse count");
+        else
+            cli_err(
+                "number of bricks given doesn't match "
+                "required count");
 
-        if (!dict)
-                goto out;
+        ret = -1;
+        goto out;
+    }
 
-        if (wordcount < 3)
-                goto out;
+    /* Everything is parsed fine. start setting info in dict */
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (wordcount > 5)
-                goto out;
+    ret = dict_set_int32(dict, "type", type);
+    if (ret)
+        goto out;
 
-        volname = (char *)words[2];
+    ret = dict_set_dynstr(dict, "transport", trans_type);
+    if (ret)
+        goto out;
+    trans_type = NULL;
 
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (thin_arbiter_count == 1) {
+        ret = dict_set_dynstr(dict, "ta-brick", ta_brick);
         if (ret)
-                goto out;
-
-        if (wordcount == 3) {
-                ret = dict_set_str (dict, "key", "all");
-                if (ret)
-                        goto out;
-        }
-
-        if (wordcount >= 4) {
-                if (!strcmp ("force", (char*)words[3])) {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "key", "all");
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = dict_set_str (dict, "key", (char *)words[3]);
-                        if (ret)
-                                goto out;
-                }
-        }
+            goto out;
+    }
 
-        if (wordcount == 5) {
-                if (strcmp ("force", (char*)words[4])) {
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                }
-        }
+    ret = dict_set_int32(dict, "count", brick_count);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
+    *options = dict;
+    *brick_list = bricks;
 out:
-        if (ret && dict) {
-                dict_unref (dict);
-        }
+    if (ret) {
+        GF_FREE(bricks);
+        GF_FREE(ta_brick);
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-                return ret;
-}
+    GF_FREE(trans_type);
 
-/* Parsing global option for NFS-Ganesha config
- *  gluster nfs-ganesha enable/disable */
+    return ret;
+}
 
 int32_t
-cli_cmd_ganesha_parse (struct cli_state *state,
-                       const char **words, int wordcount,
-                       dict_t **options, char **op_errstr)
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict     =       NULL;
-        int     ret       =       -1;
-        char    *key      =       NULL;
-        char    *value    =       NULL;
-        char    *w        =       NULL;
-        char   *opwords[] =      { "enable", "disable", NULL };
-        const char      *question       =       NULL;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    dict = dict_new();
 
-        dict = dict_new ();
+    if (!dict)
+        goto out;
 
-        if (!dict)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        if (wordcount != 2)
-                goto out;
+    if (wordcount > 5)
+        goto out;
 
-        key   = (char *) words[0];
-        value = (char *) words[1];
+    volname = (char *)words[2];
 
-        if (!key || !value) {
-                cli_out ("Usage : nfs-ganesha <enable/disable>");
-                ret = -1;
-                goto out;
-        }
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = gf_strip_whitespace (value, strlen (value));
-        if (ret == -1)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (strcmp (key, "nfs-ganesha")) {
-                gf_asprintf (op_errstr, "Global option: error: ' %s '"
-                          "is not a valid global option.", key);
-                ret = -1;
-                goto out;
-        }
+    if (wordcount == 3) {
+        ret = dict_set_str(dict, "key", "all");
+        if (ret)
+            goto out;
+    }
 
-        w = str_getunamb (value, opwords);
-        if (!w) {
-                cli_out ("Invalid global option \n"
-                         "Usage : nfs-ganesha <enable/disable>");
-                ret = -1;
+    if (wordcount >= 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
                 goto out;
-        }
-
-        question = "Enabling NFS-Ganesha requires Gluster-NFS to be"
-                   " disabled across the trusted pool. Do you "
-                   "still want to continue?\n";
-
-        if (strcmp (value, "enable") == 0) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        gf_log ("cli", GF_LOG_ERROR, "Global operation "
-                                "cancelled, exiting");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        cli_out ("This will take a few minutes to complete. Please wait ..");
-
-        ret = dict_set_str (dict, "key", key);
-        if (ret) {
-               gf_log (THIS->name, GF_LOG_ERROR, "dict set on key failed");
+            ret = dict_set_str(dict, "key", "all");
+            if (ret)
                 goto out;
-        }
-
-        ret = dict_set_str (dict, "value", value);
-        if (ret) {
-               gf_log (THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        } else {
+            ret = dict_set_str(dict, "key", (char *)words[3]);
+            if (ret)
                 goto out;
         }
+    }
 
-        ret = dict_set_str (dict, "globalname", "All");
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set on global"
-                        " key failed.");
+    if (wordcount == 5) {
+        if (strcmp("force", (char *)words[4])) {
+            ret = -1;
+            goto out;
+        } else {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
                 goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set on global key "
-                        "failed.");
-                goto out;
-        }
+    *options = dict;
 
-        *options = dict;
 out:
-        if (ret)
-                dict_unref (dict);
+    if (ret && dict) {
+        dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_get_state_parse (struct cli_state *state,
-                         const char **words, int wordcount,
-                         dict_t **options, char **op_errstr)
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t    *dict                 = NULL;
-        int        ret                  = -1;
-        char      *odir                 = NULL;
-        char      *filename             = NULL;
-        char      *daemon_name          = NULL;
-        int        count                = 0;
-
-        GF_VALIDATE_OR_GOTO ("cli", options, out);
-        GF_VALIDATE_OR_GOTO ("cli", words, out);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount < 1 || wordcount > 6) {
-                *op_errstr = gf_strdup ("Problem parsing arguments."
-                                        " Check usage.");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *odir = NULL;
+    char *filename = NULL;
+    char *daemon_name = NULL;
+    int count = 0;
+    uint32_t cmd = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", options, out);
+    GF_VALIDATE_OR_GOTO("cli", words, out);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount >= 1) {
-                gf_asprintf (&daemon_name, "%s", "glusterd");
-
-                for (count = 1; count < wordcount; count++) {
-                        if (strcmp (words[count], "odir") == 0 ||
-                                        strcmp (words[count], "file") == 0) {
-                                if (strcmp (words[count], "odir") == 0) {
-                                        if (++count < wordcount) {
-                                                odir = (char *) words[count];
-                                                continue;
-                                        } else {
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                } else if (strcmp (words[count], "file") == 0) {
-                                        if (++count < wordcount) {
-                                                filename = (char *) words[count];
-                                                continue;
-                                        } else {
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                }
-                        } else {
-                                if (count > 1) {
-                                        *op_errstr = gf_strdup ("Problem "
-                                                        "parsing arguments. "
-                                                        "Check usage.");
-                                        ret = -1;
-                                        goto out;
-
-                                }
-                                if (strcmp (words[count], "glusterd") == 0) {
-                                        continue;
-                                } else {
-                                        *op_errstr = gf_strdup ("glusterd is "
-                                                 "the only supported daemon.");
-                                        ret = -1;
-                                        goto out;
-                                }
+    if (wordcount < 1 || wordcount > 7) {
+        *op_errstr = gf_strdup(
+            "Problem parsing arguments."
+            " Check usage.");
+        goto out;
+    }
+
+    if (wordcount >= 1) {
+        gf_asprintf(&daemon_name, "%s", "glusterd");
+
+        for (count = 1; count < wordcount; count++) {
+            if (strcmp(words[count], "odir") == 0 ||
+                strcmp(words[count], "file") == 0) {
+                if (strcmp(words[count], "odir") == 0) {
+                    if (++count < wordcount) {
+                        odir = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                } else if (strcmp(words[count], "file") == 0) {
+                    if (++count < wordcount) {
+                        filename = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            } else {
+                if (count > 1) {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
                         }
-                }
-
-                ret = dict_set_str (dict, "daemon", daemon_name);
-                if (ret) {
-                        *op_errstr = gf_strdup ("Command failed. Please check "
-                                                " log file for more details.");
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Setting daemon name to dictionary failed");
+                    } else {
+                        *op_errstr = gf_strdup(
+                            "Problem"
+                            " parsing arguments. "
+                            "Check usage.");
+                        ret = -1;
                         goto out;
+                    }
                 }
-
-                if (odir) {
-                        ret = dict_set_str (dict, "odir", odir);
-                        if (ret) {
-                                *op_errstr = gf_strdup ("Command failed. Please"
-                                                        " check log file for"
-                                                        " more details.");
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Setting output directory to"
-                                        "dictionary failed");
-                                goto out;
+                if (strcmp(words[count], "glusterd") == 0) {
+                    continue;
+                } else {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
                         }
-                }
+                    }
 
-                if (filename) {
-                        ret = dict_set_str (dict, "filename", filename);
-                        if (ret) {
-                                *op_errstr = gf_strdup ("Command failed. Please"
-                                                        " check log file for"
-                                                        " more  details.");
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Setting filename to dictionary failed");
-                                goto out;
-                        }
+                    *op_errstr = gf_strdup(
+                        "glusterd is "
+                        "the only supported daemon.");
+                    ret = -1;
+                    goto out;
                 }
+            }
         }
 
- out:
-        if (dict)
-                *options = dict;
+        ret = dict_set_dynstr(dict, "daemon", daemon_name);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Command failed. Please check "
+                " log file for more details.");
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Setting daemon name to dictionary failed");
+            goto out;
+        }
+        daemon_name = NULL;
+
+        if (odir) {
+            ret = dict_set_str(dict, "odir", odir);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting output directory to"
+                       "dictionary failed");
+                goto out;
+            }
+        }
+
+        if (filename) {
+            ret = dict_set_str(dict, "filename", filename);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more  details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting filename to dictionary failed");
+                goto out;
+            }
+        }
+
+        if (cmd) {
+            ret = dict_set_uint32(dict, "getstate-cmd", cmd);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting "
+                       "get-state command type to dictionary "
+                       "failed");
+                goto out;
+            }
+        }
+    }
 
-        if (ret && dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        *options = dict;
 
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+
+    GF_FREE(daemon_name);
+
+    return ret;
 }
 
 int32_t
-cli_cmd_inode_quota_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        if (wordcount != 4)
-                goto out;
+    if (wordcount != 4)
+        goto out;
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret < 0)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
 
-        if (strcmp (words[3], "enable") != 0) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = -1;
-                goto out;
-        }
+    if (strcmp(words[3], "enable") != 0) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "type",
-                              GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
-        if (ret < 0)
-                goto out;
+    ret = dict_set_int32(dict, "type", GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
+    if (ret < 0)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret < 0) {
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
-        int              i       = -1;
-        char             key[20] = {0, };
-        int64_t          value   = 0;
-        gf_quota_type    type    = GF_QUOTA_OPTION_TYPE_NONE;
-        char           *opwords[] = { "enable", "disable", "limit-usage",
-                                      "remove", "list", "alert-time",
-                                      "soft-timeout", "hard-timeout",
-                                      "default-soft-limit", "limit-objects",
-                                      "list-objects", "remove-objects", NULL};
-        char            *w       = NULL;
-        uint32_t         time    = 0;
-        double           percent = 0;
-        char            *end_ptr = NULL;
-        int64_t          limit   = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
-
-        if (wordcount < 4)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int i = -1;
+    char key[20] = {
+        0,
+    };
+    int64_t value = 0;
+    gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE;
+    static char *opwords[] = {"enable",
+                              "disable",
+                              "limit-usage",
+                              "remove",
+                              "list",
+                              "alert-time",
+                              "soft-timeout",
+                              "hard-timeout",
+                              "default-soft-limit",
+                              "limit-objects",
+                              "list-objects",
+                              "remove-objects",
+                              NULL};
+    char *w = NULL;
+    uint32_t time = 0;
+    double percent = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
+    if (wordcount < 4) {
+        if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+            ret = 1;
         }
+        goto out;
+    }
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret < 0)
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = - 1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
 
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (strcmp (w, "limit-usage") == 0) {
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
-        } else if (strcmp (w, "limit-objects") == 0) {
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
-            type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
-
-                if (wordcount < 6 || wordcount > 7) {
-                        ret = -1;
-                        goto out;
-                }
+    if (strcmp(w, "limit-usage") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+    } else if (strcmp(w, "limit-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret)
-                        goto out;
-
-                if (!words[5]) {
-                        cli_err ("Please enter the limit value to be set");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
-                        ret = gf_string2bytesize_int64 (words[5], &value);
-                        if (ret != 0 || value <= 0) {
-                                if (errno == ERANGE || value <= 0) {
-                                        ret = -1;
-                                        cli_err ("Please enter an integer "
-                                                 "value in the range of "
-                                                 "(1 - %"PRId64 ")",
-                                                 INT64_MAX);
-                                } else
-                                        cli_err ("Please enter a correct "
-                                                 "value");
-                                goto out;
-                        }
-                } else {
-                        errno = 0;
-                        limit = strtol (words[5], &end_ptr, 10);
-                        if (errno == ERANGE || errno == EINVAL || limit <= 0
-                                            || strcmp (end_ptr, "") != 0) {
-                                ret = -1;
-                                cli_err ("Please enter an integer value in "
-                                         "the range 1 - %"PRId64, INT64_MAX);
-                                goto out;
-                        }
-                }
-
-                ret  = dict_set_str (dict, "hard-limit", (char *) words[5]);
-                if (ret < 0)
-                        goto out;
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
+        if (wordcount < 6 || wordcount > 7) {
+            ret = -1;
+            goto out;
+        }
 
-                if (wordcount == 7) {
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret)
+            goto out;
+
+        if (!words[5]) {
+            cli_err("Please enter the limit value to be set");
+            ret = -1;
+            goto out;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = gf_string2bytesize_int64(words[5], &value);
+            if (ret != 0 || value <= 0) {
+                if (errno == ERANGE || value <= 0) {
+                    ret = -1;
+                    cli_err(
+                        "Please enter an integer "
+                        "value in the range of "
+                        "(1 - %" PRId64 ")",
+                        INT64_MAX);
+                } else
+                    cli_err(
+                        "Please enter a correct "
+                        "value");
+                goto out;
+            }
+        } else {
+            errno = 0;
+            limit = strtol(words[5], &end_ptr, 10);
+            if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+                strcmp(end_ptr, "") != 0) {
+                ret = -1;
+                cli_err(
+                    "Please enter an integer value in "
+                    "the range 1 - %" PRId64,
+                    INT64_MAX);
+                goto out;
+            }
+        }
 
-                        ret = gf_string2percent (words[6], &percent);
-                        if (ret != 0 || percent > 100) {
-                                ret = -1;
-                                cli_err ("Please enter a correct value "
-                                         "in the range of 0 to 100");
-                                goto out;
-                        }
+        ret = dict_set_str(dict, "hard-limit", (char *)words[5]);
+        if (ret < 0)
+            goto out;
 
-                        ret = dict_set_str (dict, "soft-limit",
-                                            (char *) words[6]);
-                        if (ret < 0)
-                                goto out;
-                }
+        if (wordcount == 7) {
+            ret = gf_string2percent(words[6], &percent);
+            if (ret != 0 || percent > 100) {
+                ret = -1;
+                cli_err(
+                    "Please enter a correct value "
+                    "in the range of 0 to 100");
+                goto out;
+            }
 
-                goto set_type;
+            ret = dict_set_str(dict, "soft-limit", (char *)words[6]);
+            if (ret < 0)
+                goto out;
         }
 
-        if (strcmp (w, "remove") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-
-                type = GF_QUOTA_OPTION_TYPE_REMOVE;
+        goto set_type;
+    }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+    if (strcmp(w, "remove") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "remove-objects") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
+        type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
-                type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+    if (strcmp(w, "remove-objects") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
 
-                type = GF_QUOTA_OPTION_TYPE_LIST;
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                if (words[4] && words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+    if (strcmp(w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST;
 
-                        ret = dict_set_str (dict, key, (char *) words [i++]);
-                        if (ret < 0)
-                                goto out;
-                }
+        if (words[4] && words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0)
-                        goto out;
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                goto set_type;
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0)
+                goto out;
         }
 
-        if (strcmp (w, "list-objects") == 0) {
-                if (wordcount < 4) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0)
+            goto out;
 
-                type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
+        goto set_type;
+    }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+    if (strcmp(w, "list-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
 
-                        ret = dict_set_str (dict, key, (char *) words[i++]);
-                        if (ret < 0) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                        "quota patch in request dictionary");
-                                goto out;
-                        }
-                }
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set quota "
-                                "limit count in request dictionary");
-                        goto out;
-                }
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set "
+                       "quota patch in request dictionary");
+                goto out;
+            }
+        }
 
-                goto set_type;
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set quota "
+                   "limit count in request dictionary");
+            goto out;
         }
 
-        if (strcmp (w, "alert-time") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "alert-time") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
 
-        if (strcmp (w, "soft-timeout") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "soft-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
 
-        if (strcmp (w, "hard-timeout") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "hard-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "default-soft-limit") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
-        } else {
-                GF_ASSERT (!"opword mismatch");
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+    if (strcmp(w, "default-soft-limit") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
+        type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
+        ret = dict_set_str(dict, "value", (char *)words[4]);
         if (ret < 0)
-                goto out;
+            goto out;
+        goto set_type;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+    }
 
-        *options = dict;
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-        if (ret < 0) {
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-cli_is_key_spl (char *key)
+cli_is_key_spl(char *key)
 {
-        return (strcmp (key, "group") == 0);
+    return (strcmp(key, "group") == 0);
 }
 
-static int
-cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+static int32_t
+cli_add_key_group_value(dict_t *dict, const char *name, const char *value,
+                        int32_t id, char **op_errstr)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            iter_key[1024] = {0,};
-        char            iter_val[1024] = {0,};
-        char            *saveptr = NULL;
-        char            *tok_key = NULL;
-        char            *tok_val = NULL;
-        char            *dkey = NULL;
-        char            *dval = NULL;
-        char            *tagpath = NULL;
-        char            *buf = NULL;
-        char            line[PATH_MAX + 256] = {0,};
-        char            errstr[2048] = "";
-        FILE            *fp = NULL;
-
-        ret = gf_asprintf (&tagpath, "%s/groups/%s",
-                           GLUSTERD_DEFAULT_WORKDIR, value);
-        if (ret == -1) {
-                tagpath = NULL;
-                goto out;
-        }
+    char *key = NULL;
+    char *data = NULL;
+    int32_t ret = -1;
 
-        fp = fopen (tagpath, "r");
-        if (!fp) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
-                         " Error: %s", tagpath, strerror (errno));
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
-        }
+    ret = gf_asprintf(&key, "%s%d", name, id);
+    if (ret < 0) {
+        goto out;
+    }
+    data = gf_strdup(value);
+    if (data == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to allocate memory for data");
+        ret = -1;
+        goto out;
+    }
 
-        opt_count = 0;
-        buf = line;
-        while (fscanf (fp, "%s", buf) != EOF) {
+    ret = dict_set_dynstr(dict, key, data);
+    if (ret == 0) {
+        data = NULL;
+    }
 
-                opt_count++;
-                tok_key = strtok_r (line, "=", &saveptr);
-                tok_val = strtok_r (NULL, "=", &saveptr);
-                if (!tok_key || !tok_val) {
-                        ret = -1;
-                        snprintf(errstr, sizeof(errstr), "'%s' file format "
-                                 "not valid.", tagpath);
-                        if (op_errstr)
-                                *op_errstr = gf_strdup(errstr);
-                        goto out;
-                }
+out:
+    GF_FREE(key);
+    GF_FREE(data);
 
-                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
-                dkey = gf_strdup (tok_key);
-                ret = dict_set_dynstr (dict, iter_key, dkey);
-                if (ret)
-                        goto out;
-                dkey = NULL;
+    if ((ret != 0) && (op_errstr != NULL)) {
+        *op_errstr = gf_strdup("Failed to allocate memory");
+    }
 
-                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
-                dval = gf_strdup (tok_val);
-                ret = dict_set_dynstr (dict, iter_val, dval);
-                if (ret)
-                        goto out;
-                dval = NULL;
+    return ret;
+}
 
-        }
+static int
+cli_add_key_group(dict_t *dict, char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char *saveptr = NULL;
+    char *tok_key = NULL;
+    char *tok_val = NULL;
+    char *tagpath = NULL;
+    char line[PATH_MAX + 256] = {
+        0,
+    };
+    FILE *fp = NULL;
+
+    ret = gf_asprintf(&tagpath, "%s/groups/%s", GLUSTERD_DEFAULT_WORKDIR,
+                      value);
+    if (ret == -1) {
+        tagpath = NULL;
+        goto out;
+    }
 
-        if (!opt_count) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "'%s' file format "
-                         "not valid.", tagpath);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
+    fp = fopen(tagpath, "r");
+    if (!fp) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr,
+                        "Unable to open file '%s'. "
+                        "Error: %s",
+                        tagpath, strerror(errno));
         }
-        ret = dict_set_int32 (dict, "count", opt_count);
+        goto out;
+    }
+
+    opt_count = 0;
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (strlen(line) >= sizeof(line) - 1) {
+            ret = -1;
+            if (op_errstr != NULL) {
+                *op_errstr = gf_strdup("Line too long");
+            }
+            goto out;
+        }
+
+        /* Treat line that start with "#" as comments */
+        if ('#' == line[0])
+            continue;
+
+        opt_count++;
+        tok_key = strtok_r(line, "=", &saveptr);
+        tok_val = strtok_r(NULL, "\r\n", &saveptr);
+        if (!tok_key || !tok_val) {
+            ret = -1;
+            if (op_errstr) {
+                gf_asprintf(op_errstr,
+                            "'%s' file format "
+                            "not valid.",
+                            tagpath);
+            }
+            goto out;
+        }
+
+        ret = cli_add_key_group_value(dict, "key", tok_key, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+        ret = cli_add_key_group_value(dict, "value", tok_val, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (!opt_count) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr, "'%s' file format not valid.", tagpath);
+        }
+        goto out;
+    }
+    ret = dict_set_int32(dict, "count", opt_count);
 out:
 
-        GF_FREE (tagpath);
+    GF_FREE(tagpath);
 
-        if (ret) {
-                GF_FREE (dkey);
-                GF_FREE (dval);
-        }
+    if (fp)
+        fclose(fp);
 
-        if (fp)
-                fclose (fp);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_set_parse (struct cli_state *state, const char **words,
-                          int wordcount, dict_t **options, char **op_errstr)
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t                 *dict      = NULL;
-        char                   *volname   = NULL;
-        int                     ret       = -1;
-        int                     count     = 0;
-        char                   *key       = NULL;
-        char                   *value     = NULL;
-        int                     i         = 0;
-        char                    str[50]   = {0,};
-        const char             *question  = NULL;
-        gf_answer_t             answer    = GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
-
-        if (wordcount < 3)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int count = 0;
+    char *key = NULL;
+    char *value = NULL;
+    int i = 0;
+    char str[50] = {
+        0,
+    };
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 3)
+        goto out;
 
-        GF_ASSERT (volname);
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
+    GF_ASSERT(volname);
 
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (!strcmp (volname, "all")) {
-                ret = dict_set_str (dict, "globalname", "All");
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "dict set on global key failed.");
-                        goto out;
-                }
+    if (ret)
+        goto out;
 
-                ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "dict set on global key failed.");
-                        goto out;
-                }
+    if (!strcmp(volname, "all")) {
+        ret = dict_set_str(dict, "globalname", "All");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
         }
 
-        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
-            && wordcount == 3 ) {
-                ret = dict_set_str (dict, volname, volname);
-                if (ret)
-                        goto out;
-
-        } else if (wordcount < 5) {
-                ret = -1;
-                goto out;
-
-        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
-                key = (char *) words[3];
-                value = (char *) words[4];
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
-
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = cli_add_key_group (dict, key, value, op_errstr);
-                if (ret == 0)
-                        *options = dict;
-                goto out;
+        ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
         }
+    }
 
-        for (i = 3; i < wordcount; i+=2) {
-
-                key = (char *) words[i];
-                value = (char *) words[i+1];
-
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                count++;
-
-                if (fnmatch ("user.*", key, FNM_NOESCAPE) != 0) {
-                        ret = gf_strip_whitespace (value, strlen (value));
-                        if (ret == -1)
-                                goto out;
-                }
-
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (cli_is_key_spl (key)) {
-                        ret = -1;
-                        goto out;
-                }
+    if ((!strcmp(volname, "help") || !strcmp(volname, "help-xml")) &&
+        wordcount == 3) {
+        ret = dict_set_str(dict, volname, volname);
+        if (ret)
+            goto out;
 
-                sprintf (str, "key%d", count);
-                ret = dict_set_str (dict, str, key);
-                if (ret)
-                        goto out;
+    } else if (wordcount < 5) {
+        ret = -1;
+        goto out;
 
-                sprintf (str, "value%d", count);
-                ret = dict_set_str (dict, str, value);
+    } else if (wordcount == 5 && cli_is_key_spl((char *)words[3])) {
+        key = (char *)words[3];
+        value = (char *)words[4];
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                if (ret)
-                        goto out;
+        ret = gf_strip_whitespace(value, strlen(value));
+        if (ret == -1)
+            goto out;
 
-                if ((!strcmp (key, "cluster.enable-shared-storage")) &&
-                    (!strcmp (value, "disable"))) {
-                        question = "Disabling cluster.enable-shared-storage "
-                                   "will delete the shared storage volume"
-                                   "(gluster_shared_storage), which is used "
-                                   "by snapshot scheduler, geo-replication "
-                                   "and NFS-Ganesha. Do you still want to "
-                                   "continue?";
-                        answer = cli_cmd_get_confirmation (state, question);
-                        if (GF_ANSWER_NO == answer) {
-                                gf_log ("cli", GF_LOG_ERROR, "Operation "
-                                        "cancelled, exiting");
-                                *op_errstr = gf_strdup ("Aborted by user.");
-                                ret = -1;
-                                goto out;
-                        }
-                }
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "count", wordcount-3);
-
-        if (ret)
-                goto out;
+        ret = cli_add_key_group(dict, key, value, op_errstr);
+        if (ret == 0)
+            *options = dict;
+        goto out;
+    }
 
-        *options = dict;
+    for (i = 3; i < wordcount; i += 2) {
+        key = (char *)words[i];
+        value = (char *)words[i + 1];
 
-out:
-        if (ret && dict)
-                dict_unref (dict);
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-        return ret;
-}
+        count++;
 
-int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options, int *ret_type)
-{
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     brick_count = 0, brick_index = 0;
-        char    *bricks = NULL;
-        char    *opwords_cl[] = { "replica", "stripe", NULL };
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        int     arbiter_count = 0;
-        char    *w = NULL;
-        int     index;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
+        if (fnmatch("user.*", key, FNM_NOESCAPE) != 0) {
+            ret = gf_strip_whitespace(value, strlen(value));
+            if (ret == -1)
                 goto out;
+        }
 
-        if (wordcount < 3)
-                goto out;
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-        volname = (char *)words[2];
+        if (cli_is_key_spl(key)) {
+            ret = -1;
+            goto out;
+        }
 
-        GF_ASSERT (volname);
+        sprintf(str, "key%d", count);
+        ret = dict_set_str(dict, str, key);
+        if (ret)
+            goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+        sprintf(str, "value%d", count);
+        ret = dict_set_str(dict, str, value);
 
         if (ret)
-                goto out;
-
-        if (wordcount < 4) {
+            goto out;
+
+        if ((!strcmp(key, "cluster.enable-shared-storage")) &&
+            (!strcmp(value, "disable"))) {
+            question =
+                "Disabling cluster.enable-shared-storage "
+                "will delete the shared storage volume"
+                "(gluster_shared_storage), which is used "
+                "by snapshot scheduler, geo-replication "
+                "and NFS-Ganesha. Do you still want to "
+                "continue?";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
                 ret = -1;
                 goto out;
-        }
-        if (wordcount < 6) {
-                /* seems no options are given, go directly to the parse_brick */
-                brick_index = 3;
-                type = GF_CLUSTER_TYPE_NONE;
-                goto parse_bricks;
-        }
-
-        w = str_getunamb (words[3], opwords_cl);
-        if (!w) {
-                type = GF_CLUSTER_TYPE_NONE;
-                index = 3;
-        } else if ((strcmp (w, "replica")) == 0) {
-                type = GF_CLUSTER_TYPE_REPLICATE;
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("replica count should be greater than 1");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-                if (words[index] && !strcmp (words[index], "arbiter")) {
-                        arbiter_count = strtol (words[6], NULL, 0);
-                        if (arbiter_count != 1 || count != 3) {
-                                cli_err ("For arbiter configuration, replica "
-                                         "count must be 3 and arbiter count "
-                                         "must be 1. The 3rd brick of the "
-                                         "replica will be the arbiter");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "arbiter-count",
-                                              arbiter_count);
-                        if (ret)
-                                goto out;
-                        index = 7;
-                }
-        } else if ((strcmp (w, "stripe")) == 0) {
-                type = GF_CLUSTER_TYPE_STRIPE;
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("stripe count should be greater than 1");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "stripe-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                GF_ASSERT (!"opword mismatch");
+            }
+        }
+        if ((!strcmp(key, "nfs.disable")) && (!strcmp(value, "off"))) {
+            question =
+                "Gluster NFS is being deprecated in favor "
+                "of NFS-Ganesha Enter \"yes\" to continue "
+                "using Gluster NFS";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
                 ret = -1;
                 goto out;
+            }
         }
+    }
 
-        brick_index = index;
-
-parse_bricks:
-
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    ret = dict_set_int32(dict, "count", wordcount - 3);
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
-
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (dict, "count", brick_count);
-
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret_type)
-                *ret_type = type;
-
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_tier_parse (const char **words, int wordcount,
-                           dict_t **options)
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *ret_type)
 {
-        dict_t  *dict    = NULL;
-        char    *volname = NULL;
-        int      ret     = -1;
-        int32_t  command = GF_OP_CMD_NONE;
-        int32_t  is_force    = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int brick_count = 0, brick_index = 0;
+    char *bricks = NULL;
+    static char *opwords_cl[] = {"replica", "stripe", NULL};
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int count = 1;
+    int arbiter_count = 0;
+    char *w = NULL;
+    int index;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (wordcount < 3)
+        goto out;
 
-        dict = dict_new ();
+    volname = (char *)words[2];
 
-        if (!dict)
-                goto out;
+    GF_ASSERT(volname);
 
-        if (!(wordcount == 4 || wordcount == 5)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                ret = -1;
-                goto out;
-        }
-
-        volname = (char *)words[2];
+    ret = dict_set_str(dict, "volname", volname);
 
-        GF_ASSERT (volname);
-
-        ret = cli_cmd_validate_volume (volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to validate volume name");
-                goto out;
-        }
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+    if (wordcount < 6) {
+        /* seems no options are given, go directly to the parse_brick */
+        brick_index = 3;
+        type = GF_CLUSTER_TYPE_NONE;
+        goto parse_bricks;
+    }
 
+    w = str_getunamb(words[3], opwords_cl);
+    if (!w) {
+        type = GF_CLUSTER_TYPE_NONE;
+        index = 3;
+    } else if ((strcmp(w, "replica")) == 0) {
+        type = GF_CLUSTER_TYPE_REPLICATE;
+        count = strtol(words[4], NULL, 0);
+        if (!count || (count < 2)) {
+            cli_err("replica count should be greater than 1");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32(dict, "replica-count", count);
         if (ret)
+            goto out;
+        index = 5;
+        if (words[index] && !strcmp(words[index], "arbiter")) {
+            arbiter_count = strtol(words[6], NULL, 0);
+            if (arbiter_count != 1 || count != 3) {
+                cli_err(
+                    "For arbiter configuration, replica "
+                    "count must be 3 and arbiter count "
+                    "must be 1. The 3rd brick of the "
+                    "replica will be the arbiter");
+                ret = -1;
                 goto out;
-
-        volname = (char *)words[2];
-        if (wordcount == 4) {
-                if (!strcmp(words[3], "status"))
-                        command = GF_DEFRAG_CMD_STATUS_TIER;
-                else if (!strcmp(words[3], "start"))
-                        command = GF_DEFRAG_CMD_START_TIER;
-                else {
-                        ret = -1;
-                        goto out;
-                }
-        } else if (wordcount == 5) {
-                if ((!strcmp (words[3], "start")) &&
-                    (!strcmp (words[4], "force"))) {
-                        command = GF_DEFRAG_CMD_START_TIER;
-                        is_force = 1;
-                        ret = dict_set_int32 (dict, "force", is_force);
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "rebalance-command", command);
-        if (ret)
+            }
+            ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+            if (ret)
                 goto out;
-
-        *options = dict;
-out:
-
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse tier CLI");
-                if (dict)
-                        dict_unref (dict);
+            index = 7;
         }
 
-        return ret;
-}
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                question =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or "
+                    "Replica 3 to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, question);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Add brick"
+                           " cancelled, exiting");
+                    ret = -1;
+                    goto out;
+                }
+            }
+        }
+    } else if ((strcmp(w, "stripe")) == 0) {
+        cli_err("stripe option not supported");
+        goto out;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+        ret = -1;
+        goto out;
+    }
 
-int32_t
-cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
-                                  dict_t **options, int *question)
-{
-        int      ret = -1;
-        char    *word = NULL;
-        dict_t  *dict = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
+    brick_index = index;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+parse_bricks:
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        if (wordcount == 3 && !strcmp ((char *)words[2], "help")) {
-                return -1;
-        }
+    ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks, &brick_count);
+    if (ret)
+        goto out;
 
-        if (wordcount != 4) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        word = (char *)words[3];
+    ret = dict_set_int32(dict, "count", brick_count);
 
-        ret = -1;
+    if (ret)
+        goto out;
 
-        if (!strcmp(word, "start")) {
-                command = GF_OP_CMD_DETACH_START;
-        } else if (!strcmp(word, "commit")) {
-                *question = 1;
-                command = GF_OP_CMD_DETACH_COMMIT;
-        } else if (!strcmp(word, "force")) {
-                *question = 1;
-                command = GF_OP_CMD_DETACH_COMMIT_FORCE;
-        } else if (!strcmp(word, "stop"))
-                command = GF_OP_CMD_STOP_DETACH_TIER;
-        else if (!strcmp(word, "status"))
-                command = GF_OP_CMD_STATUS;
-        else
-                goto out;
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "command", command);
-        if (ret)
-                goto out;
+    *options = dict;
 
-        *options = dict;
-        ret = 0;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse detach tier CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret_type)
+        *ret_type = type;
 
-        return ret;
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question)
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *comm)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        char    key[50];
-        int     brick_count = 0, brick_index = 0;
-        int32_t tmp_index = 0;
-        int32_t j = 0;
-        char    *tmp_brick = NULL;
-        char    *tmp_brick1 = NULL;
-        char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "stop", "status",
-                               "force", NULL };
-        char    *w = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
-        long     count = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        if (wordcount < 5)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *delimiter = NULL;
+    int ret = -1;
+    char key[50];
+    int brick_index = 0;
+    int32_t tmp_index = 0;
+    int32_t j = 0;
+    char *tmp_brick = NULL;
+    char *tmp_brick1 = NULL;
+    static char *type_opword[] = {"replica", NULL};
+    static char *opwords[] = {"start",  "commit", "stop",
+                              "status", "force",  NULL};
+    char *w = NULL;
+    int32_t command = GF_OP_CMD_NONE;
+    long count = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *ques = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount < 5)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        brick_index = 3;
-        w = str_getunamb (words[3], type_opword);
-        if (w && !strcmp ("replica", w)) {
-                if (wordcount < 6) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (count < 1) {
-                        cli_err ("replica count should be greater than 0 in "
-                                 "case of remove-brick");
-                        ret = -1;
-                        goto out;
+    brick_index = 3;
+    w = str_getunamb(words[3], type_opword);
+    if (w && !strcmp("replica", w)) {
+        if (wordcount < 6) {
+            ret = -1;
+            goto out;
+        }
+        count = strtol(words[4], NULL, 0);
+        if (count < 1) {
+            cli_err(
+                "replica count should be greater than 0 in "
+                "case of remove-brick");
+            ret = -1;
+            goto out;
+        }
+
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                ques =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or Replica 3 "
+                    "to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, ques);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Remove "
+                           "brick cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
-        } else if (w) {
-                GF_ASSERT (!"opword mismatch");
+            }
         }
 
-        w = str_getunamb (words[wordcount - 1], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        brick_index = 5;
+    } else if (w) {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+    w = str_getunamb(words[wordcount - 1], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    } else {
+        /* handled this option */
+        wordcount--;
+        if (!strcmp("start", w)) {
+            command = GF_OP_CMD_START;
+            if (question)
+                *question = 1;
+        } else if (!strcmp("commit", w)) {
+            command = GF_OP_CMD_COMMIT;
+        } else if (!strcmp("stop", w)) {
+            command = GF_OP_CMD_STOP;
+        } else if (!strcmp("status", w)) {
+            command = GF_OP_CMD_STATUS;
+        } else if (!strcmp("force", w)) {
+            command = GF_OP_CMD_COMMIT_FORCE;
+            if (question)
+                *question = 1;
         } else {
-                /* handled this option */
-                wordcount--;
-                if (!strcmp ("start", w)) {
-                        command = GF_OP_CMD_START;
-                } else if (!strcmp ("commit", w)) {
-                        command = GF_OP_CMD_COMMIT;
-                        if (question)
-                                *question = 1;
-                } else if (!strcmp ("stop", w)) {
-                        command = GF_OP_CMD_STOP;
-                } else if (!strcmp ("status", w)) {
-                        command = GF_OP_CMD_STATUS;
-                } else if (!strcmp ("force", w)) {
-                        command = GF_OP_CMD_COMMIT_FORCE;
-                        if (question)
-                                *question = 1;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
-                        ret = -1;
-                        goto out;
-                }
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "command", command);
-        if (ret)
-                gf_log ("cli", GF_LOG_INFO, "failed to set 'command' %d",
-                        command);
+    ret = dict_set_int32(dict, "command", command);
+    if (ret)
+        gf_log("cli", GF_LOG_INFO, "failed to set 'command' %d", command);
 
+    tmp_index = brick_index;
+    tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
 
-        tmp_index = brick_index;
-        tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
-
-        if (!tmp_brick) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
-                goto out;
-        }
+    if (!tmp_brick) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
 
-        tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
+    tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
 
-        if (!tmp_brick1) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
+    if (!tmp_brick1) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
+
+        j = tmp_index;
+        strcpy(tmp_brick, words[brick_index]);
+        while (j < brick_index) {
+            strcpy(tmp_brick1, words[j]);
+            if (!(strcmp(tmp_brick, tmp_brick1))) {
+                gf_log("", GF_LOG_ERROR,
+                       "Duplicate bricks"
+                       " found %s",
+                       words[brick_index]);
+                cli_err("Duplicate bricks found %s", words[brick_index]);
                 ret = -1;
                 goto out;
+            }
+            j++;
         }
+        snprintf(key, 50, "brick%d", ++(*brick_count));
+        ret = dict_set_str(dict, key, (char *)words[brick_index++]);
 
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr(words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                j = tmp_index;
-                strcpy(tmp_brick, words[brick_index]);
-                while ( j < brick_index) {
-                        strcpy(tmp_brick1, words[j]);
-                        if (!(strcmp (tmp_brick, tmp_brick1))) {
-                                gf_log("",GF_LOG_ERROR, "Duplicate bricks"
-                                       " found %s", words[brick_index]);
-                                cli_err("Duplicate bricks found %s",
-                                        words[brick_index]);
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
-
-                if (ret)
-                        goto out;
-        }
+        if (ret)
+            goto out;
+    }
 
-        if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
-                ret = dict_set_int32 (dict, "count", brick_count);
-                if (ret)
-                        goto out;
-        }
+    if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
+        ret = dict_set_int32(dict, "count", *brick_count);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (tmp_brick);
-        GF_FREE (tmp_brick1);
+    GF_FREE(tmp_brick);
+    GF_FREE(tmp_brick1);
 
-        return ret;
+    *comm = command;
+
+    return ret;
 }
 
 int32_t
-cli_cmd_brick_op_validate_bricks (const char **words, dict_t *dict,
-				  int src, int dst)
+cli_cmd_brick_op_validate_bricks(const char **words, dict_t *dict, int src,
+                                 int dst)
 {
-        int             ret = -1;
-        char            *delimiter  = NULL;
-
-        if (validate_brick_name ((char *)words[src])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[src], '/');
-                ret = gf_canonicalize_path (delimiter);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_set_str (dict, "src-brick", (char *)words[src]);
+    int ret = -1;
+    char *delimiter = NULL;
+
+    if (validate_brick_name((char *)words[src])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[3]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[src], '/');
+        ret = gf_canonicalize_path(delimiter);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (dst == -1) {
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_str(dict, "src-brick", (char *)words[src]);
+    if (ret)
+        goto out;
 
-        if (validate_brick_name ((char *)words[dst])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[dst]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[dst], '/');
-                ret = gf_canonicalize_path (delimiter);
-                if (ret)
-                        goto out;
-        }
+    if (dst == -1) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "dst-brick", (char *)words[dst]);
+    if (validate_brick_name((char *)words[dst])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[dst]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[dst], '/');
+        ret = gf_canonicalize_path(delimiter);
         if (ret)
-                goto out;
-        ret = 0;
+            goto out;
+    }
+
+    ret = dict_set_str(dict, "dst-brick", (char *)words[dst]);
+    if (ret)
+        goto out;
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_reset_brick_parse (const char **words, int wordcount,
-                                  dict_t **options)
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        int                   ret        = -1;
-        char                 *volname    = NULL;
-        dict_t               *dict       = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        if (wordcount < 5 || wordcount > 7)
-                goto out;
-
-        dict = dict_new ();
+    if (wordcount < 5 || wordcount > 7)
+        goto out;
 
-        if (!dict)
-                goto out;
+    dict = dict_new();
 
-        volname = (char *)words[2];
-
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (wordcount == 5) {
-                if (strcmp (words[4], "start")) {
-                        cli_err ("Invalid option '%s' for reset-brick. Please "
-                                 "enter valid reset-brick command", words[4]);
-                        ret = -1;
-                        goto out;
-                }
+    volname = (char *)words[2];
 
-                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, -1);
-                if (ret)
-                        goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-                ret = dict_set_str (dict, "operation", "GF_RESET_OP_START");
-                if (ret)
-                        goto out;
-        } else if (wordcount == 6) {
-                if (strcmp (words[5], "commit")) {
-                        cli_err ("Invalid option '%s' for reset-brick. Please "
-                                 "enter valid reset-brick command", words[5]);
-                        ret = -1;
-                        goto out;
-                }
+    if (wordcount == 5) {
+        if (strcmp(words[4], "start")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[4]);
+            ret = -1;
+            goto out;
+        }
 
-                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
-                if (ret)
-                        goto out;
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, -1);
+        if (ret)
+            goto out;
 
-                ret = dict_set_str (dict, "operation", "GF_RESET_OP_COMMIT");
-                if (ret)
-                        goto out;
-        } else if (wordcount == 7) {
-                if (strcmp (words[5], "commit") || strcmp (words[6], "force")) {
-                        cli_err ("Invalid option '%s %s' for reset-brick. Please "
-                                 "enter valid reset-brick command",
-                                  words[5], words[6]);
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_START");
+        if (ret)
+            goto out;
+    } else if (wordcount == 6) {
+        if (strcmp(words[5], "commit")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
-                if (ret)
-                        goto out;
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT");
+        if (ret)
+            goto out;
+    } else if (wordcount == 7) {
+        if (strcmp(words[5], "commit") || strcmp(words[6], "force")) {
+            cli_err(
+                "Invalid option '%s %s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5], words[6]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-                ret = dict_set_str (dict, "operation",
-                                    "GF_RESET_OP_COMMIT_FORCE");
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT_FORCE");
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unable to parse reset-brick CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
-                                    dict_t **options)
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
+                                   dict_t **options)
 {
-        int                   ret        = -1;
-        char                 *volname    = NULL;
-        dict_t               *dict       = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (wordcount != 7) {
-                ret = -1;
-                goto out;
-        }
+    if (wordcount != 7) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
-                goto out;
-        }
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
+        goto out;
+    }
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
-        if (ret)
-                goto out;
+    ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+    if (ret)
+        goto out;
 
-        /* commit force option */
-        if (strcmp ("commit", words[5]) || strcmp ("force", words[6])) {
-                cli_err ("Invalid option '%s' '%s' for replace-brick. Please "
-                         "enter valid replace-brick command", words[5],
-                         words[6]);
-                ret = -1;
-                goto out;
-        }
+    /* commit force option */
+    if (strcmp("commit", words[5]) || strcmp("force", words[6])) {
+        cli_err(
+            "Invalid option '%s' '%s' for replace-brick. Please "
+            "enter valid replace-brick command",
+            words[5], words[6]);
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        str = (char *)words[4];
-        if (strchr (str, ':')) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
-                /* Path */
-                str = (char *)words[5];
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+    str = (char *)words[4];
+    if (strchr(str, ':')) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
         } else {
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+        /* Path */
+        str = (char *)words[5];
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    } else {
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
+cli_cmd_log_level_parse(const char **words, int worcount, dict_t **options)
 {
-        dict_t *dict            = NULL;
-        int     ret             = -1;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        /*
-         * loglevel command format:
-         *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
-         *  > volume log level colon-o posix WARNING
-         *  > volume log level colon-o replicate* DEBUG
-         *  > volume log level coon-o * TRACE
-         */
-
-        GF_ASSERT ((strncmp(words[0], "volume", 6) == 0));
-        GF_ASSERT ((strncmp(words[1], "log", 3) == 0));
-        GF_ASSERT ((strncmp(words[2], "level", 5) == 0));
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /*
+     * loglevel command format:
+     *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
+     *  > volume log level colon-o posix WARNING
+     *  > volume log level colon-o replicate* DEBUG
+     *  > volume log level coon-o * TRACE
+     */
+
+    GF_ASSERT((strncmp(words[0], "volume", 6) == 0));
+    GF_ASSERT((strncmp(words[1], "log", 3) == 0));
+    GF_ASSERT((strncmp(words[2], "level", 5) == 0));
+
+    ret = glusterd_check_log_level(words[5]);
+    if (ret == -1) {
+        cli_err("Invalid log level [%s] specified", words[5]);
+        cli_err(
+            "Valid values for loglevel: (DEBUG|WARNING|ERROR"
+            "|CRITICAL|NONE|TRACE)");
+        goto out;
+    }
 
-        ret = glusterd_check_log_level(words[5]);
-        if (ret == -1) {
-                cli_err("Invalid log level [%s] specified", words[5]);
-                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
-                        "|CRITICAL|NONE|TRACE)");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(words[3]);
+    GF_ASSERT(words[4]);
 
-        GF_ASSERT(words[3]);
-        GF_ASSERT(words[4]);
+    ret = dict_set_str(dict, "volname", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", (char *)words[3]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "xlator", (char *)words[4]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "xlator", (char *)words[4]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "loglevel", (char *)words[5]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "loglevel", (char *)words[5]);
-        if (ret)
-                goto out;
+    *options = dict;
 
-        *options = dict;
-
- out:
-        if (ret && dict)
-                dict_unref (dict);
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp ("rotate", words[3]) == 0)
-                volname = (char *)words[2];
-        else if (strcmp ("rotate", words[2]) == 0)
-                volname = (char *)words[3];
-        GF_ASSERT (volname);
+    if (strcmp("rotate", words[3]) == 0)
+        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-gsyncd_url_check (const char *w)
+gsyncd_url_check(const char *w)
 {
-        return !!strpbrk (w, ":/");
+    return !!strpbrk(w, ":/");
 }
 
 static gf_boolean_t
-gsyncd_glob_check (const char *w)
+valid_slave_gsyncd_url(const char *w)
 {
-        return !!strpbrk (w, "*?[");
+    if (strstr(w, ":::"))
+        return _gf_false;
+    else if (strstr(w, "::"))
+        return _gf_true;
+    else
+        return _gf_false;
 }
 
-static int
-config_parse (const char **words, int wordcount, dict_t *dict,
-              unsigned cmdi, unsigned glob)
+static gf_boolean_t
+gsyncd_glob_check(const char *w)
 {
-        int32_t            ret     = -1;
-        int32_t            i       = -1;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
+    return !!strpbrk(w, "*?[");
+}
 
-        switch ((wordcount - 1) - cmdi) {
+static int
+config_parse(const char **words, int wordcount, dict_t *dict, unsigned cmdi,
+             unsigned glob)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    char *append_str = NULL;
+    size_t append_len = 0;
+    char *subop = NULL;
+    char *ret_chkpt = NULL;
+    struct tm checkpoint_time;
+    char chkpt_buf[20] = "";
+
+    switch ((wordcount - 1) - cmdi) {
         case 0:
-                subop = gf_strdup ("get-all");
-                break;
+            subop = gf_strdup("get-all");
+            break;
         case 1:
-                if (words[cmdi + 1][0] == '!') {
-                        (words[cmdi + 1])++;
-                        if (gf_asprintf (&subop, "del%s",
-                                         glob ? "-glob" : "") == -1)
-                                subop = NULL;
-                } else
-                        subop = gf_strdup ("get");
+            if (words[cmdi + 1][0] == '!') {
+                (words[cmdi + 1])++;
+                if (gf_asprintf(&subop, "del%s", glob ? "-glob" : "") == -1)
+                    subop = NULL;
+            } else
+                subop = gf_strdup("get");
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
-                break;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+            break;
         default:
-                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                        subop = NULL;
+            if (gf_asprintf(&subop, "set%s", glob ? "-glob" : "") == -1)
+                subop = NULL;
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
 
-                /* join the varargs by spaces to get the op_value */
+            /* join the varargs by spaces to get the op_value */
 
-                for (i = cmdi + 2; i < wordcount; i++)
-                        append_len += (strlen (words[i]) + 1);
-                /* trailing strcat will add two bytes, make space for that */
-                append_len++;
+            for (i = cmdi + 2; i < wordcount; i++)
+                append_len += (strlen(words[i]) + 1);
+            /* trailing strcat will add two bytes, make space for that */
+            append_len++;
 
-                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
-                if (!append_str) {
-                        ret = -1;
-                        goto out;
-                }
+            /* strcat is used on this allocation and hence expected to be
+             * initiatlized to 0. So GF_CALLOC is used.
+             */
+            append_str = GF_CALLOC(1, append_len, cli_mt_append_str);
+            if (!append_str) {
+                ret = -1;
+                goto out;
+            }
+
+            for (i = cmdi + 2; i < wordcount; i++) {
+                strcat(append_str, words[i]);
+                strcat(append_str, " ");
+            }
+            append_str[append_len - 2] = '\0';
+            /* "checkpoint now" is special: we resolve that "now" */
+            if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                (strcmp(append_str, "now") == 0)) {
+                struct timeval tv = {
+                    0,
+                };
+
+                ret = gettimeofday(&tv, NULL);
+                if (ret == -1)
+                    goto out;
 
-                for (i = cmdi + 2; i < wordcount; i++) {
-                        strcat (append_str, words[i]);
-                        strcat (append_str, " ");
-                }
-                append_str[append_len - 2] = '\0';
-                /* "checkpoint now" is special: we resolve that "now" */
-                if ((strcmp (words[cmdi + 1], "checkpoint") == 0) &&
-                    (strcmp (append_str, "now") == 0)) {
-                        struct timeval tv = {0,};
-
-                        ret = gettimeofday (&tv, NULL);
-                        if (ret == -1)
-                                goto out;
-
-                        GF_FREE (append_str);
-                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-                        snprintf (append_str, 300, "%" GF_PRI_SECOND,
-                                  tv.tv_sec);
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
+                }
+                snprintf(append_str, 300, "%" GF_PRI_SECOND, tv.tv_sec);
+            } else if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                       (strcmp(append_str, "now") != 0)) {
+                memset(&checkpoint_time, 0, sizeof(struct tm));
+                ret_chkpt = strptime(append_str, "%Y-%m-%d %H:%M:%S",
+                                     &checkpoint_time);
+
+                if (ret_chkpt == NULL || *ret_chkpt != '\0') {
+                    ret = -1;
+                    cli_err(
+                        "Invalid Checkpoint label. Use format "
+                        "\"Y-m-d H:M:S\", Example: 2016-10-25 15:30:45");
+                    goto out;
+                }
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
                 }
+                strftime(chkpt_buf, sizeof(chkpt_buf), "%s", &checkpoint_time);
+                snprintf(append_str, 300, "%s", chkpt_buf);
+            }
 
-                ret = dict_set_dynstr (dict, "op_value", append_str);
-        }
+            ret = dict_set_dynstr(dict, "op_value", append_str);
+            if (ret != 0) {
+                goto out;
+            }
+            append_str = NULL;
+    }
 
-        ret = -1;
-        if (subop) {
-                ret = dict_set_dynstr (dict, "subop", subop);
-                if (!ret)
-                      subop = NULL;
-        }
+    ret = -1;
+    if (subop) {
+        ret = dict_set_dynstr(dict, "subop", subop);
+        if (!ret)
+            subop = NULL;
+    }
 
 out:
-        if (ret && append_str)
-                GF_FREE (append_str);
-
-        GF_FREE (subop);
+    GF_FREE(append_str);
+    GF_FREE(subop);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 /* ssh_port_parse: Parses and validates when ssh_port is given.
@@ -2657,1409 +2771,1467 @@ out:
  */
 
 static int32_t
-parse_ssh_port (const char **words, int wordcount, dict_t *dict,
-                unsigned *cmdi, int ssh_index, char *type) {
-
-        int        ret         = 0;
-        char      *end_ptr     = NULL;
-        int64_t    limit       = 0;
-
-        if (!strcmp ((char *)words[ssh_index], "ssh-port")) {
-                if (strcmp ((char *)words[ssh_index-1], "create")) {
-                        ret = -1;
-                        goto out;
-                }
-                (*cmdi)++;
-                limit = strtol (words[ssh_index+1], &end_ptr, 10);
-                if (errno == ERANGE || errno == EINVAL || limit <= 0
-                                    || strcmp (end_ptr, "") != 0) {
-                        ret = -1;
-                        cli_err ("Please enter an integer value for ssh_port ");
-                        goto out;
-                }
+parse_ssh_port(const char **words, int wordcount, dict_t *dict, unsigned *cmdi,
+               int ssh_index, char *type)
+{
+    int ret = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
 
-                ret = dict_set_int32 (dict, "ssh_port", limit);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
-        } else if (strcmp ((char *)words[ssh_index+1], "create")) {
-                ret = -1;
-                goto out;
+    if (!strcmp((char *)words[ssh_index], "ssh-port")) {
+        if (strcmp((char *)words[ssh_index - 1], "create")) {
+            ret = -1;
+            goto out;
+        }
+        (*cmdi)++;
+        limit = strtol(words[ssh_index + 1], &end_ptr, 10);
+        if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+            strcmp(end_ptr, "") != 0) {
+            ret = -1;
+            cli_err("Please enter an integer value for ssh_port ");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, type, 1);
+        ret = dict_set_int32(dict, "ssh_port", limit);
         if (ret)
-                goto out;
+            goto out;
         (*cmdi)++;
+    } else if (strcmp((char *)words[ssh_index + 1], "create")) {
+        ret = -1;
+        goto out;
+    }
 
- out:
-        return ret;
+    ret = dict_set_int32(dict, type, 1);
+    if (ret)
+        goto out;
+    (*cmdi)++;
+
+out:
+    return ret;
 }
 
 static int32_t
-force_push_pem_no_verify_parse (const char **words, int wordcount,
-                      dict_t *dict, unsigned *cmdi)
+force_push_pem_no_verify_parse(const char **words, int wordcount, dict_t *dict,
+                               unsigned *cmdi)
 {
-        int32_t            ret     = 0;
-
-        if (!strcmp ((char *)words[wordcount-1], "force")) {
-                if ((strcmp ((char *)words[wordcount-2], "start")) &&
-                    (strcmp ((char *)words[wordcount-2], "stop")) &&
-                    (strcmp ((char *)words[wordcount-2], "create")) &&
-                    (strcmp ((char *)words[wordcount-2], "no-verify")) &&
-                    (strcmp ((char *)words[wordcount-2], "push-pem")) &&
-                    (strcmp ((char *)words[wordcount-2], "pause")) &&
-                    (strcmp ((char *)words[wordcount-2], "resume"))) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "force",
-                                       _gf_true);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
-
-                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
-                        ret = parse_ssh_port (words, wordcount, dict, cmdi,
-                                              wordcount-4, "push_pem");
-                        if (ret)
-                                goto out;
-                } else if (!strcmp ((char *)words[wordcount-2], "no-verify")) {
-                        ret = parse_ssh_port (words, wordcount, dict, cmdi,
-                                              wordcount-4, "no_verify");
-                        if (ret)
-                                goto out;
-                }
-        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
-                ret = parse_ssh_port (words, wordcount, dict, cmdi, wordcount-3,
-                                      "push_pem");
-                if (ret)
-                        goto out;
-        } else if (!strcmp ((char *)words[wordcount-1], "no-verify")) {
-                ret = parse_ssh_port (words, wordcount, dict, cmdi, wordcount-3,
-                                      "no_verify");
-                if (ret)
-                        goto out;
+    int32_t ret = 0;
+
+    if (!strcmp((char *)words[wordcount - 1], "force")) {
+        if ((strcmp((char *)words[wordcount - 2], "start")) &&
+            (strcmp((char *)words[wordcount - 2], "stop")) &&
+            (strcmp((char *)words[wordcount - 2], "create")) &&
+            (strcmp((char *)words[wordcount - 2], "no-verify")) &&
+            (strcmp((char *)words[wordcount - 2], "push-pem")) &&
+            (strcmp((char *)words[wordcount - 2], "pause")) &&
+            (strcmp((char *)words[wordcount - 2], "resume"))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(dict, "force", SLEN("force"), 1);
+        if (ret)
+            goto out;
+        (*cmdi)++;
+
+        if (!strcmp((char *)words[wordcount - 2], "push-pem")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "push_pem");
+            if (ret)
+                goto out;
+        } else if (!strcmp((char *)words[wordcount - 2], "no-verify")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "no_verify");
+            if (ret)
+                goto out;
         }
+    } else if (!strcmp((char *)words[wordcount - 1], "push-pem")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "push_pem");
+        if (ret)
+            goto out;
+    } else if (!strcmp((char *)words[wordcount - 1], "no-verify")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "no_verify");
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **errstr)
 {
-        int32_t            ret     = -1;
-        dict_t             *dict   = NULL;
-        gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        int                i       = 0;
-        unsigned           masteri = 0;
-        unsigned           slavei  = 0;
-        unsigned           glob    = 0;
-        unsigned           cmdi    = 0;
-        char               *opwords[] = { "create", "status", "start", "stop",
-                                          "config", "force", "delete",
-                                          "ssh-port", "no-verify", "push-pem",
-                                          "detail", "pause", "resume", NULL };
-        char               *w = NULL;
-        char               *save_ptr   = NULL;
-        char               *slave_temp = NULL;
-        char               *token      = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* new syntax:
-         *
-         * volume geo-replication $m $s create [[ssh-port n] [[no-verify] | [push-pem]]] [force]
-         * volume geo-replication [$m [$s]] status [detail]
-         * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop [force]
-         * volume geo-replication $m $s delete [reset-sync-time]
-         * volume geo-replication $m $s pause [force]
-         * volume geo-replication $m $s resume [force]
-         */
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_gsync_set type = GF_GSYNC_OPTION_TYPE_NONE;
+    int i = 0;
+    unsigned masteri = 0;
+    unsigned slavei = 0;
+    unsigned glob = 0;
+    unsigned cmdi = 0;
+    static char *opwords[] = {"create",    "status",   "start",  "stop",
+                              "config",    "force",    "delete", "ssh-port",
+                              "no-verify", "push-pem", "detail", "pause",
+                              "resume",    NULL};
+    char *w = NULL;
+    char *save_ptr = NULL;
+    char *slave_temp = NULL;
+    char *token = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    /* new syntax:
+     *
+     * volume geo-replication $m $s create [[ssh-port n] [[no-verify] |
+     * [push-pem]]] [force] volume geo-replication [$m [$s]] status [detail]
+     * volume geo-replication [$m] $s config [[!]$opt [$val]]
+     * volume geo-replication $m $s start|stop [force]
+     * volume geo-replication $m $s delete [reset-sync-time]
+     * volume geo-replication $m $s pause [force]
+     * volume geo-replication $m $s resume [force]
+     */
+
+    if (wordcount < 3)
+        goto out;
 
-        for (i = 2; i <= 3 && i < wordcount - 1; i++) {
-                if (gsyncd_glob_check (words[i]))
-                        glob = i;
-                if (gsyncd_url_check (words[i])) {
-                        slavei = i;
-                        break;
-                }
+    for (i = 2; i <= 3 && i < wordcount - 1; i++) {
+        if (gsyncd_glob_check(words[i]))
+            glob = i;
+        if (gsyncd_url_check(words[i])) {
+            slavei = i;
+            break;
         }
+    }
 
-        if (glob && !slavei)
-                /* glob is allowed only for config, thus it implies there is a
-                 * slave argument; but that might have not been recognized on
-                 * the first scan as it's url characteristics has been covered
-                 * by the glob syntax.
-                 *
-                 * In this case, the slave is perforce the last glob-word -- the
-                 * upcoming one is neither glob, nor url, so it's definitely not
-                 * the slave.
-                 */
-                slavei = glob;
-        if (slavei) {
-                cmdi = slavei + 1;
-                if (slavei == 3)
-                        masteri = 2;
-        } else if (i <= 3) {
-                if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                        /* For status detail it is mandatory to provide
-                         * both master and slave */
-                        ret = -1;
-                        goto out;
-                }
-
-                /* no $s, can only be status cmd
-                 * (with either a single $m before it or nothing)
-                 * -- these conditions imply that i <= 3 after
-                 * the iteration and that i is the successor of
-                 * the (0 or 1 length) sequence of $m-s.
-                 */
-                cmdi = i;
-                if (i == 3)
-                        masteri = 2;
-        } else
-                goto out;
-
-        /* now check if input really complies syntax
-         * (in a somewhat redundant way, in favor
-         * transparent soundness)
+    if (glob && !slavei)
+        /* glob is allowed only for config, thus it implies there is a
+         * slave argument; but that might have not been recognized on
+         * the first scan as it's url characteristics has been covered
+         * by the glob syntax.
+         *
+         * In this case, the slave is perforce the last glob-word -- the
+         * upcoming one is neither glob, nor url, so it's definitely not
+         * the slave.
          */
+        slavei = glob;
+    if (slavei) {
+        cmdi = slavei + 1;
+        if (slavei == 3)
+            masteri = 2;
+    } else if (i <= 4) {
+        if (strtail("detail", (char *)words[wordcount - 1])) {
+            cmdi = wordcount - 2;
+            if (i == 4)
+                masteri = 2;
+        } else {
+            /* no $s, can only be status cmd
+             * (with either a single $m before it or nothing)
+             * -- these conditions imply that i <= 3 after
+             * the iteration and that i is the successor of
+             * the (0 or 1 length) sequence of $m-s.
+             */
+            cmdi = i;
+            if (i == 3)
+                masteri = 2;
+        }
+    } else
+        goto out;
 
-        if (masteri && gsyncd_url_check (words[masteri]))
-                goto out;
-        if (slavei && !glob && !gsyncd_url_check (words[slavei]))
-                goto out;
-
-        w = str_getunamb (words[cmdi], opwords);
-        if (!w)
-                goto out;
-
-        if (strcmp (w, "create") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CREATE;
-
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "status") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STATUS;
-
-                if (slavei && !masteri)
-                        goto out;
-        } else if (strcmp (w, "config") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CONFIG;
+    /* now check if input really complies syntax
+     * (in a somewhat redundant way, in favor
+     * transparent soundness)
+     */
 
-                if (!slavei)
-                        goto out;
-        } else if (strcmp (w, "start") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_START;
-
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "stop") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STOP;
+    if (masteri && gsyncd_url_check(words[masteri]))
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "delete") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_DELETE;
+    if (slavei && !glob && !valid_slave_gsyncd_url(words[slavei])) {
+        gf_asprintf(errstr, "Invalid slave url: %s", words[slavei]);
+        goto out;
+    }
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "pause") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_PAUSE;
+    w = str_getunamb(words[cmdi], opwords);
+    if (!w)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "resume") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_RESUME;
+    if (strcmp(w, "create") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "status") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STATUS;
+
+        if (slavei && !masteri)
+            goto out;
+    } else if (strcmp(w, "config") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CONFIG;
+
+        if (!slavei)
+            goto out;
+    } else if (strcmp(w, "start") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_START;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "stop") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STOP;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "delete") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_DELETE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "pause") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_PAUSE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "resume") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_RESUME;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = force_push_pem_no_verify_parse(words, wordcount, dict, &cmdi);
+    if (ret)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strtail("detail", (char *)words[wordcount - 1])) {
+        if (!strtail("status", (char *)words[wordcount - 2])) {
+            ret = -1;
+            goto out;
+        }
 
-        ret = force_push_pem_no_verify_parse (words, wordcount, dict, &cmdi);
+        ret = dict_set_uint32(dict, "status-detail", _gf_true);
         if (ret)
-                goto out;
+            goto out;
+        cmdi++;
+    }
 
-        if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                if (strcmp ((char *)words[wordcount-2], "status")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!slavei || !masteri) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
-                if (ret)
-                        goto out;
-                cmdi++;
-        }
-
-        if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
-            !strcmp ((char *)words[wordcount-1], "reset-sync-time")) {
-                if (strcmp ((char *)words[wordcount-2], "delete")) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "reset-sync-time", _gf_true);
-                if (ret)
-                        goto out;
-                cmdi++;
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
+        !strcmp((char *)words[wordcount - 1], "reset-sync-time")) {
+        if (strcmp((char *)words[wordcount - 2], "delete")) {
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_uint32(dict, "reset-sync-time", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
 
-        if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
-            (cmdi < wordcount - 1 || glob))
-                goto out;
+    if (type != GF_GSYNC_OPTION_TYPE_CONFIG && (cmdi < wordcount - 1 || glob)) {
+        ret = -1;
+        goto out;
+    }
 
-        /* If got so far, input is valid, assemble the message */
+    /* If got so far, input is valid, assemble the message */
 
-        ret = 0;
+    ret = 0;
 
-        if (masteri) {
-                ret = dict_set_str (dict, "master", (char *)words[masteri]);
-                if (!ret)
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[masteri]);
-        }
-        if (!ret && slavei) {
-                /* If geo-rep is created with root user using the syntax
-                 * gluster vol geo-rep <mastervol> root@<slavehost> ...
-                 * pass down only <slavehost> else pass as it is.
-                 */
-                slave_temp = gf_strdup (words[slavei]);
-                token = strtok_r (slave_temp, "@", &save_ptr);
-                if (token && !strcmp (token, "root")) {
-                        ret = dict_set_str (dict, "slave",
-                                            (char *)words[slavei]+5);
-                } else {
-                        ret = dict_set_str (dict, "slave",
-                                            (char *)words[slavei]);
-                }
-        }
+    if (masteri) {
+        ret = dict_set_str(dict, "master", (char *)words[masteri]);
         if (!ret)
-                ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
-                ret = config_parse (words, wordcount, dict, cmdi, glob);
+            ret = dict_set_str(dict, "volname", (char *)words[masteri]);
+    }
+    if (!ret && slavei) {
+        /* If geo-rep is created with root user using the syntax
+         * gluster vol geo-rep <mastervol> root@<slavehost> ...
+         * pass down only <slavehost> else pass as it is.
+         */
+        slave_temp = gf_strdup(words[slavei]);
+        if (slave_temp == NULL) {
+            ret = -1;
+            goto out;
+        }
+        token = strtok_r(slave_temp, "@", &save_ptr);
+        if (token && !strcmp(token, "root")) {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei] + 5);
+        } else {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei]);
+        }
+    }
+    if (!ret)
+        ret = dict_set_int32(dict, "type", type);
+    if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        if (!strcmp((char *)words[wordcount - 2], "ignore-deletes") &&
+            !strcmp((char *)words[wordcount - 1], "true")) {
+            question =
+                "There exists ~15 seconds delay for the option to take"
+                " effect from stime of the corresponding brick. Please"
+                " check the log for the time, the option is effective."
+                " Proceed";
+
+            answer = cli_cmd_get_confirmation(state, question);
+
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_INFO,
+                       "Operation "
+                       "cancelled, exiting");
+                *errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
 
-out:
-        if (slave_temp)
-                GF_FREE (slave_temp);
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
-        } else
-                *options = dict;
+        ret = config_parse(words, wordcount, dict, cmdi, glob);
+    }
 
+out:
+    if (slave_temp)
+        GF_FREE(slave_temp);
+    if (ret && dict)
+        dict_unref(dict);
+    else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options)
 {
-        dict_t              *dict       = NULL;
-        char                *volname    = NULL;
-        int                 ret         = -1;
-        gf1_cli_stats_op    op          = GF_CLI_STATS_NONE;
-        gf1_cli_info_op     info_op     = GF_CLI_INFO_NONE;
-        gf_boolean_t        is_peek     = _gf_false;
-
-        char      *opwords[] = { "start", "stop", "info", NULL };
-        char      *w = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+    gf_boolean_t is_peek = _gf_false;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    static char *opwords[] = {"start", "stop", "info", NULL};
+    char *w = NULL;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (wordcount < 4)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if ((strcmp (w, "start") == 0 || strcmp (w, "stop") == 0) &&
-            wordcount > 5)
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "info") == 0 && wordcount > 7)
-                goto out;
+    if ((strcmp(w, "start") == 0 || strcmp(w, "stop") == 0) && wordcount > 5) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "start") == 0) {
-                op = GF_CLI_STATS_START;
-        } else if (strcmp (w, "stop") == 0) {
-                op = GF_CLI_STATS_STOP;
-        } else if (strcmp (w, "info") == 0) {
-                op = GF_CLI_STATS_INFO;
-                info_op = GF_CLI_INFO_ALL;
-                if (wordcount > 4) {
-                        if (strcmp (words[4], "incremental") == 0) {
-                                info_op = GF_CLI_INFO_INCREMENTAL;
-                                if (wordcount > 5 &&
-                                    strcmp (words[5], "peek") == 0) {
-                                        is_peek = _gf_true;
-                                }
-                        } else if (strcmp (words[4], "cumulative") == 0) {
-                                info_op = GF_CLI_INFO_CUMULATIVE;
-                        } else if (strcmp (words[4], "clear") == 0) {
-                                info_op = GF_CLI_INFO_CLEAR;
-                        } else if (strcmp (words[4], "peek") == 0) {
-                                is_peek = _gf_true;
-                        }
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strcmp(w, "info") == 0 && wordcount > 7) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "start") == 0) {
+        op = GF_CLI_STATS_START;
+    } else if (strcmp(w, "stop") == 0) {
+        op = GF_CLI_STATS_STOP;
+    } else if (strcmp(w, "info") == 0) {
+        op = GF_CLI_STATS_INFO;
+        info_op = GF_CLI_INFO_ALL;
+        if (wordcount > 4) {
+            if (strcmp(words[4], "incremental") == 0) {
+                info_op = GF_CLI_INFO_INCREMENTAL;
+                if (wordcount > 5 && strcmp(words[5], "peek") == 0) {
+                    is_peek = _gf_true;
+                }
+            } else if (strcmp(words[4], "cumulative") == 0) {
+                info_op = GF_CLI_INFO_CUMULATIVE;
+            } else if (strcmp(words[4], "clear") == 0) {
+                info_op = GF_CLI_INFO_CLEAR;
+            } else if (strcmp(words[4], "peek") == 0) {
+                is_peek = _gf_true;
+            }
+        }
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "info-op", (int32_t)info_op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "info-op", (int32_t)info_op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "peek", is_peek);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "peek", is_peek);
+    if (!strcmp(words[wordcount - 1], "nfs")) {
+        ret = dict_set_int32(dict, "nfs", _gf_true);
         if (ret)
-                goto out;
-
-        if (!strcmp (words[wordcount - 1], "nfs")) {
-                ret = dict_set_int32 (dict, "nfs", _gf_true);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict           = NULL;
-        char    *volname        = NULL;
-        char    *value          = NULL;
-        char    *key            = NULL;
-        int      ret            = -1;
-        gf1_cli_stats_op op = GF_CLI_STATS_NONE;
-        gf1_cli_top_op    top_op = GF_CLI_TOP_NONE;
-        int32_t  list_cnt       = -1;
-        int      index          = 0;
-        int      perf           = 0;
-        int32_t  blk_size       = 0;
-        int      count          = 0;
-        gf_boolean_t nfs        = _gf_false;
-        char    *delimiter      = NULL;
-        char    *opwords[]      = { "open", "read", "write", "opendir",
-                                    "readdir", "read-perf", "write-perf",
-                                    "clear", NULL };
-        char    *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount < 4)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char *key = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    int32_t list_cnt = -1;
+    int index = 0;
+    int perf = 0;
+    int32_t blk_size = 0;
+    int count = 0;
+    gf_boolean_t nfs = _gf_false;
+    char *delimiter = NULL;
+    static char *opwords[] = {"open",       "read",    "write",
+                              "opendir",    "readdir", "read-perf",
+                              "write-perf", "clear",   NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        op = GF_CLI_STATS_TOP;
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
-        if (strcmp (w, "open") == 0) {
-                top_op = GF_CLI_TOP_OPEN;
-        } else if (strcmp (w, "read") == 0) {
-                top_op = GF_CLI_TOP_READ;
-        } else if (strcmp (w, "write") == 0) {
-                top_op = GF_CLI_TOP_WRITE;
-        } else if (strcmp (w, "opendir") == 0) {
-                top_op = GF_CLI_TOP_OPENDIR;
-        } else if (strcmp (w, "readdir") == 0) {
-                top_op = GF_CLI_TOP_READDIR;
-        } else if (strcmp (w, "read-perf") == 0) {
-                top_op = GF_CLI_TOP_READ_PERF;
-                perf = 1;
-        } else if (strcmp (w, "write-perf") == 0) {
-                top_op = GF_CLI_TOP_WRITE_PERF;
-                perf = 1;
-        } else if (strcmp (w, "clear") == 0) {
-                ret = dict_set_int32 (dict, "clear-stats", 1);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not set clear-stats in dict");
-                        goto out;
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
-        ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
-        if (ret)
-                goto out;
+    op = GF_CLI_STATS_TOP;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        if ((wordcount > 4) && !strcmp (words[4], "nfs")) {
-                nfs = _gf_true;
-                ret = dict_set_int32 (dict, "nfs", nfs);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                index = 4;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+    if (strcmp(w, "open") == 0) {
+        top_op = GF_CLI_TOP_OPEN;
+    } else if (strcmp(w, "read") == 0) {
+        top_op = GF_CLI_TOP_READ;
+    } else if (strcmp(w, "write") == 0) {
+        top_op = GF_CLI_TOP_WRITE;
+    } else if (strcmp(w, "opendir") == 0) {
+        top_op = GF_CLI_TOP_OPENDIR;
+    } else if (strcmp(w, "readdir") == 0) {
+        top_op = GF_CLI_TOP_READDIR;
+    } else if (strcmp(w, "read-perf") == 0) {
+        top_op = GF_CLI_TOP_READ_PERF;
+        perf = 1;
+    } else if (strcmp(w, "write-perf") == 0) {
+        top_op = GF_CLI_TOP_WRITE_PERF;
+        perf = 1;
+    } else if (strcmp(w, "clear") == 0) {
+        ret = dict_set_int32(dict, "clear-stats", 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not set clear-stats in dict");
+            goto out;
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
+    ret = dict_set_int32(dict, "top-op", (int32_t)top_op);
+    if (ret)
+        goto out;
 
-        for (; index < wordcount; index+=2) {
+    if ((wordcount > 4) && !strcmp(words[4], "nfs")) {
+        nfs = _gf_true;
+        ret = dict_set_int32(dict, "nfs", nfs);
+        if (ret)
+            goto out;
+        index = 5;
+    } else {
+        index = 4;
+    }
 
-                key = (char *) words[index];
-                value = (char *) words[index+1];
+    for (; index < wordcount; index += 2) {
+        key = (char *)words[index];
+        value = (char *)words[index + 1];
 
-                if ( key && !value ) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (key, "brick")) {
-                        delimiter = strchr (value, ':');
-                        if (!delimiter || delimiter == value
-                            || *(delimiter+1) != '/') {
-                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                         "<export-dir-abs-path>", value);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                ret = gf_canonicalize_path (delimiter + 1);
-                                if (ret)
-                                        goto out;
-                        }
-                        ret = dict_set_str (dict, "brick", value);
-
-                } else if (!strcmp (key, "list-cnt")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                list_cnt = atoi (value);
-                        if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_err ("list-cnt should be between 0 to 100");
-                                ret = -1;
-                                goto out;
-                        }
-                } else if (perf && !nfs && !strcmp (key, "bs")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                blk_size = atoi (value);
-                        if (ret || (blk_size <= 0)) {
-                                if (blk_size < 0)
-                                        cli_err ("block size is an invalid"
-                                                 " number");
-                                else
-                                        cli_err ("block size should be an "
-                                                 "integer greater than zero");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-size",
-                                                        (uint32_t)blk_size);
-                } else if (perf && !nfs && !strcmp (key, "count")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                count = atoi(value);
-                        if (ret || (count <= 0)) {
-                                if (count < 0)
-                                        cli_err ("count is an invalid number");
-                                else
-                                        cli_err ("count should be an integer "
-                                                 "greater than zero");
-
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-cnt", count);
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-                if (ret) {
-                        gf_log ("", GF_LOG_WARNING, "Dict set failed for "
-                                "key %s", key);
-                        goto out;
-                }
-        }
-        if (list_cnt == -1)
-                list_cnt = 100;
-        ret = dict_set_int32 (dict, "list-cnt", list_cnt);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(key, "brick")) {
+            delimiter = strchr(value, ':');
+            if (!delimiter || delimiter == value || *(delimiter + 1) != '/') {
+                cli_err(
+                    "wrong brick type: %s, use <HOSTNAME>:"
+                    "<export-dir-abs-path>",
+                    value);
+                ret = -1;
                 goto out;
-        }
-
-        if ((blk_size > 0) ^ (count > 0)) {
-                cli_err ("Need to give both 'bs' and 'count'");
+            } else {
+                ret = gf_canonicalize_path(delimiter + 1);
+                if (ret)
+                    goto out;
+            }
+            ret = dict_set_str(dict, "brick", value);
+
+        } else if (!strcmp(key, "list-cnt")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                list_cnt = atoi(value);
+            if (ret || (list_cnt < 0) || (list_cnt > 100)) {
+                cli_err("list-cnt should be between 0 to 100");
+                ret = -1;
+                goto out;
+            }
+        } else if (perf && !nfs && !strcmp(key, "bs")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                blk_size = atoi(value);
+            if (ret || (blk_size <= 0)) {
+                if (blk_size < 0)
+                    cli_err(
+                        "block size is an invalid"
+                        " number");
+                else
+                    cli_err(
+                        "block size should be an "
+                        "integer greater than zero");
                 ret = -1;
                 goto out;
-        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
-                cli_err ("'bs * count' value %"PRIu64" is greater than "
-                         "maximum allowed value of 10GB",
-                         ((uint64_t)blk_size * count));
+            }
+            ret = dict_set_uint32(dict, "blk-size", (uint32_t)blk_size);
+        } else if (perf && !nfs && !strcmp(key, "count")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                count = atoi(value);
+            if (ret || (count <= 0)) {
+                if (count < 0)
+                    cli_err("count is an invalid number");
+                else
+                    cli_err(
+                        "count should be an integer "
+                        "greater than zero");
+
                 ret = -1;
                 goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-cnt", count);
+        } else {
+            ret = -1;
+            goto out;
         }
+        if (ret) {
+            gf_log("", GF_LOG_WARNING,
+                   "Dict set failed for "
+                   "key %s",
+                   key);
+            goto out;
+        }
+    }
+    if (list_cnt == -1)
+        list_cnt = 100;
+    ret = dict_set_int32(dict, "list-cnt", list_cnt);
+    if (ret) {
+        gf_log("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        goto out;
+    }
 
-        *options = dict;
+    if ((blk_size > 0) ^ (count > 0)) {
+        cli_err("Need to give both 'bs' and 'count'");
+        ret = -1;
+        goto out;
+    } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+        cli_err("'bs * count' value %" PRIu64
+                " is greater than "
+                "maximum allowed value of 10GB",
+                ((uint64_t)blk_size * count));
+        ret = -1;
+        goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 uint32_t
-cli_cmd_get_statusop (const char *arg)
+cli_cmd_get_statusop(const char *arg)
 {
-        int        i         = 0;
-        uint32_t   ret       = GF_CLI_STATUS_NONE;
-        char      *w         = NULL;
-        char      *opwords[] = {"detail", "mem", "clients", "fd",
-                                "inode", "callpool", "tasks", NULL};
-        struct {
-                char      *opname;
-                uint32_t   opcode;
-        } optable[] = {
-                { "detail",   GF_CLI_STATUS_DETAIL   },
-                { "mem",      GF_CLI_STATUS_MEM      },
-                { "clients",  GF_CLI_STATUS_CLIENTS  },
-                { "fd",       GF_CLI_STATUS_FD       },
-                { "inode",    GF_CLI_STATUS_INODE    },
-                { "callpool", GF_CLI_STATUS_CALLPOOL },
-                { "tasks",    GF_CLI_STATUS_TASKS    },
-                { NULL }
-        };
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG,
-                        "Not a status op  %s", arg);
-                goto out;
-        }
+    int i = 0;
+    uint32_t ret = GF_CLI_STATUS_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"detail",   "mem",   "clients",     "fd", "inode",
+                              "callpool", "tasks", "client-list", NULL};
+    static struct {
+        char *opname;
+        uint32_t opcode;
+    } optable[] = {{"detail", GF_CLI_STATUS_DETAIL},
+                   {"mem", GF_CLI_STATUS_MEM},
+                   {"clients", GF_CLI_STATUS_CLIENTS},
+                   {"fd", GF_CLI_STATUS_FD},
+                   {"inode", GF_CLI_STATUS_INODE},
+                   {"callpool", GF_CLI_STATUS_CALLPOOL},
+                   {"tasks", GF_CLI_STATUS_TASKS},
+                   {"client-list", GF_CLI_STATUS_CLIENT_LIST},
+                   {NULL}};
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Not a status op  %s", arg);
+        goto out;
+    }
 
-        for (i = 0; optable[i].opname; i++) {
-                if (!strcmp (w, optable[i].opname)) {
-                        ret = optable[i].opcode;
-                        break;
-                }
+    for (i = 0; optable[i].opname; i++) {
+        if (!strcmp(w, optable[i].opname)) {
+            ret = optable[i].opcode;
+            break;
         }
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_status_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t    *dict            = NULL;
-        int        ret             = -1;
-        uint32_t   cmd             = 0;
-
-        GF_ASSERT (options);
+    dict_t *dict = NULL;
+    int ret = -1;
+    uint32_t cmd = 0;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(options);
 
-        switch (wordcount) {
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
+    switch (wordcount) {
         case 2:
-                cmd = GF_CLI_STATUS_ALL;
-                ret = 0;
-                break;
+            cmd = GF_CLI_STATUS_ALL;
+            ret = 0;
+            break;
 
         case 3:
-                if (!strcmp (words[2], "all")) {
-                        cmd = GF_CLI_STATUS_ALL;
-                        ret = 0;
+            if (!strcmp(words[2], "all")) {
+                cmd = GF_CLI_STATUS_ALL;
+                ret = 0;
 
-                } else {
-                        cmd = GF_CLI_STATUS_VOL;
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
-                }
+            } else {
+                cmd = GF_CLI_STATUS_VOL;
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+            }
 
-                break;
+            break;
 
         case 4:
-                cmd = cli_cmd_get_statusop (words[3]);
-
-                if (!strcmp (words[2], "all")) {
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                cli_err ("%s is not a valid status option",
-                                         words[3]);
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_ALL;
-                        ret  = 0;
-
-                } else {
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[2]);
-                        if (ret)
-                                goto out;
-
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                if (!strcmp (words[3], "nfs")) {
-                                        cmd |= GF_CLI_STATUS_NFS;
-                                } else if (!strcmp (words[3], "shd")) {
-                                        cmd |= GF_CLI_STATUS_SHD;
-                                } else if (!strcmp (words[3], "quotad")) {
-                                        cmd |= GF_CLI_STATUS_QUOTAD;
-                                } else if (!strcmp (words[3], "snapd")) {
-                                        cmd |= GF_CLI_STATUS_SNAPD;
-                                } else if (!strcmp (words[3], "bitd")) {
-                                        cmd |= GF_CLI_STATUS_BITD;
-                                } else if (!strcmp (words[3], "scrub")) {
-                                        cmd |= GF_CLI_STATUS_SCRUB;
-                                } else {
-                                        cmd = GF_CLI_STATUS_BRICK;
-                                        ret = dict_set_str (dict, "brick",
-                                                            (char *)words[3]);
-                                }
-
-                        } else {
-                                cmd |= GF_CLI_STATUS_VOL;
-                                ret  = 0;
-                        }
-                }
-
-                break;
-
-        case 5:
-                if (!strcmp (words[2], "all")) {
-                        cli_err ("Cannot specify brick/nfs for \"all\"");
-                        ret = -1;
-                        goto out;
-                }
+            cmd = cli_cmd_get_statusop(words[3]);
 
-                cmd = cli_cmd_get_statusop (words[4]);
+            if (!strcmp(words[2], "all")) {
                 if (cmd == GF_CLI_STATUS_NONE) {
-                        cli_err ("%s is not a valid status option",
-                                 words[4]);
-                        ret = -1;
-                        goto out;
+                    cli_err("%s is not a valid status option", words[3]);
+                    ret = -1;
+                    goto out;
                 }
+                cmd |= GF_CLI_STATUS_ALL;
+                ret = 0;
 
-
-                ret = dict_set_str (dict, "volname", (char *)words[2]);
+            } else {
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
                 if (ret)
-                        goto out;
+                    goto out;
 
-                if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Tasks status not available"
-                                         " for NFS Servers");
-                                ret = -1;
-                                goto out;
-                        }
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    if (!strcmp(words[3], "nfs")) {
                         cmd |= GF_CLI_STATUS_NFS;
-                } else if (!strcmp (words[3], "shd")){
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Clients/Tasks status not "
-                                         "available for Self-heal Daemons");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "shd")) {
                         cmd |= GF_CLI_STATUS_SHD;
-                } else if (!strcmp (words[3], "quotad")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for Quota Daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "quotad")) {
                         cmd |= GF_CLI_STATUS_QUOTAD;
-                } else if  (!strcmp (words[3], "snapd")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for snap daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "snapd")) {
                         cmd |= GF_CLI_STATUS_SNAPD;
+                    } else if (!strcmp(words[3], "bitd")) {
+                        cmd |= GF_CLI_STATUS_BITD;
+                    } else if (!strcmp(words[3], "scrub")) {
+                        cmd |= GF_CLI_STATUS_SCRUB;
+                    } else {
+                        cmd = GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str(dict, "brick", (char *)words[3]);
+                    }
+
                 } else {
-                        if (cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Tasks status not available for "
-                                         "bricks");
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_BRICK;
-                        ret = dict_set_str (dict, "brick", (char *)words[3]);
+                    cmd |= GF_CLI_STATUS_VOL;
+                    ret = 0;
                 }
-                break;
+            }
 
-        default:
-                goto out;
-        }
+            break;
 
-        if (ret)
+        case 5:
+            if (!strcmp(words[2], "all")) {
+                cli_err("Cannot specify brick/nfs for \"all\"");
+                ret = -1;
                 goto out;
+            }
 
-        ret = dict_set_int32 (dict, "cmd", cmd);
-        if (ret)
+            cmd = cli_cmd_get_statusop(words[4]);
+            if (cmd == GF_CLI_STATUS_NONE) {
+                cli_err("%s is not a valid status option", words[4]);
+                ret = -1;
                 goto out;
+            }
+
+            ret = dict_set_str(dict, "volname", (char *)words[2]);
+            if (ret)
+                goto out;
+
+            if (!strcmp(words[3], "nfs")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_DETAIL ||
+                    cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Tasks status not available"
+                        " for NFS Servers");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_NFS;
+            } else if (!strcmp(words[3], "shd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Clients/Tasks status not "
+                        "available for Self-heal Daemons");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SHD;
+            } else if (!strcmp(words[3], "quotad")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for Quota Daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_QUOTAD;
+            } else if (!strcmp(words[3], "snapd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for snap daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SNAPD;
+            } else {
+                if (cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Tasks status not available for "
+                        "bricks");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_BRICK;
+                ret = dict_set_str(dict, "brick", (char *)words[3]);
+            }
+            break;
 
-        *options = dict;
+        default:
+            goto out;
+    }
 
- out:
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret)
+        goto out;
 
-        return ret;
+    ret = dict_set_int32(dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 gf_boolean_t
-cli_cmd_validate_dumpoption (const char *arg, char **option)
+cli_cmd_validate_dumpoption(const char *arg, char **option)
 {
-        char    *opwords[] = {"all", "nfs", "mem", "iobuf", "callpool", "priv",
-                              "fd", "inode", "history", "inodectx", "fdctx",
-                              "quotad", NULL};
-        char    *w = NULL;
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG, "Unknown statedump option  %s",
-                        arg);
-                return _gf_false;
-        }
-        *option = w;
-        return _gf_true;
+    static char *opwords[] = {"all",   "nfs",    "mem",   "iobuf",   "callpool",
+                              "priv",  "fd",     "inode", "history", "inodectx",
+                              "fdctx", "quotad", NULL};
+    char *w = NULL;
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Unknown statedump option  %s", arg);
+        return _gf_false;
+    }
+    *option = w;
+    return _gf_true;
 }
 
 int
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options)
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options)
 {
-        int     ret = 0;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        int     option_cnt = 0;
-        char    *option = NULL;
-        char    option_str[100] = {0,};
-
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
-                        ret = -1;
-                        goto out;
-                }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
+    int ret = 0;
+    int i = 0;
+    dict_t *dict = NULL;
+    int option_cnt = 0;
+    char *option = NULL;
+    char *option_str = NULL;
+    char *tmp_str = NULL;
+    char *tmp = NULL;
+    char *ip_addr = NULL;
+    char *pid = NULL;
+
+    if ((wordcount >= 5) && ((strcmp(words[3], "client")) == 0)) {
+        tmp = gf_strdup(words[4]);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        ip_addr = strtok(tmp, ":");
+        pid = strtok(NULL, ":");
+        if (valid_internet_address(ip_addr, _gf_true, _gf_false) && pid &&
+            gf_valid_pid(pid, strlen(pid))) {
+            ret = gf_asprintf(&option_str, "%s %s %s", words[3], ip_addr, pid);
+            if (ret < 0) {
+                goto out;
+            }
+            option_cnt = 3;
+        } else {
+            ret = -1;
+            goto out;
         }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
+    } else {
+        for (i = 3; i < wordcount; i++, option_cnt++) {
+            if (!cli_cmd_validate_dumpoption(words[i], &option)) {
                 ret = -1;
                 goto out;
+            }
+            tmp_str = option_str;
+            option_str = NULL;
+            ret = gf_asprintf(&option_str, "%s%s ", tmp_str ? tmp_str : "",
+                              option);
+            GF_FREE(tmp_str);
+            if (ret < 0) {
+                goto out;
+            }
         }
+        if (option_str && (strstr(option_str, "nfs")) &&
+            strstr(option_str, "quotad")) {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "options", gf_strdup (option_str));
-        if (ret)
-                goto out;
+    /* dynamic string in dict is freed up when dict is freed up, and hence
+    if option_str is NULL pass in an duplicate empty string to the same */
+    ret = dict_set_dynstr(dict, "options",
+                          (option_str ? option_str : gf_strdup("")));
+    if (ret)
+        goto out;
+    option_str = NULL;
 
-        ret = dict_set_int32 (dict, "option_cnt", option_cnt);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "option_cnt", option_cnt);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
-        return ret;
+    GF_FREE(tmp);
+    GF_FREE(option_str);
+    if (ret && dict)
+        dict_unref(dict);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
+    return ret;
 }
 
 int
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
-                                  dict_t **options)
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        int     ret = -1;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        char    *kind_opts[4] = {"blocked", "granted", "all", NULL};
-        char    *types[4] = {"inode", "entry", "posix", NULL};
-        char    *free_ptr = NULL;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    int i = 0;
+    dict_t *dict = NULL;
+    char *kind_opts[4] = {"blocked", "granted", "all", NULL};
+    char *types[4] = {"inode", "entry", "posix", NULL};
+    char *free_ptr = NULL;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (words[4], "kind"))
-                goto out;
+    if (strcmp(words[4], "kind"))
+        goto out;
 
-        for (i = 0; kind_opts[i]; i++) {
-               if (!strcmp (words[5], kind_opts[i])) {
-                       free_ptr = gf_strdup (words[5]);
-                       ret = dict_set_dynstr (dict, "kind", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    for (i = 0; kind_opts[i]; i++) {
+        if (!strcmp(words[5], kind_opts[i])) {
+            free_ptr = gf_strdup(words[5]);
+            ret = dict_set_dynstr(dict, "kind", free_ptr);
+            if (ret)
                 goto out;
+            free_ptr = NULL;
+            break;
+        }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = -1;
-        for (i = 0; types[i]; i++) {
-               if (!strcmp (words[6], types[i])) {
-                       free_ptr = gf_strdup (words[6]);
-                       ret = dict_set_dynstr (dict, "type", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    ret = -1;
+    for (i = 0; types[i]; i++) {
+        if (!strcmp(words[6], types[i])) {
+            free_ptr = gf_strdup(words[6]);
+            ret = dict_set_dynstr(dict, "type", free_ptr);
+            if (ret)
                 goto out;
-
-        if (wordcount == 8) {
-                free_ptr = gf_strdup (words[7]);
-                ret = dict_set_dynstr (dict, "opts", free_ptr);
-                if (ret)
-                        goto out;
-                free_ptr = NULL;
+            free_ptr = NULL;
+            break;
         }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = 0;
-        *options = dict;
+    if (wordcount == 8) {
+        free_ptr = gf_strdup(words[7]);
+        ret = dict_set_dynstr(dict, "opts", free_ptr);
+        if (ret)
+            goto out;
+        free_ptr = NULL;
+    }
+
+    ret = 0;
+    *options = dict;
 out:
-       if (ret) {
-               GF_FREE (free_ptr);
-               dict_unref (dict);
-       }
+    if (ret) {
+        GF_FREE(free_ptr);
+        dict_unref(dict);
+    }
 
-       return ret;
+    return ret;
 }
 
 static int
-extract_hostname_path_from_token (const char *tmp_words, char **hostname,
-                                  char **path)
+extract_hostname_path_from_token(const char *tmp_words, char **hostname,
+                                 char **path)
 {
-        int ret = 0;
-        char *delimiter = NULL;
-        char *tmp_host = NULL;
-        char *host_name = NULL;
-        char *words = NULL;
-
-        *hostname = NULL;
-        *path = NULL;
-
-        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
-        if (!words){
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    char *delimiter = NULL;
+    char *tmp_host = NULL;
+    char *host_name = NULL;
+    char *words = NULL;
+    int str_len = 0;
+    *hostname = NULL;
+    *path = NULL;
+
+    str_len = strlen(tmp_words) + 1;
+    words = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!words) {
+        ret = -1;
+        goto out;
+    }
 
-        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+    snprintf(words, str_len, "%s", tmp_words);
 
-        if (validate_brick_name (words)) {
-                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                        "<export-dir-abs-path>", words);
-                ret = -1;
-                goto out;
+    if (validate_brick_name(words)) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words, ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret) {
+            goto out;
         } else {
-                delimiter = strrchr (words, ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret) {
-                        goto out;
-                } else {
-                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
-                                           gf_common_mt_char);
-                        if (!*path) {
-                           ret = -1;
-                                goto out;
-
-                        }
-                        strncpy (*path, delimiter +1,
-                                 strlen(delimiter + 1) + 1);
-                }
-        }
-
-        tmp_host = gf_strdup (words);
-        if (!tmp_host) {
-                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-        get_host_name (tmp_host, &host_name);
-        if (!host_name) {
-                ret = -1;
-                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                        "memory");
-                goto out;
-        }
-        if (!(strcmp (host_name, "localhost") &&
-            strcmp (host_name, "127.0.0.1") &&
-            strncmp (host_name, "0.", 2))) {
-                cli_err ("Please provide a valid hostname/ip other "
-                         "than localhost, 127.0.0.1 or loopback "
-                         "address (0.0.0.0 to 0.255.255.255).");
-                ret = -1;
-                goto out;
-        }
-        if (!valid_internet_address (host_name, _gf_false)) {
-                cli_err ("internet address '%s' does not conform to "
-                          "standards", host_name);
+            str_len = strlen(delimiter + 1) + 1;
+            *path = GF_MALLOC(str_len, gf_common_mt_char);
+            if (!*path) {
                 ret = -1;
                 goto out;
+            }
+            snprintf(*path, str_len, "%s", delimiter + 1);
         }
+    }
 
-        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
-                                       gf_common_mt_char);
-        if (!*hostname) {
-                ret = -1;
-                goto out;
-        }
-        strncpy (*hostname, host_name, strlen (host_name) + 1);
-        ret = 0;
+    tmp_host = gf_strdup(words);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to allocate "
+               "memory");
+        goto out;
+    }
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+        ret = -1;
+        goto out;
+    }
+
+    str_len = strlen(host_name) + 1;
+    *hostname = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!*hostname) {
+        ret = -1;
+        goto out;
+    }
+    snprintf(*hostname, str_len, "%s", host_name);
+    ret = 0;
 
 out:
-        GF_FREE (words);
-        GF_FREE (tmp_host);
-        return ret;
+    GF_FREE(words);
+    GF_FREE(tmp_host);
+    return ret;
 }
 
 static int
-set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op)
+set_hostname_path_in_dict(const char *token, dict_t *dict, int heal_op)
 {
-        char *hostname = NULL;
-        char *path     = NULL;
-        int   ret      = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int ret = 0;
 
-        ret = extract_hostname_path_from_token (token, &hostname, &path);
-        if (ret)
-                goto out;
+    ret = extract_hostname_path_from_token(token, &hostname, &path);
+    if (ret)
+        goto out;
 
-        switch (heal_op) {
+    switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_set_dynstr (dict, "heal-source-hostname",
-                                       hostname);
-                if (ret)
-                        goto out;
-                ret = dict_set_dynstr (dict, "heal-source-brickpath",
-                                       path);
-                break;
+            ret = dict_set_dynstr(dict, "heal-source-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "heal-source-brickpath", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
-                                       hostname);
-                if (ret)
-                        goto out;
-                ret = dict_set_dynstr (dict, "per-replica-cmd-path",
-                                       path);
-                break;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-path", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
 out:
-        return ret;
+    GF_FREE(hostname);
+    GF_FREE(path);
+    return ret;
 }
 
 static int
-heal_command_type_get (const char *command)
+heal_command_type_get(const char *command)
 {
-        int     i = 0;
-        /* subcommands are set as NULL */
-        char    *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
-                [GF_SHD_OP_INVALID]                            = NULL,
-                [GF_SHD_OP_HEAL_INDEX]                         = NULL,
-                [GF_SHD_OP_HEAL_FULL]                          = "full",
-                [GF_SHD_OP_INDEX_SUMMARY]                      = "info",
-                [GF_SHD_OP_HEALED_FILES]                       = NULL,
-                [GF_SHD_OP_HEAL_FAILED_FILES]                  = NULL,
-                [GF_SHD_OP_SPLIT_BRAIN_FILES]                  = NULL,
-                [GF_SHD_OP_STATISTICS]                         = "statistics",
-                [GF_SHD_OP_STATISTICS_HEAL_COUNT]              = NULL,
-                [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA]  = NULL,
-                [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE]       = NULL,
-                [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK]             = NULL,
-                [GF_SHD_OP_HEAL_ENABLE]                        = "enable",
-                [GF_SHD_OP_HEAL_DISABLE]                       = "disable",
-        };
-
-        for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
-                if (heal_cmds[i] && (strcmp (heal_cmds[i], command) == 0))
-                        return i;
-        }
-
-        return GF_SHD_OP_INVALID;
+    int i = 0;
+    /* subcommands are set as NULL */
+    char *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = NULL,
+        [GF_SHD_OP_HEAL_INDEX] = NULL,
+        [GF_SHD_OP_HEAL_FULL] = "full",
+        [GF_SHD_OP_INDEX_SUMMARY] = "info",
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL,
+        [GF_SHD_OP_STATISTICS] = "statistics",
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK] = NULL,
+        [GF_SHD_OP_HEAL_ENABLE] = "enable",
+        [GF_SHD_OP_HEAL_DISABLE] = "disable",
+    };
+
+    for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
+        if (heal_cmds[i] && (strcmp(heal_cmds[i], command) == 0))
+            return i;
+    }
+
+    return GF_SHD_OP_INVALID;
 }
 
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options)
 {
-        int     ret = 0;
-        dict_t  *dict = NULL;
-        gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *) words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set volname");
+        goto out;
+    }
 
-        if (wordcount == 3) {
-                ret = dict_set_int32 (dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
-                goto done;
+    if (wordcount == 3) {
+        ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
+        goto done;
+    }
+
+    if (wordcount == 4) {
+        op = heal_command_type_get(words[3]);
+        if (op == GF_SHD_OP_INVALID) {
+            ret = -1;
+            goto out;
         }
 
-        if (wordcount == 4) {
-                op = heal_command_type_get (words[3]);
-                if (op == GF_SHD_OP_INVALID) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_int32(dict, "heal-op", op);
+        goto done;
+    }
 
-                ret = dict_set_int32 (dict, "heal-op", op);
+    if (wordcount == 5) {
+        if (strcmp(words[3], "info") && strcmp(words[3], "statistics") &&
+            strcmp(words[3], "granular-entry-heal")) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!strcmp(words[3], "info")) {
+            if (!strcmp(words[4], "split-brain")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_SPLIT_BRAIN_FILES);
                 goto done;
+            }
+            if (!strcmp(words[4], "summary")) {
+                ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_SUMMARY);
+                goto done;
+            }
         }
 
-        if (wordcount == 5) {
-                if (strcmp (words[3], "info") &&
-                    strcmp (words[3], "statistics")) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!strcmp(words[3], "statistics")) {
+            if (!strcmp(words[4], "heal-count")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_STATISTICS_HEAL_COUNT);
+                goto done;
+            }
+        }
 
-                if (!strcmp (words[3], "info")) {
-                        if (!strcmp (words[4], "healed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                      GF_SHD_OP_HEALED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "heal-failed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_SHD_OP_HEAL_FAILED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "split-brain")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_SHD_OP_SPLIT_BRAIN_FILES);
-                                goto done;
-                        }
-                }
+        if (!strcmp(words[3], "granular-entry-heal")) {
+            if (!strcmp(words[4], "enable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE);
+                goto done;
+            } else if (!strcmp(words[4], "disable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE);
+                goto done;
+            }
+        }
 
-                if (!strcmp (words[3], "statistics")) {
-                        if (!strcmp (words[4], "heal-count")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                               GF_SHD_OP_STATISTICS_HEAL_COUNT);
-                                goto done;
-                        }
-                }
-                ret = -1;
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 6) {
+        if (strcmp(words[3], "split-brain")) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(words[4], "bigger-file")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
                 goto out;
+            goto done;
         }
-        if (wordcount == 6) {
-                if (strcmp (words[3], "split-brain")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (words[4], "bigger-file")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                        GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "file", (char *)words[5]);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
-                if (!strcmp (words[4], "latest-mtime")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                       GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "file", (char *)words[5]);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
-                if (!strcmp (words[4], "source-brick")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
-                ret = -1;
+        if (!strcmp(words[4], "latest-mtime")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
+            if (ret)
                 goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
         }
-        if (wordcount == 7) {
-                if (!strcmp (words[3], "statistics")
-                    && !strcmp (words[4], "heal-count")
-                    && !strcmp (words[5], "replica")) {
-
-                        ret = dict_set_int32 (dict, "heal-op",
-                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        ret = set_hostname_path_in_dict (words[6], dict,
-                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        goto done;
-
-                }
-                if (!strcmp (words[3], "split-brain") &&
-                    !strcmp (words[4], "source-brick")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "file",
-                                            (char *) words[6]);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
+        if (!strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            goto done;
         }
         ret = -1;
         goto out;
+    }
+    if (wordcount == 7) {
+        if (!strcmp(words[3], "statistics") &&
+            !strcmp(words[4], "heal-count") && !strcmp(words[5], "replica")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(
+                words[6], dict, GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[3], "split-brain") &&
+            !strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[6]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+    }
+    ret = -1;
+    goto out;
 done:
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict) {
-                dict_unref (dict);
-                *options = NULL;
-        }
+    if (ret && dict) {
+        dict_unref(dict);
+        *options = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_defrag_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t                 *dict = NULL;
-        int                      ret = -1;
-        char                *option  = NULL;
-        char                *volname = NULL;
-        char                *command = NULL;
-        gf_cli_defrag_type       cmd = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (!((wordcount == 4) || (wordcount == 5)))
-                goto out;
-
-        if (wordcount == 4) {
-                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
-                    strcmp (words[3], "status"))
-                            goto out;
-        } else if ((strcmp (words[3], "tier") == 0) &&
-                   (strcmp (words[4], "start") == 0)) {
-                volname = (char *) words[2];
-                cmd = GF_DEFRAG_CMD_START_TIER;
-                goto done;
-        } else if ((strcmp (words[3], "tier") == 0) &&
-               (strcmp (words[4], "status") == 0)) {
-                volname = (char *) words[2];
-                cmd = GF_DEFRAG_CMD_STATUS_TIER;
-                goto done;
-        } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "start"))
-                        goto out;
-        }
-
-        volname = (char *) words[2];
-
-        if (wordcount == 4) {
-                command = (char *) words[3];
-        }
-        if (wordcount == 5) {
-               if ((strcmp (words[3], "fix-layout") ||
-                    strcmp (words[4], "start")) &&
-                    (strcmp (words[3], "start") ||
-                    strcmp (words[4], "force"))) {
-                        ret = -1;
-                        goto out;
-                }
-                command = (char *) words[3];
-                option = (char *) words[4];
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *option = NULL;
+    char *volname = NULL;
+    char *command = NULL;
+    gf_cli_defrag_type cmd = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (command, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                if (option && strcmp (option, "force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_FORCE;
-                        }
-                goto done;
-        }
+    if (!((wordcount == 4) || (wordcount == 5)))
+        goto out;
 
-        if (strcmp (command, "fix-layout") == 0) {
-                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                goto done;
-        }
-        if (strcmp (command, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (command, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
-        }
+    if (wordcount == 4) {
+        if (strcmp(words[3], "start") && strcmp(words[3], "stop") &&
+            strcmp(words[3], "status"))
+            goto out;
+    } else {
+        if (strcmp(words[3], "fix-layout") && strcmp(words[3], "start"))
+            goto out;
+    }
+
+    volname = (char *)words[2];
+
+    if (wordcount == 4) {
+        command = (char *)words[3];
+    }
+    if (wordcount == 5) {
+        if ((strcmp(words[3], "fix-layout") || strcmp(words[4], "start")) &&
+            (strcmp(words[3], "start") || strcmp(words[4], "force"))) {
+            ret = -1;
+            goto out;
+        }
+        command = (char *)words[3];
+        option = (char *)words[4];
+    }
+
+    if (strcmp(command, "start") == 0) {
+        cmd = GF_DEFRAG_CMD_START;
+        if (option && strcmp(option, "force") == 0) {
+            cmd = GF_DEFRAG_CMD_START_FORCE;
+        }
+        goto done;
+    }
+
+    if (strcmp(command, "fix-layout") == 0) {
+        cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+        goto done;
+    }
+    if (strcmp(command, "stop") == 0) {
+        cmd = GF_DEFRAG_CMD_STOP;
+        goto done;
+    }
+    if (strcmp(command, "status") == 0) {
+        cmd = GF_DEFRAG_CMD_STATUS;
+    }
 
 done:
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+    ret = dict_set_int32(dict, "rebalance-command", (int32_t)cmd);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_snap_create_desc_parse (dict_t *dict, const char **words,
-                            size_t wordcount, int32_t desc_opt_loc)
+cli_snap_create_desc_parse(dict_t *dict, const char **words, size_t wordcount,
+                           int32_t desc_opt_loc)
 {
-        int32_t        ret      = -1;
-        char          *desc     = NULL;
-        int32_t        desc_len = 0;
-
-        desc = GF_CALLOC (MAX_SNAP_DESCRIPTION_LEN + 1, sizeof(char),
-                          gf_common_mt_char);
-        if (!desc) {
-                ret = -1;
-                goto out;
-        }
-
-
-        if (strlen (words[desc_opt_loc]) >= MAX_SNAP_DESCRIPTION_LEN) {
-                cli_out ("snapshot create: description truncated: "
-                         "Description provided is longer than 1024 characters");
-                desc_len = MAX_SNAP_DESCRIPTION_LEN;
-        } else {
-                desc_len = strlen (words[desc_opt_loc]);
-        }
+    int32_t ret = -1;
+    char *desc = NULL;
+    int32_t desc_len = 0;
+    int len;
 
-        strncpy (desc, words[desc_opt_loc], desc_len);
-        desc[desc_len] = '\0';
-        /* Calculating the size of the description as given by the user */
-
-        ret = dict_set_dynstr (dict, "description", desc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap "
-                        "description");
-                goto out;
-        }
+    desc = GF_MALLOC(MAX_SNAP_DESCRIPTION_LEN + 1, gf_common_mt_char);
+    if (!desc) {
+        ret = -1;
+        goto out;
+    }
+
+    len = strlen(words[desc_opt_loc]);
+    if (len >= MAX_SNAP_DESCRIPTION_LEN) {
+        cli_out(
+            "snapshot create: description truncated: "
+            "Description provided is longer than 1024 characters");
+        desc_len = MAX_SNAP_DESCRIPTION_LEN;
+    } else {
+        desc_len = len;
+    }
+
+    snprintf(desc, desc_len + 1, "%s", words[desc_opt_loc]);
+    /* Calculating the size of the description as given by the user */
+
+    ret = dict_set_dynstr(dict, "description", desc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to save snap "
+               "description");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && desc)
-                GF_FREE (desc);
+    if (ret && desc)
+        GF_FREE(desc);
 
-        return ret;
+    return ret;
 }
 
 /* Function to check whether the Volume name is repeated */
 int
-cli_check_if_volname_repeated (const char **words, unsigned int start_index,
-                           uint64_t cur_index) {
-        uint64_t        i       =       -1;
-        int             ret     =        0;
+cli_check_if_volname_repeated(const char **words, unsigned int start_index,
+                              uint64_t cur_index)
+{
+    uint64_t i = -1;
+    int ret = 0;
 
-        GF_ASSERT (words);
+    GF_ASSERT(words);
 
-        for (i = start_index ; i < cur_index ; i++) {
-                if (strcmp (words[i], words[cur_index]) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+    for (i = start_index; i < cur_index; i++) {
+        if (strcmp(words[i], words[cur_index]) == 0) {
+            ret = -1;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot clone <clonename> <snapname>
@@ -4071,72 +4243,77 @@ out:
  *                 0 on success
  */
 int
-cli_snap_clone_parse (dict_t *dict, const char **words, int wordcount) {
-        uint64_t        i               =       0;
-        int             ret             =       -1;
-        char            *clonename      =       NULL;
-        unsigned int    cmdi            =       2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount == cmdi + 1) {
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Invalid number of  words for snap clone command");
-                goto out;
-        }
-
-        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
-                cli_err ("snapshot clone: failed: clonename cannot exceed "
-                         "255 characters.");
-                gf_log ("cli", GF_LOG_ERROR, "Clone name too long");
-
-                goto out;
-        }
-
-        clonename = (char *) words[cmdi];
-        for (i = 0 ; i < strlen (clonename); i++) {
-                /* Following volume name convention */
-                if (!isalnum (clonename[i]) && (clonename[i] != '_'
-                                           && (clonename[i] != '-'))) {
-                        /* TODO : Is this message enough?? */
-                        cli_err ("Clonename can contain only alphanumeric, "
-                                 "\"-\" and \"_\" characters");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "volcount", 1);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "clonename", (char *)words[cmdi]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save clone "
-                        "name(%s)", (char *)words[cmdi]);
-                goto out;
-        }
+cli_snap_clone_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    char *clonename = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount == cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR,
+               "Invalid number of  words for snap clone command");
+        goto out;
+    }
 
-        /* Filling snap name in the dictionary */
-        ret = dict_set_str (dict, "snapname", (char *)words[cmdi+1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not "
-                        "save snap name(%s)", (char *)words[cmdi+1]);
-                goto out;
-        }
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot clone: failed: clonename cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Clone name too long");
 
+        goto out;
+    }
+
+    clonename = (char *)words[cmdi];
+    for (i = 0; i < strlen(clonename); i++) {
+        /* Following volume name convention */
+        if (!isalnum(clonename[i]) &&
+            (clonename[i] != '_' && (clonename[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Clonename can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "volcount", 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "clonename", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save clone "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling snap name in the dictionary */
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi + 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not "
+               "save snap name(%s)",
+               (char *)words[cmdi + 1]);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 /* snapshot create <snapname> <vol-name(s)> [description <description>]
  *                                           [force]
  * @arg-0, dict     : Request Dictionary to be sent to server side.
@@ -4147,172 +4324,181 @@ out:
  *                 0 on success
  */
 int
-cli_snap_create_parse (dict_t *dict, const char **words, int wordcount) {
-        uint64_t        i               =       0;
-        int             ret             =       -1;
-        uint64_t        volcount        =       0;
-        char            key[PATH_MAX]   =       "";
-        char            *snapname       =       NULL;
-        unsigned int    cmdi            =       2;
-        int             flags           =       0;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount <= cmdi + 1) {
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Too less words for snap create command");
-                goto out;
-        }
+cli_snap_create_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    uint64_t volcount = 0;
+    char key[PATH_MAX] = "";
+    char *snapname = NULL;
+    unsigned int cmdi = 2;
+    int flags = 0;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount <= cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Too less words for snap create command");
+        goto out;
+    }
 
-        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
-                cli_err ("snapshot create: failed: snapname cannot exceed "
-                         "255 characters.");
-                gf_log ("cli", GF_LOG_ERROR, "Snapname too long");
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot create: failed: snapname cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Snapname too long");
 
-                goto out;
+        goto out;
+    }
+
+    snapname = (char *)words[cmdi];
+    for (i = 0; i < strlen(snapname); i++) {
+        /* Following volume name convention */
+        if (!isalnum(snapname[i]) &&
+            (snapname[i] != '_' && (snapname[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Snapname can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save snap "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling volume name in the dictionary */
+    for (i = cmdi + 1;
+         i < wordcount && (strcmp(words[i], "description")) != 0 &&
+         (strcmp(words[i], "force") != 0) &&
+         (strcmp(words[i], "no-timestamp") != 0);
+         i++) {
+        volcount++;
+        /* volume index starts from 1 */
+        ret = snprintf(key, sizeof(key), "volname%" PRIu64, volcount);
+        if (ret < 0) {
+            goto out;
         }
 
-        snapname = (char *) words[cmdi];
-        for (i = 0 ; i < strlen (snapname); i++) {
-                /* Following volume name convention */
-                if (!isalnum (snapname[i]) && (snapname[i] != '_'
-                                           && (snapname[i] != '-'))) {
-                        /* TODO : Is this message enough?? */
-                        cli_err ("Snapname can contain only alphanumeric, "
-                                 "\"-\" and \"_\" characters");
-                        goto out;
-                }
+        ret = dict_set_str(dict, key, (char *)words[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not "
+                   "save volume name(%s)",
+                   (char *)words[i]);
+            goto out;
         }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                        "name(%s)", (char *)words[cmdi]);
-                goto out;
+        if (i >= cmdi + 2) {
+            ret = -1;
+            cli_err(
+                "Creating multiple volume snapshot is not "
+                "supported as of now");
+            goto out;
         }
+        /* TODO : remove this above condition check once
+         * multiple volume snapshot is supported */
+    }
 
-        /* Filling volume name in the dictionary */
-        for (i = cmdi + 1 ; i < wordcount
-                            && (strcmp (words[i], "description")) != 0
-                            && (strcmp (words[i], "force") != 0)
-                            && (strcmp (words[i], "no-timestamp") != 0);
-                            i++) {
-                volcount++;
-                /* volume index starts from 1 */
-                ret = snprintf (key, sizeof (key), "volname%"PRIu64, volcount);
-                if (ret < 0) {
-                        goto out;
-                }
+    if (volcount == 0) {
+        ret = -1;
+        cli_err("Please provide the volume name");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-                ret = dict_set_str (dict, key, (char *)words[i]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                "save volume name(%s)", (char *)words[i]);
-                        goto out;
-                }
+    ret = dict_set_int32(dict, "volcount", volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    /* Verify how we got out of "for" loop,
+     * if it is by reaching wordcount limit then goto "out",
+     * because we need not parse for "description","force" and
+     * "no-timestamp" after this.
+     */
+    if (i == wordcount) {
+        goto out;
+    }
 
-                if (i >= cmdi + 2) {
-                        ret = -1;
-                        cli_err("Creating multiple volume snapshot is not "
-                                "supported as of now");
-                        goto out;
-                }
-                /* TODO : remove this above condition check once
-                 * multiple volume snapshot is supported */
+    if (strcmp(words[i], "no-timestamp") == 0) {
+        ret = dict_set_int32n(dict, "no-timestamp", SLEN("no-timestamp"), 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "time-stamp option");
         }
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+    }
 
-        if (volcount == 0) {
-                ret = -1;
-                cli_err ("Please provide the volume name");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+    if ((strcmp(words[i], "description")) == 0) {
+        ++i;
+        if (i > (wordcount - 1)) {
+            ret = -1;
+            cli_err("Please provide a description");
+            gf_log("cli", GF_LOG_ERROR, "Description not provided");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "volcount", volcount);
+        ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
-
-        /* Verify how we got out of "for" loop,
-         * if it is by reaching wordcount limit then goto "out",
-         * because we need not parse for "description","force" and
-         * "no-timestamp" after this.
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save snap "
+                   "description");
+            goto out;
+        }
+
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+        /* point the index to next word.
+         * As description might be follwed by force option.
+         * Before that, check if wordcount limit is reached
          */
-        if (i == wordcount) {
-                goto out;
-        }
+    }
 
-        if (strcmp (words[i], "no-timestamp") == 0) {
-                ret = dict_set_str (dict, "no-timestamp", "true");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "time-stamp option");
-                }
-                if (i == (wordcount-1))
-                        goto out;
-                i++;
-        }
+    if (strcmp(words[i], "force") == 0) {
+        flags = GF_CLI_FLAG_OP_FORCE;
 
-        if ((strcmp (words[i], "description")) == 0) {
-                ++i;
-                if (i > (wordcount - 1)) {
-                        ret = -1;
-                        cli_err ("Please provide a description");
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Description not provided");
-                        goto out;
-                }
-
-                ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                                "description");
-                        goto out;
-                }
-
-                if (i == (wordcount - 1))
-                        goto out;
-                i++;
-                /* point the index to next word.
-                 * As description might be follwed by force option.
-                 * Before that, check if wordcount limit is reached
-                 */
-        }
-
-        if (strcmp (words[i], "force") == 0) {
-                flags = GF_CLI_FLAG_OP_FORCE;
-
-        } else {
-                ret = -1;
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    } else {
+        ret = -1;
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        /* Check if the command has anything after "force" keyword */
-        if (++i < wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    /* Check if the command has anything after "force" keyword */
+    if (++i < wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if(ret == 0) {
-                /*Adding force flag in either of the case i.e force set
-                 * or unset*/
-                ret = dict_set_int32 (dict, "flags", flags);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "snap force option");
-                }
+    if (ret == 0) {
+        /*Adding force flag in either of the case i.e force set
+         * or unset*/
+        ret = dict_set_int32(dict, "flags", flags);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "snap force option");
         }
-        return ret;
+    }
+    return ret;
 }
 
 /* snapshot list [volname]
@@ -4324,30 +4510,30 @@ out:
  *                 0 on success
  */
 int
-cli_snap_list_parse (dict_t *dict, const char **words, int wordcount) {
-        int             ret     =       -1;
+cli_snap_list_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if (wordcount < 2 || wordcount > 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount < 2 || wordcount > 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                ret = 0;
-                goto out;
-        }
+    if (wordcount == 2) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to save volname in dictionary");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save volname in dictionary");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot info [(snapname |  volume <volname>)]
@@ -4359,87 +4545,88 @@ out:
  *                 0 on success
  */
 int
-cli_snap_info_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_info_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
 
-        int             ret             =       -1;
-        int32_t         cmd             =       GF_SNAP_INFO_TYPE_ALL;
-        unsigned int    cmdi            =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* If 3rd word is not "volume", then it must
+     * be snapname.
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* If 3rd word is not "volume", then it must
-         * be snapname.
+        /* Once snap name is parsed, if we encounter any other
+         * word then fail it. Invalid Syntax.
+         * example : snapshot info <snapname> word
          */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[cmdi]);
-                        goto out;
-                }
-
-                /* Once snap name is parsed, if we encounter any other
-                 * word then fail it. Invalid Syntax.
-                 * example : snapshot info <snapname> word
-                 */
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                cmd = GF_SNAP_INFO_TYPE_SNAP;
-                ret = 0;
-                goto out;
-                /* No need to continue the parsing once we
-                 * get the snapname
-                 */
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word
-         * is present. As, "snapshot info volume" is an
-         * invalid command.
+        cmd = GF_SNAP_INFO_TYPE_SNAP;
+        ret = 0;
+        goto out;
+        /* No need to continue the parsing once we
+         * get the snapname
          */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_INFO_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word
+     * is present. As, "snapshot info volume" is an
+     * invalid command.
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_INFO_TYPE_VOL;
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "sub-cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "type of snapshot info");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "type of snapshot info");
         }
-        return ret;
+    }
+    return ret;
 }
 
-
-
 /* snapshot restore <snapname>
  * @arg-0, dict     : Request Dictionary to be sent to server side.
  * @arg-1, words    : Contains individual words of CLI command.
@@ -4449,42 +4636,43 @@ out:
  *                 0 on success
  */
 int
-cli_snap_restore_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_restore_parse(dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state)
 {
+    int ret = -1;
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if (wordcount != 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
-
-        question = "Restore operation will replace the "
-                   "original volume with the snapshotted volume. "
-                   "Do you still want to continue?";
+    if (wordcount != 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_ERROR, "User cancelled a snapshot "
-                        "restore operation for snap %s", (char *)words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    question =
+        "Restore operation will replace the "
+        "original volume with the snapshotted volume. "
+        "Do you still want to continue?";
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "User cancelled a snapshot "
+               "restore operation for snap %s",
+               (char *)words[2]);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot activate <snapname> [force]
@@ -4496,43 +4684,41 @@ out:
  *                 0 on success
  */
 int
-cli_snap_activate_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_activate_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int flags = 0;
 
-        int ret = -1;
-        int flags = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if ((wordcount < 3) || (wordcount > 4)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if ((wordcount < 3) || (wordcount > 4)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", (char *)words[3])) {
-                        flags = GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid option");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save force option");
-                goto out;
-        }
+    if (wordcount == 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            flags = GF_CLI_FLAG_OP_FORCE;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "Invalid option");
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save force option");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot deactivate <snapname>
@@ -4545,42 +4731,41 @@ out:
  *                 1 if user cancelled the request
  */
 int
-cli_snap_deactivate_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_deactivate_parse(dict_t *dict, const char **words, int wordcount,
+                          struct cli_state *state)
 {
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Deactivating snap will make its "
+        "data inaccessible. Do you want to "
+        "continue?";
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount != 3)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        int             ret             = -1;
-        gf_answer_t     answer          = GF_ANSWER_NO;
-        const char     *question        = "Deactivating snap will make its "
-                                          "data inaccessible. Do you want to "
-                                          "continue?";
-
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if ((wordcount != 3)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
-
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot deactivate operation");
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled "
+               "snapshot deactivate operation");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot delete (all | snapname | volume <volname>)
@@ -4593,78 +4778,84 @@ out:
  *                 1 if user cancel the operation
  */
 int
-cli_snap_delete_parse (dict_t *dict, const char **words, int wordcount,
-                       struct cli_state *state) {
-
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        int32_t         cmd             =       -1;
-        unsigned int    cmdi            =       2;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
+cli_snap_delete_parse(dict_t *dict, const char **words, int wordcount,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    int32_t cmd = -1;
+    unsigned int cmdi = 2;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if (wordcount > 4 || wordcount <= cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        question = "Deleting snap will erase all the information about "
-                   "the snap. Do you still want to continue?";
+    if (wordcount > 4 || wordcount <= cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (strcmp (words [cmdi], "all") == 0) {
-                ret = 0;
-                cmd = GF_SNAP_DELETE_TYPE_ALL;
-        } else if (strcmp (words [cmdi], "volume") == 0) {
-                if (++cmdi == wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+    question =
+        "Deleting snap will erase all the information about "
+        "the snap. Do you still want to continue?";
 
-                ret = dict_set_str (dict, "volname",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "volume name %s", words[wordcount - 1]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_VOL;
-        } else {
-                ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[2]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    if (strcmp(words[cmdi], "all") == 0) {
+        ret = 0;
+        cmd = GF_SNAP_DELETE_TYPE_ALL;
+    } else if (strcmp(words[cmdi], "volume") == 0) {
+        if (++cmdi == wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        if ((cmdi + 1) != wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "volume name %s",
+                   words[wordcount - 1]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_VOL;
+    } else {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[2]);
+            goto out;
         }
+        cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    }
 
-        if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                                "snapshot delete operation for snap %s",
-                                (char *)words[2]);
-                        goto out;
-                }
-        }
+    if ((cmdi + 1) != wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "sub-cmd", cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "type of snapshot delete");
-        }
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot delete operation for snap %s",
+                   (char *)words[2]);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "sub-cmd", cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "type of snapshot delete");
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot status [(snapname | volume <volname>)]
@@ -4676,132 +4867,138 @@ out:
  *                 0 on success
  */
 int
-cli_snap_status_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_status_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_STATUS_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
 
-        int             ret  =        -1;
-        int32_t         cmd  =       GF_SNAP_STATUS_TYPE_ALL;
-        unsigned int    cmdi =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* if 3rd word is not "volume", then it must be "snapname"
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Count not save "
+                   "snap name %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* if 3rd word is not "volume", then it must be "snapname"
-        */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                                "snap name %s", words[cmdi]);
-                        goto out;
-                }
-
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                ret = 0;
-                cmd = GF_SNAP_STATUS_TYPE_SNAP;
-                goto out;
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word is present.
-         * As, "snapshot info volume" is an invalid command
-         */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+        ret = 0;
+        cmd = GF_SNAP_STATUS_TYPE_SNAP;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words [wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_STATUS_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word is present.
+     * As, "snapshot info volume" is an invalid command
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Count not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_STATUS_TYPE_VOL;
 
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "sub-cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save cmd "
-                                "of snapshot status");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save cmd "
+                   "of snapshot status");
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 /* return value:
  *      -1  in case of failure.
  *       0  in case of success.
  */
 int32_t
-cli_snap_config_limit_parse (const char **words, dict_t *dict,
-                             unsigned int wordcount, unsigned int index,
-                             char *key)
+cli_snap_config_limit_parse(const char **words, dict_t *dict,
+                            unsigned int wordcount, unsigned int index,
+                            char *key)
 {
-        int             ret             = -1;
-        int             limit           = 0;
-        char            *end_ptr        = NULL;
+    int ret = -1;
+    int limit = 0;
+    char *end_ptr = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (key);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
 
-        if (index >= wordcount) {
-                ret = -1;
-                cli_err ("Please provide a value for %s.", key);
-                gf_log ("cli", GF_LOG_ERROR, "Value not provided for %s", key);
-                goto out;
-        }
-
-        limit = strtol (words[index], &end_ptr, 10);
+    if (index >= wordcount) {
+        ret = -1;
+        cli_err("Please provide a value for %s.", key);
+        gf_log("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+        goto out;
+    }
 
-        if (limit <= 0 || strcmp (end_ptr, "") != 0) {
-                ret = -1;
-                cli_err("Please enter an integer value "
-                        "greater than zero for %s", key);
-                goto out;
-        }
+    limit = strtol(words[index], &end_ptr, 10);
 
-        ret = dict_set_int32 (dict, key, limit);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set "
-                        "%s in dictionary", key);
-                goto out;
-        }
+    if (limit <= 0 || strcmp(end_ptr, "") != 0) {
+        ret = -1;
+        cli_err(
+            "Please enter an integer value "
+            "greater than zero for %s",
+            key);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, key, limit);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not set "
+               "%s in dictionary",
+               key);
+        goto out;
+    }
 
-        ret = dict_set_dynstr_with_alloc (dict, "globalname", "All");
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set global key");
-                goto out;
-        }
-        ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set global locks");
-                goto out;
-        }
+    ret = dict_set_dynstr_with_alloc(dict, "globalname", "All");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global key");
+        goto out;
+    }
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global locks");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* function cli_snap_config_parse
@@ -4817,766 +5014,933 @@ out:
   NOTE : snap-max-soft-limit can only be set for system.
 */
 int32_t
-cli_snap_config_parse (const char **words, int wordcount, dict_t *dict,
-                       struct cli_state *state)
+cli_snap_config_parse(const char **words, int wordcount, dict_t *dict,
+                      struct cli_state *state)
 {
-        int                            ret                 = -1;
-        gf_answer_t                    answer              = GF_ANSWER_NO;
-        gf_boolean_t                   vol_presence        = _gf_false;
-        struct snap_config_opt_vals_  *conf_vals           = NULL;
-        int8_t                         hard_limit          = 0;
-        int8_t                         soft_limit          = 0;
-        int8_t                         config_type         = -1;
-        const char                    *question            = NULL;
-        unsigned int                   cmdi                = 2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (state);
-
-        if ((wordcount < 2) || (wordcount > 7)) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Invalid wordcount(%d)", wordcount);
-                goto out;
-        }
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    gf_boolean_t vol_presence = _gf_false;
+    struct snap_config_opt_vals_ *conf_vals = NULL;
+    int8_t hard_limit = 0;
+    int8_t soft_limit = 0;
+    int8_t config_type = -1;
+    const char *question = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(state);
+
+    if ((wordcount < 2) || (wordcount > 7)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid wordcount(%d)", wordcount);
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                config_type = GF_SNAP_CONFIG_DISPLAY;
-                ret = 0;
-                goto set;
+    if (wordcount == 2) {
+        config_type = GF_SNAP_CONFIG_DISPLAY;
+        ret = 0;
+        goto set;
+    }
+
+    /* auto-delete cannot be a volume name */
+    /* Check whether the 3rd word is volname */
+    if (strcmp(words[cmdi], "snap-max-hard-limit") != 0 &&
+        strcmp(words[cmdi], "snap-max-soft-limit") != 0 &&
+        strcmp(words[cmdi], "auto-delete") != 0 &&
+        strcmp(words[cmdi], "activate-on-create") != 0) {
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set volname");
+            goto out;
         }
+        cmdi++;
+        vol_presence = _gf_true;
 
-        /* auto-delete cannot be a volume name */
-        /* Check whether the 3rd word is volname */
-        if (strcmp (words[cmdi], "snap-max-hard-limit") != 0
-             && strcmp (words[cmdi], "snap-max-soft-limit") != 0
-             && strcmp (words[cmdi], "auto-delete") != 0
-             && strcmp (words[cmdi], "activate-on-create") != 0) {
-                ret = dict_set_str (dict, "volname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set volname");
-                        goto out;
-                }
-                cmdi++;
-                vol_presence = _gf_true;
-
-                if (cmdi == wordcount) {
-                        config_type = GF_SNAP_CONFIG_DISPLAY;
-                        ret = 0;
-                        goto set;
-                }
+        if (cmdi == wordcount) {
+            config_type = GF_SNAP_CONFIG_DISPLAY;
+            ret = 0;
+            goto set;
         }
+    }
 
-        config_type = GF_SNAP_CONFIG_TYPE_SET;
-
-        if (strcmp (words[cmdi], "snap-max-hard-limit") == 0) {
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-hard-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config hard limit");
-                        goto out;
-                }
-                hard_limit = 1;
+    config_type = GF_SNAP_CONFIG_TYPE_SET;
 
-                if (++cmdi == wordcount) {
-                        ret = 0;
-                        goto set;
-                }
+    if (strcmp(words[cmdi], "snap-max-hard-limit") == 0) {
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-hard-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config hard limit");
+            goto out;
         }
+        hard_limit = 1;
 
-        if (strcmp (words[cmdi], "snap-max-soft-limit") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("Soft limit cannot be set to individual "
-                                  "volumes.");
-                        gf_log ("cli", GF_LOG_ERROR, "Soft limit cannot be "
-                                "set to volumes");
-                        goto out;
-                }
-
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-soft-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config soft limit");
-                        goto out;
-                }
-
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-                soft_limit = 1;
+        if (++cmdi == wordcount) {
+            ret = 0;
+            goto set;
         }
+    }
 
-        if (hard_limit || soft_limit)
-                goto set;
+    if (strcmp(words[cmdi], "snap-max-soft-limit") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "Soft limit cannot be set to individual "
+                "volumes.");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Soft limit cannot be "
+                   "set to volumes");
+            goto out;
+        }
 
-        if (strcmp(words[cmdi], "auto-delete") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, auto-delete option cannot be set "
-                                "to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "auto-delete option "
-                                "cannot be set to volumes");
-                        goto out;
-                }
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-soft-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config soft limit");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+        soft_limit = 1;
+    }
 
-                ret = dict_set_str (dict, "auto-delete", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                "value of auto-delete in request "
-                                "dictionary");
-                        goto out;
-                }
+    if (hard_limit || soft_limit)
+        goto set;
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, activate-on-create option "
-                                 "cannot be set to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "activate-on-create "
-                                "option cannot be set to volumes");
-                        goto out;
-                }
+    if (strcmp(words[cmdi], "auto-delete") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, auto-delete option cannot be set "
+                "to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "auto-delete option "
+                   "cannot be set to volumes");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
 
-                ret = dict_set_str (dict, "snap-activate-on-create",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set value "
-                                "of activate-on-create in request dictionary");
-                        goto out;
-                }
+        ret = dict_set_str(dict, "auto-delete", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set "
+                   "value of auto-delete in request "
+                   "dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, activate-on-create option "
+                "cannot be set to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "activate-on-create "
+                   "option cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "snap-activate-on-create",
+                           (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set value "
+                   "of activate-on-create in request dictionary");
+            goto out;
+        }
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
+    } else {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0; /* Success */
+    ret = 0; /* Success */
 
 set:
-        ret = dict_set_int32 (dict, "config-command", config_type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                        "config-command");
-                goto out;
-        }
+    ret = dict_set_int32(dict, "config-command", config_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to set "
+               "config-command");
+        goto out;
+    }
 
-        if (config_type == GF_SNAP_CONFIG_TYPE_SET &&
-           (hard_limit || soft_limit)) {
-                conf_vals = snap_confopt_vals;
-                if (hard_limit && soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
-                } else if (soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
-                } else if (hard_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
-                }
+    if (config_type == GF_SNAP_CONFIG_TYPE_SET && (hard_limit || soft_limit)) {
+        conf_vals = snap_confopt_vals;
+        if (hard_limit && soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+        } else if (soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+        } else if (hard_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+        }
 
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot config operation");
-                }
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot config operation");
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-validate_op_name (const char *op, const char *opname, char **opwords) {
-        int     ret     =       -1;
-        int     i       =       0;
+validate_op_name(const char *op, const char *opname, char **opwords)
+{
+    int ret = -1;
+    int i = 0;
 
-        GF_ASSERT (opname);
-        GF_ASSERT (opwords);
+    GF_ASSERT(opname);
+    GF_ASSERT(opwords);
 
-        for (i = 0 ; opwords[i] != NULL; i++) {
-                if (strcmp (opwords[i], opname) == 0) {
-                        cli_out ("\"%s\" cannot be a %s", opname, op);
-                        goto out;
-                }
+    for (i = 0; opwords[i] != NULL; i++) {
+        if (strcmp(opwords[i], opname) == 0) {
+            cli_out("\"%s\" cannot be a %s", opname, op);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
-                        struct cli_state *state)
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state)
 {
-        int32_t            ret        = -1;
-        dict_t             *dict      = NULL;
-        gf1_cli_snapshot   type       = GF_SNAP_OPTION_TYPE_NONE;
-        char               *w         = NULL;
-        char               *opwords[] = {"create", "delete", "restore",
-                                        "activate", "deactivate", "list",
-                                        "status", "config", "info", "clone",
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_snapshot type = GF_SNAP_OPTION_TYPE_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"create",     "delete", "restore", "activate",
+                              "deactivate", "list",   "status",  "config",
+                              "info",       "clone",  NULL};
+    static char *invalid_snapnames[] = {"description", "force", "volume", "all",
                                         NULL};
-        char               *invalid_snapnames[] = {"description", "force",
-                                                  "volume", "all", NULL};
-        char               *invalid_volnames[]  = {"volume", "type",
-                                                   "subvolumes", "option",
-                                                   "end-volume", "all",
-                                                   "volume_not_in_ring",
-                                                   "description", "force",
-                                                   "snap-max-hard-limit",
-                                                   "snap-max-soft-limit",
-                                                   "auto-delete",
-                                                   "activate-on-create", NULL};
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-        GF_ASSERT (state);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* Lowest wordcount possible */
-        if (wordcount < 2) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Invalid command: Not enough arguments");
-                goto out;
-        }
-
-        w = str_getunamb (words[1], opwords);
-        if (!w) {
-                /* Checks if the operation is a valid operation */
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
-                goto out;
-        }
+    static char *invalid_volnames[] = {"volume",
+                                       "type",
+                                       "subvolumes",
+                                       "option",
+                                       "end-volume",
+                                       "all",
+                                       "volume_not_in_ring",
+                                       "description",
+                                       "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit",
+                                       "auto-delete",
+                                       "activate-on-create",
+                                       NULL};
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+    GF_ASSERT(state);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (!strcmp (w, "create")) {
-                type = GF_SNAP_OPTION_TYPE_CREATE;
-        } else if (!strcmp (w, "list")) {
-                type = GF_SNAP_OPTION_TYPE_LIST;
-        } else if (!strcmp (w, "info")) {
-                type = GF_SNAP_OPTION_TYPE_INFO;
-        } else if (!strcmp (w, "delete")) {
-                type = GF_SNAP_OPTION_TYPE_DELETE;
-        } else if (!strcmp (w, "config")) {
-                type = GF_SNAP_OPTION_TYPE_CONFIG;
-        } else if (!strcmp (w, "restore")) {
-                type = GF_SNAP_OPTION_TYPE_RESTORE;
-        } else if (!strcmp (w, "status")) {
-                type = GF_SNAP_OPTION_TYPE_STATUS;
-        } else if (!strcmp (w, "activate")) {
-                type = GF_SNAP_OPTION_TYPE_ACTIVATE;
-        } else if (!strcmp (w, "deactivate")) {
-                type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
-        } else if (!strcmp(w, "clone")) {
-                type = GF_SNAP_OPTION_TYPE_CLONE;
-        }
-
-        if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
-            type != GF_SNAP_OPTION_TYPE_STATUS) {
-                ret = dict_set_int32 (dict, "hold_snap_locks", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to set hold-snap-locks value "
-                                "as _gf_true");
-                        goto out;
-                }
-        }
+    /* Lowest wordcount possible */
+    if (wordcount < 2) {
+        gf_log("", GF_LOG_ERROR, "Invalid command: Not enough arguments");
+        goto out;
+    }
 
-        /* Following commands does not require volume locks  */
-        if (type == GF_SNAP_OPTION_TYPE_STATUS ||
-            type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
-            type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
-                ret = dict_set_int32 (dict, "hold_vol_locks", _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Setting volume lock "
-                                "flag failed");
-                        goto out;
-                }
+    w = str_getunamb(words[1], opwords);
+    if (!w) {
+        /* Checks if the operation is a valid operation */
+        gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+        goto out;
+    }
+
+    if (!strcmp(w, "create")) {
+        type = GF_SNAP_OPTION_TYPE_CREATE;
+    } else if (!strcmp(w, "list")) {
+        type = GF_SNAP_OPTION_TYPE_LIST;
+    } else if (!strcmp(w, "info")) {
+        type = GF_SNAP_OPTION_TYPE_INFO;
+    } else if (!strcmp(w, "delete")) {
+        type = GF_SNAP_OPTION_TYPE_DELETE;
+    } else if (!strcmp(w, "config")) {
+        type = GF_SNAP_OPTION_TYPE_CONFIG;
+    } else if (!strcmp(w, "restore")) {
+        type = GF_SNAP_OPTION_TYPE_RESTORE;
+    } else if (!strcmp(w, "status")) {
+        type = GF_SNAP_OPTION_TYPE_STATUS;
+    } else if (!strcmp(w, "activate")) {
+        type = GF_SNAP_OPTION_TYPE_ACTIVATE;
+    } else if (!strcmp(w, "deactivate")) {
+        type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
+    } else if (!strcmp(w, "clone")) {
+        type = GF_SNAP_OPTION_TYPE_CLONE;
+    }
+
+    if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
+        type != GF_SNAP_OPTION_TYPE_STATUS) {
+        ret = dict_set_int32(dict, "hold_snap_locks", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to set hold-snap-locks value "
+                   "as _gf_true");
+            goto out;
+        }
+    }
+
+    /* Following commands does not require volume locks  */
+    if (type == GF_SNAP_OPTION_TYPE_STATUS ||
+        type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
+        type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
+        ret = dict_set_int32(dict, "hold_vol_locks", _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Setting volume lock "
+                   "flag failed");
+            goto out;
         }
+    }
 
-        /* Check which op is intended */
-        switch (type) {
+    /* Check which op is intended */
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                /* Syntax :
-                 * gluster snapshot create <snapname> <vol-name(s)>
-                 *                         [no-timestamp]
-                 *                         [description <description>]
-                 *                         [force]
-                 */
-                /* In cases where the snapname is not given then
-                 * parsing fails & snapname cannot be "description",
-                 * "force" and "volume", that check is made here
-                 */
-                if (wordcount == 2){
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+            /* Syntax :
+             * gluster snapshot create <snapname> <vol-name(s)>
+             *                         [no-timestamp]
+             *                         [description <description>]
+             *                         [force]
+             */
+            /* In cases where the snapname is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
 
-                ret = validate_op_name ("snapname", words[2],
-                                        invalid_snapnames);
-                if (ret) {
-                        goto out;
-                }
+            ret = validate_op_name("snapname", words[2], invalid_snapnames);
+            if (ret) {
+                goto out;
+            }
 
-                ret = cli_snap_create_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "create command parsing failed.");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_create_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "create command parsing failed.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                /* Syntax :
-                 * gluster snapshot clone <clonename> <snapname>
-                 */
-                /* In cases where the clonename is not given then
-                 * parsing fails & snapname cannot be "description",
-                 * "force" and "volume", that check is made here
-                 */
-                if (wordcount == 2) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                ret = validate_op_name ("clonename", words[2],
-                                        invalid_volnames);
-                if (ret) {
-                        goto out;
-                }
+            /* Syntax :
+             * gluster snapshot clone <clonename> <snapname>
+             */
+            /* In cases where the clonename is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
 
-                ret = cli_snap_clone_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "clone command parsing failed.");
-                        goto out;
-                }
-                break;
+            ret = validate_op_name("clonename", words[2], invalid_volnames);
+            if (ret) {
+                goto out;
+            }
 
+            ret = cli_snap_clone_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "clone command parsing failed.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_INFO:
-                /* Syntax :
-                 * gluster snapshot info [(snapname] | [vol <volname>)]
-                 */
-                ret = cli_snap_info_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot info command");
-                        goto out;
-                }
-                break;
+            /* Syntax :
+             * gluster snapshot info [(snapname] | [vol <volname>)]
+             */
+            ret = cli_snap_info_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot info command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_LIST:
-                /* Syntax :
-                 * gluster snaphsot list [volname]
-                 */
+            /* Syntax :
+             * gluster snaphsot list [volname]
+             */
 
-                ret = cli_snap_list_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot list command");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_list_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot list command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_DELETE:
-                /* Syntax :
-                 * snapshot delete (all | snapname | volume <volname>)
-                 */
-                ret = cli_snap_delete_parse (dict, words, wordcount, state);
-                if (ret) {
-                        /* A positive ret value means user cancelled
-                        * the command */
-                        if (ret < 0) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot delete command");
-                        }
-                        goto out;
+            /* Syntax :
+             * snapshot delete (all | snapname | volume <volname>)
+             */
+            ret = cli_snap_delete_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse "
+                           "snapshot delete command");
                 }
-                break;
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                /* snapshot config [volname]  [snap-max-hard-limit <count>]
-                 *                            [snap-max-soft-limit <percent>] */
-                ret = cli_snap_config_parse (words, wordcount, dict, state);
-                if (ret) {
-                        if (ret < 0)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "config command parsing failed.");
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                                "config type");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        case GF_SNAP_OPTION_TYPE_STATUS:
-                {
-                        /* Syntax :
-                         * gluster snapshot status [(snapname |
-                         *                         volume <volname>)]
-                         */
-                        ret = cli_snap_status_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot status command");
-                                goto out;
-                        }
-                        break;
-                }
+            /* snapshot config [volname]  [snap-max-hard-limit <count>]
+             *                            [snap-max-soft-limit <percent>] */
+            ret = cli_snap_config_parse(words, wordcount, dict, state);
+            if (ret) {
+                if (ret < 0)
+                    gf_log("cli", GF_LOG_ERROR,
+                           "config command parsing failed.");
+                goto out;
+            }
 
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                /* Syntax:
-                 * snapshot restore <snapname>
-                 */
-                ret = cli_snap_restore_parse (dict, words, wordcount, state);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "restore command");
-                        goto out;
-                }
-                break;
-
-                case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                        /* Syntax:
-                        * snapshot activate <snapname> [force]
-                        */
-                        ret = cli_snap_activate_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "start command");
-                                goto out;
-                        }
-                        break;
-                case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                        /* Syntax:
-                        * snapshot deactivate <snapname>
-                        */
-                        ret = cli_snap_deactivate_parse (dict, words, wordcount,
-                                state);
-                        if (ret) {
-                                /* A positive ret value means user cancelled
-                                 * the command */
-                                if (ret < 0) {
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Failed to parse deactivate "
-                                                "command");
-                                }
-                                goto out;
-                        }
-                        break;
+            ret = dict_set_int32(dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Unable to set "
+                       "config type");
+                ret = -1;
+                goto out;
+            }
+            break;
 
-        default:
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+        case GF_SNAP_OPTION_TYPE_STATUS: {
+            /* Syntax :
+             * gluster snapshot status [(snapname |
+             *                         volume <volname>)]
+             */
+            ret = cli_snap_status_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot status command");
                 goto out;
+            }
+            break;
         }
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Failed to set type.");
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            /* Syntax:
+             * snapshot restore <snapname>
+             */
+            ret = cli_snap_restore_parse(dict, words, wordcount, state);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "restore command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            /* Syntax:
+             * snapshot activate <snapname> [force]
+             */
+            ret = cli_snap_activate_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "start command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            /* Syntax:
+             * snapshot deactivate <snapname>
+             */
+            ret = cli_snap_deactivate_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse deactivate "
+                           "command");
+                }
                 goto out;
-        }
-        /* If you got so far, input is valid */
-        ret = 0;
+            }
+            break;
+
+        default:
+            ret = -1;
+            gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+            goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Failed to set type.");
+        goto out;
+    }
+    /* If you got so far, input is valid */
+    ret = 0;
 out:
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
-        } else
-                *options = dict;
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+    } else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_validate_volume (char *volname)
+cli_cmd_validate_volume(char *volname)
 {
-        int       i        =  0;
-        int       ret      = -1;
+    int i = 0;
+    int ret = -1;
+    int volname_len;
 
+    if (volname[0] == '-')
+        return ret;
 
-        if (volname[0] == '-')
-                return ret;
+    if (!strcmp(volname, "all")) {
+        cli_err("\"all\" cannot be the name of a volume.");
+        return ret;
+    }
 
-        if (!strcmp (volname, "all")) {
-                cli_err ("\"all\" cannot be the name of a volume.");
-                return ret;
-        }
+    if (strchr(volname, '/')) {
+        cli_err("Volume name should not contain \"/\" character.");
+        return ret;
+    }
 
-        if (strchr (volname, '/')) {
-                cli_err ("Volume name should not contain \"/\" character.");
-                return ret;
-        }
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volname can not exceed %d characters.", GD_VOLUME_NAME_MAX);
+        return ret;
+    }
 
-        if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                cli_err ("Volname can not exceed %d characters.",
-                        GD_VOLUME_NAME_MAX);
-                return ret;
+    for (i = 0; i < volname_len; i++)
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            return ret;
         }
 
-        for (i = 0; i < strlen (volname); i++)
-                if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                    (volname[i] != '-')) {
-                        cli_err ("Volume name should not contain \"%c\""
-                                 " character.\nVolume names can only"
-                                 "contain alphanumeric, '-' and '_' "
-                                 "characters.", volname[i]);
-                        return ret;
-                }
-
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_bitrot_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **options)
 {
-        int32_t            ret                    = -1;
-        char               *w                     = NULL;
-        char               *volname               = NULL;
-        char               *opwords[]             = {"enable", "disable",
-                                                     "scrub-throttle",
-                                                     "scrub-frequency", "scrub",
-                                                     "signing-time", NULL};
-        char               *scrub_throt_values[]  = {"lazy", "normal",
-                                                     "aggressive", NULL};
-        char               *scrub_freq_values[]   = {"hourly",
-                                                     "daily", "weekly",
-                                                     "biweekly", "monthly",
-                                                     "minute",  NULL};
-        char               *scrub_values[]        = {"pause", "resume",
-                                                     "status", "ondemand",
-                                                     NULL};
-        dict_t             *dict                  = NULL;
-        gf_bitrot_type     type                   = GF_BITROT_OPTION_TYPE_NONE;
-        int32_t            expiry_time            = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int32_t ret = -1;
+    char *w = NULL;
+    char *volname = NULL;
+    static char *opwords[] = {"enable",          "disable", "scrub-throttle",
+                              "scrub-frequency", "scrub",   "signing-time",
+                              "signer-threads",  NULL};
+    static char *scrub_throt_values[] = {"lazy", "normal", "aggressive", NULL};
+    static char *scrub_freq_values[] = {
+        "hourly", "daily", "weekly", "biweekly", "monthly", "minute", NULL};
+    static char *scrub_values[] = {"pause", "resume", "status", "ondemand",
+                                   NULL};
+    dict_t *dict = NULL;
+    gf_bitrot_type type = GF_BITROT_OPTION_TYPE_NONE;
+    int32_t expiry_time = 0;
+    int32_t signer_th_count = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /* Hack to print out bitrot help properly */
+    if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+        ret = 1;
+        return ret;
+    }
 
-        if (wordcount < 4 || wordcount > 5) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid syntax");
-                goto out;
-        }
+    if (wordcount < 4 || wordcount > 5) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = cli_cmd_validate_volume (volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to validate volume name");
-                goto out;
-        }
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                cli_out ("Failed to set volume name in dictionary ");
-                goto out;
+    ret = cli_cmd_validate_volume(volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to validate volume name");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        cli_out("Failed to set volume name in dictionary ");
+        goto out;
+    }
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid bit rot option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid bit rot option : %s", words[3]);
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!strcmp(w, "scrub-throttle")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing scrub-throttle value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_throt_values);
+            if (!w) {
+                cli_err(
+                    "Invalid scrub-throttle option for "
+                    "bitrot");
                 ret = -1;
                 goto out;
-        }
-
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_BITROT_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
+                ret = dict_set_str(dict, "scrub-throttle-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set scrub-throttle "
+                        "value in the dict");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_BITROT_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
+    if (!strcmp(words[3], "scrub-frequency")) {
+        if (!words[4]) {
+            cli_err("Missing scrub-frequency value");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_freq_values);
+            if (!w) {
+                cli_err("Invalid frequency option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
+                ret = dict_set_str(dict, "scrub-frequency-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (!strcmp (w, "scrub-throttle")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub-throttle value for bitrot "
-                                 "option");
-                        ret = -1;
-                        goto out;
+    if (!strcmp(words[3], "scrub")) {
+        if (!words[4]) {
+            cli_err("Missing scrub value for bitrot option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_values);
+            if (!w) {
+                cli_err("Invalid scrub option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                if (strcmp(words[4], "status") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_STATUS;
+                } else if (strcmp(words[4], "ondemand") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_ONDEMAND;
                 } else {
-                        w = str_getunamb (words[4], scrub_throt_values);
-                        if (!w) {
-                                cli_err ("Invalid scrub-throttle option for "
-                                         "bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
-                                ret =  dict_set_str (dict,
-                                                     "scrub-throttle-value",
-                                                     (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set scrub-throttle "
-                                                 "value in the dict");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
+                    type = GF_BITROT_OPTION_TYPE_SCRUB;
                 }
-        }
-
-        if (!strcmp (words[3], "scrub-frequency")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub-frequency value");
-                        ret = -1;
-                        goto out;
-                } else {
-                        w = str_getunamb (words[4], scrub_freq_values);
-                        if (!w) {
-                                cli_err ("Invalid frequency option for bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
-                                ret = dict_set_str (dict,
-                                                    "scrub-frequency-value",
-                                                    (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set dict for "
-                                                 "bitrot");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
+                ret = dict_set_str(dict, "scrub-value", (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (!strcmp (words[3], "scrub")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub value for bitrot option");
-                        ret = -1;
-                        goto out;
-                } else {
-                        w = str_getunamb (words[4], scrub_values);
-                        if (!w) {
-                                cli_err ("Invalid scrub option for bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                if (strcmp (words[4], "status") == 0) {
-                                        type = GF_BITROT_CMD_SCRUB_STATUS;
-                                } else if (strcmp (words[4], "ondemand") == 0) {
-                                        type = GF_BITROT_CMD_SCRUB_ONDEMAND;
-                                } else {
-                                        type = GF_BITROT_OPTION_TYPE_SCRUB;
-                                }
-                                ret =  dict_set_str (dict, "scrub-value",
-                                                    (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set dict for "
-                                                 "bitrot");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
-                }
-        }
+    if (!strcmp(words[3], "signing-time")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signing-time value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
 
-        if (!strcmp (words[3], "signing-time")) {
-                if (!words[4]) {
-                        cli_err ("Missing signing-time value for bitrot "
-                                 "option");
-                        ret = -1;
-                        goto out;
-                } else {
-                        type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
-
-                        expiry_time = strtol (words[4], NULL, 0);
-                        if (expiry_time < 1) {
-                                cli_err ("Expiry time  value should not be less"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
+            expiry_time = strtol(words[4], NULL, 0);
+            if (expiry_time < 1) {
+                cli_err(
+                    "Expiry time  value should not be less"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
 
-                        ret = dict_set_uint32 (dict, "expiry-time",
-                                               (unsigned int) expiry_time);
-                        if (ret) {
-                                cli_out ("Failed to set dict for bitrot");
-                                goto out;
-                        }
-                        goto set_type;
-                }
+            ret = dict_set_uint32(dict, "expiry-time",
+                                  (unsigned int)expiry_time);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
+                goto out;
+            }
+            goto set_type;
+        }
+    } else if (!strcmp(words[3], "signer-threads")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signer-thread value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
         } else {
-                cli_err ("Invalid option %s for bitrot. Please enter valid "
-                         "bitrot option", words[3]);
+            type = GF_BITROT_OPTION_TYPE_SIGNER_THREADS;
+
+            signer_th_count = strtol(words[4], NULL, 0);
+            if (signer_th_count < 1) {
+                cli_err("signer-thread count should not be less than 1");
                 ret = -1;
                 goto out;
-        }
+            }
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret < 0)
+            ret = dict_set_uint32(dict, "signer-threads",
+                                  (unsigned int)signer_th_count);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
                 goto out;
+            }
+            goto set_type;
+        }
+    } else {
+        cli_err(
+            "Invalid option %s for bitrot. Please enter valid "
+            "bitrot option",
+            words[3]);
+        ret = -1;
+        goto out;
+    }
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
-                if (dict)
-                        dict_unref (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
+}
+
+/* Parsing global option for NFS-Ganesha config
+ *  gluster nfs-ganesha enable/disable */
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char *w = NULL;
+    static char *opwords[] = {"enable", "disable", NULL};
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount != 2)
+        goto out;
+
+    key = (char *)words[0];
+    value = (char *)words[1];
+
+    if (!key || !value) {
+        cli_out("Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_strip_whitespace(value, strlen(value));
+    if (ret == -1)
+        goto out;
+
+    if (strcmp(key, "nfs-ganesha")) {
+        gf_asprintf(op_errstr,
+                    "Global option: error: ' %s '"
+                    "is not a valid global option.",
+                    key);
+        ret = -1;
+        goto out;
+    }
+
+    w = str_getunamb(value, opwords);
+    if (!w) {
+        cli_out(
+            "Invalid global option \n"
+            "Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        question =
+            "Enabling NFS-Ganesha requires Gluster-NFS to be "
+            "disabled across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else if (strcmp(value, "disable") == 0) {
+        question =
+            "Disabling NFS-Ganesha will tear down the entire "
+            "ganesha cluster across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else {
+        ret = -1;
+        goto out;
+    }
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Global operation "
+               "cancelled, exiting");
+        ret = -1;
+        goto out;
+    }
+    cli_out("This will take a few minutes to complete. Please wait ..");
+
+    ret = dict_set_str(dict, "key", key);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "value", value);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "globalname", "All");
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global"
+               " key failed.");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global key "
+               "failed.");
+        goto out;
+    }
+
+    *options = dict;
+out:
+    if (ret)
+        dict_unref(dict);
+
+    return ret;
 }
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index 4802f71eca8..084998701d8 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -18,295 +18,300 @@
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
 #include "protocol-common.h"
-#include "events.h"
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+int
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount);
 
 int
-cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_peer_probe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!(wordcount == 3)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        ret = valid_internet_address ((char *) words[2], _gf_false);
-        if (ret == 1) {
-                ret = 0;
-        } else {
-                cli_out ("%s is an invalid address", words[2]);
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                ret = -1;
-                goto out;
-        }
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!(wordcount == 3)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = valid_internet_address((char *)words[2], _gf_false, _gf_false);
+    if (ret == 1) {
+        ret = 0;
+    } else {
+        cli_out("%s is an invalid address", words[2]);
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer probe failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer probe failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        if (ret == 0) {
-                gf_event (EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
-        }
+    if (ret == 0) {
+        gf_event(EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_peer_deprobe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                   ret   = -1;
-        rpc_clnt_procedure_t *proc  = NULL;
-        call_frame_t         *frame = NULL;
-        dict_t               *dict  = NULL;
-        int                  flags = 0;
-        int                  sent = 0;
-        int                  parse_error = 0;
-        cli_local_t          *local = NULL;
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3]))
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int flags = 0;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    question =
+        "All clients mounted through the peer which is getting detached need "
+        "to be remounted using one of the other active peers in the trusted "
+        "storage pool to ensure client gets notification on any changes done "
+        "on the gluster configuration and if the same has been done do you "
+        "want to proceed?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
+
+    dict = dict_new();
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3]))
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret)
+        goto out;
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer detach failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer detach failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        if (ret == 0) {
-                gf_event (EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
-        }
+    if (ret == 0) {
+        gf_event(EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_peer_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_PEERS);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer status failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer status failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_pool_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS,
-                                (void *)GF_CLI_LIST_POOL_NODES);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_POOL_NODES);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("pool list: command execution failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("pool list: command execution failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd cli_probe_cmds[] = {
-        { "peer probe { <HOSTNAME> | <IP-address> }",
-          cli_cmd_peer_probe_cbk,
-          "probe peer specified by <HOSTNAME>"},
+    {"peer probe { <HOSTNAME> | <IP-address> }", cli_cmd_peer_probe_cbk,
+     "probe peer specified by <HOSTNAME>"},
 
-        { "peer detach { <HOSTNAME> | <IP-address> } [force]",
-          cli_cmd_peer_deprobe_cbk,
-          "detach peer specified by <HOSTNAME>"},
+    {"peer detach { <HOSTNAME> | <IP-address> } [force]",
+     cli_cmd_peer_deprobe_cbk, "detach peer specified by <HOSTNAME>"},
 
-        { "peer status",
-          cli_cmd_peer_status_cbk,
-          "list status of peers"},
+    {"peer status", cli_cmd_peer_status_cbk, "list status of peers"},
 
-	{ "peer help",
-           cli_cmd_peer_help_cbk,
-           "Help command for peer "},
+    {"peer help", cli_cmd_peer_help_cbk, "display help for peer commands"},
 
-        { "pool list",
-          cli_cmd_pool_list_cbk,
-          "list all the nodes in the pool (including localhost)"},
+    {"pool list", cli_cmd_pool_list_cbk,
+     "list all the nodes in the pool (including localhost)"},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *probe_cmd = NULL;
-        int                   count     = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *probe_cmd = NULL;
+    int count = 0;
 
-        cmd = GF_CALLOC (1, sizeof (cli_probe_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, cli_probe_cmds, sizeof (cli_probe_cmds));
-        count = (sizeof (cli_probe_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cli_out("\ngluster peer commands");
+    cli_out("======================\n");
 
+    cmd = GF_MALLOC(sizeof(cli_probe_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_probe_cmds, sizeof(cli_probe_cmds));
+    count = (sizeof(cli_probe_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
+    for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
+        cli_out("%s - %s", probe_cmd->pattern, probe_cmd->desc);
 
-        for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
-                cli_out ("%s - %s", probe_cmd->pattern, probe_cmd->desc);
+    GF_FREE(cmd);
 
-        GF_FREE (cmd);
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
 int
-cli_cmd_probe_register (struct cli_state *state)
+cli_cmd_probe_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
index e79128c5a59..859d6b2e40d 100644
--- a/cli/src/cli-cmd-snapshot.c
+++ b/cli/src/cli-cmd-snapshot.c
@@ -17,148 +17,119 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                           const char **words, int wordcount);
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount);
 
 int
-cli_cmd_snapshot_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_snapshot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        dict_t                  *options   = NULL;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        cli_local_t             *local     = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_SNAP];
-        if (proc == NULL) {
-               ret = -1;
-                goto out;
-        }
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SNAP];
+
+    /* Parses the command entered by the user */
+    ret = cli_cmd_snapshot_parse(words, wordcount, &options, state);
+    if (ret) {
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+        } else {
+            /* User might have cancelled the snapshot operation */
+            ret = 0;
         }
+        goto out;
+    }
 
-        /* Parses the command entered by the user */
-        ret = cli_cmd_snapshot_parse (words, wordcount, &options, state);
-        if (ret) {
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                } else {
-                        /* User might have cancelled the snapshot operation */
-                        ret = 0;
-                }
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out ("Snapshot command failed");
+    if (ret && parse_err == 0)
+        cli_out("Snapshot command failed");
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd snapshot_cmds[] = {
-        { "snapshot help",
-          cli_cmd_snapshot_help_cbk,
-          "display help for snapshot commands"
-        },
-        { "snapshot create <snapname> <volname> [no-timestamp] "
-                "[description <description>] [force]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Create."
-        },
-        { "snapshot clone <clonename> <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Clone."
-        },
-        { "snapshot restore <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Restore."
-        },
-        { "snapshot status [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Status."
-        },
-        { "snapshot info [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Info."
-        },
-        { "snapshot list [volname]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot List."
-        },
-        {"snapshot config [volname] ([snap-max-hard-limit <count>] "
-         "[snap-max-soft-limit <percent>]) "
-         "| ([auto-delete <enable|disable>])"
-         "| ([activate-on-create <enable|disable>])",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Config."
-        },
-        {"snapshot delete (all | snapname | volume <volname>)",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Delete."
-        },
-        {"snapshot activate <snapname> [force]",
-          cli_cmd_snapshot_cbk,
-          "Activate snapshot volume."
-        },
-        {"snapshot deactivate <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Deactivate snapshot volume."
-        },
-        { NULL, NULL, NULL }
-};
+    {"snapshot help", cli_cmd_snapshot_help_cbk,
+     "display help for snapshot commands"},
+    {"snapshot create <snapname> <volname> [no-timestamp] "
+     "[description <description>] [force]",
+     cli_cmd_snapshot_cbk, "Snapshot Create."},
+    {"snapshot clone <clonename> <snapname>", cli_cmd_snapshot_cbk,
+     "Snapshot Clone."},
+    {"snapshot restore <snapname>", cli_cmd_snapshot_cbk, "Snapshot Restore."},
+    {"snapshot status [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Status."},
+    {"snapshot info [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Info."},
+    {"snapshot list [volname]", cli_cmd_snapshot_cbk, "Snapshot List."},
+    {"snapshot config [volname] ([snap-max-hard-limit <count>] "
+     "[snap-max-soft-limit <percent>]) "
+     "| ([auto-delete <enable|disable>])"
+     "| ([activate-on-create <enable|disable>])",
+     cli_cmd_snapshot_cbk, "Snapshot Config."},
+    {"snapshot delete (all | snapname | volume <volname>)",
+     cli_cmd_snapshot_cbk, "Snapshot Delete."},
+    {"snapshot activate <snapname> [force]", cli_cmd_snapshot_cbk,
+     "Activate snapshot volume."},
+    {"snapshot deactivate <snapname>", cli_cmd_snapshot_cbk,
+     "Deactivate snapshot volume."},
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state,
-                           struct cli_cmd_word *in_word,
-                           const char **words,
-                           int wordcount)
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd      = NULL;
-        struct cli_cmd        *snap_cmd = NULL;
-        int                   count     = 0;
-
-        cmd = GF_CALLOC (1, sizeof (snapshot_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, snapshot_cmds, sizeof (snapshot_cmds));
-        count = (sizeof (snapshot_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
-
-        for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
-                if (_gf_false == snap_cmd->disable)
-                        cli_out ("%s - %s", snap_cmd->pattern, snap_cmd->desc);
-        GF_FREE (cmd);
-        return 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *snap_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(snapshot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, snapshot_cmds, sizeof(snapshot_cmds));
+    count = (sizeof(snapshot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster snapshot commands");
+    cli_out("=========================\n");
+
+    for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
+        if (_gf_false == snap_cmd->disable)
+            cli_out("%s - %s", snap_cmd->pattern, snap_cmd->desc);
+    cli_out("\n");
+
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_snapshot_register (struct cli_state *state)
+cli_cmd_snapshot_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 93aac0b60cc..801e8f4efed 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -18,585 +18,607 @@
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                             const char **words, int wordcount);
-
-int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount);
+int
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
-int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount);
+int
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount);
 
 int
-cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_getspec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volid", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volid", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_unref (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[2]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[2]);
+    }
 
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_pmap_b2p_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_pmap_b2p_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "brick", (char *)words[3]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "brick", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_unref (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[3]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[3]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_fsm_log_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_fsm_log_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        char                            *name = "";
-
-        if ((wordcount != 4) && (wordcount != 3)) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        if (wordcount == 4)
-                name = (char*)words[3];
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, (void*)name);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *name = "";
+
+    if ((wordcount != 4) && (wordcount != 3)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    if (wordcount == 4)
+        name = (char *)words[3];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, (void *)name);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_getwd_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_getwd_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static dict_t *
-make_seq_dict (int argc, char **argv)
+make_seq_dict(int argc, char **argv)
 {
-        char index[] = "4294967296"; // 1<<32
-        int i        = 0;
-        int ret      = 0;
-        dict_t *dict = dict_new ();
-
-        if (!dict)
-                return NULL;
-
-        for (i = 0; i < argc; i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_set_str (dict, index, argv[i]);
-                if (ret == -1)
-                        break;
-        }
-
-        if (ret) {
-                dict_unref (dict);
-                dict = NULL;
-        }
-
-        return dict;
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    int len;
+    int ret = 0;
+    dict_t *dict = dict_new();
+
+    if (!dict)
+        return NULL;
+
+    for (i = 0; i < argc; i++) {
+        len = snprintf(index, sizeof(index), "%d", i);
+        ret = dict_set_strn(dict, index, len, argv[i]);
+        if (ret == -1)
+            break;
+    }
+
+    if (ret) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+
+    return dict;
 }
 
 int
-cli_cmd_mount_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_mount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-        void *dataa[]              = {NULL, NULL};
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = make_seq_dict (wordcount - 3, (char **)words + 3);
-        if (!dict)
-                goto out;
-
-        dataa[0] = (void *)words[2];
-        dataa[1] = dict;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dataa);
-        }
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    void *dataa[] = {NULL, NULL};
+
+    if (wordcount < 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = make_seq_dict(wordcount - 3, (char **)words + 3);
+    if (!dict)
+        goto out;
+
+    dataa[0] = (void *)words[2];
+    dataa[1] = dict;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, dataa);
+    }
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Mount command failed");
+    if (!proc && ret)
+        cli_out("Mount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_umount_cbk(struct cli_state *state, struct cli_cmd_word *word,
                    const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-
-        if (!(wordcount == 3 ||
-              (wordcount == 4 && strcmp (words[3], "lazy") == 0))) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!(wordcount == 3 ||
+          (wordcount == 4 && strcmp(words[3], "lazy") == 0))) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "path", (char *)words[2]);
+    if (ret != 0)
+        goto out;
+    ret = dict_set_int32(dict, "lazy", wordcount == 4);
+    if (ret != 0)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        ret = dict_set_str (dict, "path", (char *)words[2]);
-        if (ret != 0)
-                goto out;
-        ret = dict_set_int32 (dict, "lazy", wordcount == 4);
-        if (ret != 0)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dict);
-        }
-
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Umount command failed");
+    if (!proc && ret)
+        cli_out("Umount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_uuid_get_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict  = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        cli_local_t             *local = NULL;
-        xlator_t                *this  = NULL;
-
-        this = THIS;
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        if (proc->fn)
-                ret = proc->fn (frame, this, dict);
+    int ret = -1;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    if (proc->fn)
+        ret = proc->fn(frame, this, dict);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid get failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid get failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_uuid_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        char                    *question = NULL;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict  = NULL;
-        xlator_t                *this  = NULL;
-
-        question = "Resetting uuid changes the uuid of local glusterd. "
-                   "Do you want to continue?";
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        answer = cli_cmd_get_confirmation (state, question);
-
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
-
-        //send NULL as argument since no dictionary is sent to glusterd
-        if (proc->fn) {
-                ret = proc->fn (frame, this, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char *question = NULL;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    question =
+        "Resetting uuid changes the uuid of local glusterd. "
+        "Do you want to continue?";
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+    this = THIS;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    // send NULL as argument since no dictionary is sent to glusterd
+    if (proc->fn) {
+        ret = proc->fn(frame, this, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid reset failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid reset failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-struct cli_cmd cli_system_cmds[] = {
-        { "system:: getspec <VOLNAME>",
-          cli_cmd_getspec_cbk,
-          "fetch the volume file for the volume <VOLNAME>"},
+static struct cli_cmd cli_system_cmds[] = {
+    {"system:: getspec <VOLNAME>", cli_cmd_getspec_cbk,
+     "fetch the volume file for the volume <VOLNAME>"},
 
-        { "system:: portmap brick2port <BRICK>",
-          cli_cmd_pmap_b2p_cbk,
-          "query which port <BRICK> listens on"},
+    {"system:: portmap brick2port <BRICK>", cli_cmd_pmap_b2p_cbk,
+     "query which port <BRICK> listens on"},
 
-        { "system:: fsm log [<peer-name>]",
-          cli_cmd_fsm_log_cbk,
-          "display fsm transitions"},
+    {"system:: fsm log [<peer-name>]", cli_cmd_fsm_log_cbk,
+     "display fsm transitions"},
 
-        { "system:: getwd",
-          cli_cmd_getwd_cbk,
-          "query glusterd work directory"},
+    {"system:: getwd", cli_cmd_getwd_cbk, "query glusterd work directory"},
 
-        { "system:: mount <label> <args...>",
-          cli_cmd_mount_cbk,
-          "request a mount"},
+    {"system:: mount <label> <args...>", cli_cmd_mount_cbk, "request a mount"},
 
-        { "system:: umount <path> [lazy]",
-          cli_cmd_umount_cbk,
-          "request an umount"},
+    {"system:: umount <path> [lazy]", cli_cmd_umount_cbk, "request an umount"},
 
-        { "system:: uuid get",
-          cli_cmd_uuid_get_cbk,
-          "get uuid of glusterd"},
+    {"system:: uuid get", cli_cmd_uuid_get_cbk, "get uuid of glusterd"},
 
-        { "system:: uuid reset",
-          cli_cmd_uuid_reset_cbk,
-          "reset the uuid of glusterd"},
+    {"system:: uuid reset", cli_cmd_uuid_reset_cbk,
+     "reset the uuid of glusterd"},
 
-        { "system:: help",
-           cli_cmd_system_help_cbk,
-           "display help for system commands"},
+    {"system:: help", cli_cmd_system_help_cbk,
+     "display help for system commands"},
 
-        { "system:: copy file [<filename>]",
-           cli_cmd_copy_file_cbk,
-           "Copy file from current node's $working_dir to "
-           "$working_dir of all cluster nodes"},
+    {"system:: copy file [<filename>]", cli_cmd_copy_file_cbk,
+     "Copy file from current node's $working_dir to "
+     "$working_dir of all cluster nodes"},
 
-        { "system:: execute <command> <args>",
-           cli_cmd_sys_exec_cbk,
-           "Execute the command on all the nodes "
-           "in the cluster and display their output."},
+    {"system:: execute <command> <args>", cli_cmd_sys_exec_cbk,
+     "Execute the command on all the nodes "
+     "in the cluster and display their output."},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                  *command                = NULL;
-        char                  *saveptr                = NULL;
-        char                  *tmp                    = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    in_cmd_args_count      = 0;
-        rpc_clnt_procedure_t  *proc                   = NULL;
-        call_frame_t          *frame                  = NULL;
-        dict_t                *dict                   = NULL;
-        cli_local_t           *local                  = NULL;
-
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        command = strtok_r ((char *)words[2], " ", &saveptr);
-        do {
-                tmp = strtok_r (NULL, " ", &saveptr);
-                if (tmp) {
-                        in_cmd_args_count++;
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", in_cmd_args_count);
-                        ret = dict_set_str (dict, cmd_arg_name, tmp);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to set "
-                                        "%s in dict", cmd_arg_name);
-                                goto out;
-                        }
-                }
-        } while (tmp);
-
-        cmd_args_count = wordcount - 3;
-
-        ret = dict_set_str (dict, "command", command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+    char cmd_arg_name[PATH_MAX] = "";
+    char *command = NULL;
+    char *saveptr = NULL;
+    char *tmp = NULL;
+    int ret = -1;
+    int i = -1;
+    int len;
+    int cmd_args_count = 0;
+    int in_cmd_args_count = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if ((wordcount < 3) || (words[2] == NULL)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    command = strtok_r((char *)words[2], " ", &saveptr);
+    if (command == NULL) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to parse command");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    do {
+        tmp = strtok_r(NULL, " ", &saveptr);
+        if (tmp) {
+            in_cmd_args_count++;
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                     in_cmd_args_count);
+            ret = dict_set_str(dict, cmd_arg_name, tmp);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Unable to set "
+                       "%s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    } while (tmp);
 
-        for (i=1; i <= cmd_args_count; i++) {
-                in_cmd_args_count++;
-                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                          "cmd_arg_%d", in_cmd_args_count);
-                ret = dict_set_str (dict, cmd_arg_name,
-                                    (char *)words[2+i]);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
-                               cmd_arg_name);
-                        goto out;
-                }
-        }
+    cmd_args_count = wordcount - 3;
 
-        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to set cmd_args_count in dict");
-                goto out;
-        }
+    ret = dict_set_str(dict, "command", command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set command in dict");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", "N/A");
+    for (i = 1; i <= cmd_args_count; i++) {
+        in_cmd_args_count++;
+        len = snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                       in_cmd_args_count);
+        ret = dict_set_strn(dict, cmd_arg_name, len, (char *)words[2 + i]);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+            gf_log("", GF_LOG_ERROR, "Unable to set %s in dict", cmd_arg_name);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "cmd_args_count", in_cmd_args_count);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set cmd_args_count in dict");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+
+        /* proc->fn is processed synchronously, which means that the
+         * execution flow won't return here until the operation is
+         * fully processed, including any related callback. For this
+         * reason, it's safe to destroy the stack here, since no one
+         * can still be using it. Additionally, it's not easy to move
+         * the stack destroy to the callback executed after completion
+         * of the operation because there are multiple things than can
+         * fail even before having queued the callback, so we would
+         * still need to destroy the stack if proc->fn returns an
+         * error. */
+        CLI_STACK_DESTROY(frame);
+        dict = NULL;
+    }
 out:
-        return ret;
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                    ret      = -1;
-        rpc_clnt_procedure_t  *proc     = NULL;
-        call_frame_t          *frame    = NULL;
-        char                  *filename = "";
-        dict_t                *dict     = NULL;
-        cli_local_t           *local    = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        filename = (char*)words[3];
-        ret = dict_set_str (dict, "source", filename);
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
-
-        ret = dict_set_str (dict, "volname", "N/A");
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *filename = "";
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    filename = (char *)words[3];
+    ret = dict_set_str(dict, "source", filename);
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount)
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *system_cmd = NULL;
-        int                   count     = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *system_cmd = NULL;
+    int count = 0;
 
-        cmd = GF_CALLOC (1, sizeof (cli_system_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, cli_system_cmds, sizeof (cli_system_cmds));
-        count = (sizeof (cli_system_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cmd = GF_MALLOC(sizeof(cli_system_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_system_cmds, sizeof(cli_system_cmds));
+    count = (sizeof(cli_system_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
-                cli_out ("%s - %s", system_cmd->pattern, system_cmd->desc);
+    for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
+        cli_out("%s - %s", system_cmd->pattern, system_cmd->desc);
 
-        GF_FREE (cmd);
-        return 0;
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_system_register (struct cli_state *state)
+cli_cmd_system_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index d78ac0ee3af..f238851586e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -23,3291 +23,3256 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
-#include "common-utils.h"
-#include "events.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-extern struct rpc_clnt *global_quotad_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
 extern rpc_clnt_prog_t cli_quotad_clnt;
 
-int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...);
 
 int
-cli_cmd_volume_info_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
-{
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        cli_cmd_volume_get_ctx_t        ctx = {0,};
-        cli_local_t                     *local = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+int
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount);
 
-        if ((wordcount == 2)  || (wordcount == 3 &&
-                                  !strcmp (words[2], "all"))) {
-                ctx.flags = GF_CLI_GET_NEXT_VOLUME;
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
-        } else if (wordcount == 3) {
-                ctx.flags = GF_CLI_GET_VOLUME;
-                ctx.volname = (char *)words[2];
-                if (strlen (ctx.volname) > GD_VOLUME_NAME_MAX) {
-                        cli_out ("Invalid volume name");
-                        goto out;
-                }
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
-        } else {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                return -1;
+int
+cli_cmd_volume_info_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_cmd_volume_get_ctx_t ctx = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+
+    if ((wordcount == 2) || (wordcount == 3 && !strcmp(words[2], "all"))) {
+        ctx.flags = GF_CLI_GET_NEXT_VOLUME;
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
+    } else if (wordcount == 3) {
+        ctx.flags = GF_CLI_GET_VOLUME;
+        ctx.volname = (char *)words[2];
+        if (strlen(ctx.volname) > GD_VOLUME_NAME_MAX) {
+            cli_out("Invalid volume name");
+            goto out;
         }
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+    } else {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        return -1;
+    }
 
-        local = cli_local_get ();
+    local = cli_local_get();
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        local->get_vol.flags = ctx.flags;
-        if (ctx.volname)
-                local->get_vol.volname = gf_strdup (ctx.volname);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        frame->local = local;
+    local->get_vol.flags = ctx.flags;
+    if (ctx.volname)
+        local->get_vol.volname = gf_strdup(ctx.volname);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &ctx);
-        }
+    frame->local = local;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting Volume information failed!");
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, &ctx);
+    }
 
-        CLI_STACK_DESTROY (frame);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting Volume information failed!");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_sync_volume_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = "Sync volume may make data "
-                                            "inaccessible while the sync "
-                                            "is in progress. Do you want "
-                                            "to continue?";
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Sync volume may make data "
+        "inaccessible while the sync "
+        "is in progress. Do you want "
+        "to continue?";
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if ((wordcount == 3) || !strcmp(words[3], "all")) {
-                ret = dict_set_int32 (dict, "flags", (int32_t)
-                                      GF_CLI_SYNC_ALL);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set"
-                                "flag");
-                        goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "volname", (char *) words[3]);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set "
-                                "volume");
-                        goto out;
-                }
+    if ((wordcount == 3) || !strcmp(words[3], "all")) {
+        ret = dict_set_int32(dict, "flags", (int32_t)GF_CLI_SYNC_ALL);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set"
+                   "flag");
+            goto out;
         }
-
-        ret = dict_set_str (dict, "hostname", (char *) words[2]);
+    } else {
+        ret = dict_set_str(dict, "volname", (char *)words[3]);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set hostname");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "volume");
+            goto out;
         }
+    }
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set hostname");
+        goto out;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume sync failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume sync failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_create_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        char                    *trans_type = NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_create_parse (state, words, wordcount, &options);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    char *trans_type = NULL;
+    char *bricks = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
+
+    ret = cli_cmd_volume_create_parse(state, words, wordcount, &options,
+                                      &bricks);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    ret = dict_get_str(options, "transport", &trans_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
+        goto out;
+    }
 
-        ret = dict_get_str (options, "transport", &trans_type);
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume create failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-        if (ret == 0) {
-                gf_event (EVENT_VOLUME_CREATE, "name=%s", (char *)words[2]);
-        }
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume create failed");
+    }
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_CREATE, "name=%s;bricks=%s", (char *)words[2],
+                 bricks);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-
 int
-cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_delete_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *volname = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict = NULL;
-
-        question = "Deleting volume will erase all information about the volume. "
-                   "Do you want to continue?";
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    question =
+        "Deleting volume will erase all information about the volume. "
+        "Do you want to continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Deleting the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Deleting the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume delete failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume delete failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        if (ret == 0 && GF_ANSWER_YES == answer) {
-                gf_event (EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
-        }
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_start_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        int                     flags = 0;
-        cli_local_t             *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    int flags = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3 || wordcount > 4) {
-               cli_usage_out (word->pattern);
-                parse_error = 1;
-               goto out;
-        }
+    if (!words[2])
+        goto out;
 
-        dict = dict_new ();
-        if (!dict) {
-                goto out;
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (!words[2])
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                 gf_log (THIS->name, GF_LOG_ERROR,
-                         "dict set failed");
-                 goto out;
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume start failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume start failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        if (ret == 0) {
-                gf_event (EVENT_VOLUME_START, "name=%s;force=%d",
-                          (char *)words[2], (flags & GF_CLI_FLAG_OP_FORCE));
-        }
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_START, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
 
-        return ret;
+    return ret;
 }
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question)
+cli_cmd_get_confirmation(struct cli_state *state, const char *question)
 {
-        char                    answer[5] = {'\0', };
-        char                    flush = '\0';
-        size_t			len;
+    char answer[5] = {
+        '\0',
+    };
+    int flush = '\0';
+    size_t len;
 
-        if (state->mode & GLUSTER_MODE_SCRIPT)
-                return GF_ANSWER_YES;
+    if (state->mode & GLUSTER_MODE_SCRIPT)
+        return GF_ANSWER_YES;
 
-	printf ("%s (y/n) ", question);
+    printf("%s (y/n) ", question);
 
-        if (fgets (answer, 4, stdin) == NULL) {
-                cli_out("gluster cli read error");
-                goto out;
-        }
+    if (fgets(answer, 4, stdin) == NULL) {
+        cli_out("gluster cli read error");
+        goto out;
+    }
 
-	len = strlen (answer);
+    len = strlen(answer);
 
-	if (len && answer [len - 1] == '\n'){
-		answer [--len] = '\0';
-	} else {
-		do{
-			flush = getchar ();
-		}while (flush != '\n');
-	}
+    if (len && answer[len - 1] == '\n') {
+        answer[--len] = '\0';
+    } else {
+        do {
+            flush = getchar();
+        } while (flush != '\n');
+    }
 
-	if (len > 3)
-		goto out;
+    if (len > 3)
+        goto out;
 
-	if (!strcasecmp (answer, "y") || !strcasecmp (answer, "yes"))
-		return GF_ANSWER_YES;
+    if (!strcasecmp(answer, "y") || !strcasecmp(answer, "yes"))
+        return GF_ANSWER_YES;
 
-	else if (!strcasecmp (answer, "n") || !strcasecmp (answer, "no"))
-		return GF_ANSWER_NO;
+    else if (!strcasecmp(answer, "n") || !strcasecmp(answer, "no"))
+        return GF_ANSWER_NO;
 
 out:
-	cli_out ("Invalid input, please enter y/n");
+    cli_out("Invalid input, please enter y/n");
 
-	return GF_ANSWER_NO;
+    return GF_ANSWER_NO;
 }
 
 int
-cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_stop_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     flags   = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Stopping volume will make its data inaccessible. "
-                               "Do you want to continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3 || wordcount > 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        volname = (char*) words[2];
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int flags = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    cli_local_t *local = NULL;
+
+    const char *question =
+        "Stopping volume will make its data inaccessible. "
+        "Do you want to continue?";
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    volname = (char *)words[2];
 
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Stopping the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
+    dict = dict_new();
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Stopping the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "dict set failed");
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(state, question);
 
-        answer = cli_cmd_get_confirmation (state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume stop on '%s' failed", volname);
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        if (ret == 0 && GF_ANSWER_YES == answer) {
-                gf_event (EVENT_VOLUME_STOP, "name=%s;force=%d",
-                          (char *)words[2], (flags & GF_CLI_FLAG_OP_FORCE));
-        }
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume stop on '%s' failed", volname);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    if (dict)
+        dict_unref(dict);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_STOP, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_rename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_rename_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "old-volname", (char *)words[2]);
+    ret = dict_set_str(dict, "old-volname", (char *)words[2]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "new-volname", (char *)words[3]);
+    ret = dict_set_str(dict, "new-volname", (char *)words[3]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rename on '%s' failed", (char *)words[2]);
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rename on '%s' failed", (char *)words[2]);
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_defrag_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret     = -1;
-        rpc_clnt_procedure_t *proc    = NULL;
-        call_frame_t         *frame   = NULL;
-        dict_t               *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t          *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 #if (USE_EVENTS)
-        eventtypes_t         event = EVENT_LAST;
+    eventtypes_t event = EVENT_LAST;
 #endif
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_defrag_parse(words, wordcount, &dict);
 
-        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rebalance failed");
-        } else {
-
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rebalance failed");
+    } else {
 #if (USE_EVENTS)
-                if (!(strcmp (words[wordcount-1], "start")) ||
-                        !(strcmp (words[wordcount-1], "force")))  {
-                        event = EVENT_VOLUME_REBALANCE_START;
-                } else if (!strcmp (words[wordcount-1], "stop")) {
-                        event = EVENT_VOLUME_REBALANCE_STOP;
-                }
+        if (!(strcmp(words[wordcount - 1], "start")) ||
+            !(strcmp(words[wordcount - 1], "force"))) {
+            event = EVENT_VOLUME_REBALANCE_START;
+        } else if (!strcmp(words[wordcount - 1], "stop")) {
+            event = EVENT_VOLUME_REBALANCE_STOP;
+        }
 
-                if (event != EVENT_LAST)
-                        gf_event (event, "volume=%s", (char *)words[2]);
+        if (event != EVENT_LAST)
+            gf_event(event, "volume=%s", (char *)words[2]);
 #endif
-        }
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
 #if (USE_EVENTS)
-        int                      ret1    = -1;
-        char                    *tmp_opt = NULL;
+    int ret1 = -1;
+    char *tmp_opt = NULL;
 #endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset failed");
+    }
 
 #if (USE_EVENTS)
-        if (ret == 0) {
-                ret1 = dict_get_str (options, "key", &tmp_opt);
-                if (ret1)
-                        tmp_opt = "";
-
-                gf_event (EVENT_VOLUME_RESET, "name=%s;option=%s",
-                          (char *)words[2],
-                          tmp_opt);
-        }
+    if (ret == 0) {
+        ret1 = dict_get_str(options, "key", &tmp_opt);
+        if (ret1)
+            tmp_opt = "";
+
+        gf_event(EVENT_VOLUME_RESET, "name=%s;option=%s", (char *)words[2],
+                 tmp_opt);
+    }
 #endif
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_profile_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
+    ret = cli_cmd_volume_profile_parse(words, wordcount, &options);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume profile failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume profile failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
 
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
-        char                    *op_errstr = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
 
 #if (USE_EVENTS)
-        int                      ret1          = -1;
-        int                      i             = 1;
-        char                     dict_key[50]  = {0,};
-        char                    *tmp_opt       = NULL;
-        char                    *opts_str      = NULL;
-        int                      num_options   = 0;
+    int ret1 = -1;
+    int i = 1;
+    char dict_key[50] = {
+        0,
+    };
+    char *tmp_opt = NULL;
+    char *opts_str = NULL;
+    int num_options = 0;
 #endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_set_parse(state, words, wordcount, &options,
+                                   &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
 
-        ret = cli_cmd_volume_set_parse (state, words, wordcount,
-                                        &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume set failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume set failed");
+    }
 
 #if (USE_EVENTS)
-        if (ret == 0) {
-                ret1 = dict_get_int32 (options, "count", &num_options);
-                if (ret1)
-                        num_options = 0;
-                else
-                        num_options = num_options/2;
+    if (ret == 0 && strcmp(words[2], "help") != 0) {
+        ret1 = dict_get_int32(options, "count", &num_options);
+        if (ret1) {
+            num_options = 0;
+            goto end;
+        } else {
+            num_options = num_options / 2;
+        }
 
-                /* Initialize opts_str */
-                opts_str = gf_strdup ("");
+        char *free_list_key[num_options];
+        char *free_list_val[num_options];
+        for (i = 0; i < num_options; i++) {
+            free_list_key[i] = NULL;
+            free_list_val[i] = NULL;
+        }
+        /* Initialize opts_str */
+        opts_str = "";
 
-                /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
-                for (i = 1; i <= num_options; i++) {
-                        sprintf (dict_key, "key%d", i);
-                        ret1 = dict_get_str (options, dict_key, &tmp_opt);
-                        if (ret1)
-                                tmp_opt = "";
+        /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
+        for (i = 1; i <= num_options; i++) {
+            sprintf(dict_key, "key%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-                        gf_asprintf (&opts_str, "%s,%s", opts_str, tmp_opt);
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_key[i - 1] = opts_str;
 
-                        sprintf (dict_key, "value%d", i);
-                        ret1 = dict_get_str (options, dict_key, &tmp_opt);
-                        if (ret1)
-                                tmp_opt = "";
+            sprintf(dict_key, "value%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-                        gf_asprintf (&opts_str, "%s,%s", opts_str, tmp_opt);
-                }
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_val[i - 1] = opts_str;
+        }
 
-                gf_event (EVENT_VOLUME_SET, "name=%s;options=%s",
-                          (char *)words[2],
-                          opts_str);
+        gf_event(EVENT_VOLUME_SET, "name=%s;options=%s", (char *)words[2],
+                 opts_str);
 
-                /* Allocated by gf_strdup and gf_asprintf */
-                GF_FREE (opts_str);
+        /* Allocated by gf_strdup and gf_asprintf */
+        for (i = 0; i < num_options; i++) {
+            GF_FREE(free_list_key[i]);
+            GF_FREE(free_list_val[i]);
         }
+    }
 #endif
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+end:
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
-static
-int
-cli_event_remove_brick_str (dict_t *options, char **event_str,
-                            eventtypes_t *event)
+static int
+cli_event_remove_brick_str(dict_t *options, char **event_str,
+                           eventtypes_t *event)
 {
-        int           ret             = -1;
-        char          *bricklist      = NULL;
-        char          *brick          = NULL;
-        char          *volname        = NULL;
-        char           key[256]       = {0,};
-        const char    *eventstrformat = "volume=%s;bricks=%s";
-        int32_t        command        = 0;
-        int32_t        i              = 1;
-        int32_t        count          = 0;
-        int32_t        eventstrlen    = 1;
-        char          *tmp_ptr        = NULL;
-
-        if (!options || !event_str || !event)
-                goto out;
+    int ret = -1;
+    char *bricklist = NULL;
+    char *brick = NULL;
+    char *volname = NULL;
+    char key[256] = {
+        0,
+    };
+    const char *eventstrformat = "volume=%s;bricks=%s";
+    int32_t command = 0;
+    int32_t i = 1;
+    int32_t count = 0;
+    int32_t eventstrlen = 1;
+    int bricklen = 0;
+    char *tmp_ptr = NULL;
+
+    if (!options || !event_str || !event)
+        goto out;
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret || !volname) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch volname");
-                ret = -1;
-                goto out;
-        }
-        /* Get the list of bricks for the event */
-        ret = dict_get_int32 (options, "command", &command);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch command");
-                ret = -1;
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+    /* Get the list of bricks for the event */
+    ret = dict_get_int32(options, "command", &command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch command");
+        ret = -1;
+        goto out;
+    }
 
-        switch (command) {
+    switch (command) {
         case GF_OP_CMD_START:
-                *event = EVENT_VOLUME_REMOVE_BRICK_START;
-                break;
+            *event = EVENT_VOLUME_REMOVE_BRICK_START;
+            break;
         case GF_OP_CMD_COMMIT:
-                *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
-                break;
+            *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
+            break;
         case GF_OP_CMD_COMMIT_FORCE:
-                *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
-                break;
+            *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
+            break;
         case GF_OP_CMD_STOP:
-                *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
-                break;
+            *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
+            break;
         default:
-                *event = EVENT_LAST;
-                break;
-        }
+            *event = EVENT_LAST;
+            break;
+    }
 
-        ret = -1;
+    ret = -1;
 
-        if (*event == EVENT_LAST) {
-                goto out;
-        }
+    if (*event == EVENT_LAST) {
+        goto out;
+    }
 
-        /* I could just get this from words[] but this is cleaner in case the
-         * format changes  */
-        while (i) {
-                snprintf (key, sizeof (key), "brick%d", i);
-                ret = dict_get_str (options, key, &brick);
-                if (ret) {
-                        break;
-                }
-                eventstrlen += strlen (brick) + 1;
-                i++;
+    /* I could just get this from words[] but this is cleaner in case the
+     * format changes  */
+    while (i) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
         }
+        eventstrlen += strlen(brick) + 1;
+        i++;
+    }
 
-        count = --i;
+    count = --i;
 
-        eventstrlen += 1;
+    eventstrlen += 1;
 
-        bricklist = GF_CALLOC (eventstrlen, sizeof (char), gf_common_mt_char);
-        if (!bricklist) {
-                goto out;
-        }
+    bricklist = GF_CALLOC(eventstrlen, sizeof(char), gf_common_mt_char);
+    if (!bricklist) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "memory allocation failed for"
+               "bricklist");
+        ret = -1;
+        goto out;
+    }
 
-        tmp_ptr = bricklist;
+    tmp_ptr = bricklist;
 
-        i = 1;
-        while (i <= count) {
-                snprintf (key, sizeof (key), "brick%d", i);
-                ret = dict_get_str (options, key, &brick);
-                if (ret) {
-                        break;
-                }
-                snprintf (tmp_ptr, eventstrlen, "%s ", brick);
-                eventstrlen -= (strlen (brick) + 1);
-                tmp_ptr += (strlen (brick) + 1);
-                i++;
+    i = 1;
+    while (i <= count) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
         }
+        snprintf(tmp_ptr, eventstrlen, "%s ", brick);
+        bricklen = strlen(brick);
+        eventstrlen -= (bricklen + 1);
+        tmp_ptr += (bricklen + 1);
+        i++;
+    }
 
-        if (!ret) {
-                gf_asprintf (event_str, eventstrformat, volname,
-                             bricklist);
-        } else {
-                gf_asprintf (event_str, eventstrformat, volname,
-                             "<unavailable>");
-        }
+    if (!ret) {
+        gf_asprintf(event_str, eventstrformat, volname, bricklist);
+    } else {
+        gf_asprintf(event_str, eventstrformat, volname, "<unavailable>");
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        GF_FREE (bricklist);
-        return ret;
+    GF_FREE(bricklist);
+    return ret;
 }
 
-
 int
-cli_cmd_volume_add_brick_cbk (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
+cli_cmd_volume_add_brick_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
 
 #if (USE_EVENTS)
-        char                    *event_str = NULL;
-        char                    *bricks = NULL;
-        const char              *eventstrformat = "volume=%s;bricks=%s";
+    char *event_str = NULL;
+    char *bricks = NULL;
+    const char *eventstrformat = "volume=%s;bricks=%s";
 #endif
 
-        const char *question = "Changing the 'stripe count' of the volume is "
-                "not a supported feature. In some cases it may result in data "
-                "loss on the volume. Also there may be issues with regular "
-                "filesystem operations on the volume after the change. Do you "
-                "really want to continue with 'stripe' count option ? ";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options, 0);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    const char *question =
+        "Changing the 'stripe count' of the volume is "
+        "not a supported feature. In some cases it may result in data "
+        "loss on the volume. Also there may be issues with regular "
+        "filesystem operations on the volume after the change. Do you "
+        "really want to continue with 'stripe' count option ? ";
+
+    ret = cli_cmd_volume_add_brick_parse(state, words, wordcount, &options, 0);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        /* TODO: there are challenges in supporting changing of
-           stripe-count, until it is properly supported give warning to user */
-        if (dict_get (options, "stripe-count")) {
-                answer = cli_cmd_get_confirmation (state, question);
+    /* TODO: there are challenges in supporting changing of
+       stripe-count, until it is properly supported give warning to user */
+    if (dict_get(options, "stripe-count")) {
+        answer = cli_cmd_get_confirmation(state, question);
 
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
 #if (USE_EVENTS)
-        /* Get the list of bricks for the event */
+    /* Get the list of bricks for the event */
 
-        ret = dict_get_str (options, "bricks", &bricks);
+    ret = dict_get_str(options, "bricks", &bricks);
 
-        if (!ret) {
-                gf_asprintf (&event_str, eventstrformat, (char *)words[2],
-                            &bricks[1] /*Skip leading space*/);
-        } else {
-                gf_asprintf (&event_str, eventstrformat, (char *)words[2],
-                            "<unavailable>");
-        }
+    if (!ret) {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    &bricks[1] /*Skip leading space*/);
+    } else {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    "<unavailable>");
+    }
 #endif
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume add-brick failed");
-        } else {
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume add-brick failed");
+    } else {
 #if (USE_EVENTS)
-                gf_event (EVENT_VOLUME_ADD_BRICK, event_str);
-                GF_FREE (event_str);
+        gf_event(EVENT_VOLUME_ADD_BRICK, "%s", event_str);
+#endif
+    }
+#if (USE_EVENTS)
+    GF_FREE(event_str);
 #endif
-        }
-
-        CLI_STACK_DESTROY (frame);
-        return ret;
-}
-
-int
-cli_tier_validate_replica_type (dict_t *dict, int type)
-{
-
-        int             brick_count             = -1;
-        int             replica_count           = 1;
-        int             ret                     = -1;
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get brick count");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_DEBUG, "Failed to get replica count. "
-                        "Defaulting to one");
-                replica_count = 1;
-        }
 
-        /*
-         * Change the calculation of sub_count once attach-tier support
-         * disperse volume.
-         * sub_count = disperse_count for disperse volume
-         * */
-
-
-        if (brick_count % replica_count) {
-             if (type == GF_CLUSTER_TYPE_REPLICATE)
-                     cli_err ("number of bricks is not a multiple of "
-                              "replica count");
-             else if (type == GF_CLUSTER_TYPE_DISPERSE)
-                     cli_err ("number of bricks is not a multiple of "
-                              "disperse count");
-             else
-                     cli_err ("number of bricks given doesn't match "
-                              "required count");
-
-             ret = -1;
-             goto out;
-     }
-        ret = 0;
-out:
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-do_cli_cmd_volume_attach_tier (struct cli_state *state,
-                                struct cli_cmd_word *word, const char **words,
-                                int wordcount)
+cli_get_soft_limit(dict_t *options, const char **words, dict_t *xdata)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        int                     type = 0;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options, &type);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        /*
-         * Merge this check when attach-tier has it's own cli parse function.
-         */
-        ret = cli_tier_validate_replica_type (options, type);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (options, "attach-tier", 1);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (options, "hot-type", type);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ATTACH_TIER];
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    char *default_sl = NULL;
+    char *default_sl_dup = NULL;
+    int ret = -1;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    // We need a ref on @options to prevent CLI_STACK_DESTROY
+    // from destroying it prematurely.
+    dict_ref(options);
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    ret = proc->fn(frame, THIS, options);
+
+    ret = dict_get_str(options, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get default soft limit");
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    default_sl_dup = gf_strdup(default_sl);
+    if (!default_sl_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    ret = dict_set_dynstr(xdata, "default-soft-limit", default_sl_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set default soft limit");
+        GF_FREE(default_sl_dup);
+        goto out;
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("attach-tier failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-int
-do_cli_cmd_volume_detach_tier (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
+/* Checks if at least one limit has been set on the volume
+ *
+ * Returns true if at least one limit is set. Returns false otherwise.
+ */
+gf_boolean_t
+_limits_set_on_volume(char *volname, int type)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
-        int                     need_question = 0;
-
-        const char *question = "Removing tier can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_detach_tier_parse(words, wordcount, &options,
-                                               &need_question);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (options, "force", 1);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (options, "count", 0);
-        if (ret)
-                goto out;
+    gf_boolean_t limits_set = _gf_false;
+    int ret = -1;
+    char quota_conf_file[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+    char buf[16] = {
+        0,
+    };
+    float version = 0.0f;
+    char gfid_type_stored = 0;
+    char gfid_type = 0;
+
+    /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
+     * to fetch working directory
+     */
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1)
+        goto out;
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DETACH_TIER];
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
+    else
+        gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    /* Try to read at least one gfid  of type 'gfid_type' */
+    while (1) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type_stored, version);
+        if (ret <= 0)
+            break;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+        if (gfid_type_stored == gfid_type) {
+            limits_set = _gf_true;
+            break;
         }
-
+    }
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume detach tier failed");
-        }
+    if (fd != -1)
+        sys_close(fd);
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    return limits_set;
 }
 
 int
-cli_cmd_volume_tier_cbk (struct cli_state *state,
-                         struct cli_cmd_word *word, const char **words,
-                         int wordcount)
+cli_cmd_quota_handle_list_all(const char **words, dict_t *options)
 {
-        int                      ret     = -1;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        cli_local_t             *local   = NULL;
-        int                      i       = 0;
-        eventtypes_t            event    = EVENT_LAST;
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                if (wordcount == 3 && !strcmp(words[2], "help"))
-                        ret = 0;
-                goto out;
-        }
-
-        if (!strcmp(words[1], "detach-tier")) {
-                ret = do_cli_cmd_volume_detach_tier (state, word,
-                                                     words, wordcount);
-                goto out;
-        } else if (!strcmp(words[3], "detach")) {
-                for (i = 3; i < wordcount; i++)
-                        words[i] = words[i+1];
-
-                ret = do_cli_cmd_volume_detach_tier (state, word,
-                                                     words, wordcount-1);
-                if (!strcmp (words[wordcount-2], "commit")) {
-                        event = EVENT_TIER_DETACH_COMMIT;
-                } else if (!strcmp (words[wordcount-2], "start")) {
-                        event = EVENT_TIER_DETACH_START;
-                } else if (!strcmp (words[wordcount-2], "stop")) {
-                        event = EVENT_TIER_DETACH_STOP;
-                } else if (!strcmp (words[wordcount-2], "force")) {
-                        event = EVENT_TIER_DETACH_FORCE;
-                }
-                goto out;
-
-        } else if (!strcmp(words[1], "attach-tier")) {
-                ret = do_cli_cmd_volume_attach_tier (state, word,
-                                                     words, wordcount);
-                goto out;
-        } else if (!strcmp(words[3], "attach")) {
-                for (i = 3; i < wordcount; i++)
-                        words[i] = words[i+1];
-
-                ret = do_cli_cmd_volume_attach_tier (state, word,
-                                                     words, wordcount-1);
-                if (!strcmp (words[wordcount-2], "force")) {
-                        event = EVENT_TIER_ATTACH_FORCE;
-                } else {
-                        event = EVENT_TIER_ATTACH;
-                }
-                goto out;
-        }
-
-        ret = cli_cmd_volume_tier_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TIER];
+    int all_failed = 1;
+    int count = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *xdata = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+    char *volname = NULL;
+    char *volname_dup = NULL;
+    unsigned char buf[16] = {0};
+    int fd = -1;
+    char quota_conf_file[PATH_MAX] = {0};
+    gf_boolean_t xml_err_flag = _gf_false;
+    char err_str[NAME_MAX] = {
+        0,
+    };
+    int32_t type = 0;
+    char gfid_type = 0;
+    float version = 0.0f;
+    int32_t max_count = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota option type");
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    ret = dict_set_int32(xdata, "type", type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set type in xdata");
+        goto out;
+    }
 
-out:
-        if (ret) {
-                cli_out ("Tier command failed");
+    ret = cli_get_soft_limit(options, words, xdata);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to fetch default "
+               "soft-limit");
+        goto out;
+    }
+
+    /* Check if at least one limit is set on volume. No need to check for
+     * quota enabled as cli_get_soft_limit() handles that
+     */
+    if (!_limits_set_on_volume(volname, type)) {
+        snprintf(err_str, sizeof(err_str),
+                 "No%s quota configured on"
+                 " volume %s",
+                 (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode", volname);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            xml_err_flag = _gf_true;
         } else {
-                if (event != EVENT_LAST) {
-                        gf_event (event, "vol=%s", words[2]);
-                }
+            cli_out("quota: %s", err_str);
         }
-        if (options)
-                dict_unref (options);
-
-        return ret;
-}
-
-int
-cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata)
-{
-        call_frame_t            *frame          = NULL;
-        cli_local_t             *local          = NULL;
-        rpc_clnt_procedure_t    *proc           = NULL;
-        char                    *default_sl     = NULL;
-        char                    *default_sl_dup = NULL;
-        int                      ret  = -1;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        //We need a ref on @options to prevent CLI_STACK_DESTROY
-        //from destroying it prematurely.
-        dict_ref (options);
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        ret = proc->fn (frame, THIS, options);
-
-        ret = dict_get_str (options, "default-soft-limit", &default_sl);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get default soft limit");
-                goto out;
-        }
-
-        default_sl_dup = gf_strdup (default_sl);
-        if (!default_sl_dup) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (xdata, "default-soft-limit", default_sl_dup);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set default soft limit");
-                GF_FREE (default_sl_dup);
-                goto out;
-        }
-
-out:
-        CLI_STACK_DESTROY (frame);
-        return ret;
-}
+        ret = 0;
+        goto out;
+    }
 
-/* Checks if at least one limit has been set on the volume
- *
- * Returns true if at least one limit is set. Returns false otherwise.
- */
-gf_boolean_t
-_limits_set_on_volume (char *volname, int type) {
-        gf_boolean_t    limits_set                = _gf_false;
-        int             ret                       = -1;
-        char            quota_conf_file[PATH_MAX] = {0,};
-        int             fd                        = -1;
-        char            buf[16]                   = {0,};
-        float           version                   = 0.0f;
-        char            gfid_type_stored          = 0;
-        char            gfid_type                 = 0;
-
-        /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
-         * to fetch working directory
-         */
-        snprintf (quota_conf_file, sizeof quota_conf_file,
-                  "%s/vols/%s/quota.conf",
-                  GLUSTERD_DEFAULT_WORKDIR,
-                  volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1)
-                goto out;
+    volname_dup = gf_strdup(volname);
+    if (!volname_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = quota_conf_read_version (fd, &version);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(xdata, "volume-uuid", volname_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
+        GF_FREE(volname_dup);
+        goto out;
+    }
+
+    // TODO: fix hardcoding; Need to perform an RPC call to glusterd
+    // to fetch working directory
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1) {
+        // This may because no limits were yet set on the volume
+        gf_log("cli", GF_LOG_TRACE,
+               "Unable to open "
+               "quota.conf");
+        ret = 0;
+        goto out;
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
-        else
-                gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
-
-        /* Try to read atleast one gfid  of type 'gfid_type' */
-        while (1) {
-                ret = quota_conf_read_gfid (fd, buf, &gfid_type_stored,
-                                            version);
-                if (ret <= 0)
-                        break;
-
-                if (gfid_type_stored == gfid_type) {
-                        limits_set = _gf_true;
-                        break;
-                }
-        }
-out:
-        if (fd != -1)
-                sys_close (fd);
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        return limits_set;
-}
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-int
-cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
-{
-        int                      all_failed = 1;
-        int                      count      = 0;
-        int                      ret        = -1;
-        rpc_clnt_procedure_t    *proc       = NULL;
-        cli_local_t             *local      = NULL;
-        call_frame_t            *frame      = NULL;
-        dict_t                  *xdata      = NULL;
-        char                    *gfid_str   = NULL;
-        char                    *volname    = NULL;
-        char                    *volname_dup = NULL;
-        unsigned char            buf[16]   = {0};
-        int                      fd        = -1;
-        char                     quota_conf_file[PATH_MAX] = {0};
-        gf_boolean_t             xml_err_flag   = _gf_false;
-        char                     err_str[NAME_MAX] = {0,};
-        int32_t                  type       = 0;
-        char                     gfid_type  = 0;
-        float                    version    = 0.0f;
-        int32_t                  max_count  = 0;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
+    CLI_LOCAL_INIT(local, words, frame, xdata);
+    proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        max_count++;
+    }
+    ret = dict_set_int32(xdata, "max_count", max_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set max_count");
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0L, SEEK_SET);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "failed to move offset to "
+               "the beginning: %s",
+               strerror(errno));
+        goto out;
+    }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota option type");
-                goto out;
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
         }
 
-        ret = dict_set_int32 (xdata, "type", type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set type in xdata");
-                goto out;
-        }
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
 
-        ret = cli_get_soft_limit (options, words, xdata);
+        uuid_utoa_r(buf, gfid_str);
+        ret = dict_set_str(xdata, "gfid", gfid_str);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch default "
-                        "soft-limit");
-                goto out;
-        }
-
-        /* Check if at least one limit is set on volume. No need to check for
-         * quota enabled as cli_get_soft_limit() handles that
-         */
-        if (!_limits_set_on_volume (volname, type)) {
-                snprintf (err_str, sizeof (err_str), "No%s quota configured on"
-                          " volume %s",
-                          (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode",
-                          volname);
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        xml_err_flag = _gf_true;
-                } else {
-                        cli_out ("quota: %s", err_str);
-                }
-                ret = 0;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to set gfid");
+            goto out;
         }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        volname_dup = gf_strdup (volname);
-        if (!volname_dup) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (xdata, "volume-uuid", volname_dup);
+        ret = proc->fn(frame, THIS, xdata);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
-                GF_FREE (volname_dup);
-                goto out;
-        }
-
-        //TODO: fix hardcoding; Need to perform an RPC call to glusterd
-        //to fetch working directory
-        snprintf (quota_conf_file, sizeof quota_conf_file,
-                  "%s/vols/%s/quota.conf",
-                  GLUSTERD_DEFAULT_WORKDIR,
-                  volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1) {
-                //This may because no limits were yet set on the volume
-                gf_log ("cli", GF_LOG_TRACE, "Unable to open "
-                        "quota.conf");
-                ret = 0;
-                goto out;
-         }
-
-        ret = quota_conf_read_version (fd, &version);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, xdata);
-        proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
-
-        gfid_str = GF_CALLOC (1, gf_common_mt_char, 64);
-        if (!gfid_str) {
-                ret = -1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to get quota "
+                   "limits for %s",
+                   uuid_utoa((unsigned char *)buf));
         }
 
-        for (count = 0;; count++) {
-                ret = quota_conf_read_gfid (fd, buf, &gfid_type, version);
-                if (ret == 0) {
-                        break;
-                } else if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
-                }
-
-                if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
-                    (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
-                        continue;
+        dict_del(xdata, "gfid");
+        all_failed = all_failed && ret;
+    }
 
-                max_count++;
-        }
-        ret = dict_set_int32 (xdata, "max_count", max_count);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_end(local);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set max_count");
-                goto out;
-        }
-
-        ret = sys_lseek (fd, 0L, SEEK_SET);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to move offset to "
-                        "the beginning: %s", strerror (errno));
-                goto out;
-        }
-        ret = quota_conf_read_version (fd, &version);
-        if (ret)
-                goto out;
-
-        for (count = 0;; count++) {
-                ret = quota_conf_read_gfid (fd, buf, &gfid_type, version);
-                if (ret == 0) {
-                        break;
-                } else if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
-                }
-
-                if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
-                    (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
-                        continue;
-
-                uuid_utoa_r (buf, gfid_str);
-                ret = dict_set_str (xdata, "gfid", gfid_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set gfid");
-                        goto out;
-                }
-
-                ret = proc->fn (frame, THIS, xdata);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get quota "
-                                "limits for %s", uuid_utoa ((unsigned char*)buf));
-                }
-
-                dict_del (xdata, "gfid");
-                all_failed = all_failed && ret;
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_quota_limit_list_end (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error in printing "
-                                "xml output");
-                        goto out;
-                }
-        }
-
-        if (count > 0) {
-                ret = all_failed? -1: 0;
-        } else {
-                ret = 0;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error in printing "
+                   "xml output");
+            goto out;
         }
+    }
 
+    if (count > 0) {
+        ret = all_failed ? -1 : 0;
+    } else {
+        ret = 0;
+    }
 
 out:
-        if (xml_err_flag) {
-                ret = cli_xml_output_str ("volQuota", NULL, -1, 0, err_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting in "
-                                "xml format");
-                }
-        }
-
-        if (fd != -1) {
-                sys_close (fd);
-        }
-
-        GF_FREE (gfid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch and display quota"
-                        " limits");
-        }
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (xml_err_flag) {
+        ret = cli_xml_output_str("volQuota", NULL, -1, 0, err_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error outputting in "
+                   "xml format");
+        }
+    }
+    if (xdata)
+        dict_unref(xdata);
+
+    if (fd != -1) {
+        sys_close(fd);
+    }
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch and display quota"
+               " limits");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_bitrot_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                    const char **words, int wordcount)
+cli_cmd_bitrot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                   const char **words, int wordcount)
 {
-
-        int                     ret        = -1;
-        int                     parse_err  = 0;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        cli_local_t             *local     = NULL;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        int                     sent       = 0;
+    int ret = -1;
+    int parse_err = 0;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
 #if (USE_EVENTS)
-        int                     cmd_type        = -1;
-        int                     ret1            = -1;
-        int                     event_type      = -1;
-        char                    *tmp            = NULL;
-        char                    *events_str     = NULL;
+    int cmd_type = -1;
+    int ret1 = -1;
+    int event_type = -1;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    char *volname = NULL;
 #endif
 
-        ret = cli_cmd_bitrot_parse (words, wordcount, &options);
-        if (ret < 0) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
-        }
+    ret = cli_cmd_bitrot_parse(words, wordcount, &options);
+    if (ret < 0) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    if (ret == 1) {
+        /* this is 'volume bitrot help' */
+        cli_cmd_bitrot_help_cbk(state, word, words, wordcount);
+        ret = 0;
+        goto out2;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_err == 0))
-                    cli_err ("Bit rot command failed. Please check the cli "
-                             "logs for more details");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err(
+                "Bit rot command failed. Please check the cli "
+                "logs for more details");
+    }
 
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+        else {
+            ret1 = dict_get_str(options, "volname", &volname);
+            if (ret1)
+                volname = "";
         }
 
-#if (USE_EVENTS)
-        if (ret == 0) {
-                ret1 = dict_get_int32 (options, "type", &cmd_type);
+        switch (cmd_type) {
+            case GF_BITROT_OPTION_TYPE_ENABLE:
+                event_type = EVENT_BITROT_ENABLE;
+                break;
+            case GF_BITROT_OPTION_TYPE_DISABLE:
+                event_type = EVENT_BITROT_DISABLE;
+                break;
+            case GF_BITROT_CMD_SCRUB_ONDEMAND:
+                event_type = EVENT_BITROT_SCRUB_ONDEMAND;
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+                event_type = EVENT_BITROT_SCRUB_THROTTLE;
+                ret1 = dict_get_str(options, "scrub-throttle-value", &tmp);
                 if (ret1)
-                        cmd_type = -1;
-                else {
-                        ret1 = dict_get_str (options, "volname", &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "name=%s", tmp);
-                }
-
-                switch (cmd_type) {
-                case GF_BITROT_OPTION_TYPE_ENABLE:
-                        event_type = EVENT_BITROT_ENABLE;
-                        break;
-                case GF_BITROT_OPTION_TYPE_DISABLE:
-                        event_type = EVENT_BITROT_DISABLE;
-                        break;
-                case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
-                        event_type = EVENT_BITROT_SCRUB_THROTTLE;
-                        ret1 = dict_get_str (options, "scrub-throttle-value",
-                                             &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "%s;value=%s", events_str,
-                                     tmp);
-                        break;
-                case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
-                        event_type = EVENT_BITROT_SCRUB_FREQ;
-                        ret1 = dict_get_str (options, "scrub-frequency-value",
-                                             &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "%s;value=%s", events_str,
-                                     tmp);
-                        break;
-                case GF_BITROT_OPTION_TYPE_SCRUB:
-                        event_type = EVENT_BITROT_SCRUB_OPTION;
-                        ret1 = dict_get_str (options, "scrub-value", &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "%s;value=%s", events_str,
-                                     tmp);
-                        break;
-                default:
-                        break;
-                }
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+                event_type = EVENT_BITROT_SCRUB_FREQ;
+                ret1 = dict_get_str(options, "scrub-frequency-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB:
+                event_type = EVENT_BITROT_SCRUB_OPTION;
+                ret1 = dict_get_str(options, "scrub-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            default:
+                break;
+        }
 
-                if (event_type > -1)
-                        gf_event (event_type, "%s", events_str);
+        if (event_type > -1)
+            gf_event(event_type, "%s", events_str);
 
-                if (events_str)
-                        GF_FREE (events_str);
-        }
+        if (events_str)
+            GF_FREE(events_str);
+    }
 #endif
 
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    CLI_STACK_DESTROY(frame);
+out2:
+    return ret;
 }
 
 int
-cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quota_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        int32_t                  type      = 0;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        gf_answer_t              answer    = GF_ANSWER_NO;
-        cli_local_t             *local     = NULL;
-        int                      sent      = 0;
-        char                    *volname   = NULL;
-        const char *question = "Disabling quota will delete all the quota "
-                               "configuration. Do you want to continue?";
-
-        //parse **words into options dictionary
-        if (strcmp (words[1], "inode-quota") == 0) {
-                ret = cli_cmd_inode_quota_parse (words, wordcount, &options);
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                        goto out;
-                }
-        } else {
-                ret = cli_cmd_quota_parse (words, wordcount, &options);
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                        goto out;
-                }
+    int ret = 0;
+    int parse_err = 0;
+    int32_t type = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+    int sent = 0;
+    char *volname = NULL;
+    const char *question =
+        "Disabling quota will delete all the quota "
+        "configuration. Do you want to continue?";
+
+    // parse **words into options dictionary
+    if (strcmp(words[1], "inode-quota") == 0) {
+        ret = cli_cmd_inode_quota_parse(words, wordcount, &options);
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    } else {
+        ret = cli_cmd_quota_parse(words, wordcount, &options);
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get opcode");
-                goto out;
+        if (ret == 1) {
+            cli_cmd_quota_help_cbk(state, word, words, wordcount);
+            ret = 0;
+            goto out;
+        }
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    }
 
-        //handle quota-disable and quota-list-all different from others
-        switch (type) {
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get opcode");
+        goto out;
+    }
+
+    // handle quota-disable and quota-list-all different from others
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_DISABLE:
-                answer = cli_cmd_get_confirmation (state, question);
-                if (answer == GF_ANSWER_NO)
-                        goto out;
-                break;
+            answer = cli_cmd_get_confirmation(state, question);
+            if (answer == GF_ANSWER_NO)
+                goto out;
+            break;
         case GF_QUOTA_OPTION_TYPE_LIST:
         case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                if (wordcount != 4)
-                        break;
-                ret = cli_cmd_quota_handle_list_all (words, options);
-                goto out;
-        default:
+            if (wordcount != 4)
                 break;
-        }
+            ret = cli_cmd_quota_handle_list_all(words, options);
+            goto out;
+        default:
+            break;
+    }
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0 && parse_err == 0)
-                        cli_out ("Quota command failed. Please check the cli "
-                                 "logs for more details");
-        }
-
-        /* Events for Quota */
-         if (ret == 0) {
-                switch (type) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                        gf_event (EVENT_QUOTA_ENABLE, "volume=%s", volname);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_DISABLE:
-                        gf_event (EVENT_QUOTA_DISABLE, "volume=%s", volname);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                        gf_event (EVENT_QUOTA_SET_USAGE_LIMIT, "volume=%s;"
-                                  "path=%s;limit=%s", volname, words[4],
-                                  words[5]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                        gf_event (EVENT_QUOTA_SET_OBJECTS_LIMIT, "volume=%s;"
-                                  "path=%s;limit=%s", volname, words[4],
-                                  words[5]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_REMOVE:
-                        gf_event (EVENT_QUOTA_REMOVE_USAGE_LIMIT, "volume=%s;"
-                                  "path=%s", volname, words[4]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                        gf_event (EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
-                                  "volume=%s;" "path=%s", volname, words[4]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
-                        gf_event (EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s",
-                                  volname, words[4]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
-                        gf_event (EVENT_QUOTA_SOFT_TIMEOUT, "volume=%s;"
-                                  "soft-timeout=%s", volname, words[4]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
-                        gf_event (EVENT_QUOTA_HARD_TIMEOUT, "volume=%s;"
-                                  "hard-timeout=%s", volname, words[4]);
-                        break;
-                case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                        gf_event (EVENT_QUOTA_DEFAULT_SOFT_LIMIT, "volume=%s;"
-                                  "default-soft-limit=%s", volname, words[4]);
-                        break;
-                }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0 && parse_err == 0)
+            cli_out(
+                "Quota command failed. Please check the cli "
+                "logs for more details");
+    }
+    if (options)
+        dict_unref(options);
+
+    /* Events for Quota */
+    if (ret == 0) {
+        switch (type) {
+            case GF_QUOTA_OPTION_TYPE_ENABLE:
+                gf_event(EVENT_QUOTA_ENABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DISABLE:
+                gf_event(EVENT_QUOTA_DISABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                gf_event(EVENT_QUOTA_SET_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+                gf_event(EVENT_QUOTA_SET_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE:
+                gf_event(EVENT_QUOTA_REMOVE_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                gf_event(EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                gf_event(EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s", volname,
+                         words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                gf_event(EVENT_QUOTA_SOFT_TIMEOUT,
+                         "volume=%s;"
+                         "soft-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                gf_event(EVENT_QUOTA_HARD_TIMEOUT,
+                         "volume=%s;"
+                         "hard-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                gf_event(EVENT_QUOTA_DEFAULT_SOFT_LIMIT,
+                         "volume=%s;"
+                         "default-soft-limit=%s",
+                         volname, words[4]);
+                break;
         }
+    }
 
-
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
-                                 struct cli_cmd_word *word, const char **words,
-                                 int wordcount)
+cli_cmd_volume_remove_brick_cbk(struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     need_question = 0;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int brick_count = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int need_question = 0;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
 #if (USE_EVENTS)
-        eventtypes_t            event = EVENT_LAST;
-        char                    *event_str = NULL;
-        int                     event_ret = -1;
+    eventtypes_t event = EVENT_LAST;
+    char *event_str = NULL;
+    int event_ret = -1;
 #endif
-        const char *question = "Removing brick(s) can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
-                                                 &need_question);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret || !volname) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch volname");
-                ret = -1;
-                goto out;
-        }
+    int32_t command = GF_OP_CMD_NONE;
+    char *question = NULL;
+
+    ret = cli_cmd_volume_remove_brick_parse(state, words, wordcount, &options,
+                                            &need_question, &brick_count,
+                                            &command);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (command == GF_OP_CMD_COMMIT_FORCE) {
+        question =
+            "Remove-brick force will not migrate files from the "
+            "removed bricks, so they will no longer be available"
+            " on the volume.\nDo you want to continue?";
+    } else if (command == GF_OP_CMD_START) {
+        question =
+            "It is recommended that remove-brick be run with"
+            " cluster.force-migration option disabled to prevent"
+            " possible data corruption. Doing so will ensure that"
+            " files that receive writes during migration will not"
+            " be migrated and will need to be manually copied"
+            " after the remove-brick commit operation. Please"
+            " check the value of the option and update accordingly."
+            " \nDo you want to continue with your current"
+            " cluster.force-migration settings?";
+    }
+
+    if (!brick_count) {
+        cli_err("No bricks specified");
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
 
 #if (USE_EVENTS)
-        event_ret = cli_event_remove_brick_str (options, &event_str, &event);
+    event_ret = cli_event_remove_brick_str(options, &event_str, &event);
 #endif
 
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Removing brick from the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
-                need_question = _gf_true;
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Removing brick from the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+        need_question = _gf_true;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume remove-brick failed");
-        } else {
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume remove-brick failed");
+    }
 #if (USE_EVENTS)
-                if (!event_ret) {
-                        gf_event (event, event_str);
-                        GF_FREE (event_str);
-                }
-#endif
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (!ret && !event_ret)
+        gf_event(event, "%s", event_str);
+    if (event_str)
+        GF_FREE(event_str);
 
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    if (options)
+        dict_unref(options);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_brick_cbk (struct cli_state *state,
-                                struct cli_cmd_word *word,
-                                const char **words,
-                                int wordcount)
+cli_cmd_volume_reset_brick_cbk(struct cli_state *state,
+                               struct cli_cmd_word *word, const char **words,
+                               int wordcount)
 {
-        int                      ret          = -1;
-        rpc_clnt_procedure_t    *proc         = NULL;
-        call_frame_t            *frame        = NULL;
-        dict_t                  *options      = NULL;
-        int                      sent         = 0;
-        int                      parse_error  = 0;
-        cli_local_t             *local        = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_reset_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
+    if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+        ret = dict_set_int32(options, "ignore-partition", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set ignore-"
+                   "partition option");
+            goto out;
         }
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset-brick failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset-brick failed");
+    } else {
+        if (wordcount > 5) {
+            gf_event(EVENT_BRICK_RESET_COMMIT,
+                     "Volume=%s;source-brick=%s;"
+                     "destination-brick=%s",
+                     (char *)words[2], (char *)words[3], (char *)words[4]);
         } else {
-                if (wordcount > 5) {
-                        gf_event (EVENT_BRICK_RESET_COMMIT,
-                                  "Volume=%s;source-brick=%s;"
-                                  "destination-brick=%s",
-                                  (char *)words[2], (char *)words[3],
-                                  (char *)words[4]);
-                } else {
-                        gf_event (EVENT_BRICK_RESET_START,
-                                  "Volume=%s;source-brick=%s",
-                                  (char *)words[2], (char *)words[3]);
-                }
+            gf_event(EVENT_BRICK_RESET_START, "Volume=%s;source-brick=%s",
+                     (char *)words[2], (char *)words[3]);
         }
-        CLI_STACK_DESTROY (frame);
+    }
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words,
-                                  int wordcount)
+cli_cmd_volume_replace_brick_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
-        int                      ret          = -1;
-        rpc_clnt_procedure_t    *proc         = NULL;
-        call_frame_t            *frame        = NULL;
-        dict_t                  *options = NULL;
-        int                      sent = 0;
-        int                      parse_error = 0;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_replace_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume replace-brick failed");
-        } else {
-                gf_event (EVENT_BRICK_REPLACE,
-                          "Volume=%s;source-brick=%s;destination-brick=%s",
-                          (char *)words[2], (char *)words[3], (char *)words[4]);
-        }
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume replace-brick failed");
+    } else {
+        gf_event(EVENT_BRICK_REPLACE,
+                 "Volume=%s;source-brick=%s;destination-brick=%s",
+                 (char *)words[2], (char *)words[3], (char *)words[4]);
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_set_transport_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words, int wordcount)
+cli_cmd_volume_set_transport_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
-        cli_cmd_broadcast_response (0);
-        return 0;
+    cli_cmd_broadcast_response(0);
+    return 0;
 }
 
 int
-cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_top_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_top_parse(words, wordcount, &options);
+
+    if (ret) {
+        parse_error = 1;
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        int                     sent     = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        ret = cli_cmd_volume_top_parse (words, wordcount, &options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
 
-        if (ret) {
-                parse_error = 1;
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume top failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume top failed");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
-
 int
-cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_log_rotate_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!((wordcount == 4) || (wordcount == 5))) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (!((strcmp ("rotate", words[2]) == 0) ||
-              (strcmp ("rotate", words[3]) == 0))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (!(strcmp("rotate", words[3]) == 0)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_log_rotate_parse(words, wordcount, &options);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log rotate failed");
-        }
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume log rotate failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 #if (SYNCDAEMON_COMPILE)
 static int
-cli_check_gsync_present ()
+cli_check_gsync_present()
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char                *ptr = NULL;
-        int                 ret = 0;
-
-        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
-        if (-1 == ret) {
-                gf_log ("", GF_LOG_WARNING, "setenv syscall failed, hence could"
-                        "not assert if geo-replication is installed");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_INFO, "geo-replication not installed");
-                goto out;
-        }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (-1 == ret) {
+        gf_log("", GF_LOG_WARNING,
+               "setenv syscall failed, hence could"
+               "not assert if geo-replication is installed");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_log("", GF_LOG_INFO, "geo-replication not installed");
+        goto out;
+    }
 
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret  = -1;
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                goto out;
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        ret = -1;
+        goto out;
+    }
 
-        ret = runner_end (&runner);
+    ret = runner_end(&runner);
 
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "geo-replication not installed");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "geo-replication not installed");
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret ? -1 : 0;
-
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret ? -1 : 0;
 }
 
 void
-cli_cmd_check_gsync_exists_cbk (struct cli_cmd *this)
+cli_cmd_check_gsync_exists_cbk(struct cli_cmd *this)
 {
+    int ret = 0;
 
-        int             ret = 0;
-
-        ret = cli_check_gsync_present ();
-        if (ret)
-                this->disable = _gf_true;
-
+    ret = cli_check_gsync_present();
+    if (ret)
+        this->disable = _gf_true;
 }
 #endif
 
 int
-cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_gsync_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                      ret     = 0;
-        int                      parse_err = 0;
-        dict_t                  *options = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        cli_local_t             *local   = NULL;
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char *errstr = NULL;
 #if (USE_EVENTS)
-        int                      ret1       = -1;
-        int                      cmd_type   = -1;
-        int                      tmpi       = 0;
-        char                    *tmp        = NULL;
-        char                    *events_str = NULL;
-        int                      event_type = -1;
+    int ret1 = -1;
+    int cmd_type = -1;
+    int tmpi = 0;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    int event_type = -1;
 #endif
 
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GSYNC_SET];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
+    ret = cli_cmd_gsync_set_parse(state, words, wordcount, &options, &errstr);
+    if (ret) {
+        if (errstr) {
+            cli_err("%s", errstr);
+            GF_FREE(errstr);
+        } else {
+            cli_usage_out(word->pattern);
         }
+        parse_err = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_gsync_set_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out (GEOREP" command failed");
+    if (ret && parse_err == 0)
+        cli_out(GEOREP " command failed");
 
 #if (USE_EVENTS)
-        if (ret == 0) {
-                events_str = gf_strdup ("");
-
-                /* Type of Geo-rep Action - Create, Start etc */
-                ret1 = dict_get_int32 (options, "type", &cmd_type);
+    if (ret == 0) {
+        events_str = gf_strdup("");
+
+        /* Type of Geo-rep Action - Create, Start etc */
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+
+        /* Only capture Events for modification commands */
+        switch (cmd_type) {
+            case GF_GSYNC_OPTION_TYPE_CREATE:
+                event_type = EVENT_GEOREP_CREATE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_START:
+                event_type = EVENT_GEOREP_START;
+                break;
+            case GF_GSYNC_OPTION_TYPE_STOP:
+                event_type = EVENT_GEOREP_STOP;
+                break;
+            case GF_GSYNC_OPTION_TYPE_PAUSE:
+                event_type = EVENT_GEOREP_PAUSE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_RESUME:
+                event_type = EVENT_GEOREP_RESUME;
+                break;
+            case GF_GSYNC_OPTION_TYPE_DELETE:
+                event_type = EVENT_GEOREP_DELETE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret1 = dict_get_str(options, "subop", &tmp);
                 if (ret1)
-                        cmd_type = -1;
-
-                /* Only capture Events for modification commands */
-                switch (cmd_type) {
-                case GF_GSYNC_OPTION_TYPE_CREATE:
-                        event_type = EVENT_GEOREP_CREATE;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_START:
-                        event_type = EVENT_GEOREP_START;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_STOP:
-                        event_type = EVENT_GEOREP_STOP;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_PAUSE:
-                        event_type = EVENT_GEOREP_PAUSE;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_RESUME:
-                        event_type = EVENT_GEOREP_RESUME;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_DELETE:
-                        event_type = EVENT_GEOREP_DELETE;
-                        break;
-                case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret1 = dict_get_str (options, "subop", &tmp);
-                        if (ret1)
-                                tmp = "";
-
-                        /* For Config Set additionally capture key and value */
-                        /* For Config Reset capture key */
-                        if (strcmp (tmp, "set") == 0) {
-                                event_type = EVENT_GEOREP_CONFIG_SET;
-
-                                ret1 = dict_get_str (options, "op_name", &tmp);
-                                if (ret1)
-                                        tmp = "";
-
-                                gf_asprintf (&events_str, "%soption=%s;",
-                                             events_str, tmp);
-
-                                ret1 = dict_get_str (options, "op_value", &tmp);
-                                if (ret1)
-                                        tmp = "";
-
-                                gf_asprintf (&events_str, "%svalue=%s;",
-                                             events_str, tmp);
-                        } else if (strcmp (tmp, "del") == 0) {
-                                event_type = EVENT_GEOREP_CONFIG_RESET;
-
-                                ret1 = dict_get_str (options, "op_name", &tmp);
-                                if (ret1)
-                                        tmp = "";
-
-                                gf_asprintf (&events_str, "%soption=%s;",
-                                             events_str, tmp);
-                        }
-                        break;
-                default:
-                        break;
-                }
-
-                if (event_type > -1) {
-                        /* Capture all optional arguments used */
-                        ret1 = dict_get_int32 (options, "force", &tmpi);
-                        if (ret1 == 0)
-                                gf_asprintf (&events_str, "%sforce=%d;",
-                                             events_str, tmpi);
-
-                        ret1 = dict_get_int32 (options, "push_pem", &tmpi);
-                        if (ret1 == 0)
-                                gf_asprintf (&events_str, "%spush_pem=%d;",
-                                             events_str, tmpi);
-
-                        ret1 = dict_get_int32 (options, "no_verify", &tmpi);
-                        if (ret1 == 0)
-                                gf_asprintf (&events_str, "%sno_verify=%d;",
-                                             events_str, tmpi);
-
-                        ret1 = dict_get_int32 (options, "ssh_port", &tmpi);
-                        if (ret1 == 0)
-                                gf_asprintf (&events_str, "%sssh_port=%d;",
-                                             events_str, tmpi);
-
-                        ret1 = dict_get_int32 (options, "reset-sync-time",
-                                               &tmpi);
-                        if (ret1 == 0)
-                                gf_asprintf (&events_str,
-                                             "%sreset_sync_time=%d;",
-                                             events_str, tmpi);
-
-                        /* Capture Master and Slave Info */
-                        ret1 = dict_get_str (options, "master", &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "%smaster=%s;",
-                                     events_str, tmp);
-
-                        ret1 = dict_get_str (options, "slave", &tmp);
-                        if (ret1)
-                                tmp = "";
-                        gf_asprintf (&events_str, "%sslave=%s",
-                                     events_str, tmp);
-
-                        gf_event (event_type, "%s", events_str);
-                }
-
-                /* Allocated by gf_strdup and gf_asprintf */
-                GF_FREE (events_str);
-        }
-#endif
+                    tmp = "";
 
-        CLI_STACK_DESTROY (frame);
+                /* For Config Set additionally capture key and value */
+                /* For Config Reset capture key */
+                if (strcmp(tmp, "set") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_SET;
 
-        return ret;
-}
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
 
-int
-cli_cmd_volume_status_cbk (struct cli_state *state,
-                           struct cli_cmd_word *word,
-                           const char **words, int wordcount)
-{
-        int                   ret         = -1;
-        rpc_clnt_procedure_t *proc        = NULL;
-        call_frame_t         *frame       = NULL;
-        dict_t               *dict        = NULL;
-        uint32_t              cmd         = 0;
-        cli_local_t          *local       = NULL;
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
 
-        ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
+                    ret1 = dict_get_str(options, "op_value", &tmp);
+                    if (ret1)
+                        tmp = "";
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+                    gf_asprintf_append(&events_str, "%svalue=%s;", events_str,
+                                       tmp);
+                } else if (strcmp(tmp, "del") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_RESET;
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
 
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                /* for one volume or brick */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
-        } else {
-                /* volume status all or all detail */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+                }
+                break;
+            default:
+                break;
         }
 
-        if (!proc->fn)
-                goto out;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        ret = proc->fn (frame, THIS, dict);
+        if (event_type > -1) {
+            /* Capture all optional arguments used */
+            ret1 = dict_get_int32(options, "force", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sforce=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "push_pem", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%spush_pem=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "no_verify", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sno_verify=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "ssh_port", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sssh_port=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "reset-sync-time", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sreset_sync_time=%d;",
+                                   events_str, tmpi);
+            }
+            /* Capture Master and Slave Info */
+            ret1 = dict_get_str(options, "master", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%smaster=%s;", events_str, tmp);
+
+            ret1 = dict_get_str(options, "slave", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%sslave=%s", events_str, tmp);
+
+            gf_event(event_type, "%s", events_str);
+        }
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-out:
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
+cli_cmd_volume_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        uint64_t                   free            = 0;
-        uint64_t                   total           = 0;
-        char                       key[1024]       = {0};
-        int                        ret             = 0;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", i);
-        ret = dict_get_uint64 (dict, key, &free);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    uint32_t cmd = 0;
+    cli_local_t *local = NULL;
 
-        status->free = gf_uint64_2human_readable (free);
-        if (!status->free)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total", i);
-        ret = dict_get_uint64 (dict, key, &total);
-
-        status->total = gf_uint64_2human_readable (total);
-        if (!status->total)
-                goto out;
+    ret = cli_cmd_volume_status_parse(words, wordcount, &dict);
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", i);
-        ret = dict_get_str (dict, key, &(status->device));
-        if (ret)
-                status->device = NULL;
-#endif
+    if (ret) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", i);
-        ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret) {
-                ret = 0;
-                status->block_size = 0;
-        }
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", i);
-        ret = dict_get_str (dict, key, &(status->mount_options));
-        if (ret)
-                status->mount_options = NULL;
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        /* for one volume or brick */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
+    } else {
+        /* volume status all or all detail */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", i);
-        ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret) {
-                ret = 0;
-                status->fs_name = NULL;
-        }
+    if (!proc->fn) {
+        ret = -1;
+        goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", i);
-        ret = dict_get_str (dict, key, &(status->inode_size));
-        if (ret)
-                status->inode_size = NULL;
-#endif /* GF_LINUX_HOST_OS */
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-        ret = dict_get_uint64 (dict, key,
-                        &(status->total_inodes));
-        if (ret)
-                status->total_inodes = 0;
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-        if (ret) {
-                ret = 0;
-                status->free_inodes = 0;
-        }
+    ret = proc->fn(frame, THIS, dict);
 
+out:
+    CLI_STACK_DESTROY(frame);
 
- out:
-        return ret;
+    return ret;
 }
 
-void
-cli_print_detailed_status (cli_volume_status_t *status)
+int
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status)
 {
-        cli_out ("%-20s : %-20s", "Brick", status->brick);
+    uint64_t free = 0;
+    uint64_t total = 0;
+    char key[1024] = {0};
+    int ret = 0;
 
-        if (status->online) {
-                cli_out ("%-20s : %-20d", "TCP Port", status->port);
-                cli_out ("%-20s : %-20d", "RDMA Port", status->rdma_port);
-        } else {
-                cli_out ("%-20s : %-20s", "TCP Port", "N/A");
-                cli_out ("%-20s : %-20s", "RDMA Port", "N/A");
-        }
+    snprintf(key, sizeof(key), "brick%d.free", i);
+    ret = dict_get_uint64(dict, key, &free);
 
-        cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
-        cli_out ("%-20s : %-20s", "Pid", status->pid_str);
+    status->free = gf_uint64_2human_readable(free);
+    if (!status->free)
+        goto out;
 
-#ifdef GF_LINUX_HOST_OS
-        if (status->fs_name)
-                cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        else
-                cli_out ("%-20s : %-20s", "File System", "N/A");
+    snprintf(key, sizeof(key), "brick%d.total", i);
+    ret = dict_get_uint64(dict, key, &total);
 
-        if (status->device)
-                cli_out ("%-20s : %-20s", "Device", status->device);
-        else
-                cli_out ("%-20s : %-20s", "Device", "N/A");
+    status->total = gf_uint64_2human_readable(total);
+    if (!status->total)
+        goto out;
 
-        if (status->mount_options) {
-                cli_out ("%-20s : %-20s", "Mount Options",
-                         status->mount_options);
-        } else {
-                cli_out ("%-20s : %-20s", "Mount Options", "N/A");
-        }
+    snprintf(key, sizeof(key), "brick%d.device", i);
+    ret = dict_get_str(dict, key, &(status->device));
+    if (ret)
+        status->device = NULL;
 
-        if (status->inode_size) {
-                cli_out ("%-20s : %-20s", "Inode Size",
-                         status->inode_size);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Size", "N/A");
-        }
-#endif
-        if (status->free)
-                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        else
-                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
+    snprintf(key, sizeof(key), "brick%d.block_size", i);
+    ret = dict_get_uint64(dict, key, &(status->block_size));
+    if (ret) {
+        ret = 0;
+        status->block_size = 0;
+    }
 
-        if (status->total)
-                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
-        else
-                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
+    snprintf(key, sizeof(key), "brick%d.mnt_options", i);
+    ret = dict_get_str(dict, key, &(status->mount_options));
+    if (ret)
+        status->mount_options = NULL;
 
+    snprintf(key, sizeof(key), "brick%d.fs_name", i);
+    ret = dict_get_str(dict, key, &(status->fs_name));
+    if (ret) {
+        ret = 0;
+        status->fs_name = NULL;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.inode_size", i);
+    ret = dict_get_str(dict, key, &(status->inode_size));
+    if (ret)
+        status->inode_size = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.total_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->total_inodes));
+    if (ret)
+        status->total_inodes = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->free_inodes));
+    if (ret) {
+        ret = 0;
+        status->free_inodes = 0;
+    }
 
-        if (status->total_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Inode Count",
-                         status->total_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Count", "N/A");
-        }
+out:
+    return ret;
+}
 
-        if (status->free_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Free Inodes",
-                         status->free_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Free Inodes", "N/A");
-        }
+void
+cli_print_detailed_status(cli_volume_status_t *status)
+{
+    cli_out("%-20s : %-20s", "Brick", status->brick);
+
+    if (status->online) {
+        cli_out("%-20s : %-20d", "TCP Port", status->port);
+        cli_out("%-20s : %-20d", "RDMA Port", status->rdma_port);
+    } else {
+        cli_out("%-20s : %-20s", "TCP Port", "N/A");
+        cli_out("%-20s : %-20s", "RDMA Port", "N/A");
+    }
+
+    cli_out("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
+    cli_out("%-20s : %-20s", "Pid", status->pid_str);
+
+    if (status->fs_name)
+        cli_out("%-20s : %-20s", "File System", status->fs_name);
+    else
+        cli_out("%-20s : %-20s", "File System", "N/A");
+
+    if (status->device)
+        cli_out("%-20s : %-20s", "Device", status->device);
+    else
+        cli_out("%-20s : %-20s", "Device", "N/A");
+
+    if (status->mount_options) {
+        cli_out("%-20s : %-20s", "Mount Options", status->mount_options);
+    } else {
+        cli_out("%-20s : %-20s", "Mount Options", "N/A");
+    }
+
+    if (status->inode_size) {
+        cli_out("%-20s : %-20s", "Inode Size", status->inode_size);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Size", "N/A");
+    }
+    if (status->free)
+        cli_out("%-20s : %-20s", "Disk Space Free", status->free);
+    else
+        cli_out("%-20s : %-20s", "Disk Space Free", "N/A");
+
+    if (status->total)
+        cli_out("%-20s : %-20s", "Total Disk Space", status->total);
+    else
+        cli_out("%-20s : %-20s", "Total Disk Space", "N/A");
+
+    if (status->total_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Inode Count",
+                status->total_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Count", "N/A");
+    }
+
+    if (status->free_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Free Inodes",
+                status->free_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Free Inodes", "N/A");
+    }
 }
 
 int
-cli_print_brick_status (cli_volume_status_t *status)
+cli_print_brick_status(cli_volume_status_t *status)
 {
-        int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        int  bricklen = 0;
-        char *p = NULL;
-        int  num_spaces = 0;
-
-        p = status->brick;
-        bricklen = strlen (p);
-        while (bricklen > 0) {
-                if (bricklen > fieldlen) {
-                        cli_out ("%.*s", fieldlen, p);
-                        p += fieldlen;
-                        bricklen -= fieldlen;
-                } else {
-                        num_spaces = (fieldlen - bricklen) + 1;
-                        printf ("%s", p);
-                        while (num_spaces-- != 0)
-                                printf (" ");
-                        if (status->port || status->rdma_port) {
-                                if (status->online)
-                                        cli_out ("%-10d%-11d%-8c%-5s",
-                                                 status->port,
-                                                 status->rdma_port,
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                                else
-                                        cli_out ("%-10s%-11s%-8c%-5s",
-                                                 "N/A",
-                                                 "N/A",
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                        }
-                        else
-                                cli_out ("%-10s%-11s%-8c%-5s",
-                                         "N/A", "N/A", status->online?'Y':'N',
-                                         status->pid_str);
-                        bricklen = 0;
-                }
+    int fieldlen = CLI_VOL_STATUS_BRICK_LEN;
+    int bricklen = 0;
+    char *p = NULL;
+    int num_spaces = 0;
+
+    p = status->brick;
+    bricklen = strlen(p);
+    while (bricklen > 0) {
+        if (bricklen > fieldlen) {
+            cli_out("%.*s", fieldlen, p);
+            p += fieldlen;
+            bricklen -= fieldlen;
+        } else {
+            num_spaces = (fieldlen - bricklen) + 1;
+            printf("%s", p);
+            while (num_spaces-- != 0)
+                printf(" ");
+            if (status->port || status->rdma_port) {
+                if (status->online)
+                    cli_out("%-10d%-11d%-8c%-5s", status->port,
+                            status->rdma_port, status->online ? 'Y' : 'N',
+                            status->pid_str);
+                else
+                    cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                            status->online ? 'Y' : 'N', status->pid_str);
+            } else
+                cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                        status->online ? 'Y' : 'N', status->pid_str);
+            bricklen = 0;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-#define NEEDS_GLFS_HEAL(op) ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \
-                             (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||\
-                             (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||      \
-                             (op == GF_SHD_OP_INDEX_SUMMARY) ||               \
-                             (op == GF_SHD_OP_SPLIT_BRAIN_FILES))
+#define NEEDS_GLFS_HEAL(op)                                                    \
+    ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) ||                         \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||                        \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||                               \
+     (op == GF_SHD_OP_INDEX_SUMMARY) || (op == GF_SHD_OP_SPLIT_BRAIN_FILES) || \
+     (op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||                           \
+     (op == GF_SHD_OP_HEAL_SUMMARY))
 
 int
-cli_launch_glfs_heal (int heal_op, dict_t *options)
+cli_launch_glfs_heal(int heal_op, dict_t *options)
 {
-        char      buff[PATH_MAX] = {0};
-        runner_t  runner         = {0};
-        char      *filename      = NULL;
-        char      *hostname      = NULL;
-        char      *path          = NULL;
-        char      *volname       = NULL;
-        char      *out           = NULL;
-        int        ret           = 0;
-
-        runinit (&runner);
-        ret = dict_get_str (options, "volname", &volname);
-        runner_add_args (&runner, SBIN_DIR"/glfsheal", volname, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-
-        switch (heal_op) {
+    char buff[PATH_MAX] = {0};
+    runner_t runner = {0};
+    char *filename = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    char *out = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    ret = dict_get_str(options, "volname", &volname);
+    runner_add_args(&runner, GLFSHEAL_PREFIX "/glfsheal", volname, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    switch (heal_op) {
         case GF_SHD_OP_INDEX_SUMMARY:
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        runner_add_args (&runner, "xml", NULL);
-                }
-                break;
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                ret = dict_get_str (options, "file", &filename);
-                runner_add_args (&runner, "bigger-file", filename, NULL);
-                break;
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "bigger-file", filename, NULL);
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-                ret = dict_get_str (options, "file", &filename);
-                runner_add_args (&runner, "latest-mtime", filename, NULL);
-                break;
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "latest-mtime", filename, NULL);
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (options, "heal-source-hostname",
-                                    &hostname);
-                ret = dict_get_str (options, "heal-source-brickpath",
-                                    &path);
-                runner_add_args (&runner, "source-brick", NULL);
-                runner_argprintf (&runner, "%s:%s", hostname, path);
-                if (dict_get_str (options, "file", &filename) == 0)
-                        runner_argprintf (&runner, filename);
-                break;
+            ret = dict_get_str(options, "heal-source-hostname", &hostname);
+            ret = dict_get_str(options, "heal-source-brickpath", &path);
+            runner_add_args(&runner, "source-brick", NULL);
+            runner_argprintf(&runner, "%s:%s", hostname, path);
+            if (dict_get_str(options, "file", &filename) == 0)
+                runner_argprintf(&runner, "%s", filename);
+            break;
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-                runner_add_args (&runner, "split-brain-info", NULL);
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        runner_add_args (&runner, "xml", NULL);
-                }
-                break;
+            runner_add_args(&runner, "split-brain-info", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            runner_add_args(&runner, "granular-entry-heal-op", NULL);
+            break;
+        case GF_SHD_OP_HEAL_SUMMARY:
+            runner_add_args(&runner, "info-summary", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
         default:
-                ret = -1;
-        }
-        ret = runner_start (&runner);
-        if (ret == -1)
-                goto out;
-        while ((out = fgets (buff, sizeof(buff),
-                             runner_chio (&runner, STDOUT_FILENO)))) {
-                printf ("%s", out);
-        }
-        ret = runner_end (&runner);
-        ret = WEXITSTATUS (ret);
+            ret = -1;
+            goto out;
+    }
+    if (global_state->mode & GLUSTER_MODE_GLFSHEAL_NOLOG)
+        runner_add_args(&runner, "--nolog", NULL);
+    ret = runner_start(&runner);
+    if (ret == -1)
+        goto out;
+    while ((
+        out = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO)))) {
+        printf("%s", out);
+    }
+    ret = runner_end(&runner);
 
 out:
-        return ret;
+    return ret;
 }
+
 int
-cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_heal_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *options = NULL;
-        xlator_t                *this = NULL;
-        cli_local_t             *local = NULL;
-        int                     heal_op = 0;
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3) {
-               cli_usage_out (word->pattern);
-               parse_error = 1;
-               goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *options = NULL;
+    xlator_t *this = NULL;
+    cli_local_t *local = NULL;
+    int heal_op = 0;
+
+    this = THIS;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_heal_options_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-        ret = dict_get_int32 (options, "heal-op", &heal_op);
+    ret = cli_cmd_volume_heal_options_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    ret = dict_get_int32(options, "heal-op", &heal_op);
+    if (ret < 0)
+        goto out;
+    if (NEEDS_GLFS_HEAL(heal_op)) {
+        ret = cli_launch_glfs_heal(heal_op, options);
         if (ret < 0)
-                goto out;
-        if (NEEDS_GLFS_HEAL (heal_op)) {
-                ret = cli_launch_glfs_heal (heal_op, options);
-                if (ret == -1)
-                        goto out;
-        }
-        else {
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+            goto out;
+        if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+            goto out;
+    }
 
-                CLI_LOCAL_INIT (local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
 
-                if (proc->fn) {
-                        ret = proc->fn (frame, THIS, options);
-                }
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0) &&
-                    !(global_state->mode & GLUSTER_MODE_XML)) {
-                        cli_out ("Volume heal failed.");
-                }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0) &&
+            !(global_state->mode & GLUSTER_MODE_XML)) {
+            cli_out("Volume heal failed.");
         }
+    }
 
-        CLI_STACK_DESTROY (frame);
+    if (options)
+        dict_unref(options);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_statedump_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+    if (wordcount >= 3) {
+        ret = cli_cmd_volume_statedump_options_parse(words, wordcount,
+                                                     &options);
+        if (ret) {
+            parse_error = 1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error parsing "
+                   "statedump options");
+            cli_out("Error parsing options");
+            cli_usage_out(word->pattern);
         }
+    }
 
-        if (wordcount >= 3) {
-               ret = cli_cmd_volume_statedump_options_parse (words, wordcount,
-                                                              &options);
-               if (ret) {
-                       parse_error = 1;
-                       gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                               "statedump options");
-                       cli_out ("Error parsing options");
-                       cli_usage_out (word->pattern);
-               }
-        }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume statedump failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume statedump failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_volume_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        call_frame_t            *frame = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        int                     sent = 0;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
         if (!frame)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, NULL);
-        }
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0)
-                        cli_out ("Volume list failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0)
+            cli_out("Volume list failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
-                               struct cli_cmd_word *word,
-                               const char **words, int wordcount)
+cli_cmd_volume_clearlocks_cbk(struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 7 || wordcount > 8) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-       ret = cli_cmd_volume_clrlks_opts_parse (words, wordcount, &options);
-       if (ret) {
-               parse_error = 1;
-               gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                       "clear-locks options");
-               cli_out ("Error parsing options");
-               cli_usage_out (word->pattern);
-       }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 7 || wordcount > 8) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_clrlks_opts_parse(words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Error parsing "
+               "clear-locks options");
+        cli_out("Error parsing options");
+        cli_usage_out(word->pattern);
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "path", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "path", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume clear-locks failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume clear-locks failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_barrier_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                            const char **words, int wordcount)
+cli_cmd_volume_barrier_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int ret = -1;
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame = NULL;
-        dict_t *options = NULL;
-        int sent = 0;
-        int parse_error = 0;
-        cli_local_t *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    options = dict_new();
+    if (!options) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        options = dict_new();
-        if (!options) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str(options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "barrier", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "barrier", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("Volume barrier failed");
-        }
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("Volume barrier failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_getopt_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_getopt_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret       = -1;
-        rpc_clnt_procedure_t *proc      = NULL;
-        call_frame_t         *frame     = NULL;
-        dict_t               *options   = NULL;
-        int                   sent      = 0;
-        int                   parse_err = 0;
-        cli_local_t          *local     = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_err = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        options = dict_new ();
-        if (!options)
-                goto out;
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "key", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "key", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_err == 0))
-                        cli_err ("Volume get option failed");
-        }
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err("Volume get option failed");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
+/* This is a bit of a hack to display the help. The current bitrot cmd
+ * format does not work well when registering the cmds.
+ * Ideally the should have been of the form
+ * gluster volume bitrot <subcommand> <volumename> ...
+ */
+
+struct cli_cmd bitrot_cmds[] = {
+
+    {"volume bitrot help", cli_cmd_bitrot_help_cbk,
+     "display help for volume bitrot commands"},
+
+    {"volume bitrot <VOLNAME> {enable|disable}", NULL, /*cli_cmd_bitrot_cbk,*/
+     "Enable/disable bitrot for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> signing-time <time-in-secs>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Waiting time for an object after last fd is closed to start signing "
+     "process"},
+
+    {"volume bitrot <VOLNAME> signer-threads <count>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Number of signing process threads. Usually set to number of available "
+     "cores"},
+
+    {"volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the speed of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the frequency of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Pause/resume the scrubber for <VOLNAME>. Status displays the status of "
+     "the scrubber. ondemand starts the scrubber immediately."},
+
+    {"volume bitrot <VOLNAME> {enable|disable}\n"
+     "volume bitrot <VOLNAME> signing-time <time-in-secs>\n"
+     "volume bitrot <VOLNAME> signer-threads <count>\n"
+     "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive}\n"
+     "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}\n"
+     "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
+     cli_cmd_bitrot_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
+struct cli_cmd quota_cmds[] = {
+
+    /* Quota commands */
+    {"volume quota help", cli_cmd_quota_help_cbk,
+     "display help for volume quota commands"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}",
+     cli_cmd_quota_cbk, "Enable/disable and configure quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}",
+     cli_cmd_quota_cbk, "Set maximum size for <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}",
+     cli_cmd_quota_cbk,
+     "Set the maximum number of entries allowed in <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, "Set quota timeout for <VOLNAME>"},
+
+    {"volume inode-quota <VOLNAME> enable", cli_cmd_quota_cbk,
+     "Enable/disable inode-quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}\n"
+     "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}\n"
+     "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}\n"
+     "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
 struct cli_cmd volume_cmds[] = {
-        { "volume info [all|<VOLNAME>]",
-          cli_cmd_volume_info_cbk,
-          "list information of all volumes"},
-
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] "
-          "[replica <COUNT> [arbiter <COUNT>]] "
-          "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
-          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
-#ifdef HAVE_BD_XLATOR
-          "?<vg_name>"
-#endif
-          "... [force]",
-
-          cli_cmd_volume_create_cbk,
-          "create a new volume of specified type with mentioned bricks"},
-
-        { "volume delete <VOLNAME>",
-          cli_cmd_volume_delete_cbk,
-          "delete volume specified by <VOLNAME>"},
-
-        { "volume start <VOLNAME> [force]",
-          cli_cmd_volume_start_cbk,
-          "start volume specified by <VOLNAME>"},
-
-        { "volume stop <VOLNAME> [force]",
-          cli_cmd_volume_stop_cbk,
-          "stop volume specified by <VOLNAME>"},
-
-        /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
-          cli_cmd_volume_rename_cbk,
-          "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
-
-#if !defined(__NetBSD__)
-        { "volume tier <VOLNAME> status\n"
-        "volume tier <VOLNAME> start [force]\n"
-        "volume tier <VOLNAME> attach [<replica COUNT>] <NEW-BRICK>... [force]\n"
-        "volume tier <VOLNAME> detach <start|stop|status|commit|force>\n",
-        cli_cmd_volume_tier_cbk,
-        "Tier translator specific operations."},
-
-        { "volume attach-tier <VOLNAME> [<replica COUNT>] <NEW-BRICK>...",
-        cli_cmd_volume_tier_cbk,
-          "NOTE: this is old syntax, will be depreciated in next release. "
-          "Please use gluster volume tier <vol> attach "
-          "[<replica COUNT>] <NEW-BRICK>..."},
-
-        { "volume detach-tier <VOLNAME> "
-          " <start|stop|status|commit|force>",
-        cli_cmd_volume_tier_cbk,
-          "NOTE: this is old syntax, will be depreciated in next release. "
-          "Please use gluster volume tier <vol> detach "
-          "{start|stop|commit} [force]"},
-#endif
+    {"volume help", cli_cmd_volume_help_cbk,
+     "display help for volume commands"},
+
+    {"volume info [all|<VOLNAME>]", cli_cmd_volume_info_cbk,
+     "list information of all volumes"},
+
+    {"volume create <NEW-VOLNAME> [stripe <COUNT>] "
+     "[[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] "
+     "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
+     "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> <TA-BRICK>"
+     "... [force]",
 
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
-          "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
-          cli_cmd_volume_add_brick_cbk,
-          "add brick to volume <VOLNAME>"},
+     cli_cmd_volume_create_cbk,
+     "create a new volume of specified type with mentioned bricks"},
 
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
-          " <start|stop|status|commit|force>",
-          cli_cmd_volume_remove_brick_cbk,
-          "remove brick from volume <VOLNAME>"},
+    {"volume delete <VOLNAME>", cli_cmd_volume_delete_cbk,
+     "delete volume specified by <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> {{fix-layout start} | {start [force]|stop|status}}",
-          cli_cmd_volume_defrag_cbk,
-          "rebalance operations"},
+    {"volume start <VOLNAME> [force]", cli_cmd_volume_start_cbk,
+     "start volume specified by <VOLNAME>"},
 
-        { "volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
-          "{commit force}",
-          cli_cmd_volume_replace_brick_cbk,
-          "replace-brick operations"},
+    {"volume stop <VOLNAME> [force]", cli_cmd_volume_stop_cbk,
+     "stop volume specified by <VOLNAME>"},
 
-        /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ...",
-          cli_cmd_volume_set_transport_cbk,
-          "set transport type for volume <VOLNAME>"},*/
+    /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
+      cli_cmd_volume_rename_cbk,
+      "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume set <VOLNAME> <KEY> <VALUE>",
-          cli_cmd_volume_set_cbk,
-         "set options for volume <VOLNAME>"},
+    {"volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
+     "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
+     cli_cmd_volume_add_brick_cbk, "add brick to volume <VOLNAME>"},
 
-        { "volume help",
-          cli_cmd_volume_help_cbk,
-          "display help for the volume command"},
+    {"volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
+     " <start|stop|status|commit|force>",
+     cli_cmd_volume_remove_brick_cbk, "remove brick from volume <VOLNAME>"},
 
-        { "volume log <VOLNAME> rotate [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"},
+    {"volume rebalance <VOLNAME> {{fix-layout start} | {start "
+     "[force]|stop|status}}",
+     cli_cmd_volume_defrag_cbk, "rebalance operations"},
 
-        { "volume log rotate <VOLNAME> [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"
-         " NOTE: This is an old syntax, will be deprecated from next release."},
+    {"volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
+     "{commit force}",
+     cli_cmd_volume_replace_brick_cbk, "replace-brick operations"},
 
-        { "volume sync <HOSTNAME> [all|<VOLNAME>]",
-          cli_cmd_sync_volume_cbk,
-         "sync the volume information from a peer"},
+    /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>]
+      ...", cli_cmd_volume_set_transport_cbk, "set transport type for volume
+      <VOLNAME>"},*/
 
-         { "volume reset <VOLNAME> [option] [force]",
-         cli_cmd_volume_reset_cbk,
-         "reset all the reconfigured options"},
+    {"volume set <VOLNAME> <KEY> <VALUE>", cli_cmd_volume_set_cbk,
+     "set options for volume <VOLNAME>"},
+
+    {"volume set <VOLNAME> group <GROUP>", cli_cmd_volume_set_cbk,
+     "This option can be used for setting multiple pre-defined volume options "
+     "where group_name is a file under /var/lib/glusterd/groups containing one "
+     "key value pair per line"},
+
+    {"volume log <VOLNAME> rotate [BRICK]", cli_cmd_log_rotate_cbk,
+     "rotate the log file for corresponding volume/brick"},
+
+    {"volume sync <HOSTNAME> [all|<VOLNAME>]", cli_cmd_sync_volume_cbk,
+     "sync the volume information from a peer"},
+
+    {"volume reset <VOLNAME> [option] [force]", cli_cmd_volume_reset_cbk,
+     "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [[ssh-port n] [[no-verify]|[push-pem]]] [force]"
-         "|start [force]|stop [force]|pause [force]|resume [force]|config|status [detail]|delete [reset-sync-time]} [options...]",
-         cli_cmd_volume_gsync_set_cbk,
-         "Geo-sync operations",
-         cli_cmd_check_gsync_exists_cbk},
+    {"volume " GEOREP " [<MASTER-VOLNAME>] [<SLAVE-IP>]::[<SLAVE-VOLNAME>] {"
+     "\\\n create [[ssh-port n] [[no-verify] \\\n | [push-pem]]] [force] \\\n"
+     " | start [force] \\\n | stop [force] \\\n | pause [force] \\\n | resume "
+     "[force] \\\n"
+     " | config [[[\\!]<option>] [<value>]] \\\n | status "
+     "[detail] \\\n | delete [reset-sync-time]} ",
+     cli_cmd_volume_gsync_set_cbk, "Geo-sync operations",
+     cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs]",
-           cli_cmd_volume_profile_cbk,
-           "volume profile operations"},
-
-        { "volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
-          "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
-          "default-soft-limit <percent>} |\n"
-          "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]} |\n"
-          "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]} |\n"
-          "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
-
-        { "volume inode-quota <VOLNAME> enable",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
-
-         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
-           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
-           cli_cmd_volume_top_cbk,
-           "volume top operations"},
-
-        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
-          " [detail|clients|mem|inode|fd|callpool|tasks]",
-          cli_cmd_volume_status_cbk,
-          "display status of all or specified volume(s)/brick"},
-
-        { "volume heal <VOLNAME> [enable | disable | full |"
-          "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
-          "info [healed | heal-failed | split-brain] |"
-          "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
-                       "source-brick <HOSTNAME:BRICKNAME> [<FILE>]}]",
-          cli_cmd_volume_heal_cbk,
-          "self-heal commands on volume specified by <VOLNAME>"},
-
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
-         cli_cmd_volume_statedump_cbk,
-         "perform statedump on bricks"},
-
-        {"volume list",
-         cli_cmd_volume_list_cbk,
-         "list all volumes in cluster"},
-
-        {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
-          "{inode [range]|entry [basename]|posix [range]}",
-          cli_cmd_volume_clearlocks_cbk,
-          "Clear locks held on path"
-        },
-        {"volume barrier <VOLNAME> {enable|disable}",
-         cli_cmd_volume_barrier_cbk,
-         "Barrier/unbarrier file operations on a volume"
-        },
-        {"volume get <VOLNAME> <key|all>",
-         cli_cmd_volume_getopt_cbk,
-         "Get the value of the all options or given option for volume <VOLNAME>"
-        },
-        {"volume bitrot <VOLNAME> {enable|disable} |\n"
-         "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive} |\n"
-         "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
-         "|monthly} |\n"
-         "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
-         cli_cmd_bitrot_cbk,
-         "Bitrot translator specific operation. For more information about "
-         "bitrot command type  'man gluster'"
-        },
-        { "volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
-          " {<NEW-BRICK> commit}}",
-          cli_cmd_volume_reset_brick_cbk,
-          "reset-brick operations"},
-
-        { NULL, NULL, NULL }
-};
+    {"volume profile <VOLNAME> {start|info [peek|incremental "
+     "[peek]|cumulative|clear]|stop} [nfs]",
+     cli_cmd_volume_profile_cbk, "volume profile operations"},
+
+    {"volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick "
+     "<brick>] [list-cnt <value>] | "
+     "{read-perf|write-perf} [bs <size> count <count>] "
+     "[brick <brick>] [list-cnt <value>]",
+     cli_cmd_volume_top_cbk, "volume top operations"},
+
+    {"volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
+     " [detail|clients|mem|inode|fd|callpool|tasks|client-list]",
+     cli_cmd_volume_status_cbk,
+     "display status of all or specified volume(s)/brick"},
+
+    {"volume heal <VOLNAME> [enable | disable | full |"
+     "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
+     "info [summary | split-brain] |"
+     "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
+     "source-brick <HOSTNAME:BRICKNAME> [<FILE>]} |"
+     "granular-entry-heal {enable | disable}]",
+     cli_cmd_volume_heal_cbk,
+     "self-heal commands on volume specified by <VOLNAME>"},
+
+    {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+     "priv|fd|inode|history]... | [client <hostname:process-id>]]",
+     cli_cmd_volume_statedump_cbk, "perform statedump on bricks"},
+
+    {"volume list", cli_cmd_volume_list_cbk, "list all volumes in cluster"},
+
+    {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+     "{inode [range]|entry [basename]|posix [range]}",
+     cli_cmd_volume_clearlocks_cbk, "Clear locks held on path"},
+    {"volume barrier <VOLNAME> {enable|disable}", cli_cmd_volume_barrier_cbk,
+     "Barrier/unbarrier file operations on a volume"},
+    {"volume get <VOLNAME|all> <key|all>", cli_cmd_volume_getopt_cbk,
+     "Get the value of the all options or given option for volume <VOLNAME>"
+     " or all option. gluster volume get all all is to get all global "
+     "options"},
+
+    {"volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+     " {<NEW-BRICK> commit}}",
+     cli_cmd_volume_reset_brick_cbk, "reset-brick operations"},
+
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *quota_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(quota_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, quota_cmds, sizeof(quota_cmds));
+    count = (sizeof(quota_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster quota commands");
+    cli_out("=======================\n");
+
+    for (quota_cmd = cmd; quota_cmd->pattern; quota_cmd++)
+        if ((_gf_false == quota_cmd->disable) && (quota_cmd->desc))
+            cli_out("%s - %s", quota_cmd->pattern, quota_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
+}
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd     = NULL;
-        struct cli_cmd        *vol_cmd = NULL;
-        int                   count    = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *bitrot_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(bitrot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, bitrot_cmds, sizeof(bitrot_cmds));
+    count = (sizeof(bitrot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        cmd = GF_CALLOC (1, sizeof (volume_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, volume_cmds, sizeof (volume_cmds));
-        count = (sizeof (volume_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cli_out("\ngluster bitrot commands");
+    cli_out("========================\n");
 
-        for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
-                if (_gf_false == vol_cmd->disable)
-                        cli_out ("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+    for (bitrot_cmd = cmd; bitrot_cmd->pattern; bitrot_cmd++)
+        if ((_gf_false == bitrot_cmd->disable) && (bitrot_cmd->desc))
+            cli_out("%s - %s", bitrot_cmd->pattern, bitrot_cmd->desc);
 
-        GF_FREE (cmd);
-        return 0;
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
 }
 
 int
-cli_cmd_volume_register (struct cli_state *state)
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *vol_cmd = NULL;
+    int count = 0;
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++) {
+    cmd = GF_MALLOC(sizeof(volume_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, volume_cmds, sizeof(volume_cmds));
+    count = (sizeof(volume_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster volume commands");
+    cli_out("========================\n");
+
+    for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
+        if (_gf_false == vol_cmd->disable)
+            cli_out("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_volume_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = volume_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = bitrot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = quota_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
+}
+
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...)
+{
+    va_list arg;
+    int rv = 0;
+    char *tmp = *string_ptr;
+
+    va_start(arg, format);
+    rv = gf_vasprintf(string_ptr, format, arg);
+    va_end(arg);
+
+    if (tmp)
+        GF_FREE(tmp);
+
+    return rv;
 }
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index 8a750414108..2d458b16a56 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -22,396 +22,390 @@
 
 static int cmd_done;
 static int cmd_sent;
-static pthread_cond_t      cond  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     cond_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t      conn  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     conn_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t conn = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-int    cli_op_ret = 0;
-int    connected = 0;
-
-int cli_cmd_log_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+int cli_op_ret = 0;
+static gf_boolean_t connected = _gf_false;
 
 static unsigned
-cli_cmd_needs_connection (struct cli_cmd_word *word)
+cli_cmd_needs_connection(struct cli_cmd_word *word)
 {
-        if (!strcasecmp ("quit", word->word))
-                return 0;
+    if (!strcasecmp("quit", word->word))
+        return 0;
 
-        if (!strcasecmp ("help", word->word))
-                return 0;
+    if (!strcasecmp("help", word->word))
+        return 0;
 
-        if (!strcasecmp ("getwd", word->word))
-                return 1;
+    if (!strcasecmp("getwd", word->word))
+        return 1;
 
-        if (!strcasecmp ("exit", word->word))
-                return 0;
+    if (!strcasecmp("exit", word->word))
+        return 0;
 
-        return cli_default_conn_timeout;
+    return cli_default_conn_timeout;
 }
 
 int
-cli_cmd_status_reset (void)
+cli_cmd_status_reset(void)
 {
-        int ret = 0;
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0) {
-                        cmd_sent = 0;
-                        cmd_done = 0;
-                }
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
 
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0) {
+            cmd_sent = 0;
+            cmd_done = 0;
+        }
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_sent_status_get (int *status)
+cli_cmd_sent_status_get(int *status)
 {
-        int ret = 0;
-        GF_ASSERT (status);
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0)
-                        *status = cmd_sent;
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
+    GF_ASSERT(status);
+
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0)
+            *status = cmd_sent;
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_process (struct cli_state *state, int argc, char **argv)
+cli_cmd_process(struct cli_state *state, int argc, char **argv)
 {
-        int                  ret = 0;
-        struct cli_cmd_word *word = NULL;
-        struct cli_cmd_word *next = NULL;
-        int                  i = 0;
+    int ret = 0;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    int i = 0;
 
-        word = &state->tree.root;
+    word = &state->tree.root;
 
-        if (!argc)
-                return 0;
+    if (!argc)
+        return 0;
 
-        for (i = 0; i < argc; i++) {
-                next = cli_cmd_nextword (word, argv[i]);
+    for (i = 0; i < argc; i++) {
+        next = cli_cmd_nextword(word, argv[i]);
 
-                word = next;
-                if (!word)
-                        break;
+        word = next;
+        if (!word)
+            break;
 
-                if (word->cbkfn)
-                        break;
-        }
+        if (word->cbkfn)
+            break;
+    }
 
-        if (!word) {
-                cli_out ("unrecognized word: %s (position %d)",
-                         argv[i], i);
-                return -1;
-        }
+    if (!word) {
+        cli_out("unrecognized word: %s (position %d)\n", argv[i], i);
+        usage();
+        return -1;
+    }
 
-        if (!word->cbkfn) {
-                cli_out ("unrecognized command");
-                return -1;
-        }
+    if (!word->cbkfn) {
+        cli_out("unrecognized command\n");
+        usage();
+        return -1;
+    }
 
-	if ( strcmp (word->word,"help")==0 )
-		goto callback;
+    if (strcmp(word->word, "help") == 0)
+        goto callback;
 
-        state->await_connected = cli_cmd_needs_connection (word);
+    state->await_connected = cli_cmd_needs_connection(word);
 
-        ret = cli_cmd_await_connected (state->await_connected);
-        if (ret) {
-                cli_out ("Connection failed. Please check if gluster "
-                          "daemon is operational.");
-                gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-                exit (ret);
-        }
+    ret = cli_cmd_await_connected(state->await_connected);
+    if (ret) {
+        cli_out(
+            "Connection failed. Please check if gluster "
+            "daemon is operational.");
+        gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+        exit(ret);
+    }
 
 callback:
-        ret = word->cbkfn (state, word, (const char **)argv, argc);
-        (void) cli_cmd_status_reset ();
-        return ret;
+    ret = word->cbkfn(state, word, (const char **)argv, argc);
+    (void)cli_cmd_status_reset();
+    return ret;
 }
 
 int
-cli_cmd_input_token_count (const char *text)
+cli_cmd_input_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-
 int
-cli_cmd_process_line (struct cli_state *state, const char *text)
+cli_cmd_process_line(struct cli_state *state, const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-        int     ret = -1;
-
-        count = cli_cmd_input_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return -1;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
-
-        ret = cli_cmd_process (state, count, tokens);
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+    int ret = -1;
+
+    count = cli_cmd_input_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return -1;
+
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    ret = cli_cmd_process(state, count, tokens);
 out:
-        free (copy);
+    free(copy);
 
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmds_register (struct cli_state *state)
+cli_cmds_register(struct cli_state *state)
 {
-        int  ret = 0;
-
-        ret = cli_cmd_volume_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_probe_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_system_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_misc_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_snapshot_register (state);
-        if (ret)
-                goto out;
-        ret = cli_cmd_global_register (state);
-        if (ret)
-                goto out;
+    int ret = 0;
+
+    ret = cli_cmd_volume_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_probe_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_system_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_misc_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_snapshot_register(state);
+    if (ret)
+        goto out;
+    ret = cli_cmd_global_register(state);
+    if (ret)
+        goto out;
 out:
-        return ret;
-}
-
-int
-cli_cmd_cond_init ()
-{
-
-       pthread_mutex_init (&cond_mutex, NULL);
-       pthread_cond_init (&cond, NULL);
-
-       pthread_mutex_init (&conn_mutex, NULL);
-       pthread_cond_init (&conn, NULL);
-
-       return 0;
+    return ret;
 }
 
 int
-cli_cmd_lock ()
+cli_cmd_lock()
 {
-       pthread_mutex_lock (&cond_mutex);
-       return 0;
+    pthread_mutex_lock(&cond_mutex);
+    return 0;
 }
 
 int
-cli_cmd_unlock ()
+cli_cmd_unlock()
 {
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 static void
-seconds_from_now (unsigned secs, struct timespec *ts)
+seconds_from_now(unsigned secs, struct timespec *ts)
 {
-        struct timeval tv = {0,};
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        ts->tv_sec = tv.tv_sec + secs;
-        ts->tv_nsec = tv.tv_usec * 1000;
+    ts->tv_sec = tv.tv_sec + secs;
+    ts->tv_nsec = tv.tv_usec * 1000;
 }
 
 int
-cli_cmd_await_response (unsigned time)
+cli_cmd_await_response(unsigned time)
 {
-        struct  timespec        ts = {0,};
-        int                     ret = 0;
+    struct timespec ts = {
+        0,
+    };
+    int ret = 0;
 
-        cli_op_ret = -1;
+    cli_op_ret = -1;
 
-        seconds_from_now (time, &ts);
-        while (!cmd_done && !ret) {
-                ret = pthread_cond_timedwait (&cond, &cond_mutex,
-                                        &ts);
-        }
+    seconds_from_now(time, &ts);
+    while (!cmd_done && !ret) {
+        ret = pthread_cond_timedwait(&cond, &cond_mutex, &ts);
+    }
 
-        if (!cmd_done) {
-                if (ret == ETIMEDOUT)
-                        cli_out ("Error : Request timed out");
-                else
-                        cli_out ("Error : Command returned with error code:%d",
-                                 ret);
-        }
-        cmd_done = 0;
+    if (!cmd_done) {
+        if (ret == ETIMEDOUT)
+            cli_out("Error : Request timed out");
+        else
+            cli_out("Error : Command returned with error code:%d", ret);
+    }
+    cmd_done = 0;
 
-        return cli_op_ret;
+    return cli_op_ret;
 }
 
 /* This function must be called _only_ after all actions associated with
  * command processing is complete. Otherwise, gluster process may exit before
  * reporting results to stdout/stderr. */
 int
-cli_cmd_broadcast_response (int32_t status)
+cli_cmd_broadcast_response(int32_t status)
 {
-
-        pthread_mutex_lock (&cond_mutex);
-        {
-                if (!cmd_sent)
-                        goto out;
-                cmd_done = 1;
-                cli_op_ret = status;
-                pthread_cond_broadcast (&cond);
-        }
-
+    pthread_mutex_lock(&cond_mutex);
+    {
+        if (!cmd_sent)
+            goto out;
+        cmd_done = 1;
+        cli_op_ret = status;
+        pthread_cond_broadcast(&cond);
+    }
 
 out:
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 int32_t
-cli_cmd_await_connected (unsigned conn_timo)
+cli_cmd_await_connected(unsigned conn_timo)
 {
-        int32_t                 ret = 0;
-        struct  timespec        ts = {0,};
-
-        if (!conn_timo)
-                return 0;
-
-        pthread_mutex_lock (&conn_mutex);
-        {
-                seconds_from_now (conn_timo, &ts);
-                while (!connected && !ret) {
-                        ret = pthread_cond_timedwait (&conn, &conn_mutex,
-                                                      &ts);
-                }
-        }
-        pthread_mutex_unlock (&conn_mutex);
+    int32_t ret = 0;
+    struct timespec ts = {
+        0,
+    };
+
+    if (!conn_timo)
+        return 0;
 
+    pthread_mutex_lock(&conn_mutex);
+    {
+        seconds_from_now(conn_timo, &ts);
+        while (!connected && !ret) {
+            ret = pthread_cond_timedwait(&conn, &conn_mutex, &ts);
+        }
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_broadcast_connected ()
+cli_cmd_broadcast_connected(gf_boolean_t status)
 {
-        pthread_mutex_lock (&conn_mutex);
-        {
-                connected = 1;
-                pthread_cond_broadcast (&conn);
-        }
-
-        pthread_mutex_unlock (&conn_mutex);
-
-        return 0;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        connected = status;
+        pthread_cond_broadcast(&conn);
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return 0;
 }
 
-int
-cli_cmd_submit (struct rpc_clnt* rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gf_boolean_t
+cli_cmd_connected(void)
 {
-        int             ret = -1;
-        unsigned        timeout = 0;
-
-        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
-            (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
-            (GLUSTER_CLI_GANESHA == procnum))
-                timeout = cli_ten_minutes_timeout;
-        else
-                timeout = cli_default_conn_timeout;
-
-        cli_cmd_lock ();
-        cmd_sent = 0;
-        ret = cli_submit_request (rpc, req, frame, prog,
-                                  procnum, NULL, this, cbkfn, xdrproc);
+    gf_boolean_t status;
 
-        if (!ret) {
-                cmd_sent = 1;
-                ret = cli_cmd_await_response (timeout);
-        }
+    pthread_mutex_lock(&conn_mutex);
+    {
+        status = connected;
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        cli_cmd_unlock ();
+    return status;
+}
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    unsigned timeout = 0;
+
+    if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+        (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
+        (GLUSTER_CLI_GANESHA == procnum))
+        timeout = cli_ten_minutes_timeout;
+    else
+        timeout = cli_default_conn_timeout;
+
+    cli_cmd_lock();
+    cmd_sent = 0;
+    ret = cli_submit_request(rpc, req, frame, prog, procnum, NULL, this, cbkfn,
+                             xdrproc);
+
+    if (!ret) {
+        cmd_sent = 1;
+        ret = cli_cmd_await_response(timeout);
+    }
+
+    cli_cmd_unlock();
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_cmd_pattern_cmp (void *a, void *b)
+cli_cmd_pattern_cmp(void *a, void *b)
 {
-        struct cli_cmd *ia = NULL;
-        struct cli_cmd *ib = NULL;
-        int            ret = 0;
-
-        ia = a;
-        ib = b;
-        if (strcmp (ia->pattern, ib->pattern) > 0)
-                ret = 1;
-        else if (strcmp (ia->pattern, ib->pattern) < 0)
-                ret = -1;
-        else
-                ret = 0;
-        return ret;
+    struct cli_cmd *ia = NULL;
+    struct cli_cmd *ib = NULL;
+    int ret = 0;
+
+    ia = a;
+    ib = b;
+    if (strcmp(ia->pattern, ib->pattern) > 0)
+        ret = 1;
+    else if (strcmp(ia->pattern, ib->pattern) < 0)
+        ret = -1;
+    else
+        ret = 0;
+    return ret;
 }
 
 void
-cli_cmd_sort (struct cli_cmd *cmd, int count)
+cli_cmd_sort(struct cli_cmd *cmd, int count)
 {
-        gf_array_insertionsort (cmd, 1, count - 2, sizeof(struct cli_cmd),
-                                cli_cmd_pattern_cmp);
+    gf_array_insertionsort(cmd, 1, count - 2, sizeof(struct cli_cmd),
+                           cli_cmd_pattern_cmp);
 }
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 54e3686c9e1..c1c068c7085 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -13,107 +13,117 @@
 #include <netdb.h>
 
 #include "cli.h"
-#include "list.h"
-
-#define GLUSTER_SHARED_STORAGE      "gluster_shared_storage"
-
-#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
-        do {                                                 \
-                local = cli_local_get ();                    \
-                                                             \
-                if (local) {                                 \
-                        local->words = words;                \
-                        if (dictionary)                      \
-                                local->dict = dictionary;    \
-                        if (frame)                           \
-                                frame->local = local;        \
-                }                                            \
-        } while (0)
-
-#define CLI_STACK_DESTROY(_frame)                                       \
-        do {                                                            \
-                if (_frame) {                                           \
-                        if (_frame->local) {                            \
-                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
-                                        "is not NULL (%p)", _frame->local); \
-                                cli_local_wipe (_frame->local);         \
-                                _frame->local = NULL;                   \
-                        }                                               \
-                        STACK_DESTROY (_frame->root);                   \
-                }                                                       \
-        } while (0);
-
-typedef enum {
-        GF_ANSWER_YES = 1,
-        GF_ANSWER_NO  = 2
-} gf_answer_t;
+#include <glusterfs/list.h>
+
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+
+#define CLI_LOCAL_INIT(local, words, frame, dictionary)                        \
+    do {                                                                       \
+        local = cli_local_get();                                               \
+                                                                               \
+        if (local) {                                                           \
+            local->words = words;                                              \
+            if (dictionary)                                                    \
+                local->dict = dictionary;                                      \
+            if (frame)                                                         \
+                frame->local = local;                                          \
+        }                                                                      \
+    } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        if (_frame) {                                                          \
+            if (_frame->local) {                                               \
+                gf_log("cli", GF_LOG_DEBUG,                                    \
+                       "frame->local "                                         \
+                       "is not NULL (%p)",                                     \
+                       _frame->local);                                         \
+                cli_local_wipe(_frame->local);                                 \
+                _frame->local = NULL;                                          \
+            }                                                                  \
+            STACK_DESTROY(_frame->root);                                       \
+        }                                                                      \
+    } while (0);
+
+typedef enum { GF_ANSWER_YES = 1, GF_ANSWER_NO = 2 } gf_answer_t;
 
 struct cli_cmd {
-        const char         *pattern;
-        cli_cmd_cbk_t      *cbk;
-        const char         *desc;
-        cli_cmd_reg_cbk_t  *reg_cbk; /* callback to check in runtime if the   *
-                                      * command should be enabled or disabled */
-        gf_boolean_t       disable;
+    const char *pattern;
+    cli_cmd_cbk_t *cbk;
+    const char *desc;
+    cli_cmd_reg_cbk_t *reg_cbk; /* callback to check in runtime if the   *
+                                 * command should be enabled or disabled */
+    gf_boolean_t disable;
 };
 
 struct cli_cmd_volume_get_ctx_ {
-        char            *volname;
-        int             flags;
+    char *volname;
+    int flags;
 };
 
 typedef struct cli_profile_info_ {
-        uint64_t fop_hits;
-        double min_latency;
-        double max_latency;
-        double avg_latency;
-        char   *fop_name;
-        double percentage_avg_latency;
+    uint64_t fop_hits;
+    double min_latency;
+    double max_latency;
+    double avg_latency;
+    char *fop_name;
+    double percentage_avg_latency;
 } cli_profile_info_t;
 
 typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
 
-int cli_cmd_volume_register (struct cli_state *state);
-
-int cli_cmd_probe_register (struct cli_state *state);
+int
+cli_cmd_volume_register(struct cli_state *state);
 
-int cli_cmd_system_register (struct cli_state *state);
+int
+cli_cmd_probe_register(struct cli_state *state);
 
-int cli_cmd_snapshot_register (struct cli_state *state);
+int
+cli_cmd_system_register(struct cli_state *state);
 
-int cli_cmd_global_register (struct cli_state *state);
+int
+cli_cmd_snapshot_register(struct cli_state *state);
 
-int cli_cmd_misc_register (struct cli_state *state);
+int
+cli_cmd_global_register(struct cli_state *state);
 
-struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
-                                       const char *text);
-void cli_cmd_tokens_destroy (char **tokens);
+int
+cli_cmd_misc_register(struct cli_state *state);
 
-int cli_cmd_await_response (unsigned time);
+struct cli_cmd_word *
+cli_cmd_nextword(struct cli_cmd_word *word, const char *text);
+void
+cli_cmd_tokens_destroy(char **tokens);
 
-int cli_cmd_broadcast_response (int32_t status);
+int
+cli_cmd_await_response(unsigned time);
 
-int cli_cmd_cond_init ();
+int
+cli_cmd_broadcast_response(int32_t status);
 
-int cli_cmd_lock ();
+int
+cli_cmd_lock();
 
-int cli_cmd_unlock ();
+int
+cli_cmd_unlock();
 
 int
-cli_cmd_submit (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
-int cli_cmd_pattern_cmp (void *a, void *b);
+int
+cli_cmd_pattern_cmp(void *a, void *b);
 
-void cli_cmd_sort (struct cli_cmd *cmd, int count);
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count);
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question);
-int cli_cmd_sent_status_get (int *status);
+cli_cmd_get_confirmation(struct cli_state *state, const char *question);
+int
+cli_cmd_sent_status_get(int *status);
 
 gf_boolean_t
-_limits_set_on_volume (char *volname, int type);
+_limits_set_on_volume(char *volname, int type);
 
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 5468b25cc0c..b42b4dd86c2 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -10,21 +10,21 @@
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define CLI_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum cli_mem_types_ {
-        cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
-        cli_mt_xlator_t,
-        cli_mt_xlator_cmdline_option_t,
-        cli_mt_char,
-        cli_mt_call_pool_t,
-        cli_mt_cli_local_t,
-        cli_mt_cli_get_vol_ctx_t,
-        cli_mt_append_str,
-        cli_mt_cli_cmd,
-        cli_mt_end
+    cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
+    cli_mt_xlator_t,
+    cli_mt_xlator_cmdline_option_t,
+    cli_mt_char,
+    cli_mt_call_pool_t,
+    cli_mt_cli_local_t,
+    cli_mt_cli_get_vol_ctx_t,
+    cli_mt_append_str,
+    cli_mt_cli_cmd,
+    cli_mt_end
 
 };
 
diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c
index 5be9c80c858..772b8f75bd9 100644
--- a/cli/src/cli-quotad-client.c
+++ b/cli/src/cli-quotad-client.c
@@ -10,140 +10,137 @@
 
 #include "cli-quotad-client.h"
 
-extern struct rpc_clnt global_quotad_rpc;
-extern struct rpc_clnt_program cli_quotad_clnt;
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc)
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (&global_quotad_rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    /* Send the msg */
+    ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data)
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        this = mydata;
+    this = mydata;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-                break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options)
+cli_quotad_clnt_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        int              ret  = -1;
-
-
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
-
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (rpc, cli_quotad_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
-
-        rpc_clnt_start (rpc);
+    struct rpc_clnt *rpc = NULL;
+    int ret = -1;
+
+    ret = dict_set_nstrn(options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport-type", SLEN("transport-type"),
+                         "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport.socket.connect-path",
+                         SLEN("transport.socket.connect-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(rpc, cli_quotad_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
+    return rpc;
 }
-
diff --git a/cli/src/cli-quotad-client.h b/cli/src/cli-quotad-client.h
index aa0b42af38d..71a44e5916b 100644
--- a/cli/src/cli-quotad-client.h
+++ b/cli/src/cli-quotad-client.h
@@ -8,26 +8,22 @@
    cases as published by the Free Software Foundation.
 */
 #include "cli.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "cli-cmd.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include "cli-mem-types.h"
 
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc);
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options);
+cli_quotad_clnt_init(xlator_t *this, dict_t *options);
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data);
-
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data);
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index bca37d9c509..7a38a0b882a 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -17,7 +17,7 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "event.h"
+#include <glusterfs/gf-event.h>
 
 #include <fnmatch.h>
 
@@ -27,384 +27,376 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 
-
 int
-cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        printf ("\r%*s\r", (int)strlen (state->prompt), "");
+    printf("\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vprintf (fmt, ap);
+    ret = vprintf(fmt, ap);
 
-        printf ("\n");
-        fflush(stdout);
+    printf("\n");
+    fflush(stdout);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+    fprintf(stderr, "\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vfprintf (stderr, fmt, ap);
+    ret = vfprintf(stderr, fmt, ap);
 
-        fprintf (stderr, "\n");
-        fflush(stderr);
+    fprintf(stderr, "\n");
+    fflush(stderr);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-cli_rl_process_line (char *line)
+cli_rl_process_line(char *line)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
-
-        state = global_state;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state->rl_processing = 1;
-        {
-                ret = cli_cmd_process_line (state, line);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to process line");
+    state = global_state;
 
-                add_history (line);
-        }
-        state->rl_processing = 0;
+    state->rl_processing = 1;
+    {
+        ret = cli_cmd_process_line(state, line);
+        if (ret)
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to process line");
 
+        add_history(line);
+    }
+    state->rl_processing = 0;
 }
 
-
-int
-cli_rl_stdin (int fd, int idx, void *data,
-              int poll_out, int poll_in, int poll_err)
+void
+cli_rl_stdin(int fd, int idx, int gen, void *data, int poll_out, int poll_in,
+             int poll_err, char event_thread_died)
 {
-        rl_callback_read_char ();
+    struct cli_state *state = NULL;
 
-        return 0;
-}
+    state = data;
+
+    rl_callback_read_char();
 
+    gf_event_handled(state->ctx->event_pool, fd, idx, gen);
+
+    return;
+}
 
 char *
-cli_rl_autocomplete_entry (const char *text, int times)
+cli_rl_autocomplete_entry(const char *text, int times)
 {
-        struct cli_state  *state = NULL;
-        char              *retp = NULL;
+    struct cli_state *state = NULL;
+    char *retp = NULL;
 
-        state = global_state;
+    state = global_state;
 
-        if (!state->matchesp)
-                return NULL;
+    if (!state->matchesp)
+        return NULL;
 
-        retp = *state->matchesp;
+    retp = *state->matchesp;
 
-        state->matchesp++;
+    state->matchesp++;
 
-        return retp ? strdup (retp) : NULL;
+    return retp ? strdup(retp) : NULL;
 }
 
-
 int
-cli_rl_token_count (const char *text)
+cli_rl_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        if (is_spc)
-                /* what needs to be autocompleted is a full
-                   new word, and not extend the last word
-                */
-                count++;
+    if (is_spc)
+        /* what needs to be autocompleted is a full
+           new word, and not extend the last word
+        */
+        count++;
 
-        return count;
+    return count;
 }
 
-
 char **
-cli_rl_tokenize (const char *text)
+cli_rl_tokenize(const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-
-        count = cli_rl_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return NULL;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    count = cli_rl_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return NULL;
 
-        if (i < count) {
-                /* symbolize that what needs to be autocompleted is
-                   the full set of possible nextwords, and not extend
-                   the last word
-                */
-                *tokenp = strdup ("");
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-        }
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    if (i < count) {
+        /* symbolize that what needs to be autocompleted is
+           the full set of possible nextwords, and not extend
+           the last word
+        */
+        *tokenp = strdup("");
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
 
 out:
-        free (copy);
+    free(copy);
 
-        if (i < count) {
-                cli_cmd_tokens_destroy (tokens);
-                tokens = NULL;
-        }
+    if (i < count) {
+        cli_cmd_tokens_destroy(tokens);
+        tokens = NULL;
+    }
 
-        return tokens;
+    return tokens;
 }
 
-
 char **
-cli_rl_get_matches (struct cli_state *state, struct cli_cmd_word *word,
-                    const char *text)
+cli_rl_get_matches(struct cli_state *state, struct cli_cmd_word *word,
+                   const char *text)
 {
-        char                 **matches = NULL;
-        char                 **matchesp = NULL;
-        struct cli_cmd_word  **next = NULL;
-        int                    count = 0;
-        int                    len = 0;
+    char **matches = NULL;
+    char **matchesp = NULL;
+    struct cli_cmd_word **next = NULL;
+    int count = 0;
+    int len = 0;
 
-        len = strlen (text);
+    len = strlen(text);
 
-        if (!word->nextwords)
-                return NULL;
+    if (!word->nextwords)
+        return NULL;
 
-        for (next = word->nextwords; *next; next++)
-                count++;
+    for (next = word->nextwords; *next; next++)
+        count++;
 
-        matches = calloc (count + 1, sizeof (*matches));
-        matchesp = matches;
+    matches = calloc(count + 1, sizeof(*matches));
+    matchesp = matches;
 
-        for (next = word->nextwords; *next; next++) {
-                if ((*next)->match) {
-                        continue;
-                }
+    for (next = word->nextwords; *next; next++) {
+        if ((*next)->match) {
+            continue;
+        }
 
-                if (strncmp ((*next)->word, text, len) == 0) {
-                        *matchesp = strdup ((*next)->word);
-                        matchesp++;
-                }
+        if (strncmp((*next)->word, text, len) == 0) {
+            *matchesp = strdup((*next)->word);
+            matchesp++;
         }
+    }
 
-        return matches;
+    return matches;
 }
 
-
 int
-cli_rl_autocomplete_prepare (struct cli_state *state, const char *text)
+cli_rl_autocomplete_prepare(struct cli_state *state, const char *text)
 {
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-        char                 **tokens = NULL;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        char                 **matches = NULL;
-
-        tokens = cli_rl_tokenize (text);
-        if (!tokens)
-                return 0;
-
-        word = &state->tree.root;
-
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!*(tokenp+1)) {
-                        /* last word */
-                        break;
-                }
-
-                next = cli_cmd_nextword (word, token);
-                word = next;
-                if (!word)
-                        break;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char **matches = NULL;
+
+    tokens = cli_rl_tokenize(text);
+    if (!tokens)
+        return 0;
+
+    word = &state->tree.root;
+
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!*(tokenp + 1)) {
+            /* last word */
+            break;
         }
 
+        next = cli_cmd_nextword(word, token);
+        word = next;
         if (!word)
-                goto out;
+            break;
+    }
+
+    if (!word || !token)
+        goto out;
 
-        matches = cli_rl_get_matches (state, word, token);
+    matches = cli_rl_get_matches(state, word, token);
 
-        state->matches = matches;
-        state->matchesp = matches;
+    state->matches = matches;
+    state->matchesp = matches;
 
 out:
-        cli_cmd_tokens_destroy (tokens);
-        return 0;
+    cli_cmd_tokens_destroy(tokens);
+    return 0;
 }
 
-
 int
-cli_rl_autocomplete_cleanup (struct cli_state *state)
+cli_rl_autocomplete_cleanup(struct cli_state *state)
 {
-        if (state->matches)
-                cli_cmd_tokens_destroy (state->matches);
+    if (state->matches)
+        cli_cmd_tokens_destroy(state->matches);
 
-        state->matches = NULL;
-        state->matchesp = NULL;
+    state->matches = NULL;
+    state->matchesp = NULL;
 
-        return 0;
+    return 0;
 }
 
-
 char **
-cli_rl_autocomplete (const char *text, int start, int end)
+cli_rl_autocomplete(const char *text, int start, int end)
 {
-        struct cli_state  *state = NULL;
-        char             **matches = NULL;
-        char               save = 0;
+    struct cli_state *state = NULL;
+    char **matches = NULL;
+    char save = 0;
 
-        state = global_state;
+    state = global_state;
 
-        /* hack to make the autocompletion code neater */
-        /* fake it as though the cursor is at the end of line */
+    /* hack to make the autocompletion code neater */
+    /* fake it as though the cursor is at the end of line */
 
-        save = rl_line_buffer[rl_point];
-        rl_line_buffer[rl_point] = 0;
+    save = rl_line_buffer[rl_point];
+    rl_line_buffer[rl_point] = 0;
 
-        cli_rl_autocomplete_prepare (state, rl_line_buffer);
+    cli_rl_autocomplete_prepare(state, rl_line_buffer);
 
-        matches = rl_completion_matches (text, cli_rl_autocomplete_entry);
+    matches = rl_completion_matches(text, cli_rl_autocomplete_entry);
 
-        cli_rl_autocomplete_cleanup (state);
+    cli_rl_autocomplete_cleanup(state);
 
-        rl_line_buffer[rl_point] = save;
+    rl_line_buffer[rl_point] = save;
 
-        return matches;
+    return matches;
 }
 
-
 static char *
-complete_none (const char *txt, int times)
+complete_none(const char *txt, int times)
 {
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_rl_input (void *_data)
+cli_rl_input(void *_data)
 {
-        struct cli_state *state = NULL;
-        char             *line = NULL;
+    struct cli_state *state = NULL;
+    char *line = NULL;
 
-        state = _data;
+    state = _data;
 
-        for (;;) {
-                line = readline (state->prompt);
-                if (!line)
-                        exit(0);  //break;
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        line = readline(state->prompt);
+        if (!line)
+            exit(0);  // break;
 
-                if (*line)
-                        cli_rl_process_line (line);
+        if (*line)
+            cli_rl_process_line(line);
 
-                free (line);
-        }
+        free(line);
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        int ret = 0;
-
-        rl_pre_input_hook = NULL;
-        rl_attempted_completion_function = cli_rl_autocomplete;
-        rl_completion_entry_function = complete_none;
-
-        if (!state->rl_async) {
-                ret = pthread_create (&state->input, NULL,
-                                      cli_rl_input, state);
-                if (ret == 0)
-                        state->rl_enabled = 1;
-                goto out;
-        }
+    int ret = 0;
 
-        ret = event_register (state->ctx->event_pool, 0, cli_rl_stdin, state,
-                              1, 0);
-        if (ret == -1)
-                goto out;
+    rl_pre_input_hook = NULL;
+    rl_attempted_completion_function = cli_rl_autocomplete;
+    rl_completion_entry_function = complete_none;
 
-        state->rl_enabled = 1;
-        rl_callback_handler_install (state->prompt, cli_rl_process_line);
+    if (!state->rl_async) {
+        ret = pthread_create(&state->input, NULL, cli_rl_input, state);
+        if (ret == 0)
+            state->rl_enabled = 1;
+        goto out;
+    }
+
+    ret = gf_event_register(state->ctx->event_pool, 0, cli_rl_stdin, state, 1,
+                            0, 0);
+    if (ret == -1)
+        goto out;
+
+    state->rl_enabled = 1;
+    rl_callback_handler_install(state->prompt, cli_rl_process_line);
 
 out:
-        return state->rl_enabled;
+    return state->rl_enabled;
 }
 
 #else /* HAVE_READLINE */
 
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        return 0;
+    return 0;
 }
 
 #endif /* HAVE_READLINE */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index f217e13abdd..9b6b0c7fa50 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -14,54 +14,49 @@
  */
 #define VOL_TOP_PERF_FILENAME_DEF_WIDTH 47
 #define VOL_TOP_PERF_FILENAME_ALT_WIDTH 44
-#define VOL_TOP_PERF_SPEED_WIDTH        4
-#define VOL_TOP_PERF_TIME_WIDTH         26
+#define VOL_TOP_PERF_SPEED_WIDTH 4
+#define VOL_TOP_PERF_TIME_WIDTH 26
 
 #define INDENT_MAIN_HEAD "%-25s %s "
 
+#define RETURNING "Returning %d"
+#define XML_ERROR "Error outputting to xml"
+#define XDR_DECODE_FAIL "Failed to decode xdr response"
+#define DICT_SERIALIZE_FAIL "Failed to serialize to data to dictionary"
+#define DICT_UNSERIALIZE_FAIL "Failed to unserialize the dictionary"
+
+/* Do not show estimates if greater than this number */
+#define REBAL_ESTIMATE_SEC_UPPER_LIMIT (60 * 24 * 3600)
+#define REBAL_ESTIMATE_START_TIME 600
+
 #include "cli.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "cli-cmd.h"
 #include <sys/uio.h>
 #include <stdlib.h>
 #include <sys/mount.h>
-#include "cli1-xdr.h"
-#include "xdr-generic.h"
-#include "protocol-common.h"
+#include <glusterfs/compat.h>
 #include "cli-mem-types.h"
-#include "compat.h"
-#include "upcall-utils.h"
-
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
-#include "cli-quotad-client.h"
-#include "run.h"
-#include "quota-common-utils.h"
-#include "events.h"
+#include <glusterfs/run.h>
+#include <glusterfs/events.h>
 
-enum gf_task_types {
-        GF_TASK_TYPE_REBALANCE,
-        GF_TASK_TYPE_REMOVE_BRICK
-};
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
 
-extern struct rpc_clnt *global_quotad_rpc;
-extern rpc_clnt_prog_t cli_quotad_clnt;
-extern rpc_clnt_prog_t *cli_rpc_prog;
-extern int              cli_op_ret;
-extern int              connected;
-
-int32_t
-gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
-                     void *data);
+rpc_clnt_prog_t cli_quotad_clnt;
 
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data);
 
-char *cli_vol_status_str[] = {"Created",
-                              "Started",
-                              "Stopped",
-                             };
+char *cli_vol_status_str[] = {
+    "Created",
+    "Started",
+    "Stopped",
+};
 
 char *cli_vol_task_status_str[] = {"not started",
                                    "in progress",
@@ -72,9520 +67,8582 @@ char *cli_vol_task_status_str[] = {"not started",
                                    "fix-layout stopped",
                                    "fix-layout completed",
                                    "fix-layout failed",
-                                   "unknown"
-};
+                                   "unknown"};
 
-int32_t
-gf_cli_snapshot (call_frame_t *frame, xlator_t *this, void *data);
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data);
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data);
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data);
 
-int
-cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
-                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
-                 rpc_clnt_prog_t *prog, struct iobref *iobref);
-
-rpc_clnt_prog_t cli_handshake_prog = {
-        .progname  = "cli handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref);
+
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict);
+
+static rpc_clnt_prog_t cli_handshake_prog = {
+    .progname = "cli handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
 };
 
-rpc_clnt_prog_t cli_pmap_prog = {
-        .progname   = "cli portmap",
-        .prognum    = GLUSTER_PMAP_PROGRAM,
-        .progver    = GLUSTER_PMAP_VERSION,
+static rpc_clnt_prog_t cli_pmap_prog = {
+    .progname = "cli portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
 };
 
-int
-gf_cli_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static void
+gf_free_xdr_cli_rsp(gf_cli_rsp rsp)
 {
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char                  msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (rsp.dict.dict_val) {
+        free(rsp.dict.dict_val);
+    }
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+}
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+static void
+gf_free_xdr_getspec_rsp(gf_getspec_rsp rsp)
+{
+    if (rsp.spec) {
+        free(rsp.spec);
+    }
+    if (rsp.xdata.xdata_val) {
+        free(rsp.xdata.xdata_val);
+    }
+}
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
+static void
+gf_free_xdr_fsm_log_rsp(gf1_cli_fsm_log_rsp rsp)
+{
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+    if (rsp.fsm_log.fsm_log_val) {
+        free(rsp.fsm_log.fsm_log_val);
+    }
+}
 
-        if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                if (rsp.op_ret)
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
+static int
+gf_cli_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+
+    if (rsp.op_errstr && rsp.op_errstr[0] != '\0') {
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        if (rsp.op_ret) {
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
         }
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer probe: success. %s", msg);
-        else
-                cli_err ("peer probe: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer probe: %s", msg);
+    else
+        cli_err("peer probe: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char              msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to deprobe");
-
-        if (rsp.op_ret) {
-                if (strlen (rsp.op_errstr) > 0) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                        gf_log ("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
-                }
-        } else {
-                snprintf (msg, sizeof (msg), "success");
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_deprobe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to deprobe");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr[0] != '\0') {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            gf_log("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer detach: %s", msg);
-        else
-                cli_err ("peer detach: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer detach: %s", msg);
+    else
+        cli_err("peer detach: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_output_peer_hostnames (dict_t *dict, int count, char *prefix)
-{
-        int ret = -1;
-        char key[256] = {0,};
-        int i = 0;
-        char *hostname = NULL;
-
-        cli_out ("Other names:");
-        /* Starting from friend.hostname1, as friend.hostname0 will be the same
-         * as friend.hostname
-         */
-        for (i = 1; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        break;
-                cli_out ("%s", hostname);
-                hostname = NULL;
-        }
+static int
+gf_cli_output_peer_hostnames(dict_t *dict, int count, const char *prefix)
+{
+    int ret = -1;
+    char key[512] = {
+        0,
+    };
+    int i = 0;
+    char *hostname = NULL;
+
+    cli_out("Other names:");
+    /* Starting from friend.hostname1, as friend.hostname0 will be the same
+     * as friend.hostname
+     */
+    for (i = 1; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
+        if (ret)
+            break;
+        cli_out("%s", hostname);
+        hostname = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_peer_status (dict_t *dict, int count)
-{
-        int                        ret   = -1;
-        char                       *uuid_buf = NULL;
-        char                       *hostname_buf = NULL;
-        int32_t                    i = 1;
-        char                       key[256] = {0,};
-        char                       *state = NULL;
-        int32_t                    connected = 0;
-        char                       *connected_str = NULL;
-        int                        hostname_count = 0;
-
-        cli_out ("Number of Peers: %d", count);
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
-
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+static int
+gf_cli_output_peer_status(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t i = 1;
+    char key[256] = {
+        0,
+    };
+    int keylen;
+    char *state = NULL;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+    int hostname_count = 0;
+
+    cli_out("Number of Peers: %d", count);
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                snprintf (key, 256, "friend%d.state", i);
-                ret = dict_get_str (dict, key, &state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_strn(dict, key, keylen, &state);
+        if (ret)
+            goto out;
 
-                cli_out ("\nHostname: %s\nUuid: %s\nState: %s (%s)",
-                         hostname_buf, uuid_buf, state, connected_str);
+        cli_out("\nHostname: %s\nUuid: %s\nState: %s (%s)", hostname_buf,
+                uuid_buf, state, connected_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname_count", i);
-                ret = dict_get_int32 (dict, key, &hostname_count);
-                /* Print other addresses only if there are more than 1.
-                 */
-                if ((ret == 0) && (hostname_count > 1)) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "friend%d", i);
-                        ret = gf_cli_output_peer_hostnames (dict,
-                                                            hostname_count,
-                                                            key);
-                }
-                i++;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &hostname_count);
+        /* Print other addresses only if there are more than 1.
+         */
+        if ((ret == 0) && (hostname_count > 1)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = gf_cli_output_peer_hostnames(dict, hostname_count, key);
+            if (ret) {
+                gf_log("cli", GF_LOG_WARNING,
+                       "error outputting peer other names");
+                goto out;
+            }
         }
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_pool_list (dict_t *dict, int count)
-{
-        int                         ret           = -1;
-        char                       *uuid_buf      = NULL;
-        char                       *hostname_buf  = NULL;
-        int32_t                     hostname_len  = 8; /*min len 8 chars*/
-        int32_t                     i             = 1;
-        char                        key[256]      = {0,};
-        int32_t                     connected     = 0;
-        char                       *connected_str = NULL;
-
-        if (count <= 0)
-                goto out;
+static int
+gf_cli_output_pool_list(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t hostname_len = 8; /*min len 8 chars*/
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+
+    if (count <= 0)
+        goto out;
+
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-        while (i <= count) {
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+        ret = strlen(hostname_buf);
+        if (ret > hostname_len)
+            hostname_len = ret;
 
-                ret = strlen(hostname_buf);
-                if (ret > hostname_len)
-                        hostname_len = ret;
+        i++;
+    }
 
-                i++;
-        }
+    cli_out("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
 
-        cli_out ("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
-
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                cli_out ("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
-                         connected_str);
-                i++;
-        }
+        cli_out("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
+                connected_str);
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* function pointer for gf_cli_output_{pool_list,peer_status} */
-typedef int (*cli_friend_output_fn) (dict_t*, int);
+typedef int (*cli_friend_output_fn)(dict_t *, int);
 
-int
-gf_cli_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_list_friends_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gf1_cli_peer_list_rsp      rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                    *dict = NULL;
-        char                       msg[1024] = {0,};
-        char                      *cmd = NULL;
-        cli_friend_output_fn       friend_output_fn;
-        call_frame_t              *frame = NULL;
-        unsigned long              flags = 0;
-
-        GF_ASSERT (myframe);
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    const char *cmd = NULL;
+    cli_friend_output_fn friend_output_fn;
+    call_frame_t *frame = NULL;
+    unsigned long flags = 0;
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        flags = (long)frame->local;
+    GF_ASSERT(myframe);
 
-        if (flags == GF_CLI_LIST_POOL_NODES) {
-                cmd = "pool list";
-                friend_output_fn = &gf_cli_output_pool_list;
-        } else {
-                cmd = "peer status";
-                friend_output_fn = &gf_cli_output_peer_status;
-        }
+    frame = myframe;
 
-        /* 'free' the flags set by gf_cli_list_friends */
-        frame->local = NULL;
+    flags = (long)frame->local;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        cmd = "pool list";
+        friend_output_fn = &gf_cli_output_pool_list;
+    } else {
+        cmd = "peer status";
+        friend_output_fn = &gf_cli_output_peer_status;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+    /* 'free' the flags set by gf_cli_list_friends */
+    frame->local = NULL;
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to list: %d",
-                rsp.op_ret);
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = rsp.op_ret;
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to list: %d", rsp.op_ret);
 
-        if (!rsp.op_ret) {
-
-                if (!rsp.friends.friends_len) {
-                        snprintf (msg, sizeof (msg),
-                                  "%s: No peers present", cmd);
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                ret = cli_xml_output_peer_status (dict,
-                                                                  rsp.op_ret,
-                                                                  rsp.op_errno,
-                                                                  msg);
-                                if (ret)
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Error outputting to xml");
-                                goto out;
-                        }
-                        cli_err ("%s", msg);
-                        ret = 0;
-                        goto out;
-                }
+    if (!rsp.op_ret) {
+        if (!rsp.friends.friends_len) {
+            snprintf(msg, sizeof(msg), "%s: No peers present", cmd);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                                 msg);
+                if (ret)
+                    gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+            cli_err("%s", msg);
+            ret = 0;
+            goto out;
+        }
 
-                dict = dict_new ();
+        dict = dict_new();
 
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_unserialize (rsp.friends.friends_val,
-                                        rsp.friends.friends_len,
-                                        &dict);
+        ret = dict_unserialize(rsp.friends.friends_val, rsp.friends.friends_len,
+                               &dict);
 
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                        goto out;
-                }
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
 
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, msg);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        goto out;
-                }
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             msg);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
 
-                ret = dict_get_int32 (dict, "count", &count);
-                if (ret) {
-                        goto out;
-                }
+        ret = dict_get_int32_sizen(dict, "count", &count);
+        if (ret) {
+            goto out;
+        }
 
-                ret = friend_output_fn (dict, count);
-                if (ret) {
-                        goto out;
-                }
+        ret = friend_output_fn(dict, count);
+        if (ret) {
+            goto out;
+        }
+    } else {
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             NULL);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         } else {
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, NULL);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                } else {
-                        ret = -1;
-                }
-                goto out;
+            ret = -1;
         }
+        goto out;
+    }
 
-
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret)
-                cli_err ("%s: failed", cmd);
+    if (ret)
+        cli_err("%s: failed", cmd);
 
-        cli_cmd_broadcast_response (ret);
+    cli_cmd_broadcast_response(ret);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    if (rsp.friends.friends_val) {
+        free(rsp.friends.friends_val);
+    }
+    return ret;
 }
 
-int
-gf_cli_get_state_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
-{
-        gf_cli_rsp           rsp            = {0,};
-        int                  ret            = -1;
-        dict_t               *dict          = NULL;
-        char                 *daemon_name   = NULL;
-        char                 *ofilepath     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("cli", myframe, out);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+static int
+gf_cli_get_state_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *daemon_name = NULL;
+    char *ofilepath = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", myframe, out);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Failed to get daemon state: %s", rsp.op_errstr);
+        else
+            cli_err(
+                "Failed to get daemon state. Check glusterd"
+                " log file for more details");
+    } else {
+        ret = dict_get_str_sizen(dict, "daemon", &daemon_name);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get daemon name");
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("Failed to get daemon state: %s", rsp.op_errstr);
-                else
-                        cli_err ("Failed to get daemon state. Check glusterd"
-                                 " log file for more details");
-        } else {
-                ret = dict_get_str (dict, "daemon", &daemon_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get daemon name");
+        ret = dict_get_str_sizen(dict, "ofilepath", &ofilepath);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get filepath");
 
-                ret = dict_get_str (dict, "ofilepath", &ofilepath);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get filepath");
+        if (daemon_name && ofilepath)
+            cli_out("%s state dumped to %s", daemon_name, ofilepath);
+    }
 
-                if (daemon_name && ofilepath)
-                        cli_out ("%s state dumped to %s",
-                                 daemon_name, ofilepath);
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        cli_cmd_broadcast_response (ret);
-
-        return ret;
+    return ret;
 }
 
-void
-cli_out_options ( char *substr, char *optstr, char *valstr)
+static void
+cli_out_options(char *substr, char *optstr, char *valstr)
 {
-        char                    *ptr1 = NULL;
-        char                    *ptr2 = NULL;
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
 
-        ptr1 = substr;
-        ptr2 = optstr;
+    ptr1 = substr;
+    ptr2 = optstr;
 
-        while (ptr1)
-        {
- /* Avoiding segmentation fault. */
-                if (!ptr2)
-                        return;
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
-        }
+    while (ptr1) {
+        /* Avoiding segmentation fault. */
+        if (!ptr2)
+            return;
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+    }
 
-        if (*ptr2 == '\0')
-                return;
-        cli_out ("%s: %s",ptr2 , valstr);
+    if (*ptr2 == '\0')
+        return;
+    cli_out("%s: %s", ptr2, valstr);
 }
 
 static int
-_gf_cli_output_volinfo_opts (dict_t *d, char *k,
-                             data_t *v, void *tmp)
-{
-        int     ret   = 0;
-        char   *key   = NULL;
-        char   *ptr   = NULL;
-        data_t *value = NULL;
-
-        key = tmp;
-
-        ptr = strstr (k, "option.");
-        if (ptr) {
-                value = v;
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                cli_out_options (key, k, v->data);
+_gf_cli_output_volinfo_opts(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    int ret = 0;
+    char *key = NULL;
+    char *ptr = NULL;
+    data_t *value = NULL;
+
+    key = tmp;
+
+    ptr = strstr(k, "option.");
+    if (ptr) {
+        value = v;
+        if (!value) {
+            ret = -1;
+            goto out;
         }
+        cli_out_options(key, k, v->data);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-print_brick_details (dict_t *dict, int volcount, int start_index,
-                     int end_index, int replica_count)
-{
-        char           key[1024]   = {0,};
-        int            index       = start_index;
-        int            isArbiter   = 0;
-        int            ret         = -1;
-        char          *brick       = NULL;
-#ifdef HAVE_BD_XLATOR
-        char          *caps        = NULL;
-#endif
-
-        while (index <= end_index) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, 1024, "volume%d.brick%d", volcount, index);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof(key));
-                snprintf (key, sizeof (key), "volume%d.brick%d.isArbiter",
+print_brick_details(dict_t *dict, int volcount, int start_index, int end_index,
+                    int replica_count)
+{
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int index = start_index;
+    int isArbiter = 0;
+    int ret = -1;
+    char *brick = NULL;
+
+    while (index <= end_index) {
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", volcount,
+                          index);
+        ret = dict_get_strn(dict, key, keylen, &brick);
+        if (ret)
+            goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter",
                           volcount, index);
-                if (dict_get (dict, key))
-                        isArbiter = 1;
-                else
-                        isArbiter = 0;
+        if (dict_getn(dict, key, keylen))
+            isArbiter = 1;
+        else
+            isArbiter = 0;
 
-                if (isArbiter)
-                        cli_out ("Brick%d: %s (arbiter)", index, brick);
-                else
-                        cli_out ("Brick%d: %s", index, brick);
-#ifdef HAVE_BD_XLATOR
-                snprintf (key, 1024, "volume%d.vg%d", volcount, index);
-                ret = dict_get_str (dict, key, &caps);
-                if (!ret)
-                        cli_out ("Brick%d VG: %s", index, caps);
-#endif
-                index++;
-        }
-        ret = 0;
+        if (isArbiter)
+            cli_out("Brick%d: %s (arbiter)", index, brick);
+        else
+            cli_out("Brick%d: %s", index, brick);
+        index++;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
-void
-gf_cli_print_number_of_bricks (int type, int brick_count, int dist_count,
-                               int stripe_count, int replica_count,
-                               int disperse_count, int redundancy_count,
-                               int arbiter_count)
-{
-       if (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
-               if (arbiter_count == 0) {
-                       cli_out ("Number of Bricks: %d x %d x %d = %d",
-                                (brick_count / dist_count),
-                                stripe_count,
-                                replica_count,
-                                brick_count);
-               } else {
-                       cli_out ("Number of Bricks: %d x %d x (%d + %d) = %d",
-                                (brick_count / dist_count),
-                                stripe_count, replica_count - arbiter_count,
-                                arbiter_count, brick_count);
-               }
-       } else if (type == GF_CLUSTER_TYPE_NONE ||
-                  type == GF_CLUSTER_TYPE_TIER) {
-               cli_out ("Number of Bricks: %d", brick_count);
-       } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
-               cli_out ("Number of Bricks: %d x (%d + %d) = %d",
-                        (brick_count / dist_count),
-                        disperse_count - redundancy_count,
-                        redundancy_count, brick_count);
-       } else {
-               /* For both replicate and stripe, dist_count is
-                  good enough */
-               if (arbiter_count == 0) {
-                       cli_out ("Number of Bricks: %d x %d = %d",
-                                (brick_count / dist_count),
-                                dist_count, brick_count);
-               } else {
-                       cli_out ("Number of Bricks: %d x (%d + %d) = %d",
-                                (brick_count / dist_count),
-                                dist_count - arbiter_count, arbiter_count,
-                                brick_count);
-               }
-       }
 
+static void
+gf_cli_print_number_of_bricks(int type, int brick_count, int dist_count,
+                              int stripe_count, int replica_count,
+                              int disperse_count, int redundancy_count,
+                              int arbiter_count)
+{
+    if (type == GF_CLUSTER_TYPE_NONE) {
+        cli_out("Number of Bricks: %d", brick_count);
+    } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                (brick_count / dist_count), disperse_count - redundancy_count,
+                redundancy_count, brick_count);
+    } else {
+        /* For both replicate and stripe, dist_count is
+           good enough */
+        if (arbiter_count == 0) {
+            cli_out("Number of Bricks: %d x %d = %d",
+                    (brick_count / dist_count), dist_count, brick_count);
+        } else {
+            cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                    (brick_count / dist_count), dist_count - arbiter_count,
+                    arbiter_count, brick_count);
+        }
+    }
 }
 
-int
-gf_cli_print_tier_info (dict_t *dict, int i, int brick_count)
-{
-
-        int                  hot_brick_count        = -1;
-        int                  cold_type              = 0;
-        int                  cold_brick_count       = 0;
-        int                  cold_replica_count     = 0;
-        int                  cold_arbiter_count     = 0;
-        int                  cold_disperse_count    = 0;
-        int                  cold_redundancy_count  = 0;
-        int                  cold_dist_count        = 0;
-        int                  hot_type               = 0;
-        int                  hot_replica_count      = 0;
-        int                  hot_dist_count         = 0;
-        int                  ret                    = -1;
-        int                  vol_type               = -1;
-        char                 key[256]               = {0,};
-
-        GF_ASSERT (dict);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_brick_count", i);
-        ret = dict_get_int32 (dict, key, &cold_brick_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_type", i);
-        ret = dict_get_int32 (dict, key, &cold_type);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_dist_count", i);
-        ret = dict_get_int32 (dict, key, &cold_dist_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_replica_count", i);
-        ret = dict_get_int32 (dict, key, &cold_replica_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_arbiter_count", i);
-        ret = dict_get_int32 (dict, key, &cold_arbiter_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_disperse_count", i);
-        ret = dict_get_int32 (dict, key, &cold_disperse_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256,
-                  "volume%d.cold_redundancy_count", i);
-        ret = dict_get_int32 (dict, key,
-                              &cold_redundancy_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_brick_count", i);
-        ret = dict_get_int32 (dict, key, &hot_brick_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_type", i);
-        ret = dict_get_int32 (dict, key, &hot_type);
-        if (ret)
-                goto out;
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_replica_count", i);
-        ret = dict_get_int32 (dict, key, &hot_replica_count);
-        if (ret)
-                goto out;
-
-        cli_out ("Hot Tier :");
-        vol_type = hot_type;
-        hot_dist_count = (hot_replica_count ?
-                          hot_replica_count : 1);
-
-        vol_type = get_vol_type (hot_type, hot_dist_count, hot_brick_count);
-        cli_out ("Hot Tier Type : %s",
-                 vol_type_str[vol_type]);
+static int
+gf_cli_get_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    int ret = -1;
+    int opt_count = 0;
+    int32_t i = 0;
+    int32_t j = 1;
+    int32_t status = 0;
+    int32_t type = 0;
+    int32_t brick_count = 0;
+    int32_t dist_count = 0;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t disperse_count = 0;
+    int32_t redundancy_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t snap_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t vol_type = 0;
+    int32_t transport = 0;
+    char *volume_id_str = NULL;
+    char *volname = NULL;
+    char *ta_brick = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    char key[64] = {0};
+    int keylen;
+    char err_str[2048] = {0};
+    gf_cli_rsp rsp = {0};
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to get vol: %d", rsp.op_ret);
+
+    if (!rsp.dict.dict_len) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
 
-        gf_cli_print_number_of_bricks (hot_type,
-                        hot_brick_count, hot_dist_count, 0,
-                        hot_replica_count, 0, 0, 0);
+    dict = dict_new();
 
-        ret = print_brick_details (dict, i, 1, hot_brick_count,
-                                   hot_replica_count);
-        if (ret)
-                goto out;
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        cli_out ("Cold Tier:");
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        vol_type = get_vol_type (cold_type, cold_dist_count, cold_brick_count);
-        cli_out ("Cold Tier Type : %s",
-                        vol_type_str[vol_type]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        gf_cli_print_number_of_bricks (cold_type,
-                cold_brick_count,
-                cold_dist_count, 0, cold_replica_count,
-                cold_disperse_count, cold_redundancy_count, cold_arbiter_count);
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret)
+        goto out;
 
-        ret = print_brick_details (dict, i, hot_brick_count+1,
-                                   brick_count, cold_replica_count);
-        if (ret)
-                goto out;
-out:
-        return ret;
-}
-
-int
-gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        int                        ret                  = -1;
-        int                        opt_count            = 0;
-        int32_t                    i                    = 0;
-        int32_t                    j                    = 1;
-        int32_t                    status               = 0;
-        int32_t                    type                 = 0;
-        int32_t                    brick_count          = 0;
-        int32_t                    dist_count           = 0;
-        int32_t                    stripe_count         = 0;
-        int32_t                    replica_count        = 0;
-        int32_t                    disperse_count       = 0;
-        int32_t                    redundancy_count     = 0;
-        int32_t                    arbiter_count        = 0;
-        int32_t                    snap_count           = 0;
-        int32_t                    vol_type             = 0;
-        int32_t                    transport            = 0;
-        char                      *volume_id_str        = NULL;
-        char                      *volname              = NULL;
-        dict_t                    *dict                 = NULL;
-        cli_local_t               *local                = NULL;
-        char                       key[1024]            = {0};
-        char                       err_str[2048]        = {0};
-        gf_cli_rsp                 rsp                  = {0};
-        char                      *caps                 = NULL;
-        int                        k __attribute__((unused)) = 0;
-        call_frame_t              *frame                = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
+    if (!count) {
+        switch (local->get_vol.flags) {
+            case GF_CLI_GET_NEXT_VOLUME:
+                GF_FREE(local->get_vol.volname);
+                local->get_vol.volname = NULL;
+                ret = 0;
                 goto out;
 
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            case GF_CLI_GET_VOLUME:
+                snprintf(err_str, sizeof(err_str), "Volume %s does not exist",
+                         local->get_vol.volname);
+                ret = -1;
+                if (!(global_state->mode & GLUSTER_MODE_XML))
+                    goto out;
         }
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to get vol: %d",
-                rsp.op_ret);
+    if (rsp.op_ret) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        ret = -1;
+        goto out;
+    }
 
-        if (!rsp.dict.dict_len) {
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-                cli_err ("No volumes present");
-                ret = 0;
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* For GET_NEXT_VOLUME output is already begun in
+         * and will also end in gf_cli_get_next_volume()
+         */
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_begin(local, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
 
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
+        if (dict) {
+            ret = cli_xml_output_vol_info(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unable to allocate memory");
-                goto out;
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_end(local);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
+    while (i < count) {
+        cli_out(" ");
+        keylen = snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
         if (ret)
-                goto out;
-
-        if (!count) {
-                switch (local->get_vol.flags) {
-
-                case GF_CLI_GET_NEXT_VOLUME:
-                        GF_FREE (local->get_vol.volname);
-                        local->get_vol.volname = NULL;
-                        ret = 0;
-                        goto out;
-
-                case GF_CLI_GET_VOLUME:
-                        memset (err_str, 0, sizeof (err_str));
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s does not exist",
-                                  local->get_vol.volname);
-                        ret = -1;
-                        if (!(global_state->mode & GLUSTER_MODE_XML))
-                                goto out;
-                }
-        }
+            goto out;
 
-        if (rsp.op_ret) {
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-                ret = -1;
-                goto out;
-        }
+        keylen = snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32n(dict, key, keylen, &type);
+        if (ret)
+            goto out;
 
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                /* For GET_NEXT_VOLUME output is already begun in
-                 * and will also end in gf_cli_get_next_volume()
-                 */
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_begin
-                                (local, rsp.op_ret, rsp.op_errno,
-                                 rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
+        keylen = snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32n(dict, key, keylen, &status);
+        if (ret)
+            goto out;
 
-                if (dict) {
-                        ret = cli_xml_output_vol_info (local, dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
+        keylen = snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &brick_count);
+        if (ret)
+            goto out;
 
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_end (local);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                }
-                goto out;
-        }
+        keylen = snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &dist_count);
+        if (ret)
+            goto out;
 
-        while ( i < count) {
-                cli_out (" ");
-                snprintf (key, 256, "volume%d.name", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &stripe_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &replica_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &disperse_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.brick_count", i);
-                ret = dict_get_int32 (dict, key, &brick_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &redundancy_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.dist_count", i);
-                ret = dict_get_int32 (dict, key, &dist_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &arbiter_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.stripe_count", i);
-                ret = dict_get_int32 (dict, key, &stripe_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32n(dict, key, keylen, &transport);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.replica_count", i);
-                ret = dict_get_int32 (dict, key, &replica_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_strn(dict, key, keylen, &volume_id_str);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.disperse_count", i);
-                ret = dict_get_int32 (dict, key, &disperse_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &snap_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.redundancy_count", i);
-                ret = dict_get_int32 (dict, key, &redundancy_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &thin_arbiter_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, sizeof(key), "volume%d.arbiter_count", i);
-                ret = dict_get_int32 (dict, key, &arbiter_count);
-                if (ret)
-                        goto out;
+        // Distributed (stripe/replicate/stripe-replica) setups
+        vol_type = get_vol_type(type, dist_count, brick_count);
 
-                snprintf (key, 256, "volume%d.transport", i);
-                ret = dict_get_int32 (dict, key, &transport);
-                if (ret)
-                        goto out;
+        cli_out("Volume Name: %s", volname);
+        cli_out("Type: %s", vol_type_str[vol_type]);
+        cli_out("Volume ID: %s", volume_id_str);
+        cli_out("Status: %s", cli_vol_status_str[status]);
+        cli_out("Snapshot Count: %d", snap_count);
 
-                snprintf (key, 256, "volume%d.volume_id", i);
-                ret = dict_get_str (dict, key, &volume_id_str);
-                if (ret)
-                        goto out;
+        gf_cli_print_number_of_bricks(
+            type, brick_count, dist_count, stripe_count, replica_count,
+            disperse_count, redundancy_count, arbiter_count);
 
-                snprintf (key, 256, "volume%d.snap_count", i);
-                ret = dict_get_int32 (dict, key, &snap_count);
-                if (ret)
-                        goto out;
-
-                // Distributed (stripe/replicate/stripe-replica) setups
-                vol_type = get_vol_type (type, dist_count, brick_count);
-
-                cli_out ("Volume Name: %s", volname);
-                cli_out ("Type: %s", vol_type_str[vol_type]);
-                cli_out ("Volume ID: %s", volume_id_str);
-                cli_out ("Status: %s", cli_vol_status_str[status]);
-                cli_out ("Snapshot Count: %d", snap_count);
-
-#ifdef HAVE_BD_XLATOR
-                k = 0;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.xlator%d", i, k);
-                ret = dict_get_str (dict, key, &caps);
-                if (ret)
-                        goto next;
-                do {
-                        j = 0;
-                        cli_out ("Xlator %d: %s", k + 1, caps);
-                        do {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.xlator%d.caps%d",
-                                          i, k, j++);
-                                ret = dict_get_str (dict, key, &caps);
-                                if (ret)
-                                        break;
-                                cli_out ("Capability %d: %s", j, caps);
-                        } while (1);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "volume%d.xlator%d", i, ++k);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (ret)
-                                break;
-                } while (1);
-
-next:
-#else
-                caps = 0; /* Avoid compiler warnings when BD not enabled */
-#endif
-                gf_cli_print_number_of_bricks (type, brick_count,
-                                dist_count, stripe_count, replica_count,
-                                disperse_count, redundancy_count,
-                                arbiter_count);
-
-                cli_out ("Transport-type: %s",
-                         ((transport == 0)?"tcp":
-                          (transport == 1)?"rdma":
-                          "tcp,rdma"));
-                j = 1;
-
-                GF_FREE (local->get_vol.volname);
-                local->get_vol.volname = gf_strdup (volname);
-
-                if (type == GF_CLUSTER_TYPE_TIER) {
-                        ret = gf_cli_print_tier_info (dict, i, brick_count);
-                        if (ret)
-                                goto out;
-
-                } else {
-                        cli_out ("Bricks:");
-                        ret = print_brick_details (dict, i, j, brick_count,
-                                                  replica_count);
-                        if (ret)
-                                goto out;
-                }
+        cli_out("Transport-type: %s",
+                ((transport == 0) ? "tcp"
+                                  : (transport == 1) ? "rdma" : "tcp,rdma"));
+        j = 1;
 
-                snprintf (key, 256, "volume%d.opt_count",i);
-                ret = dict_get_int32 (dict, key, &opt_count);
-                if (ret)
-                        goto out;
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
 
-                if (!opt_count)
-                        goto out;
+        cli_out("Bricks:");
+        ret = print_brick_details(dict, i, j, brick_count, replica_count);
+        if (ret)
+            goto out;
 
-                cli_out ("Options Reconfigured:");
+        if (thin_arbiter_count) {
+            snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick", i);
+            ret = dict_get_str(dict, key, &ta_brick);
+            if (ret)
+                goto out;
+            cli_out("Thin-arbiter-path: %s", ta_brick);
+        }
 
-                snprintf (key, 256, "volume%d.option.",i);
+        snprintf(key, sizeof(key), "volume%d.opt_count", i);
+        ret = dict_get_int32(dict, key, &opt_count);
+        if (ret)
+            goto out;
 
-                ret = dict_foreach (dict, _gf_cli_output_volinfo_opts, key);
-                if (ret)
-                        goto out;
+        if (!opt_count)
+            goto out;
 
-                i++;
-        }
+        cli_out("Options Reconfigured:");
 
+        snprintf(key, sizeof(key), "volume%d.option.", i);
 
-        ret = 0;
-out:
+        ret = dict_foreach(dict, _gf_cli_output_volinfo_opts, key);
         if (ret)
-                cli_err ("%s", err_str);
+            goto out;
+
+        i++;
+    }
 
-        cli_cmd_broadcast_response (ret);
+    ret = 0;
+out:
+    if (ret)
+        cli_err("%s", err_str);
 
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
 
-        free (rsp.dict.dict_val);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.op_errstr);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_create_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp       = {0,};
-        int                     ret       = -1;
-        cli_local_t             *local    = NULL;
-        char                    *volname  = NULL;
-        dict_t                  *rsp_dict = NULL;
-        call_frame_t            *frame    = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        frame = myframe;
+    frame = myframe;
 
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to create volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
-
-        ret = dict_get_str (local->dict, "volname", &volname);
+        ret = cli_xml_output_vol_create(rsp_dict, rsp.op_ret, rsp.op_errno,
+                                        rsp.op_errstr);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-                ret = cli_xml_output_vol_create (rsp_dict, rsp.op_ret,
-                                                 rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume create: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume create: %s: failed", volname);
+    else
+        cli_out(
+            "volume create: %s: success: "
+            "please start the volume to access data",
+            volname);
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume create: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume create: %s: failed", volname);
-        else
-                cli_out ("volume create: %s: success: "
-                         "please start the volume to access data", volname);
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
-int
-gf_cli_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_delete_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *rsp_dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        GF_ASSERT (myframe);
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
+    frame = myframe;
 
-        frame = myframe;
+    GF_ASSERT(frame->local);
 
-        GF_ASSERT (frame->local);
+    local = frame->local;
 
-        local = frame->local;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    gf_log("cli", GF_LOG_INFO, "Received resp to delete volume");
 
-        ret = dict_get_str (local->dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict get failed");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to delete volume");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+        ret = cli_xml_output_generic_volume("volDelete", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                ret = cli_xml_output_generic_volume ("volDelete", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume delete: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume delete: %s: failed", volname);
-        else
-                cli_out ("volume delete: %s: success", volname);
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume delete: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume delete: %s: failed", volname);
+    else
+        cli_out("volume delete: %s: success", volname);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_get_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli3_1_uuid_get_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        char                    *uuid_str = NULL;
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    char *uuid_str = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status)
-                goto out;
+    if (-1 == req->rpc_status)
+        goto out;
 
-        frame = myframe;
+    frame = myframe;
 
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid get");
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid get");
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to unserialize "
-                        "response for uuid get");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "uuid", &uuid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get uuid "
-                        "from dictionary");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidGenerate", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, "") == 0)
-                        cli_err ("Get uuid was unsuccessful");
-                else
-                        cli_err ("%s", rsp.op_errstr);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidGenerate", dict, rsp.op_ret,
+                                  rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        } else {
-                cli_out ("UUID: %s", uuid_str);
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "") == 0)
+            cli_err("Get uuid was unsuccessful");
+        else
+            cli_err("%s", rsp.op_errstr);
 
+    } else {
+        ret = dict_get_str_sizen(dict, "uuid", &uuid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get uuid from dictionary");
+            goto out;
         }
-        ret = rsp.op_ret;
+        cli_out("UUID: %s", uuid_str);
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_reset_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli3_1_uuid_reset_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        frame = myframe;
+    frame = myframe;
 
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid reset");
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid reset");
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidReset", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidReset", NULL, rsp.op_ret, rsp.op_errno,
+                                  rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("%s", rsp.op_errstr);
-        else
-                cli_out ("resetting the peer uuid has been %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("%s", rsp.op_errstr);
+    else
+        cli_out("resetting the peer uuid has been %s",
+                (rsp.op_ret) ? "unsuccessful" : "successful");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_start_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *rsp_dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        frame = myframe;
+    frame = myframe;
 
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = dict_get_str (local->dict, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "dict get failed");
+    gf_log("cli", GF_LOG_INFO, "Received resp to start volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to start volume");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+        ret = cli_xml_output_generic_volume("volStart", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                ret = cli_xml_output_generic_volume ("volStart", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume start: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume start: %s: failed", volname);
-        else
-                cli_out ("volume start: %s: success", volname);
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume start: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume start: %s: failed", volname);
+    else
+        cli_out("volume start: %s: success", volname);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
-int
-gf_cli_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_stop_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        cli_local_t           *local = NULL;
-        char                  *volname = NULL;
-        call_frame_t          *frame = NULL;
-        dict_t                *rsp_dict = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        frame = myframe;
+    GF_ASSERT(myframe);
 
-        GF_ASSERT (frame->local);
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        local = frame->local;
+    frame = myframe;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    GF_ASSERT(frame->local);
 
-        ret = dict_get_str (local->dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Unable to get volname from dict");
-                goto out;
-        }
+    local = frame->local;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to stop volume");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    gf_log("cli", GF_LOG_INFO, "Received resp to stop volume");
 
-                ret = cli_xml_output_generic_volume ("volStop", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume stop: %s: failed: %s", volname, rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume stop: %s: failed", volname);
-        else
-                cli_out ("volume stop: %s: success", volname);
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
-
-        return ret;
-}
+        ret = cli_xml_output_generic_volume("volStop", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-int
-gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type,
-                               gf_boolean_t is_tier)
-{
-        int                ret          = -1;
-        int                count        = 0;
-        int                i            = 1;
-        char               key[256]     = {0,};
-        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
-        uint64_t           files        = 0;
-        uint64_t           size         = 0;
-        uint64_t           lookup       = 0;
-        char               *node_name   = NULL;
-        uint64_t           failures     = 0;
-        uint64_t           skipped      = 0;
-        double             elapsed      = 0;
-        char               *status_str  = NULL;
-        char               *size_str    = NULL;
-        int                hrs          = 0;
-        int                min          = 0;
-        int                sec          = 0;
-        gf_boolean_t       down         = _gf_false;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Unable to get volname from dict");
+        goto out;
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "status-1");
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume stop: %s: failed", volname);
+    else
+        cli_out("volume stop: %s: success", volname);
 
-        ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_TRACE, "count %d %d", count, 1);
-                gf_log ("cli", GF_LOG_TRACE, "failed to get status");
-                goto out;
-        }
+    ret = rsp.op_ret;
 
-        if (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED) {
-                cli_out ("%10s %40s %18s", "Node", "status",
-                         "run time in h:m:s");
-                cli_out ("%10s %40s %18s", "---------", "-----------",
-                         "------------");
-        } else {
-                cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s",
-                         "Node", "Rebalanced-files", "size", "scanned",
-                         "failures", "skipped", "status", "run time in"
-                         " h:m:s");
-                cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s",
-                         "---------", "-----------", "-----------",
-                         "-----------", "-----------", "-----------",
-                         "------------", "--------------");
-        }
-
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name = NULL;
-                files = 0;
-                size = 0;
-                lookup = 0;
-                skipped = 0;
-                status_str = NULL;
-                elapsed = 0;
-
-                /* Check if status is NOT_STARTED, and continue early */
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-                if (ret == -ENOENT) {
-                        gf_log ("cli", GF_LOG_TRACE, "count %d %d", count, i);
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get status");
-                        gf_log ("cli", GF_LOG_ERROR, "node down and has failed"
-                                " to set dict");
-                        down = _gf_true;
-                        continue;
-                        /* skip this node if value not available*/
-                } else if (ret) {
-                        gf_log ("cli", GF_LOG_TRACE, "count %d %d", count, i);
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get status");
-                        continue;
-                        /* skip this node if value not available*/
-                }
-
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+    return ret;
+}
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "files-%d", i);
-                ret = dict_get_uint64 (dict, key, &files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get file count");
+static int
+gf_cli_print_rebalance_status(dict_t *dict, enum gf_task_types task_type)
+{
+    int ret = -1;
+    int count = 0;
+    int i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_defrag_status_t status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    char *node_name = NULL;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    double elapsed = 0;
+    char *status_str = NULL;
+    char *size_str = NULL;
+    int32_t hrs = 0;
+    uint32_t min = 0;
+    uint32_t sec = 0;
+    gf_boolean_t fix_layout = _gf_false;
+    uint64_t max_time = 0;
+    uint64_t max_elapsed = 0;
+    uint64_t time_left = 0;
+    gf_boolean_t show_estimates = _gf_false;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "count not set");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        /* If information from a node is missing we should skip
+         * the node and try to fetch information of other nodes.
+         * If information is not found for all nodes, we should
+         * error out.
+         */
+        if (!ret)
+            break;
+        if (ret && i == count) {
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            goto out;
+        }
+    }
+
+    /* Fix layout will be sent to all nodes for the volume
+       so every status should be of type
+       GF_DEFRAG_STATUS_LAYOUT_FIX*
+    */
+
+    if ((task_type == GF_TASK_TYPE_REBALANCE) &&
+        (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED)) {
+        fix_layout = _gf_true;
+    }
+
+    if (fix_layout) {
+        cli_out("%35s %41s %27s", "Node", "status", "run time in h:m:s");
+        cli_out("%35s %41s %27s", "---------", "-----------", "------------");
+    } else {
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                "status",
+                "run time in"
+                " h:m:s");
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                "-----------", "-----------", "-----------", "-----------",
+                "-----------", "------------", "--------------");
+    }
+
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        files = 0;
+        size = 0;
+        lookup = 0;
+        skipped = 0;
+        status_str = NULL;
+        elapsed = 0;
+        time_left = 0;
+
+        /* Check if status is NOT_STARTED, and continue early */
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        if (ret == -ENOENT) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            gf_log("cli", GF_LOG_ERROR, "node down and has failed to set dict");
+            continue;
+            /* skip this node if value not available*/
+        } else if (ret) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            continue;
+            /* skip this node if value not available*/
+        }
+
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
+
+        if (GF_DEFRAG_STATUS_STARTED == status_rcd)
+            show_estimates = _gf_true;
+
+        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_strn(dict, key, keylen, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "size-%d", i);
-                ret = dict_get_uint64 (dict, key, &size);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get size of xfer");
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get file count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "lookups-%d", i);
-                ret = dict_get_uint64 (dict, key, &lookup);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get lookedup file count");
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get size of xfer");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "failures-%d", i);
-                ret = dict_get_uint64 (dict, key, &failures);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get failures count");
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookup);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get lookedup file count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "skipped-%d", i);
-                ret = dict_get_uint64 (dict, key, &skipped);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get skipped count");
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get failures count");
 
-                /* For remove-brick include skipped count into failure count*/
-                if (task_type != GF_TASK_TYPE_REBALANCE) {
-                        failures += skipped;
-                        skipped = 0;
-                }
+        snprintf(key, sizeof(key), "skipped-%d", i);
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get skipped count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", i);
-                ret = dict_get_double (dict, key, &elapsed);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get run-time");
-
-                /* Check for array bound */
-                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
-                        status_rcd = GF_DEFRAG_STATUS_MAX;
-
-                status_str = cli_vol_task_status_str[status_rcd];
-                size_str = gf_uint64_2human_readable(size);
-                hrs = elapsed / 3600;
-                min = ((int) elapsed % 3600) / 60;
-                sec = ((int) elapsed % 3600) % 60;
-
-                if (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED) {
-                        cli_out ("%10s %40s %8d:%d:%d", node_name, status_str,
-                                 hrs, min, sec);
-                } else {
-                        if (size_str) {
-                                cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64
-                                          " %13" PRIu64" %13"PRIu64 " %20s "
-                                         "%8d:%d:%d", node_name, files,
-                                         size_str, lookup, failures, skipped,
-                                         status_str, hrs, min, sec);
-                        } else {
-                                cli_out ("%40s %16"PRIu64 " %13"PRIu64 " %13"
-                                         PRIu64 " %13"PRIu64" %13"PRIu64 " %20s"
-                                         " %8d:%d:%d", node_name, files, size,
-                                         lookup, failures, skipped, status_str,
-                                         hrs, min, sec);
-                        }
-                }
-                GF_FREE(size_str);
+        /* For remove-brick include skipped count into failure count*/
+        if (task_type != GF_TASK_TYPE_REBALANCE) {
+            failures += skipped;
+            skipped = 0;
         }
-        if (is_tier && down)
-                cli_out ("WARNING: glusterd might be down on one or more nodes."
-                         " Please check the nodes that are down using \'gluster"
-                         " peer status\' and start the glusterd on those nodes,"
-                         " else tier detach commit might fail!");
-out:
-        return ret;
-}
 
-int
-gf_cli_print_tier_status (dict_t *dict, enum gf_task_types task_type)
-{
-        int                ret          = -1;
-        int                count        = 0;
-        int                i            = 1;
-        uint64_t           promoted     = 0;
-        uint64_t           demoted      = 0;
-        char               key[256]     = {0,};
-        char               *node_name   = NULL;
-        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
-        char               *status_str  = NULL;
-        gf_boolean_t       down         = _gf_false;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get run-time");
 
-        cli_out ("%-20s %-20s %-20s %-20s", "Node", "Promoted files",
-                 "Demoted files", "Status");
-        cli_out ("%-20s %-20s %-20s %-20s", "---------", "---------",
-                 "---------", "---------");
-
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name = NULL;
-                promoted = 0;
-                demoted = 0;
-
-                /* Check if status is NOT_STARTED, and continue early */
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-                if (ret == -ENOENT) {
-                        gf_log ("cli", GF_LOG_TRACE, "count: %d, %d,"
-                                "failed to get status", count, i);
-                        gf_log ("cli", GF_LOG_ERROR, "node down and has failed"
-                                " to set dict");
-                        down = _gf_true;
-                        continue;
-                        /*skipping this node as value unavailable*/
-                } else if (ret) {
-                        gf_log ("cli", GF_LOG_TRACE, "count: %d, %d,"
-                                "failed to get status", count, i);
-                        continue;
-                }
+        snprintf(key, sizeof(key), "time-left-%d", i);
+        ret = dict_get_uint64(dict, key, &time_left);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get time left");
 
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
+        if (elapsed > max_elapsed)
+            max_elapsed = elapsed;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+        if (time_left > max_time)
+            max_time = time_left;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "promoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &promoted);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get promoted count");
-
-                memset (key, 0, 256);
-                snprintf (key, 256, "demoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &demoted);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get demoted count");
+        /* Check for array bound */
+        if (status_rcd >= GF_DEFRAG_STATUS_MAX)
+            status_rcd = GF_DEFRAG_STATUS_MAX;
 
-                /* Check for array bound */
-                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
-                        status_rcd = GF_DEFRAG_STATUS_MAX;
+        status_str = cli_vol_task_status_str[status_rcd];
+        size_str = gf_uint64_2human_readable(size);
+        hrs = elapsed / 3600;
+        min = ((uint64_t)elapsed % 3600) / 60;
+        sec = ((uint64_t)elapsed % 3600) % 60;
 
-                status_str = cli_vol_task_status_str[status_rcd];
-                cli_out ("%-20s %-20"PRIu64" %-20"PRIu64" %-20s",
-                         node_name, promoted, demoted, status_str);
+        if (fix_layout) {
+            cli_out("%35s %50s %8d:%d:%d", node_name, status_str, hrs, min,
+                    sec);
+        } else {
+            if (size_str) {
+                cli_out("%40s %16" PRIu64
+                        " %13s"
+                        " %13" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %20s "
+                        "%8d:%02d:%02d",
+                        node_name, files, size_str, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            } else {
+                cli_out("%40s %16" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %13" PRIu64 " %13" PRIu64
+                        " %20s"
+                        " %8d:%02d:%02d",
+                        node_name, files, size, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            }
+        }
+        GF_FREE(size_str);
+    }
+
+    /* Max time will be non-zero if rebalance is still running */
+    if (max_time) {
+        hrs = max_time / 3600;
+        min = (max_time % 3600) / 60;
+        sec = (max_time % 3600) % 60;
+
+        if (hrs < REBAL_ESTIMATE_SEC_UPPER_LIMIT) {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : %8d:%02d:%02d",
+                hrs, min, sec);
+        } else {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : > 2 months. Please try again "
+                "later.");
+        }
+    } else {
+        /* Rebalance will return 0 if it could not calculate the
+         * estimates or if it is complete.
+         */
+        if (!show_estimates) {
+            goto out;
+        }
+        if (max_elapsed <= REBAL_ESTIMATE_START_TIME) {
+            cli_out(
+                "The estimated time for rebalance to complete "
+                "will be unavailable for the first 10 "
+                "minutes.");
+        } else {
+            cli_out(
+                "Rebalance estimated time unavailable. Please "
+                "try again later.");
         }
-        if (down)
-                cli_out ("WARNING: glusterd might be down on one or more nodes."
-                         " Please check the nodes that are down using \'gluster"
-                         " peer status\' and start the glusterd on those nodes.");
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp   rsp          = {0,};
-        cli_local_t  *local       = NULL;
-        char         *volname     = NULL;
-        call_frame_t *frame       = NULL;
-        int          cmd          = 0;
-        int          ret          = -1;
-        dict_t       *dict        = NULL;
-        char         msg[1024]    = {0,};
-        char         *task_id_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        GF_ASSERT (myframe);
-
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
+static int
+gf_cli_defrag_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    int cmd = 0;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *task_id_str = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "Failed to get volname");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "rebalance-command",
+                               (int32_t *)&cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("glusterd", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        ret = dict_get_str (local->dict, "volname", &volname);
+    if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = dict_get_str_sizen(dict, GF_REBALANCE_TID_KEY, &task_id_str);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to get volname");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "rebalance-command",
-                              (int32_t *)&cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to get command");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        if (!((cmd == GF_DEFRAG_CMD_STOP) ||
-              (cmd == GF_DEFRAG_CMD_STATUS) ||
-              (cmd == GF_DEFRAG_CMD_STATUS_TIER)) &&
-             !(global_state->mode & GLUSTER_MODE_XML)) {
-                /* All other possibilites are about starting a rebalance */
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-                if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                } else {
-                         if (!rsp.op_ret) {
-                                /* append errstr in the cli msg for successful
-                                 * case since unlock failures can be highlighted
-                                 * event though rebalance command was successful
-                                 */
-                                 if (cmd == GF_DEFRAG_CMD_START_TIER) {
-                                         snprintf (msg, sizeof (msg),
-                                                  "Attach tier is successful "
-                                                  "on %s. use tier status to "
-                                                  "check the status.\nID: %s"
-                                                  "\n%s",
-                                                  volname, task_id_str,
-                                                  rsp.op_errstr);
-                                 } else {
-                                         snprintf (msg, sizeof (msg),
-                                                  "Rebalance on %s has been "
-                                                  "started successfully. Use "
-                                                  "rebalance status command to"
-                                                  " check status of the "
-                                                  "rebalance process.\nID: %s\n%s",
-                                                  volname, task_id_str,
-                                                  rsp.op_errstr);
-                                 }
-                         } else {
-                                snprintf (msg, sizeof (msg),
-                                          "Starting rebalance on volume %s has "
-                                          "been unsuccessful.", volname);
-                        }
-                }
-                goto done;
-        }
-
-        if (cmd == GF_DEFRAG_CMD_STOP) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "rebalance volume %s stop failed",
-                                          volname);
-                        goto done;
-                } else {
-                        /* append errstr in the cli msg for successful case
-                         * since unlock failures can be highlighted event though
-                         * rebalance command was successful */
-                        snprintf (msg, sizeof (msg),
-                                  "rebalance process may be in the middle of a "
-                                  "file migration.\nThe process will be fully "
-                                  "stopped once the migration of the file is "
-                                  "complete.\nPlease check rebalance process "
-                                  "for completion before doing any further "
-                                  "brick related tasks on the volume.\n%s",
-                                  rsp.op_errstr);
-                }
-        }
-        if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STATUS_TIER) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "Failed to get the status of "
-                                          "rebalance process");
-                        goto done;
-                } else {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                }
+            gf_log("cli", GF_LOG_WARNING, "failed to get %s from dict",
+                   GF_REBALANCE_TID_KEY);
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_rebalance (cmd, dict, rsp.op_ret,
-                                                    rsp.op_errno,
-                                                    rsp.op_errstr);
-                goto out;
+        if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        } else {
+            if (!rsp.op_ret) {
+                /* append errstr in the cli msg for successful
+                 * case since unlock failures can be highlighted
+                 * event though rebalance command was successful
+                 */
+                snprintf(msg, sizeof(msg),
+                         "Rebalance on %s has been "
+                         "started successfully. Use "
+                         "rebalance status command to"
+                         " check status of the "
+                         "rebalance process.\nID: %s",
+                         volname, task_id_str);
+            } else {
+                snprintf(msg, sizeof(msg),
+                         "Starting rebalance on volume %s has "
+                         "been unsuccessful.",
+                         volname);
+            }
+        }
+        goto done;
+    }
+
+    if (cmd == GF_DEFRAG_CMD_STOP) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg), "rebalance volume %s stop failed",
+                         volname);
+            goto done;
+        } else {
+            /* append errstr in the cli msg for successful case
+             * since unlock failures can be highlighted event though
+             * rebalance command was successful */
+            snprintf(msg, sizeof(msg),
+                     "rebalance process may be in the middle of a "
+                     "file migration.\nThe process will be fully "
+                     "stopped once the migration of the file is "
+                     "complete.\nPlease check rebalance process "
+                     "for completion before doing any further "
+                     "brick related tasks on the volume.\n%s",
+                     rsp.op_errstr);
+        }
+    }
+    if (cmd == GF_DEFRAG_CMD_STATUS) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg),
+                         "Failed to get the status of rebalance process");
+            goto done;
+        } else {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
         }
+    }
 
-        if (cmd == GF_DEFRAG_CMD_STATUS_TIER)
-                ret = gf_cli_print_tier_status (dict, GF_TASK_TYPE_REBALANCE);
-        else if (cmd == GF_DEFRAG_CMD_DETACH_STATUS)
-                ret = gf_cli_print_rebalance_status (dict,
-                                                     GF_TASK_TYPE_REBALANCE,
-                                                     _gf_true);
-        else
-                ret = gf_cli_print_rebalance_status (dict,
-                                                     GF_TASK_TYPE_REBALANCE,
-                                                     _gf_false);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_rebalance(cmd, dict, rsp.op_ret, rsp.op_errno,
+                                           rsp.op_errstr);
+        goto out;
+    }
 
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to print rebalance status");
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REBALANCE);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Failed to print rebalance status");
 
 done:
-        if (global_state->mode & GLUSTER_MODE_XML)
-                cli_xml_output_str ("volRebalance", msg,
-                                    rsp.op_ret, rsp.op_errno,
-                                    rsp.op_errstr);
-        else {
-                if (rsp.op_ret)
-
-                        if (cmd == GF_DEFRAG_CMD_START_TIER || cmd ==
-                            GF_DEFRAG_CMD_STATUS_TIER) {
-                                cli_err ("Tiering Migration Functionality: %s:"
-                                         " failed%s%s", volname,
-                                         strlen (msg) ? ": " : "", msg);
-                        } else
-                                cli_err ("volume rebalance: %s: failed%s%s",
-                                         volname, strlen (msg) ? ": " : "",
-                                         msg);
-                else
-                        if (cmd == GF_DEFRAG_CMD_START_TIER || cmd ==
-                            GF_DEFRAG_CMD_STATUS_TIER) {
-                                cli_out ("Tiering Migration Functionality: %s:"
-                                         " success%s%s", volname,
-                                         strlen (msg) ? ": " : "", msg);
-                        } else
-                                cli_out ("volume rebalance: %s: success%s%s",
-                                         volname, strlen (msg) ? ": " : "",
-                                         msg);
-        }
-        ret = rsp.op_ret;
-
-out:
-        free (rsp.op_errstr); //malloced by xdr
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        char                    msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-        snprintf (msg, sizeof (msg), "Rename volume %s",
-                  (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRename", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
+    if (global_state->mode & GLUSTER_MODE_XML)
+        cli_xml_output_str("volRebalance", msg, rsp.op_ret, rsp.op_errno,
+                           rsp.op_errstr);
+    else {
         if (rsp.op_ret)
-                cli_err ("volume rename: failed");
+            cli_err("volume rebalance: %s: failed%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
         else
-                cli_out ("volume rename: success");
-
-        ret = rsp.op_ret;
+            cli_out("volume rebalance: %s: success%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int
-gf_cli_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        char                 msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to reset");
-
-        if (strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "reset volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volReset", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_rename_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+    snprintf(msg, sizeof(msg), "Rename volume %s",
+             (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volRename", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume reset: failed: %s", msg);
-        else
-                cli_out ("volume reset: success: %s", msg);
+    if (rsp.op_ret)
+        cli_err("volume rename: failed");
+    else
+        cli_out("volume rename: success");
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_ganesha_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli_reset_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        dict_t               *dict = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to ganesha");
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset");
+
+    if (strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "reset volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volReset", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("nfs-ganesha: failed: %s", rsp.op_errstr);
-                else
-                        cli_err ("nfs-ganesha: failed");
-        }
+    if (rsp.op_ret)
+        cli_err("volume reset: failed: %s", msg);
+    else
+        cli_out("volume reset: success: %s", msg);
 
-        else {
-                cli_out("nfs-ganesha : success ");
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-char *
-is_server_debug_xlator (void *myframe)
-{
-        call_frame_t         *frame        = NULL;
-        cli_local_t          *local        = NULL;
-        char                 **words       = NULL;
-        char                 *key          = NULL;
-        char                 *value        = NULL;
-        char                 *debug_xlator = NULL;
-
-        frame = myframe;
-        local = frame->local;
-        words = (char **)local->words;
-
-        while (*words != NULL) {
-                if (strstr (*words, "trace") == NULL &&
-                    strstr (*words, "error-gen") == NULL) {
-                        words++;
-                        continue;
-                }
-
-                key = *words;
-                words++;
-                value = *words;
-                if (value == NULL)
-                        break;
-                if (strstr (value, "client")) {
-                        words++;
-                        continue;
-                } else {
-                        if (!(strstr (value, "posix") || strstr (value, "acl")
-                              || strstr (value, "locks") ||
-                              strstr (value, "io-threads") ||
-                              strstr (value, "marker") ||
-                              strstr (value, "index"))) {
-                                words++;
-                                continue;
-                        } else {
-                                debug_xlator = gf_strdup (key);
-                                break;
-                        }
-                }
-        }
-
-        return debug_xlator;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_ganesha_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        dict_t               *dict = NULL;
-        char                 *help_str = NULL;
-        char                 msg[1024] = {0,};
-        char                 *debug_xlator = NULL;
-        char                 tmp_str[512] = {0,};
-
-        GF_ASSERT (myframe);
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to set");
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        dict = dict_new ();
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to ganesha");
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    dict = dict_new();
 
-        /* For brick processes graph change does not happen on the fly.
-         * The process has to be restarted. So this is a check from the
-         * volume set option such that if debug xlators such as trace/errorgen
-         * are provided in the set command, warn the user.
-         */
-        debug_xlator = is_server_debug_xlator (myframe);
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        if (dict_get_str (dict, "help-str", &help_str) && !msg[0])
-                snprintf (msg, sizeof (msg), "Set volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-        if (rsp.op_ret == 0 && debug_xlator) {
-                snprintf (tmp_str, sizeof (tmp_str), "\n%s translator has been "
-                          "added to the server volume file. Please restart the"
-                          " volume for enabling the translator", debug_xlator);
-        }
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
 
-        if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
-                ret = cli_xml_output_str ("volSet", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("volume set: failed: %s", rsp.op_errstr);
-                else
-                        cli_err ("volume set: failed");
-        } else {
-                if (help_str == NULL) {
-                        if (debug_xlator == NULL)
-                                cli_out ("volume set: success");
-                        else
-                                cli_out ("volume set: success%s", tmp_str);
-                }else {
-                        cli_out ("%s", help_str);
-                }
-        }
-
-        ret = rsp.op_ret;
-
-out:
-        if (dict)
-                dict_unref (dict);
-        GF_FREE (debug_xlator);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli_attach_tier_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                  rsp   = {0,};
-        int                         ret   = -1;
-        char                        msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to attach tier");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("nfs-ganesha: failed: %s", rsp.op_errstr);
         else
-                snprintf (msg, sizeof (msg), "Attach tier %s",
-                          (rsp.op_ret) ? "unsuccessful" : "successful");
+            cli_err("nfs-ganesha: failed");
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volAttachTier", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    else {
+        cli_out("nfs-ganesha : success ");
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume attach-tier: failed: %s", msg);
-        else
-                cli_out ("volume attach-tier: success");
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int
-gf_cli_detach_tier_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static char *
+is_server_debug_xlator(void *myframe)
 {
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char **words = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *debug_xlator = NULL;
 
-        gf_cli_rsp                      rsp   = {0,};
-        int                             ret   = -1;
-        char                            msg[1024] = {0,};
-        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
-        char                           *cmd_str = "unknown";
-        cli_local_t                    *local = NULL;
-        call_frame_t                   *frame = NULL;
-        char                           *task_id_str = NULL;
-        dict_t                         *rsp_dict = NULL;
+    frame = myframe;
+    local = frame->local;
+    words = (char **)local->words;
 
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
+    while (*words != NULL) {
+        if (strstr(*words, "trace") == NULL &&
+            strstr(*words, "error-gen") == NULL) {
+            words++;
+            continue;
         }
 
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
-        if (ret) {
-                 gf_log ("", GF_LOG_ERROR, "failed to get command");
-                 goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                rsp_dict = dict_new ();
-                if (!rsp_dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to unserialize rsp_dict");
-                        goto out;
-                }
-        }
-
-        switch (cmd) {
-        case GF_OP_CMD_DETACH_START:
-                cmd_str = "start";
-
-                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY,
-                                    &task_id_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "remove-brick-id is not present in dict");
-                }
-                break;
-        case GF_OP_CMD_DETACH_COMMIT:
-                cmd_str = "commit";
-                break;
-        case GF_OP_CMD_DETACH_COMMIT_FORCE:
-                cmd_str = "commit force";
-                break;
-        default:
-                cmd_str = "unknown";
+        key = *words;
+        words++;
+        value = *words;
+        if (value == NULL)
+            break;
+        if (strstr(value, "client")) {
+            words++;
+            continue;
+        } else {
+            if (!(strstr(value, "posix") || strstr(value, "acl") ||
+                  strstr(value, "locks") || strstr(value, "io-threads") ||
+                  strstr(value, "marker") || strstr(value, "index"))) {
+                words++;
+                continue;
+            } else {
+                debug_xlator = gf_strdup(key);
                 break;
+            }
         }
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to detach tier");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Detach tier %s %s", cmd_str,
-                          (rsp.op_ret) ? "unsuccessful" : "successful");
-
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                       _gf_false, rsp_dict,
-                                                       rsp.op_ret, rsp.op_errno,
-                                                       msg, "volDetachTier");
+    return debug_xlator;
+}
 
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *help_str = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *debug_xlator = NULL;
+    char tmp_str[512] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to set");
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    /* For brick processes graph change does not happen on the fly.
+     * The process has to be restarted. So this is a check from the
+     * volume set option such that if debug xlators such as trace/errorgen
+     * are provided in the set command, warn the user.
+     */
+    debug_xlator = is_server_debug_xlator(myframe);
+
+    if (dict_get_str_sizen(dict, "help-str", &help_str) && !msg[0])
+        snprintf(msg, sizeof(msg), "Set volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+    if (rsp.op_ret == 0 && debug_xlator) {
+        snprintf(tmp_str, sizeof(tmp_str),
+                 "\n%s translator has been "
+                 "added to the server volume file. Please restart the"
+                 " volume for enabling the translator",
+                 debug_xlator);
+    }
+
+    if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
+        ret = cli_xml_output_str("volSet", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                cli_err ("volume detach tier %s: failed: %s", cmd_str,
-                         msg);
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("volume set: failed: %s", rsp.op_errstr);
+        else
+            cli_err("volume set: failed");
+    } else {
+        if (help_str == NULL) {
+            if (debug_xlator == NULL)
+                cli_out("volume set: success");
+            else
+                cli_out("volume set: success%s", tmp_str);
         } else {
-                cli_out ("volume detach tier %s: success", cmd_str);
-                if (GF_OP_CMD_DETACH_START == cmd && task_id_str != NULL)
-                        cli_out ("ID: %s", task_id_str);
-                if (GF_OP_CMD_DETACH_COMMIT == cmd)
-                        cli_out ("Check the detached bricks to ensure all files"
-                                 " are migrated.\nIf files with data are "
-                                 "found on the brick path, copy them via a "
-                                 "gluster mount point before re-purposing the "
-                                 "removed brick. ");
+            cli_out("%s", help_str);
         }
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(debug_xlator);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
 int
-gf_cli_detach_tier_status_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
-{
-        gf_cli_rsp               rsp     = {0,};
-        int                      ret     = -1;
-        dict_t                  *dict    = NULL;
-        char                     msg[1024] = {0,};
-        int32_t                  command = 0;
-        gf1_op_commands          cmd = GF_OP_CMD_NONE;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *cmd_str = "unknown";
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "command", &command);
+gf_cli_add_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to add brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Add Brick %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volAddBrick", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                goto out;
-
-        cmd = command;
-
-        switch (cmd) {
-        case GF_OP_CMD_STOP_DETACH_TIER:
-                cmd_str = "stop";
-                break;
-        case GF_OP_CMD_STATUS:
-                cmd_str = "status";
-                break;
-        default:
-                break;
-        }
-
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume tier detach %s: "
-                                  "failed: %s", cmd_str, rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume tier detach %s: "
-                                  "failed", cmd_str);
-
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-
-                cli_err ("%s", msg);
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        strncpy (msg, "failed to unserialize req-buffer to "
-                                 "dictionary", sizeof (msg));
-
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                rsp.op_ret = -1;
-                                goto xml_output;
-                        }
-
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
-        }
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr,
-                                                               "volDetachTier");
-                } else {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier
-                                                               (_gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               msg,
-                                                               "volDetachTier");
-                }
-                goto out;
-        }
-
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REMOVE_BRICK,
-                                             _gf_true);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to print remove-brick "
-                        "rebalance status");
-                goto out;
-        }
-
-        if ((cmd == GF_OP_CMD_STOP_DETACH_TIER) && (rsp.op_ret == 0)) {
-                cli_out ("'detach tier' process may be in the middle of a "
-                         "file migration.\nThe process will be fully stopped "
-                         "once the migration of the file is complete.\nPlease "
-                         "check detach tier process for completion before "
-                         "doing any further brick related tasks on the "
-                         "volume.");
-        }
+    if (rsp.op_ret)
+        cli_err("volume add-brick: failed: %s", rsp.op_errstr);
+    else
+        cli_out("volume add-brick: success");
+    ret = rsp.op_ret;
 
 out:
-        free (rsp.dict.dict_val); /* malloced by xdr */
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-
-int
-gf_cli_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli3_remove_brick_status_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        gf_cli_rsp                  rsp   = {0,};
-        int                         ret   = -1;
-        char                        msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to add brick");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Add Brick %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volAddBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret)
-                cli_err ("volume add-brick: failed: %s", rsp.op_errstr);
-        else
-                cli_out ("volume add-brick: success");
-        ret = rsp.op_ret;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int32_t command = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *cmd_str;
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
-}
+    GF_ASSERT(myframe);
 
-int
-gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
-{
-        gf_cli_rsp               rsp     = {0,};
-        int                      ret     = -1;
-        dict_t                  *dict    = NULL;
-        char                     msg[1024] = {0,};
-        int32_t                  command = 0;
-        gf1_op_commands          cmd = GF_OP_CMD_NONE;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *cmd_str = "unknown";
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        frame = myframe;
+    frame = myframe;
 
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = dict_get_int32 (local->dict, "command", &command);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(local->dict, "command", &command);
+    if (ret)
+        goto out;
 
-        cmd = command;
+    cmd = command;
 
-        switch (cmd) {
+    switch (cmd) {
         case GF_OP_CMD_STOP:
-                cmd_str = "stop";
-                break;
+            cmd_str = "stop";
+            break;
         case GF_OP_CMD_STATUS:
-                cmd_str = "status";
-                break;
+            cmd_str = "status";
+            break;
         default:
-                break;
-        }
+            cmd_str = "unknown";
+            break;
+    }
+
+    ret = rsp.op_ret;
+    if (rsp.op_ret == -1) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed: %s",
+                     cmd_str, rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed",
+                     cmd_str);
 
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed: %s", cmd_str, rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed", cmd_str);
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+        cli_err("%s", msg);
+        goto out;
+    }
 
-                cli_err ("%s", msg);
-                goto out;
-        }
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        strncpy (msg, "failed to unserialize req-buffer to "
-                                 "dictionary", sizeof (msg));
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            strncpy(msg, DICT_UNSERIALIZE_FAIL, sizeof(msg));
 
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                rsp.op_ret = -1;
-                                goto xml_output;
-                        }
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                rsp.op_ret = -1;
+                goto xml_output;
+            }
 
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
+            goto out;
         }
+    }
 
 xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr,
-                                                               "volRemoveBrick");
-                } else {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               msg,
-                                                               "volRemoveBrick");
-                }
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (strcmp(rsp.op_errstr, "")) {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr,
+                                                  "volRemoveBrick");
+        } else {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, msg,
+                                                  "volRemoveBrick");
+        }
+        goto out;
+    }
+
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REMOVE_BRICK);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to print remove-brick rebalance status");
+        goto out;
+    }
+
+    if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+        cli_out(
+            "'remove-brick' process may be in the middle of a "
+            "file migration.\nThe process will be fully stopped "
+            "once the migration of the file is complete.\nPlease "
+            "check remove-brick process for completion before "
+            "doing any further brick related tasks on the "
+            "volume.");
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
 
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REMOVE_BRICK,
-                                             _gf_false);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to print remove-brick "
-                        "rebalance status");
-                goto out;
-        }
+static int
+gf_cli_remove_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *cmd_str = "unknown";
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    char *task_id_str = NULL;
+    dict_t *rsp_dict = NULL;
 
-        if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
-                cli_out ("'remove-brick' process may be in the middle of a "
-                         "file migration.\nThe process will be fully stopped "
-                         "once the migration of the file is complete.\nPlease "
-                         "check remove-brick process for completion before "
-                         "doing any further brick related tasks on the "
-                         "volume.");
-        }
+    GF_ASSERT(myframe);
 
-out:
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
+    frame = myframe;
 
-int
-gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                      rsp   = {0,};
-        int                             ret   = -1;
-        char                            msg[1024] = {0,};
-        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
-        char                           *cmd_str = "unknown";
-        cli_local_t                    *local = NULL;
-        call_frame_t                   *frame = NULL;
-        char                           *task_id_str = NULL;
-        dict_t                         *rsp_dict = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(frame->local);
 
-        frame = myframe;
+    local = frame->local;
 
-        GF_ASSERT (frame->local);
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        local = frame->local;
+    ret = dict_get_int32_sizen(local->dict, "command", (int32_t *)&cmd);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "failed to get command");
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+    if (rsp.dict.dict_len) {
+        rsp_dict = dict_new();
+        if (!rsp_dict) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret) {
-                 gf_log ("", GF_LOG_ERROR, "failed to get command");
-                 goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        if (rsp.dict.dict_len) {
-                rsp_dict = dict_new ();
-                if (!rsp_dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to unserialize rsp_dict");
-                        goto out;
-                }
-        }
-
-        switch (cmd) {
+    switch (cmd) {
         case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
-                cmd_str = "start";
-
-                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "remove-brick-id is not present in dict");
-                }
-                break;
+            cmd_str = "start";
+
+            ret = dict_get_str_sizen(rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                     &task_id_str);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "remove-brick-id is not present in dict");
+            }
+            break;
         case GF_OP_CMD_COMMIT:
-                cmd_str = "commit";
-                break;
+            cmd_str = "commit";
+            break;
         case GF_OP_CMD_COMMIT_FORCE:
-                cmd_str = "commit force";
-                break;
+            cmd_str = "commit force";
+            break;
         default:
-                cmd_str = "unknown";
-                break;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to remove brick");
+            cmd_str = "unknown";
+            break;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to remove brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Remove Brick %s %s", cmd_str,
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_remove_brick(_gf_false, rsp_dict, rsp.op_ret,
+                                              rsp.op_errno, msg,
+                                              "volRemoveBrick");
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Remove Brick %s %s", cmd_str,
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    if (rsp.op_ret) {
+        cli_err("volume remove-brick %s: failed: %s", cmd_str, msg);
+    } else {
+        cli_out("volume remove-brick %s: success", cmd_str);
+        if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+            cli_out("ID: %s", task_id_str);
+        if (GF_OP_CMD_COMMIT == cmd)
+            cli_out(
+                "Check the removed bricks to ensure all files "
+                "are migrated.\nIf files with data are "
+                "found on the brick path, copy them via a "
+                "gluster mount point before re-purposing the "
+                "removed brick. ");
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                       _gf_false, rsp_dict,
-                                                       rsp.op_ret, rsp.op_errno,
-                                                       msg, "volRemoveBrick");
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("volume remove-brick %s: failed: %s", cmd_str,
-                         msg);
-        } else {
-                cli_out ("volume remove-brick %s: success", cmd_str);
-                if (GF_OP_CMD_START == cmd && task_id_str != NULL)
-                        cli_out ("ID: %s", task_id_str);
-                if (GF_OP_CMD_COMMIT == cmd)
-                        cli_out ("Check the removed bricks to ensure all files "
-                                 "are migrated.\nIf files with data are "
-                                 "found on the brick path, copy them via a "
-                                 "gluster mount point before re-purposing the "
-                                 "removed brick. ");
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        return ret;
-}
-
-int
-gf_cli_reset_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
-{
-        gf_cli_rsp                       rsp              = {0,};
-        int                              ret              = -1;
-        cli_local_t                     *local            = NULL;
-        call_frame_t                    *frame            = NULL;
-        char                            *rb_operation_str = NULL;
-        dict_t                          *rsp_dict         = NULL;
-        char                             msg[1024]        = {0,};
-        char                            *reset_op         = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        GF_ASSERT (frame->local);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        local = frame->local;
+    return ret;
+}
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+static int
+gf_cli_reset_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *reset_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &reset_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (strcmp(reset_op, "GF_RESET_OP_START") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_str (local->dict, "operation", &reset_op);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &rsp_dict);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize rsp buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        if (strcmp (reset_op, "GF_RESET_OP_START") &&
-            strcmp (reset_op, "GF_RESET_OP_COMMIT") &&
-            strcmp (reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
-                rb_operation_str = gf_strdup ("Unknown operation");
-                ret = -1;
-                goto out;
-        }
-
-        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = gf_strdup (rsp.op_errstr);
-        } else {
-                if (!strcmp (reset_op, "GF_RESET_OP_START")) {
-                        if (rsp.op_ret)
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "start "
-                                                              "operation "
-                                                              "failed");
-                        else
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "start "
-                                                              "operation "
-                                                              "successful");
-                } else if (!strcmp (reset_op, "GF_RESET_OP_COMMIT")) {
-
-                        if (rsp.op_ret)
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "commit "
-                                                              "operation "
-                                                              "failed");
-                        else
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "commit "
-                                                              "operation "
-                                                              "successful");
-                } else if (!strcmp (reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
-
-                        if (rsp.op_ret)
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "commit "
-                                                              "force operation "
-                                                              "failed");
-                        else
-                                rb_operation_str = gf_strdup ("reset-brick "
-                                                              "commit "
-                                                              "force operation "
-                                                              "successful");
-                }
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to reset brick");
-        snprintf (msg, sizeof (msg), "%s",
-                  rb_operation_str ? rb_operation_str : "Unknown operation");
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    } else {
+        if (!strcmp(reset_op, "GF_RESET_OP_START")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick start operation failed";
+            else
+                rb_operation_str = "reset-brick start operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit operation failed";
+            else
+                rb_operation_str = "reset-brick commit operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit force operation failed";
+            else
+                rb_operation_str =
+                    "reset-brick commit force operation successful";
+        }
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_replace_brick (rsp_dict,
-                                                        rsp.op_ret,
-                                                        rsp.op_errno, msg);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret)
-                cli_err ("volume reset-brick: failed: %s", msg);
-        else
-                cli_out ("volume reset-brick: success: %s", msg);
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume reset-brick: failed: %s", msg);
+    else
+        cli_out("volume reset-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
 out:
-        if (frame)
-                frame->local = NULL;
-
-        if (local)
-                cli_local_wipe (local);
+    if (frame)
+        frame->local = NULL;
 
-        if (rb_operation_str)
-                GF_FREE (rb_operation_str);
+    if (local)
+        cli_local_wipe(local);
 
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
-int
-gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                       rsp              = {0,};
-        int                              ret              = -1;
-        cli_local_t                     *local            = NULL;
-        call_frame_t                    *frame            = NULL;
-        char                            *rb_operation_str = NULL;
-        dict_t                          *rsp_dict         = NULL;
-        char                             msg[1024]        = {0,};
-        char                            *replace_op       = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+static int
+gf_cli_replace_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *replace_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &replace_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        ret = dict_get_str (local->dict, "operation", &replace_op);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &rsp_dict);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize rsp buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        if (!strcmp (replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
-
-                if (rsp.op_ret || ret)
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "force operation failed");
-                else
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "force operation "
-                                                      "successful");
-        } else {
-                gf_log (frame->this->name, GF_LOG_DEBUG, "Unknown operation");
-        }
-
-        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = gf_strdup (rsp.op_errstr);
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to replace brick");
-        snprintf (msg, sizeof (msg), "%s",
-                  rb_operation_str ? rb_operation_str : "Unknown operation");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_replace_brick (rsp_dict,
-                                                        rsp.op_ret,
-                                                        rsp.op_errno, msg);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret)
-                cli_err ("volume replace-brick: failed: %s", msg);
+    if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        if (rsp.op_ret || ret)
+            rb_operation_str = "replace-brick commit force operation failed";
         else
-                cli_out ("volume replace-brick: success: %s", msg);
-        ret = rsp.op_ret;
+            rb_operation_str =
+                "replace-brick commit force operation successful";
+    } else {
+        gf_log(frame->this->name, GF_LOG_DEBUG, "Unknown operation");
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to replace brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-out:
-        if (frame)
-                frame->local = NULL;
+    if (rsp.op_ret)
+        cli_err("volume replace-brick: failed: %s", msg);
+    else
+        cli_out("volume replace-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
-        if (local)
-                cli_local_wipe (local);
+out:
+    if (frame)
+        frame->local = NULL;
 
-        if (rb_operation_str)
-                GF_FREE (rb_operation_str);
+    if (local)
+        cli_local_wipe(local);
 
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-gf_cli_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        gf_cli_rsp             rsp   = {0,};
-        int                    ret   = -1;
-        char                   msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log rotate");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "log rotate %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volLogRotate", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+gf_cli_log_rotate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to log rotate");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "log rotate %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volLogRotate", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume log-rotate: failed: %s", msg);
-        else
-                cli_out ("volume log-rotate: success");
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume log-rotate: failed: %s", msg);
+    else
+        cli_out("volume log-rotate: success");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
-{
-        gf_cli_rsp                     rsp   = {0,};
-        int                            ret   = -1;
-        char                           msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "volume sync: failed: %s",
-                          rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "volume sync: %s",
-                          (rsp.op_ret) ? "failed": "success");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volSync", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+gf_cli_sync_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to sync");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "volume sync: failed: %s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "volume sync: %s",
+                 (rsp.op_ret) ? "failed" : "success");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volSync", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("%s", msg);
-        else
-                cli_out ("%s", msg);
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("%s", msg);
+    else
+        cli_out("%s", msg);
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
 static int
-print_quota_list_usage_output (cli_local_t *local, char *path, int64_t avail,
-                               char *sl_str, quota_limits_t *limits,
-                               quota_meta_t *used_space, gf_boolean_t sl,
-                               gf_boolean_t hl, double sl_num,
-                               gf_boolean_t limit_set)
-{
-        int32_t         ret          = -1;
-        char           *used_str     = NULL;
-        char           *avail_str    = NULL;
-        char           *hl_str       = NULL;
-        char           *sl_val       = NULL;
-
-        used_str = gf_uint64_2human_readable (used_space->size);
-
-        if (limit_set) {
-                hl_str = gf_uint64_2human_readable (limits->hl);
-                avail_str = gf_uint64_2human_readable (avail);
-
-                sl_val = gf_uint64_2human_readable (sl_num);
+print_quota_list_usage_output(cli_local_t *local, char *path, int64_t avail,
+                              char *sl_str, quota_limits_t *limits,
+                              quota_meta_t *used_space, gf_boolean_t sl,
+                              gf_boolean_t hl, double sl_num,
+                              gf_boolean_t limit_set)
+{
+    int32_t ret = -1;
+    char *used_str = NULL;
+    char *avail_str = NULL;
+    char *hl_str = NULL;
+    char *sl_val = NULL;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_xml_output(local, path, limits->hl, sl_str, sl_num,
+                                   used_space->size, avail, sl ? "Yes" : "No",
+                                   hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
         }
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_quota_xml_output (local, path, limits->hl,
-                                            sl_str, sl_num, used_space->size,
-                                            avail, sl ? "Yes" : "No",
-                                            hl ? "Yes" : "No", limit_set);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "output in xml format for quota "
-                                "list command");
-                }
-                goto out;
-        }
+    used_str = gf_uint64_2human_readable(used_space->size);
 
-        if (limit_set) {
-                if (!used_str) {
-                        cli_out ("%-40s %7s %7s(%s) %8"PRIu64 "%9"PRIu64""
-                                 "%15s %18s", path, hl_str, sl_str, sl_val,
-                                 used_space->size, avail,
-                                 sl ? "Yes" : "No", hl ? "Yes" : "No");
-                } else {
-                        cli_out ("%-40s %7s %7s(%s) %8s %7s %15s %20s",
-                                 path, hl_str, sl_str, sl_val, used_str,
-                                 avail_str, sl ? "Yes" : "No",
-                                 hl ? "Yes" : "No");
-                }
+    if (limit_set) {
+        hl_str = gf_uint64_2human_readable(limits->hl);
+        sl_val = gf_uint64_2human_readable(sl_num);
+
+        if (!used_str) {
+            cli_out("%-40s %7s %7s(%s) %8" PRIu64 "%9" PRIu64
+                    ""
+                    "%15s %18s",
+                    path, hl_str, sl_str, sl_val, used_space->size, avail,
+                    sl ? "Yes" : "No", hl ? "Yes" : "No");
         } else {
-                        cli_out ("%-36s %10s %10s %14s %9s %15s %18s",
-                                 path, "N/A", "N/A", used_str, "N/A",
-                                 "N/A", "N/A");
+            avail_str = gf_uint64_2human_readable(avail);
+            cli_out("%-40s %7s %7s(%s) %8s %7s %15s %20s", path, hl_str, sl_str,
+                    sl_val, used_str, avail_str, sl ? "Yes" : "No",
+                    hl ? "Yes" : "No");
         }
+    } else {
+        cli_out("%-36s %10s %10s %14s %9s %15s %18s", path, "N/A", "N/A",
+                used_str, "N/A", "N/A", "N/A");
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        GF_FREE (hl_str);
-        GF_FREE (used_str);
-        GF_FREE (avail_str);
-        GF_FREE (sl_val);
+    GF_FREE(hl_str);
+    GF_FREE(used_str);
+    GF_FREE(avail_str);
+    GF_FREE(sl_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-print_quota_list_object_output (cli_local_t *local, char *path, int64_t avail,
+print_quota_list_object_output(cli_local_t *local, char *path, int64_t avail,
                                char *sl_str, quota_limits_t *limits,
                                quota_meta_t *used_space, gf_boolean_t sl,
                                gf_boolean_t hl, double sl_num,
                                gf_boolean_t limit_set)
 {
-        int32_t         ret       = -1;
-        int64_t         sl_val    = sl_num;
+    int32_t ret = -1;
+    int64_t sl_val = sl_num;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_quota_object_xml_output (local, path, sl_str, sl_val,
-                                                   limits, used_space, avail,
-                                                   sl ? "Yes" : "No",
-                                                   hl ? "Yes" : "No",
-                                                   limit_set);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "output in xml format for quota "
-                                "list command");
-                }
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_object_xml_output(local, path, sl_str, sl_val, limits,
+                                          used_space, avail, sl ? "Yes" : "No",
+                                          hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
         }
+        goto out;
+    }
 
-        if (limit_set) {
-                cli_out ("%-40s %9"PRIu64" %9s(%"PRId64") %10"PRIu64""
-                         "%10"PRIu64" %11"PRIu64" %15s %20s",
-                         path, limits->hl, sl_str, sl_val,
-                         used_space->file_count, used_space->dir_count,
-                         avail, sl ? "Yes" : "No", hl ? "Yes" : "No");
-        } else {
-                cli_out ("%-40s %9s %9s %10"PRIu64" %10"PRIu64" %11s %15s %20s",
-                         path, "N/A", "N/A", used_space->file_count,
-                         used_space->dir_count, "N/A", "N/A", "N/A");
-        }
-        ret = 0;
+    if (limit_set) {
+        cli_out("%-40s %9" PRIu64 " %9s(%" PRId64 ") %10" PRIu64
+                ""
+                "%10" PRIu64 " %11" PRIu64 " %15s %20s",
+                path, limits->hl, sl_str, sl_val, used_space->file_count,
+                used_space->dir_count, avail, sl ? "Yes" : "No",
+                hl ? "Yes" : "No");
+    } else {
+        cli_out("%-40s %9s %9s %10" PRIu64 " %10" PRIu64 " %11s %15s %20s",
+                path, "N/A", "N/A", used_space->file_count,
+                used_space->dir_count, "N/A", "N/A", "N/A");
+    }
+    ret = 0;
 
 out:
 
-        return ret;
+    return ret;
 }
 
 static int
-print_quota_list_output (cli_local_t *local, char *path, char *default_sl,
-                         quota_limits_t *limits, quota_meta_t *used_space,
-                         int type, gf_boolean_t limit_set)
-{
-        int64_t         avail            = 0;
-        char            percent_str[20]  = {0};
-        char           *sl_final         = NULL;
-        int             ret              = -1;
-        double          sl_num           = 0;
-        gf_boolean_t    sl               = _gf_false;
-        gf_boolean_t    hl               = _gf_false;
-        int64_t         used_size        = 0;
-
-        GF_ASSERT (local);
-        GF_ASSERT (path);
-
-        if (limit_set) {
-                if (limits->sl < 0) {
-                        ret = gf_string2percent (default_sl, &sl_num);
-                        sl_num = (sl_num * limits->hl) / 100;
-                        sl_final = default_sl;
-                } else {
-                        sl_num = (limits->sl * limits->hl) / 100;
-                        snprintf (percent_str, sizeof (percent_str), "%"PRIu64"%%",
-                                        limits->sl);
-                        sl_final = percent_str;
-                }
-                if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                        used_size = used_space->size;
-                else
-                        used_size = used_space->file_count + used_space->dir_count;
-
-                if (limits->hl > used_size) {
-                        avail = limits->hl - used_size;
-                        hl = _gf_false;
-                        if (used_size > sl_num)
-                                sl = _gf_true;
-                        else
-                                sl = _gf_false;
-                } else {
-                        avail = 0;
-                        hl = sl = _gf_true;
-                }
+print_quota_list_output(cli_local_t *local, char *path, char *default_sl,
+                        quota_limits_t *limits, quota_meta_t *used_space,
+                        int type, gf_boolean_t limit_set)
+{
+    int64_t avail = 0;
+    char percent_str[20] = {0};
+    char *sl_final = NULL;
+    int ret = -1;
+    double sl_num = 0;
+    gf_boolean_t sl = _gf_false;
+    gf_boolean_t hl = _gf_false;
+    int64_t used_size = 0;
+
+    GF_ASSERT(local);
+    GF_ASSERT(path);
+
+    if (limit_set) {
+        if (limits->sl < 0) {
+            ret = gf_string2percent(default_sl, &sl_num);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "could not convert default soft limit to percent");
+                goto out;
+            }
+            sl_num = (sl_num * limits->hl) / 100;
+            sl_final = default_sl;
+        } else {
+            sl_num = (limits->sl * limits->hl) / 100;
+            ret = snprintf(percent_str, sizeof(percent_str), "%" PRIu64 "%%",
+                           limits->sl);
+            if (ret < 0)
+                goto out;
+            sl_final = percent_str;
         }
-
         if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                ret = print_quota_list_usage_output (local, path, avail,
-                                                     sl_final, limits,
-                                                     used_space, sl, hl,
-                                                     sl_num, limit_set);
+            used_size = used_space->size;
         else
-                ret = print_quota_list_object_output (local, path, avail,
-                                                      sl_final, limits,
-                                                      used_space, sl, hl,
-                                                      sl_num, limit_set);
+            used_size = used_space->file_count + used_space->dir_count;
+
+        if (limits->hl > used_size) {
+            avail = limits->hl - used_size;
+            hl = _gf_false;
+            if (used_size > sl_num)
+                sl = _gf_true;
+            else
+                sl = _gf_false;
+        } else {
+            avail = 0;
+            hl = sl = _gf_true;
+        }
+    }
 
-        return ret;
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = print_quota_list_usage_output(local, path, avail, sl_final,
+                                            limits, used_space, sl, hl, sl_num,
+                                            limit_set);
+    else
+        ret = print_quota_list_object_output(local, path, avail, sl_final,
+                                             limits, used_space, sl, hl, sl_num,
+                                             limit_set);
+out:
+    return ret;
 }
 
 static int
-print_quota_list_from_mountdir (cli_local_t *local, char *mountdir,
-                                char *default_sl, char *path, int type)
-{
-        int             ret              = -1;
-        ssize_t         xattr_size       = 0;
-        quota_limits_t  limits           = {0,};
-        quota_meta_t    used_space       = {0,};
-        char           *key              = NULL;
-        gf_boolean_t    limit_set        = _gf_true;
-
-        GF_ASSERT (local);
-        GF_ASSERT (mountdir);
-        GF_ASSERT (path);
-
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                key = QUOTA_LIMIT_KEY;
-        else
-                key = QUOTA_LIMIT_OBJECTS_KEY;
-
-
-        ret = sys_lgetxattr (mountdir, key, (void *)&limits, sizeof (limits));
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get the xattr %s "
-                        "on %s. Reason : %s", key, mountdir, strerror (errno));
-
-                switch (errno) {
+print_quota_list_from_mountdir(cli_local_t *local, char *mountdir,
+                               char *default_sl, char *path, int type)
+{
+    int ret = -1;
+    ssize_t xattr_size = 0;
+    quota_limits_t limits = {
+        0,
+    };
+    quota_meta_t used_space = {
+        0,
+    };
+    char *key = NULL;
+    gf_boolean_t limit_set = _gf_true;
+
+    GF_ASSERT(local);
+    GF_ASSERT(mountdir);
+    GF_ASSERT(path);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        key = QUOTA_LIMIT_KEY;
+    else
+        key = QUOTA_LIMIT_OBJECTS_KEY;
+
+    ret = sys_lgetxattr(mountdir, key, (void *)&limits, sizeof(limits));
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get the xattr %s on %s. Reason : %s", key, mountdir,
+               strerror(errno));
+
+        switch (errno) {
 #if defined(ENODATA)
-                case ENODATA:
+            case ENODATA:
 #endif
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                case ENOATTR:
+            case ENOATTR:
 #endif
-                        /* If it's an ENOATTR, quota/inode-quota is
-                         * configured(limit is set atleast for one directory).
-                         * The user is trying to issue 'list/list-objects'
-                         * command for a directory on which quota limit is
-                         * not set and we are showing the used-space in case
-                         * of list-usage and showing (dir_count, file_count)
-                         * in case of list-objects. Other labels are
-                         * shown "N/A".
-                         */
-
-                        limit_set = _gf_false;
-                        goto enoattr;
-                        break;
-
-                default:
-                        cli_err ("%-40s %s", path, strerror (errno));
-                        break;
-                }
-
-                goto out;
-        }
-
-        limits.hl = ntoh64 (limits.hl);
-        limits.sl = ntoh64 (limits.sl);
-
-enoattr:
-        xattr_size = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY, NULL, 0);
-        if (xattr_size < (sizeof (int64_t) * 2) &&
-            type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
-                ret = -1;
-
-                /* This can happen when glusterfs is upgraded from 3.6 to 3.7
-                 * and the xattr healing is not completed.
+                /* If it's an ENOATTR, quota/inode-quota is
+                 * configured(limit is set at least for one directory).
+                 * The user is trying to issue 'list/list-objects'
+                 * command for a directory on which quota limit is
+                 * not set and we are showing the used-space in case
+                 * of list-usage and showing (dir_count, file_count)
+                 * in case of list-objects. Other labels are
+                 * shown "N/A".
                  */
-        } else if (xattr_size > (sizeof (int64_t) * 2)) {
-                ret = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY,
-                                     &used_space, sizeof (used_space));
-        } else if (xattr_size > 0) {
-                /* This is for compatibility.
-                 * Older version had only file usage
-                 */
-                ret = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY,
-                             &(used_space.size), sizeof (used_space.size));
-                used_space.file_count = 0;
-                used_space.dir_count = 0;
-        } else {
-                ret = -1;
-        }
 
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota size "
-                        "on path %s: %s", mountdir, strerror (errno));
-                print_quota_list_empty (path, type);
-                goto out;
-        }
-
-        used_space.size = ntoh64 (used_space.size);
-        used_space.file_count = ntoh64 (used_space.file_count);
-        used_space.dir_count = ntoh64 (used_space.dir_count);
+                limit_set = _gf_false;
+                goto enoattr;
+                break;
 
-        ret = print_quota_list_output (local, path, default_sl, &limits,
-                                       &used_space, type, limit_set);
-out:
-        return ret;
-}
+            default:
+                cli_err("%-40s %s", path, strerror(errno));
+                break;
+        }
 
-int
-gf_cli_print_limit_list_from_dict (cli_local_t *local, char *volname,
-                                   dict_t *dict, char *default_sl, int count,
-                                   int op_ret, int op_errno, char *op_errstr)
-{
-        int             ret                     = -1;
-        int             i                       = 0;
-        char            key[1024]               = {0,};
-        char            mountdir[PATH_MAX]      = {0,};
-        char            *path                   = NULL;
-        gf_boolean_t    xml_err_flag            = _gf_false;
-        char            err_str[NAME_MAX]       = {0,};
-        int             type                    = -1;
-
-        if (!dict|| count <= 0)
-                goto out;
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota type");
-                goto out;
-        }
+    limits.hl = ntoh64(limits.hl);
+    limits.sl = ntoh64(limits.sl);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_quota_limit_list_begin
-                                (local, op_ret, op_errno, op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting xml begin");
-                        goto out;
-                }
-        } else {
-                print_quota_list_header (type);
-        }
+enoattr:
+    xattr_size = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, NULL, 0);
+    if (xattr_size < (sizeof(int64_t) * 2) &&
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        ret = -1;
 
-        while (count--) {
-                snprintf (key, sizeof (key), "path%d", i++);
+        /* This can happen when glusterfs is upgraded from 3.6 to 3.7
+         * and the xattr healing is not completed.
+         */
+    } else if (xattr_size > (sizeof(int64_t) * 2)) {
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &used_space,
+                            sizeof(used_space));
+    } else if (xattr_size > 0) {
+        /* This is for compatibility.
+         * Older version had only file usage
+         */
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &(used_space.size),
+                            sizeof(used_space.size));
+        used_space.file_count = 0;
+        used_space.dir_count = 0;
+    } else {
+        ret = -1;
+    }
 
-                ret = dict_get_str (dict, key, &path);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_DEBUG, "Path not present in limit"
-                                " list");
-                        continue;
-                }
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota size on path %s: %s",
+               mountdir, strerror(errno));
+        print_quota_list_empty(path, type);
+        goto out;
+    }
 
-                ret = gf_canonicalize_path (path);
-                if (ret)
-                        goto out;
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, path);
-                ret = print_quota_list_from_mountdir (local, mountdir,
-                                                      default_sl, path, type);
-        }
+    used_space.size = ntoh64(used_space.size);
+    used_space.file_count = ntoh64(used_space.file_count);
+    used_space.dir_count = ntoh64(used_space.dir_count);
 
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, limit_set);
 out:
-        if (xml_err_flag) {
-                ret = cli_xml_output_str ("volQuota", NULL, -1, 0, err_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting in xml "
-                                "format");
-                }
-        }
-        return ret;
+    return ret;
 }
 
-int
-print_quota_list_from_quotad (call_frame_t *frame, dict_t *rsp_dict)
+static int
+gluster_remove_auxiliary_mount(char *volname)
 {
-        char             *path          = NULL;
-        char             *default_sl    = NULL;
-        int              ret            = -1;
-        cli_local_t     *local          = NULL;
-        dict_t          *gd_rsp_dict    = NULL;
-        quota_meta_t     used_space     = {0, };
-        quota_limits_t   limits         = {0, };
-        quota_limits_t  *size_limits    = NULL;
-        int32_t          type           = 0;
-        int32_t          success_count  = 0;
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
 
-        GF_ASSERT (frame);
+    this = THIS;
+    GF_ASSERT(this);
 
-        local = frame->local;
-        gd_rsp_dict = local->dict;
+    GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    ret = gf_umount_lazy(this->name, mountdir, 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "umount on %s failed, reason : %s",
+               mountdir, strerror(errno));
+    }
 
-        ret = dict_get_int32 (rsp_dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
-                goto out;
-        }
-
-        ret = dict_get_str (rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "path key is not present "
-                        "in dict");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = dict_get_str (gd_rsp_dict, "default-soft-limit", &default_sl);
+static int
+gf_cli_print_limit_list_from_dict(cli_local_t *local, char *volname,
+                                  dict_t *dict, char *default_sl, int count,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+    int ret = -1;
+    int i = 0;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int type = -1;
+
+    if (!dict || count <= 0)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota type");
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_begin(local, op_ret, op_errno,
+                                                        op_errstr);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                        "get default soft limit");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
         }
+    } else {
+        print_quota_list_header(type);
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                ret = dict_get_bin (rsp_dict, QUOTA_LIMIT_KEY,
-                                    (void **)&size_limits);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING,
-                                "limit key not present in dict on %s",
-                                path);
-                        goto out;
-                }
-        } else {
-                ret = dict_get_bin (rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
-                                   (void **)&size_limits);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING,
-                                "object limit key not present in dict on %s",
-                                path);
-                        goto out;
-                }
-        }
-
-        limits.hl = ntoh64 (size_limits->hl);
-        limits.sl = ntoh64 (size_limits->sl);
-
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                ret = quota_dict_get_meta (rsp_dict, QUOTA_SIZE_KEY,
-                                           &used_space);
-        else
-                ret = quota_dict_get_inode_meta (rsp_dict, QUOTA_SIZE_KEY,
-                                                 &used_space);
+    while (count--) {
+        keylen = snprintf(key, sizeof(key), "path%d", i++);
 
+        ret = dict_get_strn(dict, key, keylen, &path);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_WARNING,
-                        "size key not present in dict");
-                print_quota_list_empty (path, type);
-                goto out;
+            gf_log("cli", GF_LOG_DEBUG, "Path not present in limit list");
+            continue;
         }
 
-        LOCK (&local->lock);
-        {
-                ret = dict_get_int32 (gd_rsp_dict, "quota-list-success-count",
-                                      &success_count);
-                if (ret)
-                        success_count = 0;
+        ret = gf_canonicalize_path(path);
+        if (ret)
+            goto out;
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, path);
+        ret = print_quota_list_from_mountdir(local, mountdir, default_sl, path,
+                                             type);
+    }
+
+out:
+    return ret;
+}
 
-                ret = dict_set_int32 (gd_rsp_dict,
-                                      "quota-list-success-count",
-                                      success_count + 1);
+static int
+print_quota_list_from_quotad(call_frame_t *frame, dict_t *rsp_dict)
+{
+    char *path = NULL;
+    char *default_sl = NULL;
+    int ret = -1;
+    cli_local_t *local = NULL;
+    dict_t *gd_rsp_dict = NULL;
+    quota_meta_t used_space = {
+        0,
+    };
+    quota_limits_t limits = {
+        0,
+    };
+    quota_limits_t *size_limits = NULL;
+    int32_t type = 0;
+    int32_t success_count = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    gd_rsp_dict = local->dict;
+
+    ret = dict_get_int32_sizen(rsp_dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "path key is not present in dict");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(gd_rsp_dict, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get default soft limit");
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_KEY, (void **)&size_limits);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "limit key not present in dict on %s",
+                   path);
+            goto out;
         }
-        UNLOCK (&local->lock);
+    } else {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
+                           (void **)&size_limits);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                        "quota-list-success-count in dict");
+            gf_log("cli", GF_LOG_WARNING,
+                   "object limit key not present in dict on %s", path);
+            goto out;
+        }
+    }
+
+    limits.hl = ntoh64(size_limits->hl);
+    limits.sl = ntoh64(size_limits->sl);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = quota_dict_get_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &used_space);
+    else
+        ret = quota_dict_get_inode_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                        SLEN(QUOTA_SIZE_KEY), &used_space);
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_WARNING, "size key not present in dict");
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   &success_count);
+        if (ret)
+            success_count = 0;
+
+        ret = dict_set_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   success_count + 1);
+    }
+    UNLOCK(&local->lock);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to set quota-list-success-count in dict");
+        goto out;
+    }
+
+    if (success_count == 0) {
+        if (!(global_state->mode & GLUSTER_MODE_XML)) {
+            print_quota_list_header(type);
+        } else {
+            ret = cli_xml_output_vol_quota_limit_list_begin(local, 0, 0, NULL);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
+    }
 
-        if (success_count == 0) {
-                if (!(global_state->mode & GLUSTER_MODE_XML)) {
-                        print_quota_list_header (type);
-                } else {
-                        ret = cli_xml_output_vol_quota_limit_list_begin
-                                (local, 0, 0, NULL);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Error in "
-                                        "printing xml output");
-                                goto out;
-                        }
-                }
-        }
-
-        ret = print_quota_list_output (local, path, default_sl, &limits,
-                                       &used_space, type, _gf_true);
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, _gf_true);
 out:
-        return ret;
+    return ret;
 }
 
-void*
-cli_cmd_broadcast_response_detached (void *opaque)
+static void *
+cli_cmd_broadcast_response_detached(void *opaque)
 {
-        int32_t ret = 0;
+    int32_t ret = 0;
 
-        ret = (intptr_t) opaque;
-        cli_cmd_broadcast_response (ret);
+    ret = (intptr_t)opaque;
+    cli_cmd_broadcast_response(ret);
 
-        return NULL;
+    return NULL;
 }
 
-int32_t
-cli_quota_compare_path (struct list_head *list1,
-                        struct list_head *list2)
+static int32_t
+cli_quota_compare_path(struct list_head *list1, struct list_head *list2)
 {
-        struct list_node  *node1  = NULL;
-        struct list_node  *node2  = NULL;
-        dict_t            *dict1  = NULL;
-        dict_t            *dict2  = NULL;
-        char              *path1  = NULL;
-        char              *path2  = NULL;
-        int                ret    = 0;
+    struct list_node *node1 = NULL;
+    struct list_node *node2 = NULL;
+    dict_t *dict1 = NULL;
+    dict_t *dict2 = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    int ret = 0;
 
-        node1 = list_entry (list1, struct list_node, list);
-        node2 = list_entry (list2, struct list_node, list);
+    node1 = list_entry(list1, struct list_node, list);
+    node2 = list_entry(list2, struct list_node, list);
 
-        dict1 = node1->ptr;
-        dict2 = node2->ptr;
+    dict1 = node1->ptr;
+    dict2 = node2->ptr;
 
-        ret = dict_get_str (dict1, GET_ANCESTRY_PATH_KEY, &path1);
-        if (ret < 0)
-                return 0;
+    ret = dict_get_str_sizen(dict1, GET_ANCESTRY_PATH_KEY, &path1);
+    if (ret < 0)
+        return 0;
 
-        ret = dict_get_str (dict2, GET_ANCESTRY_PATH_KEY, &path2);
-        if (ret < 0)
-                return 0;
+    ret = dict_get_str_sizen(dict2, GET_ANCESTRY_PATH_KEY, &path2);
+    if (ret < 0)
+        return 0;
 
-        return strcmp (path1, path2);
+    return strcmp(path1, path2);
 }
 
-int
-cli_quotad_getlimit_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+static int
+cli_quotad_getlimit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
     /*TODO: we need to gather the path, hard-limit, soft-limit and used space*/
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        struct list_node  *node        = NULL;
-        struct list_node  *tmpnode     = NULL;
-        call_frame_t      *frame       = NULL;
-        cli_local_t       *local       = NULL;
-        int32_t            list_count  = 0;
-        pthread_t          th_id       = {0, };
-        int32_t            max_count   = 0;
-
-        GF_ASSERT (myframe);
-
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        LOCK (&local->lock);
-        {
-                ret = dict_get_int32 (local->dict, "quota-list-count",
-                                      &list_count);
-                if (ret)
-                        list_count = 0;
-
-                list_count++;
-                ret = dict_set_int32 (local->dict, "quota-list-count",
-                                      list_count);
-        }
-        UNLOCK (&local->lock);
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    struct list_node *node = NULL;
+    struct list_node *tmpnode = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    int32_t list_count = 0;
+    pthread_t th_id = {
+        0,
+    };
+    int32_t max_count = 0;
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(local->dict, "quota-list-count",
+                                   &list_count);
+        if (ret)
+            list_count = 0;
+
+        list_count++;
+        ret = dict_set_int32_sizen(local->dict, "quota-list-count", list_count);
+    }
+    UNLOCK(&local->lock);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set quota-list-count in dict");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        if (list_count == 0)
+            cli_err(
+                "Connection failed. Please check if quota "
+                "daemon is operational.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("quota command failed : %s", rsp.op_errstr);
+        else
+            cli_err("quota command : failed");
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                        "quota-list-count in dict");
-                goto out;
-        }
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        if (-1 == req->rpc_status) {
-                if (list_count == 0)
-                        cli_err ("Connection failed. Please check if quota "
-                                 "daemon is operational.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("quota command : failed");
-                goto out;
+        node = list_node_add_order(dict, &local->dict_list,
+                                   cli_quota_compare_path);
+        if (node == NULL) {
+            gf_log("cli", GF_LOG_ERROR, "failed to add node to the list");
+            dict_unref(dict);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    ret = dict_get_int32_sizen(local->dict, "max_count", &max_count);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "failed to get max_count");
+        goto out;
+    }
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-
-                ret = dict_get_int32 (local->dict, "max_count",
-                                      &max_count);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to get max_count");
-                        goto out;
-                }
-
-                node = list_node_add_order (dict, &local->dict_list,
-                                            cli_quota_compare_path);
-                if (node == NULL) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to add node to the list");
-                        dict_unref (dict);
-                        goto out;
-                }
-
-                if (list_count == max_count) {
-                        list_for_each_entry_safe (node, tmpnode,
-                                                  &local->dict_list, list) {
-                                dict = node->ptr;
-                                print_quota_list_from_quotad (frame, dict);
-                                list_node_del (node);
-                                dict_unref (dict);
-                        }
-                }
-        }
-
-out:
-        /* Bad Fix: CLI holds the lock to process a command.
-         * When processing quota list command, below sequence of steps executed
-         * in the same thread and causing deadlock
-         *
-         * 1) CLI holds the lock
-         * 2) Send rpc_clnt_submit request to quotad for quota usage
-         * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
-         * 4) cbk function cli_quotad_getlimit_cbk invokes
-         *    cli_cmd_broadcast_response which tries to hold lock to broadcast
-         *    the results and hangs, because same thread has already holding
-         *    the lock
-         *
-         * Broadcasting response in a seperate thread which is not a
-         * good fix. This needs to be re-visted with better solution
-         */
-        if (ret == -1) {
-                ret = pthread_create (&th_id, NULL,
-                                cli_cmd_broadcast_response_detached,
-                                (void *)-1);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "pthread_create failed: "
-                                "%s", strerror (errno));
-        } else {
-                cli_cmd_broadcast_response (ret);
-        }
+    if (list_count == max_count) {
+        list_for_each_entry_safe(node, tmpnode, &local->dict_list, list)
+        {
+            dict = node->ptr;
+            print_quota_list_from_quotad(frame, dict);
+            list_node_del(node);
+            dict_unref(dict);
+        }
+    }
+
+out:
+    /* Bad Fix: CLI holds the lock to process a command.
+     * When processing quota list command, below sequence of steps executed
+     * in the same thread and causing deadlock
+     *
+     * 1) CLI holds the lock
+     * 2) Send rpc_clnt_submit request to quotad for quota usage
+     * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
+     * 4) cbk function cli_quotad_getlimit_cbk invokes
+     *    cli_cmd_broadcast_response which tries to hold lock to broadcast
+     *    the results and hangs, because same thread has already holding
+     *    the lock
+     *
+     * Broadcasting response in a separate thread which is not a
+     * good fix. This needs to be re-visted with better solution
+     */
+    if (ret == -1) {
+        ret = pthread_create(&th_id, NULL, cli_cmd_broadcast_response_detached,
+                             (void *)-1);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "pthread_create failed: %s",
+                   strerror(errno));
+    } else {
+        cli_cmd_broadcast_response(ret);
+    }
+    gf_free_xdr_cli_rsp(rsp);
 
-        free (rsp.dict.dict_val);
-        return ret;
+    return ret;
 }
 
-int
-cli_quotad_getlimit (call_frame_t *frame, xlator_t *this, void *data)
+static int
+cli_quotad_getlimit(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+    dict = data;
+    ret = add_cli_cmd_timeout_to_dict(dict);
 
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+
+        goto out;
+    }
 
-        ret = cli_cmd_submit (global_quotad_rpc, &req, frame, &cli_quotad_clnt,
-                              GF_AGGREGATOR_GETLIMIT, NULL,
-                              this, cli_quotad_getlimit_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(global_quotad_rpc, &req, frame, &cli_quotad_clnt,
+                         GF_AGGREGATOR_GETLIMIT, NULL, this,
+                         cli_quotad_getlimit_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-void
-gf_cli_quota_list (cli_local_t *local, char *volname, dict_t *dict,
-                   char *default_sl, int count, int op_ret,
-                   int op_errno, char *op_errstr)
+static void
+gf_cli_quota_list(cli_local_t *local, char *volname, dict_t *dict,
+                  char *default_sl, int count, int op_ret, int op_errno,
+                  char *op_errstr)
 {
-        GF_VALIDATE_OR_GOTO ("cli", volname, out);
+    if (!cli_cmd_connected())
+        goto out;
 
-        if (!connected)
-                goto out;
+    if (count > 0) {
+        GF_VALIDATE_OR_GOTO("cli", volname, out);
 
-        if (count > 0)
-                gf_cli_print_limit_list_from_dict (local, volname, dict,
-                                                   default_sl, count, op_ret,
-                                                   op_errno, op_errstr);
+        gf_cli_print_limit_list_from_dict(local, volname, dict, default_sl,
+                                          count, op_ret, op_errno, op_errstr);
+    }
 out:
-        return;
+    return;
 }
 
-int
-gf_cli_quota_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        char              *volname     = NULL;
-        int32_t            type        = 0;
-        call_frame_t      *frame       = NULL;
-        char              *default_sl  = NULL;
-        cli_local_t       *local       = NULL;
-        char              *default_sl_dup  = NULL;
-        int32_t            entry_count      = 0;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int
+gf_cli_quota_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    char *default_sl = NULL;
+    cli_local_t *local = NULL;
+    char *default_sl_dup = NULL;
+    int32_t entry_count = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-        frame = myframe;
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("quota command failed : %s", rsp.op_errstr);
+            if (rsp.op_ret == -ENOENT)
+                cli_err("please enter the path relative to the volume");
+        } else {
+            cli_err("quota command : failed");
+        }
 
-        GF_ASSERT (frame->local);
+        goto out;
+    }
 
-        local = frame->local;
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-
-                if (strcmp (rsp.op_errstr, "")) {
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                        if (rsp.op_ret == -ENOENT)
-                                cli_err ("please enter the path relative to "
-                                         "the volume");
-                } else {
-                        cli_err ("quota command : failed");
-                }
-
-                goto out;
-        }
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to quota command");
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    ret = dict_get_str_sizen(dict, "default-soft-limit", &default_sl);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE,
+               "failed to get default soft limit");
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
+    // default-soft-limit is part of rsp_dict only iff we sent
+    // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
+    if (default_sl) {
+        default_sl_dup = gf_strdup(default_sl);
+        if (!default_sl_dup) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, "default-soft-limit",
+                                    default_sl_dup);
+        if (ret) {
+            gf_log(frame->this->name, GF_LOG_TRACE,
+                   "failed to set default soft limit");
+            GF_FREE(default_sl_dup);
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to quota command");
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get volname");
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get type");
 
-        ret = dict_get_str (dict, "default-soft-limit", &default_sl);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get "
-                        "default soft limit");
+    ret = dict_get_int32_sizen(dict, "count", &entry_count);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get count");
 
-        // default-soft-limit is part of rsp_dict only iff we sent
-        // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
-        if (default_sl) {
-                default_sl_dup = gf_strdup (default_sl);
-                if (!default_sl_dup) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (local->dict, "default-soft-limit",
-                                       default_sl_dup);
-                if (ret) {
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "failed to set default soft limit");
-                        GF_FREE (default_sl_dup);
-                }
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gf_cli_quota_list(local, volname, dict, default_sl, entry_count,
+                          rsp.op_ret, rsp.op_errno, rsp.op_errstr);
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_vol_quota_limit_list_end(local);
+            if (ret < 0) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE,
-                        "failed to get type");
+xml_output:
 
-        ret = dict_get_int32 (dict, "count", &entry_count);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volQuota", NULL, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get count");
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if ((type == GF_QUOTA_OPTION_TYPE_LIST)
-            || (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
-                gf_cli_quota_list (local, volname, dict, default_sl,
-                                   entry_count, rsp.op_ret,
-                                   rsp.op_errno, rsp.op_errstr);
+    if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST &&
+        type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
+        cli_out("volume quota : success");
 
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_vol_quota_limit_list_end (local);
-                        if (ret < 0) {
-                                ret = -1;
-                                gf_log ("cli", GF_LOG_ERROR, "Error in printing"
-                                       " xml output");
-                        }
-                        goto out;
-                }
-        }
-
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volQuota", NULL, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = rsp.op_ret;
+out:
 
-        if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST
-                        && type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
-                cli_out ("volume quota : success");
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gluster_remove_auxiliary_mount(volname);
+    }
 
-        ret = rsp.op_ret;
-out:
-        cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_getspec_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_getspec_rsp          rsp   = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "getspec failed");
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "getspec failed");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getspec");
+    gf_log("cli", GF_LOG_INFO, "Received resp to getspec");
 
-        spec = GF_MALLOC (rsp.op_ret + 1, cli_mt_char);
-        if (!spec) {
-                gf_log("", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-        memcpy (spec, rsp.spec, rsp.op_ret);
-        spec[rsp.op_ret] = '\0';
-        cli_out ("%s", spec);
-        GF_FREE (spec);
+    spec = GF_MALLOC(rsp.op_ret + 1, cli_mt_char);
+    if (!spec) {
+        gf_log("", GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    memcpy(spec, rsp.spec, rsp.op_ret);
+    spec[rsp.op_ret] = '\0';
+    cli_out("%s", spec);
+    GF_FREE(spec);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_getspec_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli_pmap_b2p_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        pmap_port_by_brick_rsp rsp = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "pump_b2p failed");
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "pump_b2p failed");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to pmap b2p");
+    gf_log("cli", GF_LOG_INFO, "Received resp to pmap b2p");
 
-        cli_out ("%d", rsp.port);
-        GF_FREE (spec);
+    cli_out("%d", rsp.port);
+    GF_FREE(spec);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-
-int32_t
-gf_cli_probe (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_probe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req         req      = {{0,},};
-        int                ret      = 0;
-        dict_t            *dict     = NULL;
-        int                port     = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_probe_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_probe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_PROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_deprobe (call_frame_t *frame, xlator_t *this,
-                   void *data)
+static int32_t
+gf_cli_deprobe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req           req      = {{0,},};
-        int                  ret      = 0;
-        dict_t              *dict     = NULL;
-        int                  port     = 0;
-        int                  flags    = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+    int flags = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                ret = dict_set_int32 (dict, "flags", 0);
-                if (ret)
-                        goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "flags", &flags);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "flags", 0);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_deprobe_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                              GLUSTER_CLI_DEPROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_deprobe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEPROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_list_friends (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_list_friends(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_peer_list_req   req = {0,};
-        int                     ret = 0;
-        unsigned long           flags = 0;
+    gf1_cli_peer_list_req req = {
+        0,
+    };
+    int ret = 0;
+    unsigned long flags = 0;
 
-        if (!frame || !this) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
 
-        GF_ASSERT (frame->local == NULL);
+    GF_ASSERT(frame->local == NULL);
 
-        flags = (long)data;
-        req.flags = flags;
-        frame->local = (void*)flags;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LIST_FRIENDS, NULL,
-                              this, gf_cli_list_friends_cbk,
-                              (xdrproc_t) xdr_gf1_cli_peer_list_req);
+    flags = (long)data;
+    req.flags = flags;
+    frame->local = (void *)flags;
+    ret = cli_cmd_submit(
+        NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_LIST_FRIENDS, NULL, this,
+        gf_cli_list_friends_cbk, (xdrproc_t)xdr_gf1_cli_peer_list_req);
 
 out:
-        if (ret) {
-                /*
-                 * If everything goes fine, gf_cli_list_friends_cbk()
-                 * [invoked through cli_cmd_submit()]resets the
-                 * frame->local to NULL. In case cli_cmd_submit()
-                 * fails in between, RESET frame->local here.
-                 */
-                frame->local = NULL;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && frame) {
+        /*
+         * If everything goes fine, gf_cli_list_friends_cbk()
+         * [invoked through cli_cmd_submit()]resets the
+         * frame->local to NULL. In case cli_cmd_submit()
+         * fails in between, RESET frame->local here.
+         */
+        frame->local = NULL;
+    }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_get_state (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_get_state(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,},};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_get_state_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GET_STATE, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_state_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GET_STATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_get_next_volume (call_frame_t *frame, xlator_t *this,
-                           void *data)
+static int32_t
+gf_cli_get_next_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    cli_local_t *local = NULL;
 
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        cli_local_t                     *local = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        ctx = data;
-        local = frame->local;
+    ctx = data;
+    local = frame->local;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_begin (local, 0, 0, "");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_begin(local, 0, 0, "");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
         }
+    }
 
-        ret = gf_cli_get_volume (frame, this, data);
+    ret = gf_cli_get_volume(frame, this, data);
 
+    if (!local || !local->get_vol.volname) {
+        if ((global_state->mode & GLUSTER_MODE_XML))
+            goto end_xml;
 
-        if (!local || !local->get_vol.volname) {
-                if ((global_state->mode & GLUSTER_MODE_XML))
-                        goto end_xml;
-
-                cli_err ("No volumes present");
-                goto out;
-        }
+        cli_err("No volumes present");
+        goto out;
+    }
 
+    ctx->volname = local->get_vol.volname;
 
+    while (ctx->volname) {
+        ret = gf_cli_get_volume(frame, this, ctx);
+        if (ret)
+            goto out;
         ctx->volname = local->get_vol.volname;
-
-        while (ctx->volname) {
-                ret = gf_cli_get_volume (frame, this, ctx);
-                if (ret)
-                        goto out;
-                ctx->volname = local->get_vol.volname;
-        }
+    }
 
 end_xml:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_end (local);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_end(local);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        dict_t                          *dict = NULL;
-        int32_t                         flags = 0;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (ctx->volname) {
-                ret = dict_set_str (dict, "volname", ctx->volname);
-                if (ret)
-                        goto out;
-        }
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = data;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    if (ctx->volname) {
+        ret = dict_set_str_sizen(dict, "volname", ctx->volname);
+        if (ret)
+            goto out;
+    }
 
-        flags = ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
-                goto out;
-        }
+    flags = ctx->flags;
+    ret = dict_set_int32_sizen(dict, "flags", flags);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to set flags");
+        goto out;
+    }
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GET_VOLUME, NULL,
-                              this, gf_cli_get_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_GET_VOLUME, NULL, this,
+                         gf_cli_get_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_get (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_get(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_get_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_GET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_get_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_UUID_GET,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_reset (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_reset(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_reset_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_reset_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_create_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_create_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_delete_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_delete_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_start_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_start_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog, NULL);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_stop_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req             req = {{0,}};
-        int                    ret = 0;
-        dict_t                 *dict = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = data;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_stop_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_defrag_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req     =  {{0,}};
-        int                     ret     = 0;
-        dict_t                 *dict    = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_defrag_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_rename_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_rename_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
+    dict = data;
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
 
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_RENAME_VOLUME, NULL,
-                              this, gf_cli_rename_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_RENAME_VOLUME, NULL, this,
+                         gf_cli_rename_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
-                        void *data)
+static int32_t
+gf_cli_reset_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_ganesha (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_ganesha(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  { {0,} } ;
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_ganesha_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_GANESHA,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_ganesha_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GANESHA, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_set_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_set_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
+gf_cli_add_brick(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        int32_t                 count = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret)
-                goto out;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_add_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_attach_tier (call_frame_t *frame, xlator_t *this,
-                    void *data)
-{
-        gf_cli_req              req             =  {{0,} };
-        int                     ret             = 0;
-        dict_t                  *dict           = NULL;
-        dict_t                  *newdict        = NULL;
-        char                    *tierwords[]    = {"volume", "rebalance", "",
-                                                   "tier", "start", NULL};
-        const char              **words         = (const char **)tierwords;
-        int                     wordcount       = 5;
-        char                    *volname        = NULL;
-        cli_local_t             *local          = NULL;
-        cli_local_t             *oldlocal       = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        if (ret)
-                goto out;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_attach_tier_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_ATTACH_TIER, this,
-                               cli_rpc_prog, NULL);
-        if (ret)
-                goto out;
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto notify_cli;
-        }
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    ;
+    gf_cli_req status_req = {{
+        0,
+    }};
+    ;
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t command = 0;
+    int32_t cmd = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "command", &command);
+    if (ret)
+        goto out;
+
+    if ((command != GF_OP_CMD_STATUS) && (command != GF_OP_CMD_STOP)) {
+        ret = cli_to_glusterd(
+            &req, frame, gf_cli_remove_brick_cbk, (xdrproc_t)xdr_gf_cli_req,
+            dict, GLUSTER_CLI_REMOVE_BRICK, this, cli_rpc_prog, NULL);
+    } else {
+        /* Need rebalance status to be sent :-) */
+        if (command == GF_OP_CMD_STATUS)
+            cmd |= GF_DEFRAG_CMD_STATUS;
+        else
+            cmd |= GF_DEFRAG_CMD_STOP;
 
-        words[2] = volname;
-        ret = cli_cmd_volume_defrag_parse ((const char **)words,
-                                           wordcount, &newdict);
+        ret = dict_set_int32_sizen(dict, "rebalance-command", (int32_t)cmd);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to parse tier start "
-                        "command");
-                goto notify_cli;
+            gf_log(this->name, GF_LOG_ERROR, "Failed to set dict");
+            goto out;
         }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Sending tier start");
-
-        oldlocal = frame->local;
-        CLI_LOCAL_INIT (local, words, frame, newdict);
-        ret = gf_cli_defrag_volume (frame, this, newdict);
-        frame->local = oldlocal;
-        cli_local_wipe (local);
-
-notify_cli:
-        if (ret) {
-                cli_out ("Failed to run tier start. Please execute tier start "
-                         "command explicitly");
-                cli_out ("Usage : gluster volume rebalance <volname> tier "
-                         "start");
-        }
+        ret = cli_to_glusterd(
+            &status_req, frame, gf_cli3_remove_brick_status_cbk,
+            (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEFRAG_VOLUME, this,
+            cli_rpc_prog, NULL);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
-}
+    GF_FREE(req.dict.dict_val);
 
-int32_t
-gf_cli_tier (call_frame_t *frame, xlator_t *this,
-             void *data)
-{
-        int                       ret = 0;
-        gf_cli_req                req =  { {0,} };
-        gf_cli_req                status_req = { {0,} };
-        dict_t                    *dict = NULL;
+    GF_FREE(status_req.dict.dict_val);
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (status_req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_detach_tier (call_frame_t *frame, xlator_t *this,
-                    void *data)
-{
-        gf_cli_req                req =  { {0,} };
-        gf_cli_req                status_req = { {0,} };
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-        int32_t                   command = 0;
-        char                     *volname = NULL;
-        int32_t                   cmd = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret)
-                goto out;
-
-        if ((command != GF_OP_CMD_STATUS) &&
-            (command != GF_OP_CMD_STOP_DETACH_TIER)) {
-
-
-                ret = cli_to_glusterd (&req, frame, gf_cli_detach_tier_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_REMOVE_BRICK, this,
-                                       cli_rpc_prog, NULL);
-        } else {
-                /* Need rebalance status to be sent :-) */
-                if (command == GF_OP_CMD_STATUS)
-                        cmd |= GF_DEFRAG_CMD_DETACH_STATUS;
-                else
-                        cmd |= GF_DEFRAG_CMD_STOP_DETACH_TIER;
-
-                ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&status_req, frame,
-                                       gf_cli_detach_tier_status_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
-                                       cli_rpc_prog, NULL);
-
-                }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (status_req.dict.dict_val);
-
-        return ret;
-}
-
-
-
-int32_t
-gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
-                     void *data)
-{
-        gf_cli_req                req =  {{0,}};;
-        gf_cli_req                status_req =  {{0,}};;
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-        int32_t                   command = 0;
-        char                     *volname = NULL;
-        int32_t                   cmd = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret)
-                goto out;
-
-        if ((command != GF_OP_CMD_STATUS) &&
-            (command != GF_OP_CMD_STOP)) {
-
-
-                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_REMOVE_BRICK, this,
-                                       cli_rpc_prog, NULL);
-        } else {
-                /* Need rebalance status to be sent :-) */
-                if (command == GF_OP_CMD_STATUS)
-                        cmd |= GF_DEFRAG_CMD_STATUS;
-                else
-                        cmd |= GF_DEFRAG_CMD_STOP;
-
-                ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&status_req, frame,
-                                       gf_cli3_remove_brick_status_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
-                                       cli_rpc_prog, NULL);
-
-                }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (status_req.dict.dict_val);
-
-        return ret;
-}
-
-int32_t
-gf_cli_reset_brick (call_frame_t *frame, xlator_t *this, void *data)
-{
-        gf_cli_req                  req        =  { {0,} };
-        int                         ret        = 0;
-        dict_t                     *dict       = NULL;
-        char                       *dst_brick  = NULL;
-        char                       *src_brick  = NULL;
-        char                       *volname    = NULL;
-        char                       *op         = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "operation", &op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on volname failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
+static int32_t
+gf_cli_reset_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    char *dst_brick = NULL;
+    char *op = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_str_sizen(dict, "dst-brick", &dst_brick);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on src-brick failed");
-                goto out;
-        }
-
-        if (!strcmp (op, "GF_RESET_OP_COMMIT") ||
-            !strcmp (op, "GF_RESET_OP_COMMIT_FORCE")) {
-                ret = dict_get_str (dict, "dst-brick", &dst_brick);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "dict_get on dst-brick failed");
-                        goto out;
-                }
+            gf_log(this->name, GF_LOG_DEBUG, "dict_get on dst-brick failed");
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Received command reset-brick %s on %s.", op, src_brick);
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_reset_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf_cli_req                  req        =  {{0,}};
-        int                         ret        = 0;
-        dict_t                     *dict       = NULL;
-        char                       *src_brick  = NULL;
-        char                       *dst_brick  = NULL;
-        char                       *volname    = NULL;
-        int32_t                     op         = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+static int32_t
+gf_cli_replace_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t op = 0;
 
-        dict = data;
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on volname failed");
-                goto out;
-        }
+    dict = data;
 
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on src-brick failed");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on dst-brick failed");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Received command replace-brick %s with "
-                "%s with operation=%d", src_brick,
-                dst_brick, op);
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_replace_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-
-int32_t
-gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli_log_rotate(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                req = {{0,}};
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_log_rotate_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
-                       void *data)
+static int32_t
+gf_cli_sync_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int               ret = 0;
-        gf_cli_req        req = {{0,}};
-        dict_t            *dict = NULL;
+    int ret = 0;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_sync_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_getspec (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_getspec_req          req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        dict_t                  *op_dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volid", &req.key);
-        if (ret)
-                goto out;
-
-        op_dict = dict_new ();
-        if (!op_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (op_dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (op_dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (op_dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_handshake_prog,
-                              GF_HNDSK_GETSPEC, NULL,
-                              this, gf_cli_getspec_cbk,
-                              (xdrproc_t) xdr_gf_getspec_req);
-
-out:
-        if (op_dict) {
-                dict_unref(op_dict);
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+static int32_t
+gf_cli_getspec(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    dict_t *op_dict = NULL;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "volid", &req.key);
+    if (ret)
+        goto out;
+
+    op_dict = dict_new();
+    if (!op_dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32_sizen(op_dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(op_dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(op_dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_handshake_prog,
+                         GF_HNDSK_GETSPEC, NULL, this, gf_cli_getspec_cbk,
+                         (xdrproc_t)xdr_gf_getspec_req);
+
+out:
+    if (op_dict) {
+        dict_unref(op_dict);
+    }
+    GF_FREE(req.xdata.xdata_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
 }
 
-int32_t
-gf_cli_quota (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_quota(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
+    dict = data;
 
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_quota_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_QUOTA,
+                          this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_pmap_b2p(call_frame_t *frame, xlator_t *this, void *data)
 {
-        pmap_port_by_brick_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    pmap_port_by_brick_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(dict, "brick", &req.brick);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_pmap_prog,
-                              GF_PMAP_PORTBYBRICK, NULL,
-                              this, gf_cli_pmap_b2p_cbk,
-                              (xdrproc_t) xdr_pmap_port_by_brick_req);
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_pmap_prog, GF_PMAP_PORTBYBRICK,
+                         NULL, this, gf_cli_pmap_b2p_cbk,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf1_cli_fsm_log_rsp        rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                     *dict = NULL;
-        int                        tr_count = 0;
-        char                       key[256] = {0};
-        int                        i = 0;
-        char                       *old_state = NULL;
-        char                       *new_state = NULL;
-        char                       *event = NULL;
-        char                       *time = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("fsm log unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.fsm_log.fsm_log_val,
-                                rsp.fsm_log.fsm_log_len,
-                                &dict);
-
-        if (ret) {
-                cli_err ("bad response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &tr_count);
-        if (tr_count)
-                cli_out("number of transitions: %d", tr_count);
-        else
-                cli_err("No transitions");
-        for (i = 0; i < tr_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-old-state", i);
-                ret = dict_get_str (dict, key, &old_state);
-                if (ret)
-                        goto out;
+gf_cli_fsm_log_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf1_cli_fsm_log_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    int tr_count = 0;
+    char key[64] = {0};
+    int keylen;
+    int i = 0;
+    char *old_state = NULL;
+    char *new_state = NULL;
+    char *event = NULL;
+    char *time = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("fsm log unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.fsm_log.fsm_log_val, rsp.fsm_log.fsm_log_len,
+                           &dict);
+
+    if (ret) {
+        cli_err(DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &tr_count);
+    if (!ret && tr_count)
+        cli_out("number of transitions: %d", tr_count);
+    else
+        cli_err("No transitions");
+    for (i = 0; i < tr_count; i++) {
+        keylen = snprintf(key, sizeof(key), "log%d-old-state", i);
+        ret = dict_get_strn(dict, key, keylen, &old_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-event", i);
-                ret = dict_get_str (dict, key, &event);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-event", i);
+        ret = dict_get_strn(dict, key, keylen, &event);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-new-state", i);
-                ret = dict_get_str (dict, key, &new_state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-new-state", i);
+        ret = dict_get_strn(dict, key, keylen, &new_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-time", i);
-                ret = dict_get_str (dict, key, &time);
-                if (ret)
-                        goto out;
-                cli_out ("Old State: [%s]\n"
-                         "New State: [%s]\n"
-                         "Event    : [%s]\n"
-                         "timestamp: [%s]\n", old_state, new_state, event, time);
-        }
+        keylen = snprintf(key, sizeof(key), "log%d-time", i);
+        ret = dict_get_strn(dict, key, keylen, &time);
+        if (ret)
+            goto out;
+        cli_out(
+            "Old State: [%s]\n"
+            "New State: [%s]\n"
+            "Event    : [%s]\n"
+            "timestamp: [%s]\n",
+            old_state, new_state, event, time);
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (dict) {
+        dict_unref(dict);
+    }
+    gf_free_xdr_fsm_log_rsp(rsp);
+
+    return ret;
 }
 
-int32_t
-gf_cli_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_fsm_log(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret = -1;
-        gf1_cli_fsm_log_req        req = {0,};
+    int ret = -1;
+    gf1_cli_fsm_log_req req = {
+        0,
+    };
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        if (!frame || !this || !data)
-                goto out;
-        req.name = data;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_FSM_LOG, NULL,
-                              this, gf_cli_fsm_log_cbk,
-                              (xdrproc_t) xdr_gf1_cli_fsm_log_req);
+    if (!frame || !this || !data)
+        goto out;
+    req.name = data;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_FSM_LOG,
+                         NULL, this, gf_cli_fsm_log_cbk,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_gsync_config_command (dict_t *dict)
-{
-        runner_t runner     = {0,};
-        char *subop         = NULL;
-        char *gwd           = NULL;
-        char *slave         = NULL;
-        char *confpath      = NULL;
-        char *master        = NULL;
-        char *op_name       = NULL;
-        int   ret           = -1;
-        char  conf_path[PATH_MAX] = "";
-
-        if (dict_get_str (dict, "subop", &subop) != 0)
-                return -1;
-
-        if (strcmp (subop, "get") != 0 && strcmp (subop, "get-all") != 0) {
-                cli_out (GEOREP" config updated successfully");
-                return 0;
-        }
-
-        if (dict_get_str (dict, "glusterd_workdir", &gwd) != 0 ||
-            dict_get_str (dict, "slave", &slave) != 0)
-                return -1;
-
-        if (dict_get_str (dict, "master", &master) != 0)
-                master = NULL;
-        if (dict_get_str (dict, "op_name", &op_name) != 0)
-                op_name = NULL;
-
-        ret = dict_get_str (dict, "conf_path", &confpath);
-        if (!confpath) {
-                ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
-                conf_path[ret] = '\0';
-                confpath = conf_path;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", confpath);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        if (master)
-                runner_argprintf (&runner, ":%s", master);
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
-        if (op_name)
-                runner_add_arg (&runner, op_name);
-
-        return runner_run (&runner);
-}
-
-char*
-get_struct_variable (int mem_num, gf_gsync_status_t *sts_val)
-{
-        switch (mem_num) {
-        case 0:  return (sts_val->node);
-        case 1:  return (sts_val->master);
-        case 2:  return (sts_val->brick);
-        case 3:  return (sts_val->slave_user);
-        case 4:  return (sts_val->slave);
-        case 5:  return (sts_val->slave_node);
-        case 6:  return (sts_val->worker_status);
-        case 7:  return (sts_val->crawl_status);
-        case 8:  return (sts_val->last_synced);
-        case 9:  return (sts_val->entry);
-        case 10:  return (sts_val->data);
-        case 11:  return (sts_val->meta);
-        case 12: return (sts_val->failures);
-        case 13:  return (sts_val->checkpoint_time);
-        case 14:  return (sts_val->checkpoint_completed);
-        case 15:  return (sts_val->checkpoint_completion_time);
-        case 16: return (sts_val->brick_host_uuid);
-        case 17: return (sts_val->last_synced_utc);
-        case 18: return (sts_val->checkpoint_time_utc);
-        case 19: return (sts_val->checkpoint_completion_time_utc);
-        case 20: return (sts_val->slavekey);
-        case 21: return (sts_val->session_slave);
-        default:
-                 goto out;
-        }
-
-out:
-        return NULL;
+static int
+gf_cli_gsync_config_command(dict_t *dict)
+{
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str_sizen(dict, "subop", &subop) != 0)
+        return -1;
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        cli_out(GEOREP " config updated successfully");
+        return 0;
+    }
+
+    if (dict_get_str_sizen(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str_sizen(dict, "slave", &slave) != 0)
+        return -1;
+
+    if (dict_get_str_sizen(dict, "master", &master) != 0)
+        master = NULL;
+    if (dict_get_str_sizen(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
+
+    ret = dict_get_str_sizen(dict, "conf_path", &confpath);
+    if (ret || !confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
+
+    return runner_run(&runner);
 }
 
-int
-gf_cli_print_status (char **title_values,
-                     gf_gsync_status_t **sts_vals,
-                     int *spacing, int gsync_count,
-                     int number_of_fields, int is_detail)
-{
-        int     i                        = 0;
-        int     j                        = 0;
-        int     ret                      = 0;
-        int     status_fields            = 8; /* Indexed at 0 */
-        int     total_spacing            = 0;
-        char  **output_values            = NULL;
-        char   *tmp                      = NULL;
-        char   *hyphens                  = NULL;
-
-        /* calculating spacing for hyphens */
-        for (i = 0; i < number_of_fields; i++) {
-                /* Suppressing detail output for status */
-                if ((!is_detail) && (i > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                        continue;
-                }
-                spacing[i] += 3; /* Adding extra space to
-                                    distinguish between fields */
-                total_spacing += spacing[i];
-        }
-        total_spacing += 4; /* For the spacing between the fields */
-
-        /* char pointers for each field */
-        output_values = GF_CALLOC (number_of_fields, sizeof (char *),
-                                   gf_common_mt_char);
-        if (!output_values) {
-                ret = -1;
-                goto out;
-        }
-        for (i = 0; i < number_of_fields; i++) {
-                output_values[i] = GF_CALLOC (spacing[i] + 1, sizeof (char),
-                                              gf_common_mt_char);
-                if (!output_values[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        hyphens = GF_CALLOC (total_spacing + 1, sizeof (char),
-                             gf_common_mt_char);
-        if (!hyphens) {
+static int
+gf_cli_print_status(char **title_values, gf_gsync_status_t **sts_vals,
+                    int *spacing, int gsync_count, int number_of_fields,
+                    int is_detail)
+{
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int status_fields = 8; /* Indexed at 0 */
+    int total_spacing = 0;
+    char **output_values = NULL;
+    char *tmp = NULL;
+    char *hyphens = NULL;
+
+    /* calculating spacing for hyphens */
+    for (i = 0; i < number_of_fields; i++) {
+        /* Suppressing detail output for status */
+        if ((!is_detail) && (i > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            continue;
+        }
+        spacing[i] += 3; /* Adding extra space to
+                            distinguish between fields */
+        total_spacing += spacing[i];
+    }
+    total_spacing += 4; /* For the spacing between the fields */
+
+    /* char pointers for each field */
+    output_values = GF_MALLOC(number_of_fields * sizeof(char *),
+                              gf_common_mt_char);
+    if (!output_values) {
+        ret = -1;
+        goto out;
+    }
+    for (i = 0; i < number_of_fields; i++) {
+        output_values[i] = GF_CALLOC(spacing[i] + 1, sizeof(char),
+                                     gf_common_mt_char);
+        if (!output_values[i]) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cli_out(" ");
+
+    /* setting the title "NODE", "MASTER", etc. from title_values[]
+       and printing the same */
+    for (j = 0; j < number_of_fields; j++) {
+        if ((!is_detail) && (j > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            output_values[j][0] = '\0';
+            continue;
+        }
+        memset(output_values[j], ' ', spacing[j]);
+        memcpy(output_values[j], title_values[j], strlen(title_values[j]));
+        output_values[j][spacing[j]] = '\0';
+    }
+    cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", output_values[0],
+            output_values[1], output_values[2], output_values[3],
+            output_values[4], output_values[5], output_values[6],
+            output_values[7], output_values[8], output_values[9],
+            output_values[10], output_values[11], output_values[12],
+            output_values[13], output_values[14], output_values[15]);
+
+    hyphens = GF_MALLOC((total_spacing + 1) * sizeof(char), gf_common_mt_char);
+    if (!hyphens) {
+        ret = -1;
+        goto out;
+    }
+
+    /* setting and printing the hyphens */
+    memset(hyphens, '-', total_spacing);
+    hyphens[total_spacing] = '\0';
+    cli_out("%s", hyphens);
+
+    for (i = 0; i < gsync_count; i++) {
+        for (j = 0; j < number_of_fields; j++) {
+            if ((!is_detail) && (j > status_fields)) {
+                /* Suppressing detailed output for
+                 * status */
+                output_values[j][0] = '\0';
+                continue;
+            }
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
                 ret = -1;
                 goto out;
+            }
+            memset(output_values[j], ' ', spacing[j]);
+            memcpy(output_values[j], tmp, strlen(tmp));
+            output_values[j][spacing[j]] = '\0';
         }
 
-        cli_out (" ");
-
-        /* setting the title "NODE", "MASTER", etc. from title_values[]
-           and printing the same */
-        for (j = 0; j < number_of_fields; j++) {
-                if ((!is_detail) && (j > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                       output_values[j][0] = '\0';
-                       continue;
-                }
-                memset (output_values[j], ' ', spacing[j]);
-                memcpy (output_values[j], title_values[j],
-                        strlen(title_values[j]));
-                output_values[j][spacing[j]] = '\0';
-        }
-        cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
-                 output_values[0], output_values[1],
-                 output_values[2], output_values[3],
-                 output_values[4], output_values[5],
-                 output_values[6], output_values[7],
-                 output_values[8], output_values[9],
-                 output_values[10], output_values[11],
-                 output_values[12], output_values[13],
-                 output_values[14], output_values[15]);
-
-        /* setting and printing the hyphens */
-        memset (hyphens, '-', total_spacing);
-        hyphens[total_spacing] = '\0';
-        cli_out ("%s", hyphens);
-
-        for (i = 0; i < gsync_count; i++) {
-                for (j = 0; j < number_of_fields; j++) {
-                        if ((!is_detail) && (j > status_fields)) {
-                                /* Suppressing detailed output for
-                                 * status */
-                                output_values[j][0] = '\0';
-                                continue;
-                        }
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        memset (output_values[j], ' ', spacing[j]);
-                        memcpy (output_values[j], tmp, strlen (tmp));
-                        output_values[j][spacing[j]] = '\0';
-                }
-
-                cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
-                         output_values[0], output_values[1],
-                         output_values[2], output_values[3],
-                         output_values[4], output_values[5],
-                         output_values[6], output_values[7],
-                         output_values[8], output_values[9],
-                         output_values[10], output_values[11],
-                         output_values[12], output_values[13],
-                         output_values[14], output_values[15]);
-        }
+        cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
+                output_values[0], output_values[1], output_values[2],
+                output_values[3], output_values[4], output_values[5],
+                output_values[6], output_values[7], output_values[8],
+                output_values[9], output_values[10], output_values[11],
+                output_values[12], output_values[13], output_values[14],
+                output_values[15]);
+    }
 
 out:
-        if (output_values) {
-                for (i = 0; i < number_of_fields; i++) {
-                        if (output_values[i])
-                                GF_FREE (output_values[i]);
-                }
-                GF_FREE (output_values);
+    if (output_values) {
+        for (i = 0; i < number_of_fields; i++) {
+            if (output_values[i])
+                GF_FREE(output_values[i]);
         }
+        GF_FREE(output_values);
+    }
 
-        if (hyphens)
-                GF_FREE (hyphens);
+    if (hyphens)
+        GF_FREE(hyphens);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_gsync_status_t_comparator (const void *p, const void *q)
+gf_gsync_status_t_comparator(const void *p, const void *q)
 {
-        char *slavekey1 = NULL;
-        char *slavekey2 = NULL;
-
-        slavekey1 = get_struct_variable (20, (*(gf_gsync_status_t **)p));
-        slavekey2 = get_struct_variable (20, (*(gf_gsync_status_t **)q));
-        if (!slavekey1 || !slavekey2) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "struct member empty.");
-                return 0;
-        }
-
-        return strcmp (slavekey1, slavekey2);
-}
-
-int
-gf_cli_read_status_data (dict_t *dict,
-                         gf_gsync_status_t **sts_vals,
-                         int *spacing, int gsync_count,
-                         int number_of_fields)
-{
-        char   *tmp                    = NULL;
-        char    sts_val_name[PATH_MAX] = "";
-        int     ret                    = 0;
-        int     i                      = 0;
-        int     j                      = 0;
-
-        /* Storing per node status info in each object */
-        for (i = 0; i < gsync_count; i++) {
-                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i);
-
-                /* Fetching the values from dict, and calculating
-                   the max length for each field */
-                ret = dict_get_bin (dict, sts_val_name, (void **)&(sts_vals[i]));
-                if (ret)
-                        goto out;
-
-                for (j = 0; j < number_of_fields; j++) {
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        if (strlen (tmp) > spacing[j])
-                                spacing[j] = strlen (tmp);
-                }
-        }
+    char *slavekey1 = NULL;
+    char *slavekey2 = NULL;
 
-        /* Sort based on Session Slave */
-        qsort(sts_vals, gsync_count,
-              sizeof(gf_gsync_status_t *),
-              gf_gsync_status_t_comparator);
+    slavekey1 = get_struct_variable(20, (*(gf_gsync_status_t **)p));
+    slavekey2 = get_struct_variable(20, (*(gf_gsync_status_t **)q));
+    if (!slavekey1 || !slavekey2) {
+        gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+        return 0;
+    }
 
-out:
-        return ret;
+    return strcmp(slavekey1, slavekey2);
 }
 
-int
-gf_cli_gsync_status_output (dict_t *dict, gf_boolean_t is_detail)
-{
-        int                     gsync_count    = 0;
-        int                     i              = 0;
-        int                     ret            = 0;
-        int                     spacing[16]    = {0};
-        int                     num_of_fields  = 16;
-        char                    errmsg[1024]   = "";
-        char                   *master         = NULL;
-        char                   *slave          = NULL;
-        char                   *title_values[] = {"MASTER NODE", "MASTER VOL",
-                                                  "MASTER BRICK", "SLAVE USER",
-                                                  "SLAVE", "SLAVE NODE",
-                                                  "STATUS", "CRAWL STATUS",
-                                                  "LAST_SYNCED", "ENTRY",
-                                                  "DATA", "META", "FAILURES",
-                                                  "CHECKPOINT TIME",
-                                                  "CHECKPOINT COMPLETED",
-                                                  "CHECKPOINT COMPLETION TIME"};
-        gf_gsync_status_t     **sts_vals       = NULL;
-
-        /* Checks if any session is active or not */
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-        if (ret) {
-                ret = dict_get_str (dict, "master", &master);
-
-                ret = dict_get_str (dict, "slave", &slave);
-
-                if (master) {
-                        if (slave)
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions between %s"
-                                          " and %s", master, slave);
-                        else
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions for %s",
-                                          master);
-                } else
-                        snprintf (errmsg, sizeof(errmsg), "No active "
-                                  "geo-replication sessions");
-
-                gf_log ("cli", GF_LOG_INFO, "%s", errmsg);
-                cli_out ("%s", errmsg);
-                ret = 0;
-                goto out;
-        }
-
-        for (i = 0; i < num_of_fields; i++)
-                spacing[i] = strlen(title_values[i]);
+static int
+gf_cli_read_status_data(dict_t *dict, gf_gsync_status_t **sts_vals,
+                        int *spacing, int gsync_count, int number_of_fields)
+{
+    char *tmp = NULL;
+    char sts_val_name[PATH_MAX] = "";
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+
+    /* Storing per node status info in each object */
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+
+        /* Fetching the values from dict, and calculating
+           the max length for each field */
+        ret = dict_get_bin(dict, sts_val_name, (void **)&(sts_vals[i]));
+        if (ret)
+            goto out;
 
-        /* gsync_count = number of nodes reporting output.
-           each sts_val object will store output of each
-           node */
-        sts_vals = GF_CALLOC (gsync_count, sizeof (gf_gsync_status_t *),
-                              gf_common_mt_char);
-        if (!sts_vals) {
+        for (j = 0; j < number_of_fields; j++) {
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
                 ret = -1;
                 goto out;
+            }
+            if (strlen(tmp) > spacing[j])
+                spacing[j] = strlen(tmp);
         }
-        for (i = 0; i < gsync_count; i++) {
-                sts_vals[i] = GF_CALLOC (1, sizeof (gf_gsync_status_t),
-                                         gf_common_mt_char);
-                if (!sts_vals[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = gf_cli_read_status_data (dict, sts_vals, spacing,
-                                       gsync_count, num_of_fields);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read status data");
-                goto out;
-        }
+    }
 
-        ret = gf_cli_print_status (title_values, sts_vals, spacing, gsync_count,
-                                   num_of_fields, is_detail);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to print status output");
-                goto out;
-        }
+    /* Sort based on Session Slave */
+    qsort(sts_vals, gsync_count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
 
 out:
-        if (sts_vals)
-                GF_FREE (sts_vals);
+    return ret;
+}
 
-        return ret;
+static int
+gf_cli_gsync_status_output(dict_t *dict, gf_boolean_t is_detail)
+{
+    int gsync_count = 0;
+    int i = 0;
+    int ret = 0;
+    int spacing[16] = {0};
+    int num_of_fields = 16;
+    char errmsg[1024] = "";
+    char *master = NULL;
+    char *slave = NULL;
+    static char *title_values[] = {"MASTER NODE",
+                                   "MASTER VOL",
+                                   "MASTER BRICK",
+                                   "SLAVE USER",
+                                   "SLAVE",
+                                   "SLAVE NODE",
+                                   "STATUS",
+                                   "CRAWL STATUS",
+                                   "LAST_SYNCED",
+                                   "ENTRY",
+                                   "DATA",
+                                   "META",
+                                   "FAILURES",
+                                   "CHECKPOINT TIME",
+                                   "CHECKPOINT COMPLETED",
+                                   "CHECKPOINT COMPLETION TIME"};
+    gf_gsync_status_t **sts_vals = NULL;
+
+    /* Checks if any session is active or not */
+    ret = dict_get_int32_sizen(dict, "gsync-count", &gsync_count);
+    if (ret) {
+        ret = dict_get_str_sizen(dict, "master", &master);
+
+        ret = dict_get_str_sizen(dict, "slave", &slave);
+
+        if (master) {
+            if (slave)
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions between %s"
+                         " and %s",
+                         master, slave);
+            else
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions for %s", master);
+        } else
+            snprintf(errmsg, sizeof(errmsg),
+                     "No active geo-replication sessions");
+
+        gf_log("cli", GF_LOG_INFO, "%s", errmsg);
+        cli_out("%s", errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < num_of_fields; i++)
+        spacing[i] = strlen(title_values[i]);
+
+    /* gsync_count = number of nodes reporting output.
+       each sts_val object will store output of each
+       node */
+    sts_vals = GF_MALLOC(gsync_count * sizeof(gf_gsync_status_t *),
+                         gf_common_mt_char);
+    if (!sts_vals) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_cli_read_status_data(dict, sts_vals, spacing, gsync_count,
+                                  num_of_fields);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to read status data");
+        goto out;
+    }
+
+    ret = gf_cli_print_status(title_values, sts_vals, spacing, gsync_count,
+                              num_of_fields, is_detail);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to print status output");
+        goto out;
+    }
+
+out:
+    if (sts_vals)
+        GF_FREE(sts_vals);
+
+    return ret;
 }
 
 static int32_t
-write_contents_to_common_pem_file (dict_t *dict, int output_count)
-{
-        char            *workdir                   = NULL;
-        char             common_pem_file[PATH_MAX] = "";
-        char            *output                    = NULL;
-        char             output_name[PATH_MAX]     = "";
-        int              bytes_written             = 0;
-        int              fd                        = -1;
-        int              ret                       = -1;
-        int              i                         = -1;
-
-        ret = dict_get_str (dict, "glusterd_workdir", &workdir);
-        if (ret || !workdir) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch workdir");
+write_contents_to_common_pem_file(dict_t *dict, int output_count)
+{
+    char *workdir = NULL;
+    char common_pem_file[PATH_MAX] = "";
+    char *output = NULL;
+    char output_name[32] = "";
+    int bytes_written = 0;
+    int fd = -1;
+    int ret = -1;
+    int i = -1;
+
+    ret = dict_get_str_sizen(dict, "glusterd_workdir", &workdir);
+    if (ret || !workdir) {
+        gf_log("", GF_LOG_ERROR, "Unable to fetch workdir");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(common_pem_file, sizeof(common_pem_file),
+             "%s/geo-replication/common_secret.pem.pub", workdir);
+
+    sys_unlink(common_pem_file);
+
+    fd = open(common_pem_file, O_WRONLY | O_CREAT, 0600);
+    if (fd == -1) {
+        gf_log("", GF_LOG_ERROR, "Failed to open %s Error : %s",
+               common_pem_file, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
+        }
+        if (output) {
+            bytes_written = sys_write(fd, output, strlen(output));
+            if (bytes_written != strlen(output)) {
+                gf_log("", GF_LOG_ERROR, "Failed to write to %s",
+                       common_pem_file);
                 ret = -1;
                 goto out;
-        }
-
-        snprintf (common_pem_file, sizeof(common_pem_file),
-                  "%s/geo-replication/common_secret.pem.pub",
-                  workdir);
-
-        sys_unlink (common_pem_file);
-
-        fd = open (common_pem_file, O_WRONLY | O_CREAT, 0600);
-        if (fd == -1) {
-                gf_log ("", GF_LOG_ERROR, "Failed to open %s"
-                        " Error : %s", common_pem_file,
-                        strerror (errno));
+            }
+            /* Adding the new line character */
+            bytes_written = sys_write(fd, "\n", 1);
+            if (bytes_written != 1) {
+                gf_log("", GF_LOG_ERROR, "Failed to add new line char");
                 ret = -1;
                 goto out;
+            }
+            output = NULL;
         }
+    }
 
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        bytes_written = sys_write (fd, output, strlen(output));
-                        if (bytes_written != strlen(output)) {
-                                gf_log ("", GF_LOG_ERROR, "Failed to write "
-                                        "to %s", common_pem_file);
-                                ret = -1;
-                                goto out;
-                        }
-                        /* Adding the new line character */
-                        bytes_written = sys_write (fd, "\n", strlen("\n"));
-                        if (bytes_written != strlen("\n")) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Failed to add new line char");
-                                ret = -1;
-                                goto out;
-                        }
-                        output = NULL;
-                }
-        }
-
-        cli_out ("Common secret pub file present at %s", common_pem_file);
-        ret = 0;
+    cli_out("Common secret pub file present at %s", common_pem_file);
+    ret = 0;
 out:
-        if (fd >= 0)
-                sys_close (fd);
+    if (fd >= 0)
+        sys_close(fd);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_sys_exec_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        int                     ret     = -1;
-        int                     output_count     = -1;
-        int                     i     = -1;
-        char                   *output  = NULL;
-        char                   *command = NULL;
-        char                    output_name[PATH_MAX] = "";
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (req->rpc_status == -1) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+static int
+gf_cli_sys_exec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    int output_count = -1;
+    int i = -1;
+    char *output = NULL;
+    char *command = NULL;
+    char output_name[32] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Command failed.");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "output_count", &output_count);
+    if (ret) {
+        cli_out("Command executed successfully.");
+        ret = 0;
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Command failed.");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    ret = dict_get_str_sizen(dict, "command", &command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to get command from dict");
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "output_count", &output_count);
-        if (ret) {
-                cli_out ("Command executed successfully.");
-                ret = 0;
-                goto out;
-        }
+    if (!strcmp(command, "gsec_create")) {
+        ret = write_contents_to_common_pem_file(dict, output_count);
+        if (!ret)
+            goto out;
+    }
 
-        ret = dict_get_str (dict, "command", &command);
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get command from dict");
-                goto out;
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
         }
-
-        if (!strcmp (command, "gsec_create")) {
-                ret = write_contents_to_common_pem_file (dict, output_count);
-                if (!ret)
-                        goto out;
-        }
-
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        cli_out ("%s", output);
-                        output = NULL;
-                }
+        if (output) {
+            cli_out("%s", output);
+            output = NULL;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_copy_file_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+static int
+gf_cli_copy_file_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (req->rpc_status == -1) {
-                goto out;
-        }
+    if (req->rpc_status == -1) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-       if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Copy unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Copy unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        cli_out ("Successfully copied file.");
+    cli_out("Successfully copied file.");
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
-{
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        char                    *gsync_status = NULL;
-        char                    *master = NULL;
-        char                    *slave  = NULL;
-        int32_t                  type   = 0;
-        gf_boolean_t             status_detail = _gf_false;
-
-        GF_ASSERT (myframe);
-
-        if (req->rpc_status == -1) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
+static int
+gf_cli_gsync_set_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *gsync_status = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int32_t type = 0;
+    gf_boolean_t status_detail = _gf_false;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_gsync(dict, rsp.op_ret, rsp.op_errno,
+                                       rsp.op_errstr);
         if (ret)
-                goto out;
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_gsync (dict, rsp.op_ret, rsp.op_errno,
-                                                rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         GEOREP" command unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "gsync-status", &gsync_status);
-        if (!ret)
-                cli_out ("%s", gsync_status);
-        else
-                ret = 0;
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "failed to get type");
-                goto out;
-        }
-
-        switch (type) {
-                case GF_GSYNC_OPTION_TYPE_START:
-                case GF_GSYNC_OPTION_TYPE_STOP:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_START ?
-                                  "Starting" : "Stopping",
-                                 master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_PAUSE:
-                case GF_GSYNC_OPTION_TYPE_RESUME:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_PAUSE ?
-                                  "Pausing" : "Resuming",
-                                 master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret = gf_cli_gsync_config_command (dict);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_STATUS:
-                        status_detail = dict_get_str_boolean (dict,
-                                                              "status-detail",
-                                                              _gf_false);
-                        ret = gf_cli_gsync_status_output (dict, status_detail);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_DELETE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Deleting " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_CREATE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Creating " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
+    if (rsp.op_ret) {
+        cli_err("%s",
+                rsp.op_errstr ? rsp.op_errstr : GEOREP " command unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "gsync-status", &gsync_status);
+    if (!ret)
+        cli_out("%s", gsync_status);
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "failed to get type");
+        goto out;
+    }
+
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out(
+                "%s " GEOREP " session between %s & %s has been successful",
+                type == GF_GSYNC_OPTION_TYPE_START ? "Starting" : "Stopping",
+                master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_PAUSE:
+        case GF_GSYNC_OPTION_TYPE_RESUME:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out("%s " GEOREP " session between %s & %s has been successful",
+                    type == GF_GSYNC_OPTION_TYPE_PAUSE ? "Pausing" : "Resuming",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            ret = gf_cli_gsync_config_command(dict);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+            status_detail = dict_get_str_boolean(dict, "status-detail",
+                                                 _gf_false);
+            ret = gf_cli_gsync_status_output(dict, status_detail);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Deleting " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Creating " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
 
-                default:
-                        cli_out (GEOREP" command executed successfully");
-        }
+        default:
+            cli_out(GEOREP " command executed successfully");
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_sys_exec (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_sys_exec(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
 
+    ret = cli_to_glusterd(&req, frame, gf_cli_sys_exec_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_SYS_EXEC,
+                          this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_sys_exec_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYS_EXEC, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_copy_file (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_copy_file(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
 
+    ret = cli_to_glusterd(&req, frame, gf_cli_copy_file_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_copy_file_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_gsync_set(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_gsync_set_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-
 int
-cli_profile_info_percentage_cmp (void *a, void *b)
+cli_profile_info_percentage_cmp(void *a, void *b)
 {
-        cli_profile_info_t *ia = NULL;
-        cli_profile_info_t *ib = NULL;
-        int                ret = 0;
-
-        ia = a;
-        ib = b;
-        if (ia->percentage_avg_latency < ib->percentage_avg_latency)
-                ret = -1;
-        else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
-                ret = 1;
-        else
-                ret = 0;
-        return ret;
-}
+    cli_profile_info_t *ia = NULL;
+    cli_profile_info_t *ib = NULL;
 
+    ia = a;
+    ib = b;
+    if (ia->percentage_avg_latency < ib->percentage_avg_latency)
+        return -1;
+    else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
+        return 1;
 
-void
-cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
-{
-        char                    key[256] = {0};
-        int                     i = 0;
-        uint64_t                sec = 0;
-        uint64_t                r_count = 0;
-        uint64_t                w_count = 0;
-        uint64_t                rb_counts[32] = {0};
-        uint64_t                wb_counts[32] = {0};
-        cli_profile_info_t      profile_info[GF_FOP_MAXVALUE] = {{0}};
-        cli_profile_info_t      upcall_info[GF_UPCALL_FLAGS_MAXVALUE] = {{0},};
-        char                    output[128] = {0};
-        int                     per_line = 0;
-        char                    read_blocks[128] = {0};
-        char                    write_blocks[128] = {0};
-        int                     index = 0;
-        int                     is_header_printed = 0;
-        int                     ret = 0;
-        double                  total_percentage_latency = 0;
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-read-%d", count,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &rb_counts[i]);
-        }
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-write-%d", count, interval,
-                          (1<<i));
-                ret = dict_get_uint64 (dict, key, &wb_counts[i]);
-        }
-
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                snprintf (key, sizeof (key), "%d-%d-%d-upcall-hits", count,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &upcall_info[i].fop_hits);
-                upcall_info[i].fop_name = (char *)gf_upcall_list[i];
-        }
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-hits", count,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &profile_info[i].fop_hits);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].avg_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].min_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].max_latency);
-                profile_info[i].fop_name = (char *)gf_fop_list[i];
-
-                total_percentage_latency +=
-                       (profile_info[i].fop_hits * profile_info[i].avg_latency);
-        }
-        if (total_percentage_latency) {
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        profile_info[i].percentage_avg_latency = 100 * (
-                     (profile_info[i].avg_latency* profile_info[i].fop_hits) /
-                                total_percentage_latency);
-                }
-                gf_array_insertionsort (profile_info, 1, GF_FOP_MAXVALUE - 1,
-                                    sizeof (cli_profile_info_t),
-                                    cli_profile_info_percentage_cmp);
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-duration", count, interval);
-        ret = dict_get_uint64 (dict, key, &sec);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-read", count, interval);
-        ret = dict_get_uint64 (dict, key, &r_count);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-write", count, interval);
-        ret = dict_get_uint64 (dict, key, &w_count);
-
-        if (ret == 0) {
-        }
+    return 0;
+}
 
-        if (interval == -1)
-                cli_out ("Cumulative Stats:");
-        else
-                cli_out ("Interval %d Stats:", interval);
-        snprintf (output, sizeof (output), "%14s", "Block Size:");
-        snprintf (read_blocks, sizeof (read_blocks), "%14s", "No. of Reads:");
-        snprintf (write_blocks, sizeof (write_blocks), "%14s", "No. of Writes:");
-        index = 14;
-        for (i = 0; i < 32; i++) {
-                if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
-                        continue;
-                per_line++;
-                snprintf (output+index, sizeof (output)-index, "%19db+ ", (1<<i));
-                if (rb_counts[i]) {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21"PRId64" ", rb_counts[i]);
-                } else {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21s ", "0");
-                }
-                if (wb_counts[i]) {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21"PRId64" ", wb_counts[i]);
-                } else {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21s ", "0");
-                }
-                index += 22;
-                if (per_line == 3) {
-                        cli_out ("%s", output);
-                        cli_out ("%s", read_blocks);
-                        cli_out ("%s", write_blocks);
-                        cli_out (" ");
-                        per_line = 0;
-                        memset (output, 0, sizeof (output));
-                        memset (read_blocks, 0, sizeof (read_blocks));
-                        memset (write_blocks, 0, sizeof (write_blocks));
-                        snprintf (output, sizeof (output), "%14s", "Block Size:");
-                        snprintf (read_blocks, sizeof (read_blocks), "%14s",
-                                  "No. of Reads:");
-                        snprintf (write_blocks, sizeof (write_blocks), "%14s",
-                                  "No. of Writes:");
-                        index = 14;
-                }
+static void
+cmd_profile_volume_brick_out(dict_t *dict, int count, int interval)
+{
+    char key[256] = {0};
+    int i = 0;
+    uint64_t sec = 0;
+    uint64_t r_count = 0;
+    uint64_t w_count = 0;
+    uint64_t rb_counts[32] = {0};
+    uint64_t wb_counts[32] = {0};
+    cli_profile_info_t profile_info[GF_FOP_MAXVALUE] = {{0}};
+    cli_profile_info_t upcall_info[GF_UPCALL_FLAGS_MAXVALUE] = {
+        {0},
+    };
+    char output[128] = {0};
+    int per_line = 0;
+    char read_blocks[128] = {0};
+    char write_blocks[128] = {0};
+    int index = 0;
+    int is_header_printed = 0;
+    int ret = 0;
+    double total_percentage_latency = 0;
+
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &rb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+    }
 
-        if (per_line != 0) {
-                cli_out ("%s", output);
-                cli_out ("%s", read_blocks);
-                cli_out ("%s", write_blocks);
-        }
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (profile_info[i].fop_hits == 0)
-                        continue;
-                if (is_header_printed == 0) {
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "%-latency",
-                                 "Avg-latency", "Min-Latency", "Max-Latency",
-                                 "No. of calls", "Fop");
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "---------",
-                                 "-----------", "-----------", "-----------",
-                                 "------------", "----");
-                        is_header_printed = 1;
-                }
-                if (profile_info[i].fop_hits) {
-                        cli_out ("%10.2lf %10.2lf us %10.2lf us %10.2lf us"
-                                 " %14"PRId64" %11s",
-                                 profile_info[i].percentage_avg_latency,
-                                 profile_info[i].avg_latency,
-                                 profile_info[i].min_latency,
-                                 profile_info[i].max_latency,
-                                 profile_info[i].fop_hits,
-                                 profile_info[i].fop_name);
-                }
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &wb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+    }
 
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                if (upcall_info[i].fop_hits == 0)
-                        continue;
-                if (upcall_info[i].fop_hits) {
-                        cli_out ("%10.2lf %10.2lf us %10.2lf us %10.2lf us"
-                                 " %14"PRId64" %11s",
-                                 upcall_info[i].percentage_avg_latency,
-                                 upcall_info[i].avg_latency,
-                                 upcall_info[i].min_latency,
-                                 upcall_info[i].max_latency,
-                                 upcall_info[i].fop_hits,
-                                 upcall_info[i].fop_name);
-                }
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &upcall_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+        upcall_info[i].fop_name = (char *)gf_upcall_list[i];
+    }
 
-        cli_out (" ");
-        cli_out ("%12s: %"PRId64" seconds", "Duration", sec);
-        cli_out ("%12s: %"PRId64" bytes", "Data Read", r_count);
-        cli_out ("%12s: %"PRId64" bytes", "Data Written", w_count);
-        cli_out (" ");
-}
-
-int32_t
-gf_cli_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                            *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               interval = 0;
-        int                               i = 1;
-        int32_t                           brick_count = 0;
-        char                              *volname = NULL;
-        char                              *brick = NULL;
-        char                              str[1024] = {0,};
-        int                               stats_cleared = 0;
-        gf1_cli_info_op                   info_op = GF_CLI_INFO_NONE;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &profile_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to profile");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].avg_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].min_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].max_latency);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
-        } else {
-                dict->extra_stdfree = rsp.dict.dict_val;
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+        profile_info[i].fop_name = (char *)gf_fop_list[i];
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+        total_percentage_latency += (profile_info[i].fop_hits *
+                                     profile_info[i].avg_latency);
+    }
+    if (total_percentage_latency) {
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            profile_info[i]
+                .percentage_avg_latency = 100 * ((profile_info[i].avg_latency *
+                                                  profile_info[i].fop_hits) /
+                                                 total_percentage_latency);
+        }
+        gf_array_insertionsort(profile_info, 1, GF_FOP_MAXVALUE - 1,
+                               sizeof(cli_profile_info_t),
+                               cli_profile_info_percentage_cmp);
+    }
+    snprintf(key, sizeof(key), "%d-%d-duration", count, interval);
+    ret = dict_get_uint64(dict, key, &sec);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", count, interval);
+    ret = dict_get_uint64(dict, key, &r_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", count, interval);
+    ret = dict_get_uint64(dict, key, &w_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    if (interval == -1)
+        cli_out("Cumulative Stats:");
+    else
+        cli_out("Interval %d Stats:", interval);
+    snprintf(output, sizeof(output), "%14s", "Block Size:");
+    snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+    snprintf(write_blocks, sizeof(write_blocks), "%14s", "No. of Writes:");
+    index = 14;
+    for (i = 0; i < 32; i++) {
+        if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
+            continue;
+        per_line++;
+        snprintf(output + index, sizeof(output) - index, "%19" PRIu32 "b+ ",
+                 (1U << i));
+        if (rb_counts[i]) {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index,
+                     "%21" PRId64 " ", rb_counts[i]);
+        } else {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index, "%21s ",
+                     "0");
         }
+        if (wb_counts[i]) {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21" PRId64 " ", wb_counts[i]);
+        } else {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21s ", "0");
+        }
+        index += 22;
+        if (per_line == 3) {
+            cli_out("%s", output);
+            cli_out("%s", read_blocks);
+            cli_out("%s", write_blocks);
+            cli_out(" ");
+            per_line = 0;
+            snprintf(output, sizeof(output), "%14s", "Block Size:");
+            snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+            snprintf(write_blocks, sizeof(write_blocks), "%14s",
+                     "No. of Writes:");
+            index = 14;
+        }
+    }
+
+    if (per_line != 0) {
+        cli_out("%s", output);
+        cli_out("%s", read_blocks);
+        cli_out("%s", write_blocks);
+    }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        if (profile_info[i].fop_hits == 0)
+            continue;
+        if (is_header_printed == 0) {
+            cli_out("%10s %13s %13s %13s %14s %11s", "%-latency", "Avg-latency",
+                    "Min-Latency", "Max-Latency", "No. of calls", "Fop");
+            cli_out("%10s %13s %13s %13s %14s %11s", "---------", "-----------",
+                    "-----------", "-----------", "------------", "----");
+            is_header_printed = 1;
+        }
+        if (profile_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                profile_info[i].percentage_avg_latency,
+                profile_info[i].avg_latency, profile_info[i].min_latency,
+                profile_info[i].max_latency, profile_info[i].fop_hits,
+                profile_info[i].fop_name);
+        }
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        if (upcall_info[i].fop_hits == 0)
+            continue;
+        if (upcall_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                upcall_info[i].percentage_avg_latency,
+                upcall_info[i].avg_latency, upcall_info[i].min_latency,
+                upcall_info[i].max_latency, upcall_info[i].fop_hits,
+                upcall_info[i].fop_name);
+        }
+    }
+
+    cli_out(" ");
+    cli_out("%12s: %" PRId64 " seconds", "Duration", sec);
+    cli_out("%12s: %" PRId64 " bytes", "Data Read", r_count);
+    cli_out("%12s: %" PRId64 " bytes", "Data Written", w_count);
+    cli_out(" ");
+}
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
+static int32_t
+gf_cli_profile_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[64] = {0};
+    int len;
+    int interval = 0;
+    int i = 1;
+    int32_t brick_count = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    char str[1024] = {
+        0,
+    };
+    int stats_cleared = 0;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to profile");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
         if (ret)
-                goto out;
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                cli_err ("%s", rsp.op_errstr);
-        } else {
-                switch (op) {
-                case GF_CLI_STATS_START:
-                        cli_out ("Starting volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_STOP:
-                        cli_out ("Stopping volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_INFO:
-                        break;
-                default:
-                        cli_out ("volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                }
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+        cli_err("%s", rsp.op_errstr);
+    } else {
+        switch (op) {
+            case GF_CLI_STATS_START:
+                cli_out("Starting volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_STOP:
+                cli_out("Stopping volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_INFO:
+                break;
+            default:
+                cli_out("volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
         }
+    }
 
-        if (rsp.op_ret) {
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret) {
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        if (GF_CLI_STATS_INFO != op) {
-                ret = 0;
-                goto out;
-        }
+    if (GF_CLI_STATS_INFO != op) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "info-op", (int32_t*)&info_op);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        goto out;
+    }
 
-        if (!brick_count) {
-                cli_err ("All bricks of volume %s are down.", volname);
-                goto out;
+    ret = dict_get_int32_sizen(dict, "info-op", (int32_t *)&info_op);
+    if (ret)
+        goto out;
+
+    while (i <= brick_count) {
+        len = snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_strn(dict, key, len, &brick);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get brick name");
+            goto out;
         }
 
-        while (i <= brick_count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get brick name");
-                        goto out;
-                }
+        ret = dict_get_str_boolean(dict, "nfs", _gf_false);
 
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+        if (ret)
+            len = snprintf(str, sizeof(str), "NFS Server : %s", brick);
+        else
+            len = snprintf(str, sizeof(str), "Brick: %s", brick);
+        cli_out("%s", str);
+        memset(str, '-', len);
+        cli_out("%s", str);
+
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            len = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, len, &stats_cleared);
+            if (ret)
+                goto out;
+            cli_out(stats_cleared ? "Cleared stats."
+                                  : "Failed to clear stats.");
+        } else {
+            len = snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
 
-                if (ret)
-                        snprintf (str, sizeof (str), "NFS Server : %s", brick);
-                else
-                        snprintf (str, sizeof (str), "Brick: %s", brick);
-                cli_out ("%s", str);
-                memset (str, '-', strlen (str));
-                cli_out ("%s", str);
-
-                if (GF_CLI_INFO_CLEAR == info_op) {
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats." :
-                                                 "Failed to clear stats.");
-                } else {
-                        snprintf (key, sizeof (key), "%d-cumulative", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-
-                        snprintf (key, sizeof (key), "%d-interval", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-                }
-                i++;
+            len = snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
         }
-        ret = rsp.op_ret;
+        i++;
+    }
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_profile_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_profile_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                           *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               i = 0;
-        int32_t                           brick_count = 0;
-        char                              brick[1024];
-        int32_t                           members = 0;
-        char                             *filename;
-        char                             *bricks;
-        uint64_t                          value = 0;
-        int32_t                           j = 0;
-        gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
-        uint64_t                          nr_open = 0;
-        uint64_t                          max_nr_open = 0;
-        double                            throughput = 0;
-        double                            time = 0;
-        int32_t                           time_sec = 0;
-        long int                          time_usec = 0;
-        char                              timestr[256] = {0, };
-        char                             *openfd_str = NULL;
-        gf_boolean_t                      nfs = _gf_false;
-        gf_boolean_t                      clear_stats = _gf_false;
-        int                               stats_cleared = 0;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int32_t
+gf_cli_top_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[256] = {0};
+    int keylen;
+    int i = 0;
+    int32_t brick_count = 0;
+    char brick[1024];
+    int32_t members = 0;
+    char *filename;
+    char *bricks;
+    uint64_t value = 0;
+    int32_t j = 0;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    uint64_t nr_open = 0;
+    uint64_t max_nr_open = 0;
+    double throughput = 0;
+    double time = 0;
+    int32_t time_sec = 0;
+    long int time_usec = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *openfd_str = NULL;
+    gf_boolean_t nfs = _gf_false;
+    gf_boolean_t clear_stats = _gf_false;
+    int stats_cleared = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to top");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("volume top unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to top");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    dict = dict_new();
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("volume top unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
 
+    if (op != GF_CLI_STATS_TOP) {
+        ret = 0;
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_top(dict, rsp.op_ret, rsp.op_errno,
+                                     rsp.op_errstr);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    keylen = snprintf(key, sizeof(key), "%d-top-op", 1);
+    ret = dict_get_int32n(dict, key, keylen, (int32_t *)&top_op);
+    if (ret)
+        goto out;
 
-        if (op != GF_CLI_STATS_TOP) {
-                ret = 0;
+    clear_stats = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+
+    while (i < brick_count) {
+        i++;
+        keylen = snprintf(brick, sizeof(brick), "%d-brick", i);
+        ret = dict_get_strn(dict, brick, keylen, &bricks);
+        if (ret)
+            goto out;
+
+        nfs = dict_get_str_boolean(dict, "nfs", _gf_false);
+
+        if (clear_stats) {
+            keylen = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, keylen, &stats_cleared);
+            if (ret)
                 goto out;
+            cli_out(stats_cleared ? "Cleared stats for %s %s"
+                                  : "Failed to clear stats for %s %s",
+                    nfs ? "NFS server on" : "brick", bricks);
+            continue;
         }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_top (dict, rsp.op_ret,
-                                              rsp.op_errno,
-                                              rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+        if (nfs)
+            cli_out("NFS Server : %s", bricks);
+        else
+            cli_out("Brick: %s", bricks);
+
+        keylen = snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32n(dict, key, keylen, &members);
+
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &nr_open);
+                if (ret)
+                    break;
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_nr_open);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_strn(dict, key, keylen, &openfd_str);
+                if (ret)
+                    goto out;
+                cli_out("Current open fds: %" PRIu64 ", Max open fds: %" PRIu64
+                        ", Max openfd time: %s",
+                        nr_open, max_nr_open, openfd_str);
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
+                if (!members) {
+                    continue;
+                }
+                cli_out("Count\t\tfilename\n=======================");
+                break;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time);
                 }
+                if (!ret)
+                    cli_out("Throughput %.2f MBps time %.4f secs", throughput,
+                            time / 1e6);
+
+                if (!members) {
+                    continue;
+                }
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH, "MBps",
+                        VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
+                        VOL_TOP_PERF_TIME_WIDTH, "Time");
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                        "====", VOL_TOP_PERF_FILENAME_DEF_WIDTH,
+                        "========", VOL_TOP_PERF_TIME_WIDTH, "====");
+                break;
+            default:
                 goto out;
         }
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        snprintf (key, sizeof (key), "%d-top-op", 1);
-        ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
-        if (ret)
-                goto out;
-
-        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
-
-        while (i < brick_count) {
-                i++;
-                snprintf (brick, sizeof (brick), "%d-brick", i);
-                ret = dict_get_str (dict, brick, &bricks);
-                if (ret)
-                        goto out;
-
-                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
-
-                if (clear_stats) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
-                                 "Failed to clear stats for %s %s",
-                                 nfs ? "NFS server on" : "brick", bricks);
-                        continue;
-                }
-
-                if (nfs)
-                        cli_out ("NFS Server : %s", bricks);
+        for (j = 1; j <= members; j++) {
+            keylen = snprintf(key, sizeof(key), "%d-filename-%d", i, j);
+            ret = dict_get_strn(dict, key, keylen, &filename);
+            if (ret)
+                break;
+            snprintf(key, sizeof(key), "%d-value-%d", i, j);
+            ret = dict_get_uint64(dict, key, &value);
+            if (ret)
+                goto out;
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                keylen = snprintf(key, sizeof(key), "%d-time-sec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_sec);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-time-usec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_usec);
+                if (ret)
+                    goto out;
+                gf_time_fmt(timestr, sizeof timestr, time_sec, gf_timefmt_FT);
+                snprintf(timestr + strlen(timestr),
+                         sizeof timestr - strlen(timestr), ".%ld", time_usec);
+                if (strlen(filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
+                    cli_out("%*" PRIu64 " %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                            value, VOL_TOP_PERF_FILENAME_DEF_WIDTH, filename,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
                 else
-                        cli_out ("Brick: %s", bricks);
-
-                snprintf(key, sizeof (key), "%d-members", i);
-                ret = dict_get_int32 (dict, key, &members);
-
-                switch (top_op) {
-                case GF_CLI_TOP_OPEN:
-                        snprintf (key, sizeof (key), "%d-current-open", i);
-                        ret = dict_get_uint64 (dict, key, &nr_open);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-max-open", i);
-                        ret = dict_get_uint64 (dict, key, &max_nr_open);
-                        if (ret)
-                                goto out;
-                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
-                        ret = dict_get_str (dict, key, &openfd_str);
-                        if (ret)
-                                goto out;
-                        cli_out ("Current open fds: %"PRIu64", Max open"
-                                " fds: %"PRIu64", Max openfd time: %s", nr_open,
-                                 max_nr_open, openfd_str);
-                case GF_CLI_TOP_READ:
-                case GF_CLI_TOP_WRITE:
-                case GF_CLI_TOP_OPENDIR:
-                case GF_CLI_TOP_READDIR:
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("Count\t\tfilename\n=======================");
-                        break;
-                case GF_CLI_TOP_READ_PERF:
-                case GF_CLI_TOP_WRITE_PERF:
-                        snprintf (key, sizeof (key), "%d-throughput", i);
-                        ret = dict_get_double (dict, key, &throughput);
-                        if (!ret) {
-                                snprintf (key, sizeof (key), "%d-time", i);
-                                ret = dict_get_double (dict, key, &time);
-                        }
-                        if (!ret)
-                                cli_out ("Throughput %.2f MBps time %.4f secs", throughput,
-                                          time / 1e6);
-
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "MBps",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
-                                 VOL_TOP_PERF_TIME_WIDTH, "Time");
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "====",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "========",
-                                 VOL_TOP_PERF_TIME_WIDTH, "====");
-                        break;
-                default:
-                        goto out;
-                }
-
-                for (j = 1; j <= members; j++) {
-                        snprintf (key, sizeof (key), "%d-filename-%d", i, j);
-                        ret = dict_get_str (dict, key, &filename);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-value-%d", i, j);
-                        ret = dict_get_uint64 (dict, key, &value);
-                        if (ret)
-                                goto out;
-                        if ( top_op == GF_CLI_TOP_READ_PERF ||
-                                top_op == GF_CLI_TOP_WRITE_PERF) {
-                                snprintf (key, sizeof (key), "%d-time-sec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
-                                if (ret)
-                                        goto out;
-                                snprintf (key, sizeof (key), "%d-time-usec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
-                                if (ret)
-                                        goto out;
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             time_sec, gf_timefmt_FT);
-                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%ld", time_usec);
-                                if (strlen (filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
-                                        cli_out ("%*"PRIu64" %-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH,
-                                                 filename,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                                else
-                                        cli_out ("%*"PRIu64" ...%-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH ,
-                                                 filename + strlen (filename) -
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                        } else {
-                                cli_out ("%"PRIu64"\t\t%s", value, filename);
-                        }
-                }
+                    cli_out("%*" PRIu64 " ...%-*s %-*s",
+                            VOL_TOP_PERF_SPEED_WIDTH, value,
+                            VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            filename + strlen(filename) -
+                                VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
+            } else {
+                cli_out("%" PRIu64 "\t\t%s", value, filename);
+            }
         }
-        ret = rsp.op_ret;
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
+    cli_cmd_broadcast_response(ret);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_top_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_top_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-
-int
-gf_cli_getwd_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getwd_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf1_cli_getwd_rsp rsp   = {0,};
-        int               ret   = -1;
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                cli_err ("getwd failed");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        cli_err("getwd failed");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getwd");
+    gf_log("cli", GF_LOG_INFO, "Received resp to getwd");
 
-        cli_out ("%s", rsp.wd);
+    cli_out("%s", rsp.wd);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (rsp.wd) {
+        free(rsp.wd);
+    }
+
+    return ret;
 }
 
-int32_t
-gf_cli_getwd (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_getwd(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret = -1;
-        gf1_cli_getwd_req        req = {0,};
+    int ret = -1;
+    gf1_cli_getwd_req req = {
+        0,
+    };
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
 
-        if (!frame || !this)
-                goto out;
+    if (!frame || !this)
+        goto out;
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GETWD, NULL,
-                              this, gf_cli_getwd_cbk,
-                              (xdrproc_t) xdr_gf1_cli_getwd_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_GETWD,
+                         NULL, this, gf_cli_getwd_cbk,
+                         (xdrproc_t)xdr_gf1_cli_getwd_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-void
-cli_print_volume_status_mempool (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        int32_t         mempool_count = 0;
-        char            *name = NULL;
-        int32_t         hotcount = 0;
-        int32_t         coldcount = 0;
-        uint64_t        paddedsizeof = 0;
-        uint64_t        alloccount = 0;
-        int32_t         maxalloc = 0;
-        uint64_t        pool_misses = 0;
-        int32_t         maxstdalloc = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mempool-count",prefix);
-        ret = dict_get_int32 (dict, key, &mempool_count);
+static void
+cli_print_volume_status_mempool(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    int32_t mempool_count = 0;
+    char *name = NULL;
+    int32_t hotcount = 0;
+    int32_t coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int32_t maxalloc = 0;
+    uint64_t pool_misses = 0;
+    int32_t maxstdalloc = 0;
+    char key[128] = {
+        /* prefix is really small 'brick%d' really */
+        0,
+    };
+    int keylen;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    keylen = snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32n(dict, key, keylen, &mempool_count);
+    if (ret)
+        goto out;
+
+    cli_out("Mempool Stats\n-------------");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+            "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc", "Misses",
+            "Max-StdAlloc");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+            "---------", "------------", "----------", "--------", "--------",
+            "------------");
+
+    for (i = 0; i < mempool_count; i++) {
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_strn(dict, key, keylen, &name);
         if (ret)
-                goto out;
-
-        cli_out ("Mempool Stats\n-------------");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
-                 "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc",
-                 "Misses", "Max-StdAlloc");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
-                 "---------", "------------", "----------",
-                 "--------", "--------", "------------");
-
-        for (i = 0; i < mempool_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
-                ret = dict_get_str (dict, key, &name);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &hotcount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &coldcount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
-                          prefix, i);
-                ret = dict_get_uint64 (dict, key, &paddedsizeof);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxstdalloc);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
-                ret = dict_get_uint64 (dict, key, &pool_misses);
-                if (ret)
-                        goto out;
-
-                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d %8"PRIu64
-                         " %12d", name, hotcount, coldcount, paddedsizeof,
-                         alloccount, maxalloc, pool_misses, maxstdalloc);
-        }
-
-out:
-        return;
+            goto out;
 
-}
-
-void
-cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        char            key[1024] = {0,};
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        int             val = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &hotcount);
         if (ret)
-                goto out;
-        cli_out ("Memory status for volume : %s", volname);
+            goto out;
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &coldcount);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
+            goto out;
 
-        index_max = brick_index_max + other_count;
-
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        continue;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        continue;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
-
-                cli_out ("Mallinfo\n--------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.arena", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d","Arena", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Ordblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Smblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblkhd", val);
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Usmblks", val);
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fsmblks", val);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &maxalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Uordblks", val);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix,
+                          i);
+        ret = dict_get_int32n(dict, key, keylen, &maxstdalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fordblks", val);
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &pool_misses);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Keepcost", val);
+        cli_out("%-30s %9d %9d %12" PRIu64 " %10" PRIu64 " %8d %8" PRIu64
+                " %12d",
+                name, hotcount, coldcount, paddedsizeof, alloccount, maxalloc,
+                pool_misses, maxstdalloc);
+    }
 
-                cli_out (" ");
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d", i);
-                cli_print_volume_status_mempool (dict, key);
-        }
 out:
-        cli_out ("----------------------------------------------\n");
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             client_count = 0;
-        char            *clientname = NULL;
-        uint64_t        bytesread = 0;
-        uint64_t        byteswrite = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Client connections for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_mem(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    char key[64] = {
+        0,
+    };
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int val = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Memory status for volume : %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            continue;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
+            continue;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-        for (i = 0; i <= index_max; i++) {
-                hostname = NULL;
-                path = NULL;
-                online = -1;
-                client_count = 0;
-                clientname = NULL;
-                bytesread = 0;
-                byteswrite = 0;
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                cli_out ("----------------------------------------------");
+        cli_out("Mallinfo\n--------");
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.arena", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Arena", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Ordblks", val);
 
-                if (hostname && path) {
-                        if (notbrick)
-                                cli_out ("%s : %s", hostname, path);
-                        else
-                                cli_out ("Brick : %s:%s", hostname, path);
-                }
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Smblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.clientcount", i);
-                ret = dict_get_int32 (dict, key, &client_count);
-
-                if (hostname && path)
-                        cli_out ("Clients connected : %d", client_count);
-                if (client_count == 0)
-                        continue;
-
-                cli_out ("%-48s %15s %15s", "Hostname", "BytesRead",
-                         "BytesWritten");
-                cli_out ("%-48s %15s %15s", "--------", "---------",
-                         "------------");
-                for (j =0; j < client_count; j++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.hostname", i, j);
-                        ret = dict_get_str (dict, key, &clientname);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.bytesread", i, j);
-                        ret = dict_get_uint64 (dict, key, &bytesread);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                 "brick%d.client%d.byteswrite", i, j);
-                        ret = dict_get_uint64 (dict, key, &byteswrite);
-
-                        cli_out ("%-48s %15"PRIu64" %15"PRIu64,
-                                 clientname, bytesread, byteswrite);
-                }
-        }
-out:
-        cli_out ("----------------------------------------------\n");
-        return;
-}
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblks", val);
 
-void
-cli_print_volume_status_inode_entry (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        char            *gfid = NULL;
-        uint64_t        nlookup = 0;
-        uint32_t        ref = 0;
-        int             ia_type = 0;
-        char            inode_type;
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblkhd", val);
 
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Usmblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_get_str (dict, key, &gfid);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Fsmblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.nlookup", prefix);
-        ret = dict_get_uint64 (dict, key, &nlookup);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Uordblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ref", prefix);
-        ret = dict_get_uint32 (dict, key, &ref);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Fordblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ia_type", prefix);
-        ret = dict_get_int32 (dict, key, &ia_type);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Keepcost", val);
+
+        cli_out(" ");
+        snprintf(key, sizeof(key), "brick%d", i);
+        cli_print_volume_status_mempool(dict, key);
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
 
-        switch (ia_type) {
+static void
+cli_print_volume_status_client_list(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int client_count = 0;
+    int current_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int total = 0;
+    char *name = NULL;
+    gf_boolean_t is_fuse_done = _gf_false;
+    gf_boolean_t is_gfapi_done = _gf_false;
+    gf_boolean_t is_rebalance_done = _gf_false;
+    gf_boolean_t is_glustershd_done = _gf_false;
+    gf_boolean_t is_quotad_done = _gf_false;
+    gf_boolean_t is_snapd_done = _gf_false;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "client-count", &client_count);
+    if (ret)
+        goto out;
+
+    cli_out("%-48s %15s", "Name", "count");
+    cli_out("%-48s %15s", "-----", "------");
+    for (i = 0; i < client_count; i++) {
+        name = NULL;
+        snprintf(key, sizeof(key), "client%d.name", i);
+        ret = dict_get_str(dict, key, &name);
+
+        if (!strncmp(name, "fuse", 4)) {
+            if (!is_fuse_done) {
+                is_fuse_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "fuse-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strncmp(name, "gfapi", 5)) {
+            if (!is_gfapi_done) {
+                is_gfapi_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "gfapi-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "rebalance")) {
+            if (!is_rebalance_done) {
+                is_rebalance_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "rebalance-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "glustershd")) {
+            if (!is_glustershd_done) {
+                is_glustershd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "glustershd-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "quotad")) {
+            if (!is_quotad_done) {
+                is_quotad_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "quotad-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "snapd")) {
+            if (!is_snapd_done) {
+                is_snapd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "snapd-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        }
+
+    print:
+        cli_out("%-48s %15d", name, current_count);
+    }
+out:
+    cli_out("\ntotal clients for volume %s : %d ", volname, total);
+    cli_out(
+        "-----------------------------------------------------------------\n");
+    return;
+}
+
+static void
+cli_print_volume_status_clients(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int client_count = 0;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+    char key[128] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        hostname = NULL;
+        path = NULL;
+        online = -1;
+        client_count = 0;
+        clientname = NULL;
+        bytesread = 0;
+        byteswrite = 0;
+
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+
+        if (hostname && path) {
+            if (notbrick)
+                cli_out("%s : %s", hostname, path);
+            else
+                cli_out("Brick : %s:%s", hostname, path);
+        }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.clientcount", i);
+        ret = dict_get_int32(dict, key, &client_count);
+
+        if (hostname && path)
+            cli_out("Clients connected : %d", client_count);
+        if (client_count == 0)
+            continue;
+
+        cli_out("%-48s %15s %15s %15s", "Hostname", "BytesRead", "BytesWritten",
+                "OpVersion");
+        cli_out("%-48s %15s %15s %15s", "--------", "---------", "------------",
+                "---------");
+        for (j = 0; j < client_count; j++) {
+            snprintf(key, sizeof(key), "brick%d.client%d.hostname", i, j);
+            ret = dict_get_str(dict, key, &clientname);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.bytesread", i, j);
+            ret = dict_get_uint64(dict, key, &bytesread);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", i, j);
+            ret = dict_get_uint64(dict, key, &byteswrite);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.opversion", i, j);
+            ret = dict_get_uint32(dict, key, &opversion);
+
+            cli_out("%-48s %15" PRIu64 " %15" PRIu64 " %15" PRIu32, clientname,
+                    bytesread, byteswrite, opversion);
+        }
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
+
+#ifdef DEBUG /* this function is only used in debug */
+static void
+cli_print_volume_status_inode_entry(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char inode_type;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+
+    switch (ia_type) {
         case IA_IFREG:
-                inode_type = 'R';
-                break;
+            inode_type = 'R';
+            break;
         case IA_IFDIR:
-                inode_type = 'D';
-                break;
+            inode_type = 'D';
+            break;
         case IA_IFLNK:
-                inode_type = 'L';
-                break;
+            inode_type = 'L';
+            break;
         case IA_IFBLK:
-                inode_type = 'B';
-                break;
+            inode_type = 'B';
+            break;
         case IA_IFCHR:
-                inode_type = 'C';
-                break;
+            inode_type = 'C';
+            break;
         case IA_IFIFO:
-                inode_type = 'F';
-                break;
+            inode_type = 'F';
+            break;
         case IA_IFSOCK:
-                inode_type = 'S';
-                break;
+            inode_type = 'S';
+            break;
         default:
-                inode_type = 'I';
-                break;
-        }
+            inode_type = 'I';
+            break;
+    }
 
-        cli_out ("%-40s %14"PRIu64" %14"PRIu32" %9c",
-                 gfid, nlookup, ref, inode_type);
+    cli_out("%-40s %14" PRIu64 " %14" PRIu32 " %9c", gfid, nlookup, ref,
+            inode_type);
 
 out:
-        return;
-
+    return;
 }
+#endif
 
-void
-cli_print_volume_status_itables (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        uint32_t        active_size = 0;
-        uint32_t        lru_size = 0;
-        uint32_t        purge_size = 0;
-        int             i =0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
+static void
+cli_print_volume_status_itables(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    uint32_t lru_limit = 0;
+#ifdef DEBUG
+    int i = 0;
+#endif
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.lru_limit", prefix);
+    ret = dict_get_uint32(dict, key, &lru_limit);
+    if (ret)
+        goto out;
+    cli_out("LRU limit     : %u", lru_limit);
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (active_size != 0) {
+        cli_out("Active inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < active_size; i++) {
+        snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("Active Inodes : %u", active_size);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.active_size", prefix);
-        ret = dict_get_uint32 (dict, key, &active_size);
-        if (ret)
-                goto out;
-        if (active_size != 0) {
-                cli_out ("Active inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < active_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.active%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.lru_size", prefix);
-        ret = dict_get_uint32 (dict, key, &lru_size);
-        if (ret)
-                goto out;
-        if (lru_size != 0) {
-                cli_out ("LRU inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < lru_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (lru_size != 0) {
+        cli_out("LRU inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < lru_size; i++) {
+        snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("LRU Inodes    : %u", lru_size);
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.purge_size", prefix);
-        ret = dict_get_uint32 (dict, key, &purge_size);
-        if (ret)
-                goto out;
-        if (purge_size != 0) {
-                cli_out ("Purged inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < purge_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+#ifdef DEBUG
+    if (purge_size != 0) {
+        cli_out("Purged inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < purge_size; i++) {
+        snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+#else
+    cli_out("Purge Inodes  : %u", purge_size);
+#endif
 
 out:
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Inode tables for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_inode(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Inode tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for ( i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+            goto out;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                for (j = 0; j < conn_count; j++) {
-                        if (conn_count > 1)
-                                cli_out ("Connection %d:", j+1);
+        for (j = 0; j < conn_count; j++) {
+            if (conn_count > 1)
+                cli_out("Connection %d:", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.itable",
-                                  i, j);
-                        cli_print_volume_status_itables (dict, key);
-                        cli_out (" ");
-                }
+            snprintf(key, sizeof(key), "brick%d.conn%d.itable", i, j);
+            cli_print_volume_status_itables(dict, key);
+            cli_out(" ");
         }
+    }
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
 void
-cli_print_volume_status_fdtable (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             refcount = 0;
-        uint32_t        maxfds = 0;
-        int             firstfree = 0;
-        int             openfds = 0;
-        int             fd_pid = 0;
-        int             fd_refcount = 0;
-        int             fd_flags = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &refcount);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.maxfds", prefix);
-        ret = dict_get_uint32 (dict, key, &maxfds);
+cli_print_volume_status_fdtable(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+
+    cli_out("RefCount = %d  MaxFDs = %d  FirstFree = %d", refcount, maxfds,
+            firstfree);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    if (0 == openfds) {
+        cli_err("No open fds");
+        goto out;
+    }
+
+    cli_out("%-19s %-19s %-19s %-19s", "FD Entry", "PID", "RefCount", "Flags");
+    cli_out("%-19s %-19s %-19s %-19s", "--------", "---", "--------", "-----");
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
         if (ret)
-                goto out;
+            continue;
 
-        memset (key, 0 ,sizeof (key));
-        snprintf (key, sizeof (key), "%s.firstfree", prefix);
-        ret = dict_get_int32 (dict, key, &firstfree);
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
         if (ret)
-                goto out;
-
-        cli_out ("RefCount = %d  MaxFDs = %d  FirstFree = %d",
-                 refcount, maxfds, firstfree);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.openfds", prefix);
-        ret = dict_get_int32 (dict, key, &openfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
         if (ret)
-                goto out;
-        if (0 == openfds) {
-                cli_err ("No open fds");
-                goto out;
-        }
+            continue;
 
-        cli_out ("%-19s %-19s %-19s %-19s", "FD Entry", "PID",
-                 "RefCount", "Flags");
-        cli_out ("%-19s %-19s %-19s %-19s", "--------", "---",
-                 "--------", "-----");
-
-        for (i = 0; i < maxfds ; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_pid);
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
-                          prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_refcount);
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_flags);
-                if (ret)
-                        continue;
-
-                cli_out ("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount,
-                         fd_flags);
-        }
+        cli_out("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount, fd_flags);
+    }
 
 out:
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("FD tables for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_fd(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("FD tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                for (j = 0; j < conn_count; j++) {
-                        cli_out ("Connection %d:", j+1);
+        for (j = 0; j < conn_count; j++) {
+            cli_out("Connection %d:", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
-                                  i, j);
-                        cli_print_volume_status_fdtable (dict, key);
-                        cli_out (" ");
-                }
+            snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", i, j);
+            cli_print_volume_status_fdtable(dict, key);
+            cli_out(" ");
         }
+    }
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
-void
-cli_print_volume_status_call_frame (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             ref_count = 0;
-        char            *translator = 0;
-        int             complete = 0;
-        char            *parent = NULL;
-        char            *wind_from = NULL;
-        char            *wind_to = NULL;
-        char            *unwind_from = NULL;
-        char            *unwind_to = NULL;
+static void
+cli_print_volume_status_call_frame(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int ref_count = 0;
+    char *translator = 0;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+
+    if (!dict || !prefix)
+        return;
 
-        if (!dict || !prefix)
-                return;
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &ref_count);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_get_str (dict, key, &translator);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.complete", prefix);
-        ret = dict_get_int32 (dict, key, &complete);
-        if (ret)
-                return;
+    cli_out("  Ref Count   = %d", ref_count);
+    cli_out("  Translator  = %s", translator);
+    cli_out("  Completed   = %s", (complete ? "Yes" : "No"));
 
-        cli_out ("  Ref Count   = %d", ref_count);
-        cli_out ("  Translator  = %s", translator);
-        cli_out ("  Completed   = %s", (complete ? "Yes" : "No"));
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret)
+        cli_out("  Parent      = %s", parent);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.parent", prefix);
-        ret = dict_get_str (dict, key, &parent);
-        if (!ret)
-                cli_out ("  Parent      = %s", parent);
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret)
+        cli_out("  Wind From   = %s", wind_from);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windfrom", prefix);
-        ret = dict_get_str (dict, key, &wind_from);
-        if (!ret)
-                cli_out ("  Wind From   = %s", wind_from);
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret)
+        cli_out("  Wind To     = %s", wind_to);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windto", prefix);
-        ret = dict_get_str (dict, key, &wind_to);
-        if (!ret)
-                cli_out ("  Wind To     = %s", wind_to);
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret)
+        cli_out("  Unwind From = %s", unwind_from);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-        ret = dict_get_str (dict, key, &unwind_from);
-        if (!ret)
-                cli_out ("  Unwind From = %s", unwind_from);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindto", prefix);
-        ret = dict_get_str (dict, key, &unwind_to);
-        if (!ret)
-                cli_out ("  Unwind To   = %s", unwind_to);
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret)
+        cli_out("  Unwind To   = %s", unwind_to);
 }
 
-void
-cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             uid = 0;
-        int             gid = 0;
-        int             pid = 0;
-        uint64_t        unique = 0;
-        //char            *op = NULL;
-        int             count = 0;
-        int             i = 0;
-
-        if (!dict || !prefix)
-                return;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.uid", prefix);
-        ret = dict_get_int32 (dict, key, &uid);
-        if (ret)
-                return;
+static void
+cli_print_volume_status_call_stack(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    // char            *op = NULL;
+    int count = 0;
+    int i = 0;
+
+    if (!prefix)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gid", prefix);
-        ret = dict_get_int32 (dict, key, &gid);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", prefix);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unique", prefix);
-        ret = dict_get_uint64 (dict, key, &unique);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        return;
 
-        /*
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.op", prefix);
-        ret = dict_get_str (dict, key, &op);
-        if (ret)
-                return;
-        */
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        return;
 
+    /*
+    snprintf (key, sizeof (key), "%s.op", prefix);
+    ret = dict_get_str (dict, key, &op);
+    if (ret)
+            return;
+    */
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        return;
 
-        cli_out (" UID    : %d", uid);
-        cli_out (" GID    : %d", gid);
-        cli_out (" PID    : %d", pid);
-        cli_out (" Unique : %"PRIu64, unique);
-        //cli_out ("\tOp     : %s", op);
-        cli_out (" Frames : %d", count);
+    cli_out(" UID    : %d", uid);
+    cli_out(" GID    : %d", gid);
+    cli_out(" PID    : %d", pid);
+    cli_out(" Unique : %" PRIu64, unique);
+    // cli_out ("\tOp     : %s", op);
+    cli_out(" Frames : %d", count);
 
-        for (i = 0; i < count; i++) {
-                cli_out (" Frame %d", i+1);
+    for (i = 0; i < count; i++) {
+        cli_out(" Frame %d", i + 1);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
-                cli_print_volume_status_call_frame (dict, key);
-        }
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        cli_print_volume_status_call_frame(dict, key);
+    }
 
-        cli_out (" ");
+    cli_out(" ");
 }
 
-void
-cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             call_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Pending calls for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_callpool(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int call_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Pending calls for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
+            goto out;
 
-        index_max = brick_index_max + other_count;
-
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.callpool.count", i);
-                ret = dict_get_int32 (dict, key, &call_count);
-                if (ret)
-                        goto out;
-                cli_out ("Pending calls: %d", call_count);
+        snprintf(key, sizeof(key), "brick%d.callpool.count", i);
+        ret = dict_get_int32(dict, key, &call_count);
+        if (ret)
+            goto out;
+        cli_out("Pending calls: %d", call_count);
 
-                if (0 == call_count)
-                        continue;
+        if (0 == call_count)
+            continue;
 
-                for (j = 0; j < call_count; j++) {
-                        cli_out ("Call Stack%d", j+1);
+        for (j = 0; j < call_count; j++) {
+            cli_out("Call Stack%d", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.callpool.stack%d", i, j);
-                        cli_print_volume_status_call_stack (dict, key);
-                }
+            snprintf(key, sizeof(key), "brick%d.callpool.stack%d", i, j);
+            cli_print_volume_status_call_stack(dict, key);
         }
+    }
 
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
 static void
-cli_print_volume_status_tasks (dict_t *dict)
-{
-        int             ret         = -1;
-        int             i           = 0;
-        int             j           = 0;
-        int             count       = 0;
-        int             task_count  = 0;
-        int             status      = 0;
-        char           *op          = NULL;
-        char           *task_id_str = NULL;
-        char           *volname     = NULL;
-        char            key[1024]   = {0,};
-        char            task[1024]  = {0,};
-        char           *brick       = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+cli_print_volume_status_tasks(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+    int count = 0;
+    int task_count = 0;
+    int status = 0;
+    char *op = NULL;
+    char *task_id_str = NULL;
+    char *volname = NULL;
+    char key[64] = {
+        0,
+    };
+    char task[32] = {
+        0,
+    };
+    char *brick = NULL;
+
+    ret = dict_get_int32_sizen(dict, "tasks", &task_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get tasks count");
+        return;
+    }
 
-        ret = dict_get_int32 (dict, "tasks", &task_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get tasks count");
-                return;
-        }
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-        cli_out ("Task Status of Volume %s", volname);
-        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+    cli_out("Task Status of Volume %s", volname);
+    cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
 
-        if (task_count == 0) {
-                cli_out ("There are no active volume tasks");
-                cli_out (" ");
-                return;
-        }
+    if (task_count == 0) {
+        cli_out("There are no active volume tasks");
+        cli_out(" ");
+        return;
+    }
 
-        for (i = 0; i < task_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str(dict, key, &op);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "Task", op);
+    for (i = 0; i < task_count; i++) {
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &op);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "Task", op);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (dict, key, &task_id_str);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "ID", task_id_str);
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "ID", task_id_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        return;
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            return;
 
-                snprintf (task, sizeof (task), "task%d", i);
+        snprintf(task, sizeof(task), "task%d", i);
 
-                if (!strcmp (op, "Remove brick")) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.count", task);
-                        ret = dict_get_int32 (dict, key, &count);
-                        if (ret)
-                                goto out;
+        if (!strcmp(op, "Remove brick")) {
+            snprintf(key, sizeof(key), "%s.count", task);
+            ret = dict_get_int32(dict, key, &count);
+            if (ret)
+                goto out;
 
-                        cli_out ("%-20s", "Removed bricks:");
+            cli_out("%-20s", "Removed bricks:");
 
-                        for (j = 1; j <= count; j++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),"%s.brick%d",
-                                          task, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
+            for (j = 1; j <= count; j++) {
+                snprintf(key, sizeof(key), "%s.brick%d", task, j);
+                ret = dict_get_str(dict, key, &brick);
+                if (ret)
+                    goto out;
 
-                                cli_out ("%-20s", brick);
-                        }
-                }
-                cli_out ("%-20s : %-20s", "Status",
-                         cli_vol_task_status_str[status]);
-                cli_out (" ");
+                cli_out("%-20s", brick);
+            }
         }
+        cli_out("%-20s : %-20s", "Status", cli_vol_task_status_str[status]);
+        cli_out(" ");
+    }
 
 out:
-        return;
+    return;
 }
 
 static int
-gf_cli_status_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
-{
-        int                             ret             = -1;
-        int                             brick_index_max = -1;
-        int                             other_count     = 0;
-        int                             index_max       = 0;
-        int                             i               = 0;
-        int                             type            = -1;
-        int                             hot_brick_count = -1;
-        int                             pid             = -1;
-        uint32_t                        cmd             = 0;
-        gf_boolean_t                    notbrick        = _gf_false;
-        char                            key[1024]       = {0,};
-        char                           *hostname        = NULL;
-        char                           *path            = NULL;
-        char                           *volname         = NULL;
-        dict_t                         *dict            = NULL;
-        gf_cli_rsp                      rsp             = {0,};
-        cli_volume_status_t             status          = {0};
-        cli_local_t                    *local           = NULL;
-        gf_boolean_t                    wipe_local      = _gf_false;
-        char                            msg[1024]       = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (req->rpc_status == -1)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+gf_cli_status_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int i = 0;
+    int type = -1;
+    int hot_brick_count = -1;
+    int pid = -1;
+    uint32_t cmd = 0;
+    gf_boolean_t notbrick = _gf_false;
+    char key[64] = {
+        0,
+    };
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_volume_status_t status = {0};
+    cli_local_t *local = NULL;
+    gf_boolean_t wipe_local = _gf_false;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received response to status cmd");
+
+    local = ((call_frame_t *)myframe)->local;
+    if (!local) {
+        local = cli_local_get();
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get local");
+            goto out;
         }
+        wipe_local = _gf_true;
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to status cmd");
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg),
+                     "Unable to obtain volume status information.");
 
-        local = ((call_frame_t *) myframe)->local;
-        if (!local) {
-                local = cli_local_get ();
-                if (!local) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get local");
-                        goto out;
-                }
-                wipe_local = _gf_true;
-        }
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            if (!local->all)
+                cli_xml_output_str("volStatus", msg, rsp.op_ret, rsp.op_errno,
+                                   rsp.op_errstr);
+            ret = 0;
+            goto out;
+        }
+
+        cli_err("%s", msg);
+        if (local && local->all) {
+            ret = 0;
+            cli_out(" ");
+        } else
+            ret = -1;
+
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_ALL)) {
+        if (local && local->dict) {
+            dict_ref(dict);
+            ret = dict_set_static_ptr(local->dict, "rsp-dict", dict);
+            ret = 0;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "local not found");
+            ret = -1;
+        }
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_begin(local, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+        if (cmd & GF_CLI_STATUS_TASKS) {
+            ret = cli_xml_output_vol_status_tasks_detail(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Error outputting to xml");
+                goto xml_end;
+            }
+        } else {
+            ret = cli_xml_output_vol_status(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+
+    xml_end:
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_end(local);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        }
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
+        (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
+        (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
+        notbrick = _gf_true;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            cli_print_volume_status_mem(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENTS:
+            cli_print_volume_status_clients(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENT_LIST:
+            cli_print_volume_status_client_list(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_INODE:
+            cli_print_volume_status_inode(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_FD:
+            cli_print_volume_status_fd(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CALLPOOL:
+            cli_print_volume_status_callpool(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_TASKS:
+            cli_print_volume_status_tasks(dict);
+            goto cont;
+            break;
+        default:
+            break;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "Unable to obtain volume "
-                                  "status information.");
-
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        if (!local->all)
-                                cli_xml_output_str ("volStatus", msg,
-                                                    rsp.op_ret, rsp.op_errno,
-                                                    rsp.op_errstr);
-                        ret = 0;
-                        goto out;
-                }
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-                cli_err ("%s", msg);
-                if (local && local->all) {
-                        ret = 0;
-                        cli_out (" ");
-                } else
-                        ret = -1;
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
 
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    index_max = brick_index_max + other_count;
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        goto out;
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "hot_brick_count", &hot_brick_count);
+    if (ret)
+        goto out;
 
-        if ((cmd & GF_CLI_STATUS_ALL)) {
-                if (local && local->dict) {
-                        dict_ref (dict);
-                        ret = dict_set_static_ptr (local->dict, "rsp-dict", dict);
-                        ret = 0;
-                } else {
-                        gf_log ("cli", GF_LOG_ERROR, "local not found");
-                        ret = -1;
-                }
-                goto out;
-        }
+    cli_out("Status of volume: %s", volname);
 
-        if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
-            (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
-            (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
-                notbrick = _gf_true;
+    if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
+        cli_out("%-*s %s  %s  %s  %s", CLI_VOL_STATUS_BRICK_LEN,
+                "Gluster process", "TCP Port", "RDMA Port", "Online", "Pid");
+        cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (!local->all) {
-                        ret = cli_xml_output_vol_status_begin (local,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto xml_end;
-                        }
-                }
-                if (cmd & GF_CLI_STATUS_TASKS) {
-                        ret = cli_xml_output_vol_status_tasks_detail (local,
-                                                                      dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,"Error outputting "
-                                        "to xml");
-                                goto xml_end;
-                        }
-                } else {
-                        ret = cli_xml_output_vol_status (local, dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto xml_end;
-                        }
-                }
+    status.brick = GF_MALLOC(PATH_MAX + 256, gf_common_mt_strdup);
+    if (!status.brick) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-xml_end:
-                if (!local->all) {
-                        ret = cli_xml_output_vol_status_end (local);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        }
-                }
-                goto out;
-        }
+    for (i = 0; i <= index_max; i++) {
+        status.rdma_port = 0;
 
-        status.brick = GF_CALLOC (1, PATH_MAX + 256, gf_common_mt_strdup);
-        if (!status.brick) {
-                errno   = ENOMEM;
-                ret     = -1;
-                goto out;
-        }
-        switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_MEM:
-                        cli_print_volume_status_mem (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_CLIENTS:
-                        cli_print_volume_status_clients (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_INODE:
-                        cli_print_volume_status_inode (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_FD:
-                        cli_print_volume_status_fd (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_CALLPOOL:
-                        cli_print_volume_status_callpool (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_TASKS:
-                        cli_print_volume_status_tasks (dict);
-                        goto cont;
-                        break;
-                default:
-                        break;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
+            continue;
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
+            continue;
 
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
+        /* Brick/not-brick is handled separately here as all
+         * types of nodes are contained in the default output
+         */
+        status.brick[0] = '\0';
+        if (!strcmp(hostname, "NFS Server") ||
+            !strcmp(hostname, "Self-heal Daemon") ||
+            !strcmp(hostname, "Quota Daemon") ||
+            !strcmp(hostname, "Snapshot Daemon") ||
+            !strcmp(hostname, "Scrubber Daemon") ||
+            !strcmp(hostname, "Bitrot Daemon"))
+            snprintf(status.brick, PATH_MAX + 255, "%s on %s", hostname, path);
+        else {
+            snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+            ret = dict_get_int32(dict, key, &(status.rdma_port));
+            if (ret)
+                continue;
+            snprintf(status.brick, PATH_MAX + 255, "Brick %s:%s", hostname,
+                     path);
+        }
 
-        index_max = brick_index_max + other_count;
+        snprintf(key, sizeof(key), "brick%d.port", i);
+        ret = dict_get_int32(dict, key, &(status.port));
+        if (ret)
+            continue;
 
-        ret = dict_get_int32 (dict, "type", &type);
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &(status.online));
         if (ret)
-                goto out;
+            continue;
 
-        ret = dict_get_int32 (dict, "hot_brick_count", &hot_brick_count);
+        snprintf(key, sizeof(key), "brick%d.pid", i);
+        ret = dict_get_int32(dict, key, &pid);
         if (ret)
-                goto out;
+            continue;
+        if (pid == -1)
+            ret = gf_asprintf(&(status.pid_str), "%s", "N/A");
+        else
+            ret = gf_asprintf(&(status.pid_str), "%d", pid);
 
-        cli_out ("Status of volume: %s", volname);
+        if (ret == -1)
+            goto out;
 
-        if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
-                cli_out ("%-*s %s  %s  %s  %s", CLI_VOL_STATUS_BRICK_LEN,
-                         "Gluster process", "TCP Port", "RDMA Port",
-                         "Online", "Pid");
-                cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
-        }
-        if (type == GF_CLUSTER_TYPE_TIER) {
-                cli_out ("Hot Bricks:");
+        if ((cmd & GF_CLI_STATUS_DETAIL)) {
+            ret = cli_get_detail_status(dict, i, &status);
+            if (ret)
+                goto out;
+            cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+            cli_print_detailed_status(&status);
+        } else {
+            cli_print_brick_status(&status);
         }
-        for (i = 0; i <= index_max; i++) {
-
-                if (type == GF_CLUSTER_TYPE_TIER && i == hot_brick_count) {
-                        cli_out ("Cold Bricks:");
-                }
-                status.rdma_port = 0;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        continue;
-
-                /* Brick/not-brick is handled separately here as all
-                 * types of nodes are contained in the default output
-                 */
-                memset (status.brick, 0, PATH_MAX + 255);
-                if (!strcmp (hostname, "NFS Server") ||
-                    !strcmp (hostname, "Self-heal Daemon") ||
-                    !strcmp (hostname, "Quota Daemon") ||
-                    !strcmp (hostname, "Snapshot Daemon") ||
-                    !strcmp (hostname, "Scrubber Daemon") ||
-                    !strcmp (hostname, "Bitrot Daemon"))
-                        snprintf (status.brick, PATH_MAX + 255, "%s on %s",
-                                  hostname, path);
-                else {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.rdma_port", i);
-                        ret = dict_get_int32 (dict, key, &(status.rdma_port));
-                        if (ret)
-                                continue;
-                        snprintf (status.brick, PATH_MAX + 255, "Brick %s:%s",
-                                  hostname, path);
-                }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.port", i);
-                ret = dict_get_int32 (dict, key, &(status.port));
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &(status.online));
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.pid", i);
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret)
-                        continue;
-                if (pid == -1)
-                        ret = gf_asprintf (&(status.pid_str), "%s", "N/A");
-                else
-                        ret = gf_asprintf (&(status.pid_str), "%d", pid);
-
-                if (ret == -1)
-                        goto out;
-
-                if ((cmd & GF_CLI_STATUS_DETAIL)) {
-                        ret = cli_get_detail_status (dict, i, &status);
-                        if (ret)
-                                goto out;
-                        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
-                        cli_print_detailed_status (&status);
-                } else {
-                        cli_print_brick_status (&status);
-                }
+        /* Allocatated memory using gf_asprintf*/
+        GF_FREE(status.pid_str);
+    }
+    cli_out(" ");
 
-        }
-        cli_out (" ");
-
-        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
-                cli_print_volume_status_tasks (dict);
+    if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
+        cli_print_volume_status_tasks(dict);
 cont:
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        GF_FREE (status.brick);
-        if (local && wipe_local) {
-                cli_local_wipe (local);
-        }
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(status.brick);
+    if (local && wipe_local) {
+        cli_local_wipe(local);
+    }
 
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_status_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_status_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req  = {{0,}};
-        int                             ret  = -1;
-        dict_t                         *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
+    dict = data;
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_status_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog,
-                               NULL);
- out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_status_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int
-gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
-{
-        int              i            = 0;
-        int              ret          = -1;
-        int              vol_count    = -1;
-        uint32_t         cmd          = 0;
-        char             key[1024]    = {0};
-        char            *volname      = NULL;
-        void            *vol_dict     = NULL;
-        dict_t          *dict         = NULL;
-        cli_local_t     *local        = NULL;
-
-        if (!frame)
-                goto out;
+static int
+gf_cli_status_volume_all(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int i = 0;
+    int ret = -1;
+    int vol_count = -1;
+    uint32_t cmd = 0;
+    char key[1024] = {0};
+    char *volname = NULL;
+    void *vol_dict = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
 
-        if (!frame->local)
-                goto out;
+    if (!frame)
+        goto out;
 
-        local = frame->local;
+    if (!frame->local)
+        goto out;
 
-        ret = dict_get_uint32 (local->dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    local = frame->local;
 
-        local->all = _gf_true;
+    ret = dict_get_uint32(local->dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-        ret = gf_cli_status_volume (frame, this, data);
+    local->all = _gf_true;
 
-        if (ret)
-                goto out;
+    ret = gf_cli_status_volume(frame, this, data);
 
-        ret = dict_get_ptr (local->dict, "rsp-dict", &vol_dict);
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 ((dict_t *)vol_dict, "vol_count", &vol_count);
-        if (ret) {
-                cli_err ("Failed to get names of volumes");
-                goto out;
-        }
+    ret = dict_get_ptr(local->dict, "rsp-dict", &vol_dict);
+    if (ret)
+        goto out;
 
-        /* remove the "all" flag in cmd */
-        cmd &= ~GF_CLI_STATUS_ALL;
-        cmd |= GF_CLI_STATUS_VOL;
+    ret = dict_get_int32_sizen((dict_t *)vol_dict, "vol_count", &vol_count);
+    if (ret) {
+        cli_err("Failed to get names of volumes");
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                //TODO: Pass proper op_* values
-                ret = cli_xml_output_vol_status_begin (local, 0,0, NULL);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                        goto xml_end;
-                }
-        }
+    /* remove the "all" flag in cmd */
+    cmd &= ~GF_CLI_STATUS_ALL;
+    cmd |= GF_CLI_STATUS_VOL;
 
-        if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
-                cli_err ("No volumes present");
-                ret = 0;
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        // TODO: Pass proper op_* values
+        ret = cli_xml_output_vol_status_begin(local, 0, 0, NULL);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto xml_end;
         }
+    }
 
-        for (i = 0; i < vol_count; i++) {
+    if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
 
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
+    for (i = 0; i < vol_count; i++) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "vol%d", i);
-                ret = dict_get_str (vol_dict, key, &volname);
-                if (ret)
-                        goto out;
+        ret = snprintf(key, sizeof(key), "vol%d", i);
+        ret = dict_get_strn(vol_dict, key, ret, &volname);
+        if (ret)
+            goto out;
 
-                ret = dict_set_str (dict, "volname", volname);
-                if (ret)
-                        goto out;
+        ret = dict_set_str_sizen(dict, "volname", volname);
+        if (ret)
+            goto out;
 
-                ret = dict_set_uint32 (dict, "cmd", cmd);
-                if (ret)
-                        goto out;
+        ret = dict_set_uint32(dict, "cmd", cmd);
+        if (ret)
+            goto out;
 
-                ret = gf_cli_status_volume (frame, this, dict);
-                if (ret)
-                        goto out;
+        ret = gf_cli_status_volume(frame, this, dict);
+        if (ret)
+            goto out;
 
-                dict_unref (dict);
-        }
+        dict_unref(dict);
+    }
 
 xml_end:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_status_end (local);
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_status_end(local);
+    }
 
- out:
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "status all failed");
+out:
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "status all failed");
 
-        if (vol_dict)
-                dict_unref (vol_dict);
+    if (vol_dict)
+        dict_unref(vol_dict);
 
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        if (local)
-                cli_local_wipe (local);
+    if (local)
+        cli_local_wipe(local);
 
-        if (frame)
-                frame->local = NULL;
+    if (frame)
+        frame->local = NULL;
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_mount_cbk (struct rpc_req *req, struct iovec *iov,
-                  int count, void *myframe)
+gf_cli_mount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf1_cli_mount_rsp rsp   = {0,};
-        int               ret   = -1;
+    gf1_cli_mount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to mount");
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
 
-        if (rsp.op_ret == 0) {
-                ret = 0;
-                cli_out ("%s", rsp.path);
-        } else {
-                /* weird sounding but easy to parse... */
-                cli_err ("%d : failed with this errno (%s)",
-                         rsp.op_errno, strerror (rsp.op_errno));
-                ret = -1;
-        }
+    if (rsp.op_ret == 0) {
+        ret = 0;
+        cli_out("%s", rsp.path);
+    } else {
+        /* weird sounding but easy to parse... */
+        cli_err("%d : failed with this errno (%s)", rsp.op_errno,
+                strerror(rsp.op_errno));
+        ret = -1;
+    }
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (rsp.path) {
+        free(rsp.path);
+    }
+    return ret;
 }
 
-int32_t
-gf_cli_mount (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_mount(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_mount_req  req  = {0,};
-        int                ret  = -1;
-        void            **dataa = data;
-        char             *label = NULL;
-        dict_t            *dict = NULL;
+    gf1_cli_mount_req req = {
+        0,
+    };
+    int ret = -1;
+    void **dataa = data;
+    char *label = NULL;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
+    if (!frame || !this || !data)
+        goto out;
 
-        label = dataa[0];
-        dict  = dataa[1];
+    label = dataa[0];
+    dict = dataa[1];
 
-        req.label = label;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    req.label = label;
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_MOUNT, NULL,
-                              this, gf_cli_mount_cbk,
-                              (xdrproc_t)xdr_gf1_cli_mount_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_MOUNT,
+                         NULL, this, gf_cli_mount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_mount_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
 static int
-gf_cli_umount_cbk (struct rpc_req *req, struct iovec *iov,
-                   int count, void *myframe)
+gf_cli_umount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
 {
-        gf1_cli_umount_rsp rsp   = {0,};
-        int               ret   = -1;
+    gf1_cli_umount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (myframe);
+    GF_ASSERT(myframe);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to mount");
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
 
-        if (rsp.op_ret == 0)
-                ret = 0;
-        else {
-                cli_err ("umount failed");
-                ret = -1;
-        }
+    if (rsp.op_ret == 0)
+        ret = 0;
+    else {
+        cli_err("umount failed");
+        ret = -1;
+    }
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int32_t
-gf_cli_umount (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_umount(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_umount_req  req  = {0,};
-        int                ret  = -1;
-        dict_t            *dict = NULL;
+    gf1_cli_umount_req req = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
+    if (!frame || !this || !data)
+        goto out;
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "path", &req.path);
-        if (ret == 0)
-                ret = dict_get_int32 (dict, "lazy", &req.lazy);
+    ret = dict_get_str_sizen(dict, "path", &req.path);
+    if (ret == 0)
+        ret = dict_get_int32_sizen(dict, "lazy", &req.lazy);
 
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_UMOUNT, NULL,
-                              this, gf_cli_umount_cbk,
-                              (xdrproc_t)xdr_gf1_cli_umount_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_UMOUNT,
+                         NULL, this, gf_cli_umount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_umount_req);
 
- out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-void
-cmd_heal_volume_statistics_out (dict_t *dict, int  brick)
-{
-
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char            *hostname = NULL;
-        uint64_t        i = 0;
-        uint64_t        healed_count = 0;
-        uint64_t        split_brain_count = 0;
-        uint64_t        heal_failed_count = 0;
-        char            *start_time_str = NULL;
-        char            *end_time_str = NULL;
-        char            *crawl_type = NULL;
-        int             progress = -1;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
+static void
+cmd_heal_volume_statistics_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[256] = {0};
+    char *hostname = NULL;
+    uint64_t i = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = NULL;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    cli_out("------------------------------------------------");
+    cli_out("\nCrawl statistics for brick no %d", brick);
+    cli_out("Hostname of brick %s", hostname);
+
+    snprintf(key, sizeof key, "statistics-%d-count", brick);
+    ret = dict_get_uint64(dict, key, &num_entries);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < num_entries; i++) {
+        snprintf(key, sizeof key, "statistics_crawl_type-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_str(dict, key, &crawl_type);
         if (ret)
-                goto out;
-        cli_out ("------------------------------------------------");
-        cli_out ("\nCrawl statistics for brick no %d", brick);
-        cli_out ("Hostname of brick %s", hostname);
+            goto out;
 
-        snprintf (key, sizeof key, "statistics-%d-count", brick);
-        ret = dict_get_uint64 (dict, key, &num_entries);
+        snprintf(key, sizeof key, "statistics_healed_cnt-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_uint64(dict, key, &healed_count);
         if (ret)
-                goto out;
-
-        for (i = 0; i < num_entries; i++)
-        {
-                snprintf (key, sizeof key, "statistics_crawl_type-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key, &crawl_type);
-                if (ret)
-                        goto out;
-
-                snprintf (key, sizeof key, "statistics_healed_cnt-%d-%"PRIu64,
-                          brick,i);
-                ret = dict_get_uint64 (dict, key, &healed_count);
-                if (ret)
-                        goto out;
-
-                snprintf (key, sizeof key, "statistics_sb_cnt-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_uint64 (dict, key, &split_brain_count);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_heal_failed_cnt-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_uint64 (dict, key, &heal_failed_count);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_strt_time-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key,  &start_time_str);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_end_time-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key, &end_time_str);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_inprogress-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_int32 (dict, key, &progress);
-                if (ret)
-                        goto out;
-
-                cli_out ("\nStarting time of crawl: %s", start_time_str);
-                if (progress == 1)
-                        cli_out ("Crawl is in progress");
-                else
-                        cli_out ("Ending time of crawl: %s", end_time_str);
+            goto out;
 
-                cli_out ("Type of crawl: %s", crawl_type);
-                cli_out ("No. of entries healed: %"PRIu64,
-                         healed_count);
-                cli_out ("No. of entries in split-brain: %"PRIu64,
-                        split_brain_count);
-                cli_out ("No. of heal failed entries: %"PRIu64,
-                         heal_failed_count);
-
-        }
-
-
-out:
-        return;
-}
-
-void
-cmd_heal_volume_brick_out (dict_t *dict, int brick)
-{
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char           *hostname = NULL;
-        char           *path = NULL;
-        char           *status = NULL;
-        uint64_t        i = 0;
-        uint32_t        time = 0;
-        char            timestr[32] = {0};
-        char            *shd_status = NULL;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
+        snprintf(key, sizeof key, "statistics_sb_cnt-%d-%" PRIu64, brick, i);
+        ret = dict_get_uint64(dict, key, &split_brain_count);
         if (ret)
-                goto out;
-        snprintf (key, sizeof key, "%d-path", brick);
-        ret = dict_get_str (dict, key, &path);
+            goto out;
+        snprintf(key, sizeof key, "statistics_heal_failed_cnt-%d-%" PRIu64,
+                 brick, i);
+        ret = dict_get_uint64(dict, key, &heal_failed_count);
         if (ret)
-                goto out;
-        cli_out ("\nBrick %s:%s", hostname, path);
-
-        snprintf (key, sizeof key, "%d-status", brick);
-        ret = dict_get_str (dict, key, &status);
-        if (status && strlen (status))
-                cli_out ("Status: %s", status);
+            goto out;
+        snprintf(key, sizeof key, "statistics_strt_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &start_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_end_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &end_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_inprogress-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_int32(dict, key, &progress);
+        if (ret)
+            goto out;
 
-        snprintf (key, sizeof key, "%d-shd-status",brick);
-        ret = dict_get_str (dict, key, &shd_status);
+        cli_out("\nStarting time of crawl: %s", start_time_str);
+        if (progress == 1)
+            cli_out("Crawl is in progress");
+        else
+            cli_out("Ending time of crawl: %s", end_time_str);
 
-        if(!shd_status)
-        {
-                snprintf (key, sizeof key, "%d-count", brick);
-                ret = dict_get_uint64 (dict, key, &num_entries);
-                cli_out ("Number of entries: %"PRIu64, num_entries);
-
-
-                for (i = 0; i < num_entries; i++) {
-                        snprintf (key, sizeof key, "%d-%"PRIu64, brick, i);
-                        ret = dict_get_str (dict, key, &path);
-                        if (ret)
-                                continue;
-                        time = 0;
-                        snprintf (key, sizeof key, "%d-%"PRIu64"-time",
-                                  brick, i);
-                        ret = dict_get_uint32 (dict, key, &time);
-                        if (!time) {
-                                cli_out ("%s", path);
-                        } else {
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             time, gf_timefmt_FT);
-                                if (i == 0) {
-                                cli_out ("at                    path on brick");
-                                cli_out ("-----------------------------------");
-                                }
-                                cli_out ("%s %s", timestr, path);
-                        }
-                }
-        }
+        cli_out("Type of crawl: %s", crawl_type);
+        cli_out("No. of entries healed: %" PRIu64, healed_count);
+        cli_out("No. of entries in split-brain: %" PRIu64, split_brain_count);
+        cli_out("No. of heal failed entries: %" PRIu64, heal_failed_count);
+    }
 
 out:
-        return;
+    return;
 }
 
+static void
+cmd_heal_volume_brick_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    uint64_t i = 0;
+    uint32_t time = 0;
+    char timestr[GF_TIMESTR_SIZE] = {0};
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && status[0] != '\0')
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-count", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
+        cli_out("Number of entries: %" PRIu64, num_entries);
+
+        for (i = 0; i < num_entries; i++) {
+            snprintf(key, sizeof key, "%d-%" PRIu64, brick, i);
+            ret = dict_get_str(dict, key, &path);
+            if (ret)
+                continue;
+            time = 0;
+            snprintf(key, sizeof key, "%d-%" PRIu64 "-time", brick, i);
+            ret = dict_get_uint32(dict, key, &time);
+            if (ret || !time) {
+                cli_out("%s", path);
+            } else {
+                gf_time_fmt(timestr, sizeof timestr, time, gf_timefmt_FT);
+                if (i == 0) {
+                    cli_out("at                    path on brick");
+                    cli_out("-----------------------------------");
+                }
+                cli_out("%s %s", timestr, path);
+            }
+        }
+    }
+
+out:
+    return;
+}
 
-void
-cmd_heal_volume_statistics_heal_count_out (dict_t *dict, int brick)
-{
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char           *hostname = NULL;
-        char           *path = NULL;
-        char           *status = NULL;
-        char            *shd_status = NULL;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
-        if (ret)
-                goto out;
-        snprintf (key, sizeof key, "%d-path", brick);
-        ret = dict_get_str (dict, key, &path);
+static void
+cmd_heal_volume_statistics_heal_count_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && strlen(status))
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-hardlinks", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
         if (ret)
-                goto out;
-        cli_out ("\nBrick %s:%s", hostname, path);
-
-        snprintf (key, sizeof key, "%d-status", brick);
-        ret = dict_get_str (dict, key, &status);
-        if (status && strlen (status))
-                cli_out ("Status: %s", status);
-
-        snprintf (key, sizeof key, "%d-shd-status",brick);
-        ret = dict_get_str (dict, key, &shd_status);
-
-        if(!shd_status)
-        {
-                snprintf (key, sizeof key, "%d-hardlinks", brick);
-                ret = dict_get_uint64 (dict, key, &num_entries);
-                if (ret)
-                        cli_out ("No gathered input for this brick");
-                else
-                        cli_out ("Number of entries: %"PRIu64, num_entries);
-
-
-        }
+            cli_out("No gathered input for this brick");
+        else
+            cli_out("Number of entries: %" PRIu64, num_entries);
+    }
 
 out:
-        return;
+    return;
 }
 
-int
-gf_is_cli_heal_get_command (gf_xl_afr_op_t heal_op)
-{
-        /* If the command is get command value is 1 otherwise 0, for
-           invalid commands -1 */
-        int    get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
-                       [GF_SHD_OP_INVALID] = -1,
-                       [GF_SHD_OP_HEAL_INDEX] = 0,
-                       [GF_SHD_OP_HEAL_FULL] = 0,
-                       [GF_SHD_OP_INDEX_SUMMARY] = 1,
-                       [GF_SHD_OP_HEALED_FILES] = 1,
-                       [GF_SHD_OP_HEAL_FAILED_FILES] = 1,
-                       [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
-                       [GF_SHD_OP_STATISTICS] = 1,
-                       [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
-                       [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
-                       [GF_SHD_OP_HEAL_ENABLE] = 0,
-                       [GF_SHD_OP_HEAL_DISABLE] = 0,
-                };
-
-        if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
-                return get_cmds[heal_op] == 1;
-        return _gf_false;
+static int
+gf_is_cli_heal_get_command(gf_xl_afr_op_t heal_op)
+{
+    /* If the command is get command value is 1 otherwise 0, for
+       invalid commands -1 */
+    static int get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = -1,
+        [GF_SHD_OP_HEAL_INDEX] = 0,
+        [GF_SHD_OP_HEAL_FULL] = 0,
+        [GF_SHD_OP_INDEX_SUMMARY] = 1,
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
+        [GF_SHD_OP_STATISTICS] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
+        [GF_SHD_OP_HEAL_ENABLE] = 0,
+        [GF_SHD_OP_HEAL_DISABLE] = 0,
+    };
+
+    if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
+        return get_cmds[heal_op] == 1;
+    return _gf_false;
 }
 
 int
-gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     brick_count = 0;
-        int                     i = 0;
-        gf_xl_afr_op_t          heal_op = GF_SHD_OP_INVALID;
-        char                    *operation = NULL;
-        char                    *substr = NULL;
-        char                    *heal_op_str = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        GF_ASSERT (frame->local);
-
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "heal-op", (int32_t *)&heal_op);
-//TODO: Proper XML output
-//#if (HAVE_LIB_XML)
-//        if (global_state->mode & GLUSTER_MODE_XML) {
-//                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
-//                                           rsp.op_errno, rsp.op_errstr);
-//                if (ret)
-//                        gf_log ("cli", GF_LOG_ERROR,
-//                                "Error outputting to xml");
-//                goto out;
-//        }
-//#endif
-
-        ret = dict_get_str (local->dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get volname");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to heal volume");
-
-        operation = "Gathering ";
-        substr = "";
-        switch (heal_op) {
-        case    GF_SHD_OP_HEAL_INDEX:
-                operation   = "Launching heal operation ";
-                heal_op_str = "to perform index self heal";
-                substr      = "\nUse heal info commands to check"
-                              " status";
-                break;
-        case    GF_SHD_OP_HEAL_FULL:
-                operation   = "Launching heal operation ";
-                heal_op_str = "to perform full self heal";
-                substr      = "\nUse heal info commands to check"
-                              " status";
-                break;
-        case    GF_SHD_OP_INDEX_SUMMARY:
-                heal_op_str = "list of entries to be healed";
-                break;
-        case    GF_SHD_OP_HEALED_FILES:
-                heal_op_str = "list of healed entries";
-                break;
-        case    GF_SHD_OP_HEAL_FAILED_FILES:
-                heal_op_str = "list of heal failed entries";
-                break;
-        case    GF_SHD_OP_SPLIT_BRAIN_FILES:
-                heal_op_str = "list of split brain entries";
-                break;
-        case    GF_SHD_OP_STATISTICS:
-                heal_op_str =  "crawl statistics";
-                break;
-        case    GF_SHD_OP_STATISTICS_HEAL_COUNT:
-                heal_op_str = "count of entries to be healed";
-                break;
-        case    GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                heal_op_str = "count of entries to be healed per replica";
-                break;
-        /* The below 3 cases are never hit; they're coded only to make
-         * compiler warnings go away.*/
-        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-        case    GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                break;
-
-        case    GF_SHD_OP_INVALID:
-                heal_op_str = "invalid heal op";
-                break;
-        case    GF_SHD_OP_HEAL_ENABLE:
-                operation   = "";
-                heal_op_str = "Enable heal";
-                break;
-        case    GF_SHD_OP_HEAL_DISABLE:
-                operation   = "";
-                heal_op_str = "Disable heal";
-                break;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("%s%s on volume %s has been unsuccessful on "
-                         "bricks that are down. Please check if all brick "
-                         "processes are running.",
-                         operation, heal_op_str, volname);
-                ret = rsp.op_ret;
-                goto out;
-        } else {
-                cli_out ("%s%s on volume %s has been successful %s", operation,
-                         heal_op_str, volname, substr);
+gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int brick_count = 0;
+    int i = 0;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    const char *operation = NULL;
+    const char *substr = NULL;
+    const char *heal_op_str = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "heal-op", (int32_t *)&heal_op);
+    // TODO: Proper XML output
+    //#if (HAVE_LIB_XML)
+    //        if (global_state->mode & GLUSTER_MODE_XML) {
+    //                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
+    //                                           rsp.op_errno, rsp.op_errstr);
+    //                if (ret)
+    //                        gf_log ("cli", GF_LOG_ERROR, XML_ERROR);
+    //                goto out;
+    //        }
+    //#endif
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to heal volume");
+
+    operation = "Gathering ";
+    substr = "";
+    switch (heal_op) {
+        case GF_SHD_OP_HEAL_INDEX:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform index self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_HEAL_FULL:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform full self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            heal_op_str = "list of entries to be healed";
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            heal_op_str = "list of split brain entries";
+            break;
+        case GF_SHD_OP_STATISTICS:
+            heal_op_str = "crawl statistics";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+            heal_op_str = "count of entries to be healed";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            heal_op_str = "count of entries to be healed per replica";
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+        case GF_SHD_OP_HEAL_SUMMARY:
+        case GF_SHD_OP_HEALED_FILES:
+        case GF_SHD_OP_HEAL_FAILED_FILES:
+            /* These cases are never hit; they're coded just to silence the
+             * compiler warnings.*/
+            break;
+
+        case GF_SHD_OP_INVALID:
+            heal_op_str = "invalid heal op";
+            break;
+        case GF_SHD_OP_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable heal";
+            break;
+        case GF_SHD_OP_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable granular entry heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable granular entry heal";
+            break;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("%s%s on volume %s has been unsuccessful:", operation,
+                    heal_op_str, volname);
+            cli_err("%s", rsp.op_errstr);
         }
-
         ret = rsp.op_ret;
-        if (!gf_is_cli_heal_get_command (heal_op))
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
-        } else {
-                dict->extra_stdfree = rsp.dict.dict_val;
-        }
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-
-        if (!brick_count) {
-                cli_err ("All bricks of volume %s are down.", volname);
-                goto out;
-        }
-
-        switch (heal_op) {
+        goto out;
+    } else {
+        cli_out("%s%s on volume %s has been successful %s", operation,
+                heal_op_str, volname, substr);
+    }
+
+    ret = rsp.op_ret;
+    if (!gf_is_cli_heal_get_command(heal_op))
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        ret = -1;
+        goto out;
+    }
+
+    switch (heal_op) {
         case GF_SHD_OP_STATISTICS:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_statistics_out (dict, i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_out(dict, i);
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_statistics_heal_count_out (dict,
-                                                                   i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_heal_count_out(dict, i);
+            break;
         case GF_SHD_OP_INDEX_SUMMARY:
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_brick_out (dict, i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_brick_out(dict, i);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.op_errstr);
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int32_t
-gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_heal_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_heal_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_heal_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog, NULL);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
-{
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        char                            msg[1024] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to statedump");
-        if (rsp.op_ret)
-                snprintf (msg, sizeof(msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Volume statedump successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volStatedump", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int32_t
+gf_cli_statedump_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "Volume statedump successful";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to statedump");
+    if (rsp.op_ret)
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volStatedump", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume statedump: failed: %s", msg);
-        else
-                cli_out ("volume statedump: success");
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume statedump: failed: %s", msg);
+    else
+        cli_out("volume statedump: success");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_statedump_volume (call_frame_t *frame, xlator_t *this,
-                            void *data)
+static int32_t
+gf_cli_statedump_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
-
-        if (!frame || !this || !data)
-                goto out;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        options = data;
+    options = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_statedump_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_statedump_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
-{
-        int             ret = -1;
-        gf_cli_rsp      rsp = {0,};
-        dict_t          *dict = NULL;
-        int             vol_count = 0;;
-        char            *volname = NULL;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+static int32_t
+gf_cli_list_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int vol_count = 0;
+    ;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_list(dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("%s", rsp.op_errstr);
+    else {
+        ret = dict_get_int32_sizen(dict, "count", &vol_count);
+        if (ret)
+            goto out;
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
+        if (vol_count == 0) {
+            cli_err("No volumes present in cluster");
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_list (dict, rsp.op_ret, rsp.op_errno,
-                                               rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+        for (i = 0; i < vol_count; i++) {
+            ret = snprintf(key, sizeof(key), "volume%d", i);
+            ret = dict_get_strn(dict, key, ret, &volname);
+            if (ret)
                 goto out;
+            cli_out("%s", volname);
         }
+    }
 
-        if (rsp.op_ret)
-                cli_err ("%s", rsp.op_errstr);
-        else {
-                ret = dict_get_int32 (dict, "count", &vol_count);
-                if (ret)
-                        goto out;
-
-                if (vol_count == 0) {
-                        cli_err ("No volumes present in cluster");
-                        goto out;
-                }
-                for (i = 0; i < vol_count; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "volume%d", i);
-                        ret = dict_get_str (dict, key, &volname);
-                        if (ret)
-                                goto out;
-                        cli_out ("%s", volname);
-                }
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int32_t
-gf_cli_list_volume (call_frame_t *frame, xlator_t *this, void *data)
-{
-        int             ret = -1;
-        gf_cli_req      req = {{0,}};
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (!frame || !this)
-                goto out;
-
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LIST_VOLUME, NULL,
-                              this, gf_cli_list_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int32_t
-gf_cli_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                  int count, void *myframe)
+static int32_t
+gf_cli_list_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        char                            *lk_summary = NULL;
-        char                            *volname = NULL;
-        dict_t                          *dict = NULL;
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
 
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
-                goto out;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_LIST_VOLUME, NULL, this,
+                         gf_cli_list_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+static int32_t
+gf_cli_clearlocks_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *lk_summary = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to clear-locks");
+
+    if (rsp.op_ret) {
+        cli_err("Volume clear-locks unsuccessful");
+        cli_err("%s", rsp.op_errstr);
+
+    } else {
+        if (!rsp.dict.dict_len) {
+            cli_err("Possibly no locks cleared");
+            ret = 0;
+            goto out;
         }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to clear-locks");
 
-        if (rsp.op_ret) {
-                cli_err ("Volume clear-locks unsuccessful");
-                cli_err ("%s", rsp.op_errstr);
-
-        } else {
-                if (!rsp.dict.dict_len) {
-                        cli_err ("Possibly no locks cleared");
-                        ret = 0;
-                        goto out;
-                }
-
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
+        dict = dict_new();
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to serialize response dictionary");
-                        goto out;
-                }
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get volname "
-                                "from dictionary");
-                        goto out;
-                }
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-                ret = dict_get_str (dict, "lk-summary", &lk_summary);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get lock "
-                                "summary from dictionary");
-                        goto out;
-                }
-                cli_out ("Volume clear-locks successful");
-                cli_out ("%s", lk_summary);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
 
+        ret = dict_get_str(dict, "lk-summary", &lk_summary);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to get lock summary from dictionary");
+            goto out;
         }
+        cli_out("Volume clear-locks successful");
+        cli_out("%s", lk_summary);
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_clearlocks_volume (call_frame_t *frame, xlator_t *this,
-                             void *data)
+static int32_t
+gf_cli_clearlocks_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
+    options = data;
 
-        options = data;
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_clearlocks_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_clearlocks_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-cli_snapshot_remove_reply (gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
+static int32_t
+cli_snapshot_remove_reply(gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
 {
-        int32_t         ret             = -1;
-        char            *snap_name      = NULL;
-        int32_t         delete_cmd      = -1;
-        cli_local_t     *local          = NULL;
+    int32_t ret = -1;
+    char *snap_name = NULL;
+    int32_t delete_cmd = -1;
+    cli_local_t *local = NULL;
 
-        GF_ASSERT (frame);
-        GF_ASSERT (rsp);
-        GF_ASSERT (dict);
+    GF_ASSERT(frame);
+    GF_ASSERT(rsp);
+    GF_ASSERT(dict);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = dict_get_int32 (dict, "sub-cmd", &delete_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get sub-cmd");
-                goto end;
-        }
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
+        goto end;
+    }
 
-        if ((global_state->mode & GLUSTER_MODE_XML) &&
-            (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
-                ret = cli_xml_output_snap_delete_begin (local, rsp->op_ret,
-                                                        rsp->op_errno,
-                                                        rsp->op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for delete");
-                        goto end;
-                }
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_begin(local, rsp->op_ret,
+                                               rsp->op_errno, rsp->op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for delete");
+            goto end;
+        }
+    }
+
+    if (rsp->op_ret && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("snapshot delete: failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
+        delete_cmd == GF_SNAP_DELETE_TYPE_VOL) {
+        local = ((call_frame_t *)frame)->local;
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+            goto out;
         }
 
+        /* During first call back of snapshot delete of type
+         * ALL and VOL, We will get the snapcount and snapnames.
+         * Hence to make the subsequent rpc calls for individual
+         * snapshot delete, We need to save it in local dictionary.
+         */
+        dict_copy(dict, local->dict);
+        ret = 0;
+        goto out;
+    }
 
-        if (rsp->op_ret && !(global_state->mode & GLUSTER_MODE_XML)) {
-                        cli_err ("snapshot delete: failed: %s",
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                 "Please check log file for details");
-                ret = rsp->op_ret;
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_delete(local, dict, rsp);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot delete command");
+            goto out;
         }
-
-        if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
-            delete_cmd == GF_SNAP_DELETE_TYPE_VOL) {
-                local = ((call_frame_t *) frame) -> local;
-                if (!local) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "frame->local is NULL");
-                        goto out;
-                }
-
-                /* During first call back of snapshot delete of type
-                 * ALL and VOL, We will get the snapcount and snapnames.
-                 * Hence to make the subsequent rpc calls for individual
-                 * snapshot delete, We need to save it in local dictionary.
-                 */
-                dict_copy (dict, local->dict);
-                ret = 0;
-                goto out;
+        /* Error out in case of the op already failed */
+        if (rsp->op_ret) {
+            ret = rsp->op_ret;
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_delete (local->writer, local->doc,
-                                               dict, rsp);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot delete command");
-                        goto out;
-                }
-                /* Error out in case of the op already failed */
-                if (rsp->op_ret) {
-                        ret = rsp->op_ret;
-                        goto out;
-                }
-        } else {
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
-                        goto out;
-                }
-
-                cli_out ("snapshot delete: %s: snap removed successfully",
-                         snap_name);
+    } else {
+        ret = dict_get_str_sizen(dict, "snapname", &snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+            goto out;
         }
-        ret = 0;
+
+        cli_out("snapshot delete: %s: snap removed successfully", snap_name);
+    }
+    ret = 0;
 
 out:
-        if ((global_state->mode & GLUSTER_MODE_XML) &&
-            (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
-                ret = cli_xml_output_snap_delete_end (local);
-        }
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_end(local);
+    }
 end:
-        return ret;
+    return ret;
 }
 
-int
-cli_snapshot_config_display (dict_t *dict, gf_cli_rsp *rsp)
-{
-        char                buf[PATH_MAX]        = "";
-        char               *volname              = NULL;
-        int                 ret                  = -1;
-        int                 config_command       = 0;
-        uint64_t            value                = 0;
-        uint64_t            hard_limit           = 0;
-        uint64_t            soft_limit           = 0;
-        uint64_t            i                    = 0;
-        uint64_t            voldisplaycount      = 0;
-        char               *auto_delete          = NULL;
-        char               *snap_activate        = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp);
+static int
+cli_snapshot_config_display(dict_t *dict, gf_cli_rsp *rsp)
+{
+    char buf[PATH_MAX] = "";
+    char *volname = NULL;
+    int ret = -1;
+    int config_command = 0;
+    uint64_t value = 0;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    uint64_t i = 0;
+    uint64_t voldisplaycount = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+
+    if (rsp->op_ret) {
+        cli_err("Snapshot Config : failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* Ignore the error, as the key specified is optional */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+
+    ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+
+    ret = dict_get_str_sizen(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit &&
+        config_command != GF_SNAP_CONFIG_DISPLAY && !auto_delete &&
+        !snap_activate) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "Could not fetch config-key");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    /* Ignore the error, as volname is optional */
+
+    if (!volname) {
+        volname = "System";
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+            if (hard_limit && soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "& snap-max-soft-limit for system set successfully");
+            } else if (hard_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-soft-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (auto_delete) {
+                cli_out("snapshot config: auto-delete successfully set");
+            } else if (snap_activate) {
+                cli_out("snapshot config: activate-on-create successfully set");
+            }
+            break;
 
-        if (rsp->op_ret) {
-                cli_err ("Snapshot Config : failed: %s",
-                         rsp->op_errstr ? rsp->op_errstr :
-                         "Please check log file for details");
-                ret = rsp->op_ret;
+        case GF_SNAP_CONFIG_DISPLAY:
+            cli_out("\nSnapshot System Configuration:");
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap_max_hard_limit for %s", volname);
+                ret = -1;
                 goto out;
-        }
+            }
+            cli_out("snap-max-hard-limit : %" PRIu64, value);
 
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap-max-soft-limit for %s", volname);
+                ret = -1;
                 goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        /* Ignore the error, as volname is optional */
+            }
+            cli_out("snap-max-soft-limit : %" PRIu64 "%%", soft_limit);
 
-        if (!volname) {
-                volname = "System";
-        }
-
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
-        /* Ignore the error, as the key specified is optional */
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+            cli_out("auto-delete : %s", auto_delete);
 
-        ret = dict_get_str (dict, "auto-delete", &auto_delete);
+            cli_out("activate-on-create : %s\n", snap_activate);
 
-        ret = dict_get_str (dict, "snap-activate-on-create", &snap_activate);
+            cli_out("Snapshot Volume Configuration:");
 
-        if (!hard_limit && !soft_limit
-                        && config_command != GF_SNAP_CONFIG_DISPLAY
-                        && !auto_delete && !snap_activate) {
+            ret = dict_get_uint64(dict, "voldisplaycount", &voldisplaycount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch voldisplaycount");
                 ret = -1;
-                gf_log(THIS->name, GF_LOG_ERROR,
-                       "Could not fetch config-key");
                 goto out;
-        }
-
-        switch (config_command) {
-        case GF_SNAP_CONFIG_TYPE_SET:
-                if (hard_limit && soft_limit) {
-                        cli_out ("snapshot config: snap-max-hard-limit "
-                                "& snap-max-soft-limit for system set "
-                                 "successfully");
-                } else if (hard_limit) {
-                        cli_out ("snapshot config: snap-max-hard-limit "
-                                 "for %s set successfully",
-                                 volname);
-                } else if (soft_limit) {
-                        cli_out ("snapshot config: snap-max-soft-limit "
-                                 "for %s set successfully",
-                                 volname);
-                } else if (auto_delete) {
-                        cli_out ("snapshot config: auto-delete "
-                                 "successfully set");
-                } else if (snap_activate) {
-                        cli_out ("snapshot config: activate-on-create "
-                                 "successfully set");
-                }
-                break;
+            }
 
-        case GF_SNAP_CONFIG_DISPLAY:
-                cli_out ("\nSnapshot System Configuration:");
-                ret = dict_get_uint64 (dict, "snap-max-hard-limit",
-                                       &value);
+            for (i = 0; i < voldisplaycount; i++) {
+                ret = snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname",
+                               i);
+                ret = dict_get_strn(dict, buf, ret, &volname);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                "snap_max_hard_limit for %s", volname);
-                        ret = -1;
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
                 }
-                cli_out ("snap-max-hard-limit : %"PRIu64, value);
+                cli_out("\nVolume : %s", volname);
 
-                ret = dict_get_uint64 (dict, "snap-max-soft-limit",
-                                       &soft_limit);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                               "snap-max-soft-limit for %s", volname);
-                        ret = -1;
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
                 }
-                cli_out ("snap-max-soft-limit : %"PRIu64"%%",
-                         soft_limit);
-
-                cli_out ("auto-delete : %s", auto_delete);
-
-                cli_out ("activate-on-create : %s\n", snap_activate);
-
-                cli_out ("Snapshot Volume Configuration:");
+                cli_out("snap-max-hard-limit : %" PRIu64, value);
 
-                ret = dict_get_uint64 (dict, "voldisplaycount",
-                                       &voldisplaycount);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-active-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                               "Could not fetch voldisplaycount");
-                        ret = -1;
-                        goto out;
-                }
-
-                for (i = 0; i < voldisplaycount; i++) {
-                        snprintf (buf, sizeof(buf), "volume%"PRIu64"-volname", i);
-                        ret = dict_get_str (dict, buf, &volname);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("\nVolume : %s", volname);
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-hard-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("snap-max-hard-limit : %"PRIu64, value);
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-active-hard-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                 gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
-                                        " effective snap_max_hard_limit for "
-                                        "%s", volname);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("Effective snap-max-hard-limit : %"PRIu64,
-                                 value);
-
-                        snprintf (buf, sizeof(buf),
-                                      "volume%"PRIu64"-snap-max-soft-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("Effective snap-max-soft-limit : %"PRIu64" "
-                                 "(%"PRIu64"%%)", value, soft_limit);
-                }
-                break;
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Could not fetch"
+                           " effective snap_max_hard_limit for %s",
+                           volname);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-hard-limit : %" PRIu64, value);
+
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-soft-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-soft-limit : %" PRIu64
+                        " "
+                        "(%" PRIu64 "%%)",
+                        value, soft_limit);
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is used to print the volume related information
@@ -9594,2332 +8651,2299 @@ out:
  * arg - 0, dict       : Response Dictionary.
  * arg - 1, prefix str : snaplist.snap{0..}.vol{0..}.*
  */
-int
-cli_get_each_volinfo_in_snap (dict_t *dict, char *keyprefix,
-                              gf_boolean_t snap_driven) {
-        char             key[PATH_MAX]  =  "";
-        char            *get_buffer     =  NULL;
-        int              value          =  0;
-        int              ret            = -1;
-        char             indent[5]      = "\t";
-        char            *volname        =  NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        if (snap_driven) {
-                ret = snprintf (key, sizeof (key), "%s.volname", keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &get_buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                        "Snap Volume Name", ":", get_buffer);
-
-                ret = snprintf (key, sizeof (key),
-                                "%s.origin-volname", keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
-                        cli_out ("%-12s", "Origin:");
-                }
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                        "Origin Volume name", ":", volname);
-
-
-                ret = snprintf (key, sizeof (key), "%s.snapcount",
-                                keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s%s %s      %s %d", indent, "Snaps taken for",
-                        volname, ":", value);
-
-                ret = snprintf (key, sizeof (key), "%s.snaps-available",
-                                keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s%s %s  %s %d", indent, "Snaps available for",
-                         volname, ":", value);
-        }
+static int
+cli_get_each_volinfo_in_snap(dict_t *dict, char *keyprefix,
+                             gf_boolean_t snap_driven)
+{
+    char key[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int value = 0;
+    int ret = -1;
+    char indent[5] = "\t";
+    char *volname = NULL;
 
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
 
-        ret = snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key, &get_buffer);
+        ret = dict_get_str(dict, key, &get_buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Status",
-                 ":", get_buffer);
-out:
-        return ret;
-}
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap Volume Name", ":",
+                get_buffer);
 
-/* This function is used to print snap related information
- * arg - 0, dict       : Response dictionary.
- * arg - 1, prefix_str : snaplist.snap{0..}.*
- */
-int
-cli_get_volinfo_in_snap (dict_t *dict, char *keyprefix) {
-
-        char            key[PATH_MAX]   = "";
-        int             i               = 0;
-        int             volcount        = 0;
-        int             ret             = -1;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        ret = snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        ret = snprintf(key, sizeof(key), "%s.origin-volname", keyprefix);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, key, &volcount);
-        for (i = 1 ; i <= volcount ; i++) {
-                ret = snprintf (key, sizeof (key),
-                                "%s.vol%d", keyprefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
-                                "details of volume in a snap");
-                        goto out;
-                }
-                cli_out (" ");
+        ret = dict_get_str(dict, key, &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+            cli_out("%-12s", "Origin:");
         }
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Origin Volume name", ":",
+                volname);
 
-out:
-        return ret;
-}
-
-int
-cli_get_each_snap_info (dict_t *dict, char *prefix_str,
-                        gf_boolean_t snap_driven) {
-        char            key_buffer[PATH_MAX] = "";
-        char           *get_buffer           = NULL;
-        int             ret                  = -1;
-        char            indent[5]            = "";
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix_str);
-
-        if (!snap_driven)
-                strcat (indent, "\t");
-
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
-                        prefix_str);
+        ret = snprintf(key, sizeof(key), "%s.snapcount", keyprefix);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        ret = dict_get_int32(dict, key, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
-                        key_buffer);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot",
-                ":", get_buffer);
+        cli_out("%s%s %s      %s %d", indent, "Snaps taken for", volname, ":",
+                value);
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id",
-                        prefix_str);
+        ret = snprintf(key, sizeof(key), "%s.snaps-available", keyprefix);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        ret = dict_get_int32(dict, key, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
-                        key_buffer);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID",
-                ":", get_buffer);
+        cli_out("%s%s %s  %s %d", indent, "Snaps available for", volname, ":",
+                value);
+    }
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc",
-                        prefix_str);
-        if (ret < 0) {
-                goto out;
-        }
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
 
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
-        if (!ret) {
-                /* Ignore error for description */
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                         "Description", ":", get_buffer);
-        }
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Status", ":", get_buffer);
+out:
+    return ret;
+}
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time",
-                        prefix_str);
+/* This function is used to print snap related information
+ * arg - 0, dict       : Response dictionary.
+ * arg - 1, prefix_str : snaplist.snap{0..}.*
+ */
+static int
+cli_get_volinfo_in_snap(dict_t *dict, char *keyprefix)
+{
+    char key[PATH_MAX] = "";
+    int i = 0;
+    int volcount = 0;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.vol-count", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, key, &volcount);
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
-
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_true);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
-                        prefix_str);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list details of volume in a snap");
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Created",
-                ":", get_buffer);
+        cli_out(" ");
+    }
 
-        if (snap_driven) {
-                cli_out ("%-12s", "Snap Volumes:\n");
-                ret = cli_get_volinfo_in_snap (dict, prefix_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to list details "
-                                "of the snaps");
-                        goto out;
-                }
+out:
+    return ret;
+}
+
+static int
+cli_get_each_snap_info(dict_t *dict, char *prefix_str, gf_boolean_t snap_driven)
+{
+    char key_buffer[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int ret = -1;
+    char indent[5] = "";
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix_str);
+
+    if (!snap_driven)
+        strcat(indent, "\t");
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (!ret) {
+        /* Ignore error for description */
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Description", ":",
+                get_buffer);
+    }
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
+               prefix_str);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Created", ":", get_buffer);
+
+    if (snap_driven) {
+        cli_out("%-12s", "Snap Volumes:\n");
+        ret = cli_get_volinfo_in_snap(dict, prefix_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to list details of the snaps");
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a generic function to print snap related information.
  * arg - 0, dict : Response Dictionary
  */
-int
-cli_call_snapshot_info (dict_t *dict, gf_boolean_t bool_snap_driven) {
-        int             snap_count       =  0;
-        char            key[PATH_MAX]    =  "";
-        int             ret              =  -1;
-        int             i                =   0;
+static int
+cli_call_snapshot_info(dict_t *dict, gf_boolean_t bool_snap_driven)
+{
+    int snap_count = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
 
-        GF_ASSERT (dict);
+    GF_ASSERT(dict);
 
-        ret = dict_get_int32 (dict, "snapcount", &snap_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapcount");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "snapcount", &snap_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapcount");
+        goto out;
+    }
 
-        if (snap_count == 0) {
-                cli_out ("No snapshots present");
-        }
+    if (snap_count == 0) {
+        cli_out("No snapshots present");
+    }
 
-        for (i = 1 ; i <= snap_count ; i++) {
-                ret = snprintf (key, sizeof (key), "snap%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_snap_info (dict, key, bool_snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to print snap details");
-                        goto out;
-                }
+    for (i = 1; i <= snap_count; i++) {
+        ret = snprintf(key, sizeof(key), "snap%d", i);
+        if (ret < 0) {
+            goto out;
+        }
+        ret = cli_get_each_snap_info(dict, key, bool_snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_get_snaps_in_volume (dict_t *dict) {
-        int      ret             =  -1;
-        int      i               =  0;
-        int      count           =  0;
-        int      avail           =  0;
-        char     key[PATH_MAX]   =  "";
-        char    *get_buffer      =  NULL;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "origin-volname", &get_buffer);
+static int
+cli_get_snaps_in_volume(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int count = 0;
+    int avail = 0;
+    char key[32] = "";
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "origin-volname", &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+
+    ret = dict_get_int32_sizen(dict, "snapcount", &avail);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snapcount");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
+
+    ret = dict_get_int32_sizen(dict, "snaps-available", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+
+    for (i = 1; i <= avail; i++) {
+        snprintf(key, sizeof(key), "snap%d", i);
+        ret = cli_get_each_snap_info(dict, key, _gf_false);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
         }
-        cli_out (INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
 
-        ret = dict_get_int32 (dict, "snapcount", &avail);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snapcount");
-                goto out;
+        ret = snprintf(key, sizeof(key), "snap%d.vol1", i);
+        if (ret < 0) {
+            goto out;
         }
-        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
-
-        ret = dict_get_int32 (dict, "snaps-available", &count);
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_false);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not get volume related information");
+            goto out;
         }
-        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
-
-        for (i = 1 ; i <= avail ; i++) {
-                snprintf (key, sizeof (key), "snap%d", i);
-                ret = cli_get_each_snap_info (dict, key, _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to print snap details");
-                        goto out;
-                }
-
-                ret = snprintf (key, sizeof (key), "snap%d.vol1", i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get volume "
-                                "related information");
-                        goto out;
-                }
 
-                cli_out (" ");
-        }
+        cli_out(" ");
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_snapshot_list (dict_t *dict) {
-        int     snapcount       =       0;
-        char    key[PATH_MAX]   =       "";
-        int     ret             =       -1;
-        int     i               =       0;
-        char    *get_buffer     =       NULL;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap count");
-                goto out;
-        }
+static int
+cli_snapshot_list(dict_t *dict)
+{
+    int snapcount = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
+    char *get_buffer = NULL;
 
-        if (snapcount == 0) {
-                cli_out ("No snapshots present");
-        }
+    GF_ASSERT(dict);
 
-        for (i = 1 ; i <= snapcount ; i++) {
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = dict_get_int32_sizen(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snap count");
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &get_buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                } else {
-                        cli_out ("%s", get_buffer);
-                }
-        }
-out:
-        return ret;
-}
+    if (snapcount == 0) {
+        cli_out("No snapshots present");
+    }
 
-int
-cli_get_snap_volume_status (dict_t *dict, char *key_prefix)
-{
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        char            *buffer         =       NULL;
-        int             brickcount      =       0;
-        int             i               =       0;
-        int             pid             =       0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (key_prefix);
-
-        ret = snprintf (key, sizeof (key), "%s.brickcount", key_prefix);
+    for (i = 1; i <= snapcount; i++) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
         if (ret < 0) {
-                goto out;
-        }
-        ret = dict_get_int32 (dict, key, &brickcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
-                goto out;
+            goto out;
         }
 
-        for (i = 0 ; i < brickcount ; i++) {
-                ret = snprintf (key, sizeof (key), "%s.brick%d.path",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Brick Path");
-                        continue;
-                }
-                cli_out ("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Volume Group");
-                        cli_out ("\t%-17s %s   %s", "Volume Group", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Volume Group", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.status",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Brick Running");
-                        cli_out ("\t%-17s %s   %s", "Brick Running", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Brick Running", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get pid");
-                        cli_out ("\t%-17s %s   %s", "Brick PID", ":", "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %d", "Brick PID", ":", pid);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.data",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Data Percent");
-                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.lvsize",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO, "Unable to get LV Size");
-                        cli_out ("\t%-17s %s   %s", "LV Size", ":", "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "LV Size", ":", buffer);
-
+        ret = dict_get_strn(dict, key, ret, &get_buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            cli_out("%s", get_buffer);
         }
-
-        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-
-int
-cli_get_single_snap_status (dict_t *dict, char *keyprefix)
-{
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        int             i               =       0;
-        int             volcount        =       0;
-        char            *get_buffer     =       NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+static int
+cli_get_snap_volume_status(dict_t *dict, char *key_prefix)
+{
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(key_prefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", key_prefix);
+    if (ret < 0) {
+        goto out;
+    }
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    for (i = 0; i < brickcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", key_prefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key, &get_buffer);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
-                goto out;
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Path");
+            continue;
         }
-        cli_out ("\nSnap Name : %s", get_buffer);
+        cli_out("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
 
-        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", key_prefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key, &get_buffer);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
-                goto out;
-        }
-        cli_out ("Snap UUID : %s", get_buffer);
+            gf_log("cli", GF_LOG_INFO, "Unable to get Volume Group");
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", buffer);
 
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", key_prefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, key, &volcount);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
-                goto out;
-        }
-
-        for (i = 0 ; i < volcount ; i++) {
-                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", buffer);
 
-                ret = cli_get_snap_volume_status (dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not get snap volume status");
-                        goto out;
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", key_prefix, i);
+        if (ret < 0) {
+            goto out;
         }
-out:
-        return ret;
-}
 
-int32_t
-cli_populate_req_dict_for_delete (dict_t *snap_dict, dict_t *dict, size_t index)
-{
-        int32_t         ret             = -1;
-        char            key[PATH_MAX]   = "";
-        char            *buffer         = NULL;
-
-        GF_ASSERT (snap_dict);
-        GF_ASSERT (dict);
-
-        ret = dict_set_int32 (snap_dict, "sub-cmd",
-                              GF_SNAP_DELETE_TYPE_ITER);
+        ret = dict_get_int32(dict, key, &pid);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save command "
-                        "type in snap dictionary");
-                goto out;
-        }
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            cli_out("\t%-17s %s   %s", "Brick PID", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %d", "Brick PID", ":", pid);
 
-        ret = snprintf (key, sizeof (key), "snapname%zu", index);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", key_prefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key, &buffer);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
-                goto out;
-        }
+            gf_log("cli", GF_LOG_INFO, "Unable to get Data Percent");
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", buffer);
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "snapname", buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to save snapname");
-                goto out;
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", key_prefix, i);
+        if (ret < 0) {
+            goto out;
         }
-
-        ret = dict_set_int32 (snap_dict, "type", GF_SNAP_OPTION_TYPE_DELETE);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to save command type");
-                goto out;
+            gf_log("cli", GF_LOG_INFO, "Unable to get LV Size");
+            cli_out("\t%-17s %s   %s", "LV Size", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "LV Size", ":", buffer);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+cli_get_single_snap_status(dict_t *dict, char *keyprefix)
+{
+    int ret = -1;
+    char key[64] = ""; /* keyprefix is ""status.snap0" */
+    int i = 0;
+    int volcount = 0;
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+    cli_out("\nSnap Name : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+    cli_out("Snap UUID : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    for (i = 0; i < volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "cmd-str",
-                                       "snapshot delete");
+        ret = cli_get_snap_volume_status(dict, key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command string as delete");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_populate_req_dict_for_status (dict_t *snap_dict, dict_t *dict, int index)
+static int32_t
+cli_populate_req_dict_for_delete(dict_t *snap_dict, dict_t *dict, size_t index)
+{
+    int32_t ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
+
+    ret = dict_set_int32_sizen(snap_dict, "sub-cmd", GF_SNAP_DELETE_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save command type in snap dictionary");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "snapname%zu", index);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save snapname");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_DELETE);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save command type");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot delete");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as delete");
+        goto out;
+    }
+out:
+    return ret;
+}
+
+static int
+cli_populate_req_dict_for_status(dict_t *snap_dict, dict_t *dict, int index)
 {
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        char            *buffer         =       NULL;
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
 
-        GF_ASSERT (snap_dict);
-        GF_ASSERT (dict);
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
 
-        ret = dict_set_uint32 (snap_dict, "sub-cmd",
-                               GF_SNAP_STATUS_TYPE_ITER);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save command "
-                        "type in snap dict");
-                goto out;
-        }
+    ret = dict_set_uint32(snap_dict, "sub-cmd", GF_SNAP_STATUS_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type in snap dict");
+        goto out;
+    }
 
-        ret = snprintf (key, sizeof (key), "status.snap%d.snapname", index);
-         if (ret < 0) {
-                 goto out;
-         }
+    ret = snprintf(key, sizeof(key), "status.snap%d.snapname", index);
+    if (ret < 0) {
+        goto out;
+    }
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get snapname");
-                goto out;
-        }
+    ret = dict_get_strn(dict, key, ret, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapname");
+        goto out;
+    }
 
-        ret = dict_set_str (snap_dict, "snapname", buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save snapname "
-                        "in snap dict");
-                goto out;
+    ret = dict_set_str_sizen(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save snapname in snap dict");
+        goto out;
+    }
 
-        }
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_STATUS);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type");
+        goto out;
+    }
 
-        ret = dict_set_int32 (snap_dict, "type", GF_SNAP_OPTION_TYPE_STATUS);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command type");
-                goto out;
-        }
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot status");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as status");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "hold_vol_locks", _gf_false);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Setting volume lock flag failed");
+        goto out;
+    }
+
+out:
+    return ret;
+}
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "cmd-str",
-                                       "snapshot status");
+static int
+cli_snapshot_status(dict_t *dict, gf_cli_rsp *rsp, call_frame_t *frame)
+{
+    int ret = -1;
+    int status_cmd = -1;
+    cli_local_t *local = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+    GF_ASSERT(frame);
+
+    local = ((call_frame_t *)frame)->local;
+    if (!local) {
+        gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+        goto out;
+    }
+
+    if (rsp->op_ret) {
+        if (rsp->op_errstr) {
+            ret = dict_set_dynstr_with_alloc(local->dict, "op_err_str",
+                                             rsp->op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set op_errstr in local dictionary");
+                goto out;
+            }
+        }
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((status_cmd != GF_SNAP_STATUS_TYPE_SNAP) &&
+        (status_cmd != GF_SNAP_STATUS_TYPE_ITER)) {
+        dict_copy(dict, local->dict);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_status_single_snap(local, dict, "status.snap0");
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command string as status");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot status");
         }
-
-        ret = dict_set_int32 (snap_dict, "hold_vol_locks", _gf_false);
+    } else {
+        ret = cli_get_single_snap_status(dict, "status.snap0");
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Setting volume lock flag failed");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch status of snap");
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_snapshot_status (dict_t *dict, gf_cli_rsp *rsp,
-                             call_frame_t *frame)
-{
-        char            key[PATH_MAX]   =       "";
-        int             ret             =       -1;
-        int             status_cmd      =       -1;
-        cli_local_t     *local          =       NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp);
-        GF_ASSERT (frame);
-
-        local = ((call_frame_t *) frame) -> local;
-        if (!local) {
-                gf_log ("cli", GF_LOG_ERROR, "frame->local is NULL");
-                goto out;
-        }
-
-        if (rsp->op_ret) {
-                if (rsp->op_errstr) {
-                        ret = dict_set_dynstr_with_alloc (local->dict,
-                                                          "op_err_str",
-                                                          rsp->op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                        "op_errstr in local dictionary");
-                                goto out;
-                        }
-                }
-                ret = rsp->op_ret;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "sub-cmd", &status_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
+static int
+gf_cli_generate_snapshot_event(gf_cli_rsp *rsp, dict_t *dict, int32_t type,
+                               char *snap_name, char *volname, char *snap_uuid,
+                               char *clone_name)
+{
+    int ret = -1;
+    int config_command = 0;
+    int32_t delete_cmd = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+    char msg[PATH_MAX] = {
+        0,
+    };
+    char option[512] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+    GF_VALIDATE_OR_GOTO("cli", rsp, out);
+
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
                 goto out;
-        }
+            }
 
-        if ((status_cmd != GF_SNAP_STATUS_TYPE_SNAP) &&
-            (status_cmd != GF_SNAP_STATUS_TYPE_ITER)) {
-                dict_copy (dict, local->dict);
+            if (!volname) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
                 goto out;
-        }
+            }
 
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CREATE_FAILED,
+                         "snapshot_name=%s;volume_name=%s;error=%s", snap_name,
+                         volname,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-        ret = snprintf (key, sizeof (key), "status.snap0");
-        if (ret < 0) {
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_status_single_snap (local, dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot status");
-                        goto out;
-                }
-        } else {
-                ret = cli_get_single_snap_status (dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                "status of snap");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-gf_cli_generate_snapshot_event (gf_cli_rsp *rsp, dict_t *dict,
-                                int32_t type, char *snap_name,
-                                char *volname, char *snap_uuid,
-                                char *clone_name)
-{
-        int         ret               = -1;
-        int         config_command    = 0;
-        int32_t     delete_cmd        = -1;
-        uint64_t    hard_limit        = 0;
-        uint64_t    soft_limit        = 0;
-        char       *auto_delete       = NULL;
-        char       *snap_activate     = NULL;
-        char        msg[PATH_MAX]     = {0, };
-        char        option[PATH_MAX]  = {0, };
-
-        GF_VALIDATE_OR_GOTO ("cli", dict, out);
-        GF_VALIDATE_OR_GOTO ("cli", rsp, out);
-
-        switch (type) {
-        case GF_SNAP_OPTION_TYPE_CREATE:
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            }
 
-                if (!volname) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get volume name");
-                        goto out;
-                }
+            gf_event(EVENT_SNAPSHOT_CREATED,
+                     "snapshot_name=%s;volume_name=%s;snapshot_uuid=%s",
+                     snap_name, volname, snap_uuid);
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_CREATE_FAILED,
-                                  "snapshot_name=%s;volume_name=%s;error=%s",
-                                  snap_name, volname,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            ret = 0;
+            break;
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
-
-                gf_event (EVENT_SNAPSHOT_CREATED, "snapshot_name=%s;"
-                          "volume_name=%s;snapshot_uuid=%s", snap_name,
-                          volname, snap_uuid);
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_ACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_ACTIVATE_FAILED,
-                                  "snapshot_name=%s;error=%s", snap_name,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            gf_event(EVENT_SNAPSHOT_ACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
+            ret = 0;
+            break;
 
-                gf_event (EVENT_SNAPSHOT_ACTIVATED, "snapshot_name=%s;"
-                          "snapshot_uuid=%s", snap_name, snap_uuid);
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DEACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_DEACTIVATE_FAILED,
-                                  "snapshot_name=%s;error=%s", snap_name,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            gf_event(EVENT_SNAPSHOT_DEACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
+            ret = 0;
+            break;
 
-                gf_event (EVENT_SNAPSHOT_DEACTIVATED, "snapshot_name=%s;"
-                          "snapshot_uuid=%s", snap_name, snap_uuid);
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_RESTORE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_RESTORE_FAILED,
-                                  "snapshot_name=%s;error=%s", snap_name,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volname");
+                goto out;
+            }
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
+            gf_event(EVENT_SNAPSHOT_RESTORED,
+                     "snapshot_name=%s;snapshot_uuid=%s;volume_name=%s",
+                     snap_name, snap_uuid, volname);
 
-                if (!volname) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get volname");
-                        goto out;
-                }
+            ret = 0;
+            break;
 
-                gf_event (EVENT_SNAPSHOT_RESTORED, "snapshot_name=%s;"
-                          "snapshot_uuid=%s;volume_name=%s",
-                          snap_name, snap_uuid, volname);
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
+                goto out;
+            }
 
+            /*
+             * Need not generate any event (success or failure) for delete *
+             * all, as it will trigger individual delete for all snapshots *
+             */
+            if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL) {
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = dict_get_int32 (dict, "sub-cmd", &delete_cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get sub-cmd");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                /*
-                 * Need not generate any event (success or failure) for delete *
-                 * all, as it will trigger individual delete for all snapshots *
-                 */
-                if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL) {
-                        ret = 0;
-                        break;
-                }
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DELETE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_DELETE_FAILED,
-                                  "snapshot_name=%s;error=%s", snap_name,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            gf_event(EVENT_SNAPSHOT_DELETED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+                goto out;
+            }
 
-                gf_event (EVENT_SNAPSHOT_DELETED, "snapshot_name=%s;"
-                          "snapshot_uuid=%s", snap_name, snap_uuid);
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
+                goto out;
+            }
 
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CLONE_FAILED,
+                         "snapshot_name=%s;clone_name=%s;error=%s", snap_name,
+                         clone_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_CLONE:
-                if (!clone_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get clone name");
-                        goto out;
-                }
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snapname name");
-                        goto out;
-                }
+            gf_event(EVENT_SNAPSHOT_CLONED,
+                     "snapshot_name=%s;clone_name=%s;clone_uuid=%s", snap_name,
+                     clone_name, snap_uuid);
 
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_CLONE_FAILED,
-                                  "snapshot_name=%s;clone_name=%s;"
-                                  "error=%s", snap_name, clone_name,
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            ret = 0;
+            break;
 
-                if (!snap_uuid) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                        goto out;
-                }
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CONFIG_UPDATE_FAILED, "error=%s",
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-                gf_event (EVENT_SNAPSHOT_CLONED, "snapshot_name=%s;"
-                          "clone_name=%s;clone_uuid=%s",
-                          snap_name, clone_name, snap_uuid);
+            ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+                goto out;
+            }
 
+            if (config_command == GF_SNAP_CONFIG_DISPLAY) {
                 ret = 0;
                 break;
+            }
 
-        case GF_SNAP_OPTION_TYPE_CONFIG:
-                if (rsp->op_ret != 0) {
-                        gf_event (EVENT_SNAPSHOT_CONFIG_UPDATE_FAILED,
-                                  "error=%s",
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                  "Please check log file for details");
-                        ret = 0;
-                        break;
-                }
+            /* These are optional parameters therefore ignore the error */
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+            ret = dict_get_str_sizen(dict, "snap-activate-on-create",
+                                     &snap_activate);
 
-                ret = dict_get_int32 (dict, "config-command", &config_command);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not fetch config type");
-                        goto out;
-                }
+            if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR,
+                       "At least one option from "
+                       "snap-max-hard-limit, snap-max-soft-limit, "
+                       "auto-delete and snap-activate-on-create "
+                       "should be set");
+                goto out;
+            }
 
-                if (config_command == GF_SNAP_CONFIG_DISPLAY) {
-                        ret = 0;
-                        break;
-                }
+            if (hard_limit || soft_limit) {
+                snprintf(option, sizeof(option), "%s=%" PRIu64,
+                         hard_limit ? "hard_limit" : "soft_limit",
+                         hard_limit ? hard_limit : soft_limit);
+            } else if (auto_delete || snap_activate) {
+                snprintf(option, sizeof(option), "%s=%s",
+                         auto_delete ? "auto-delete" : "snap-activate",
+                         auto_delete ? auto_delete : snap_activate);
+            }
 
-                /* These are optional parameters therefore ignore the error */
-                ret = dict_get_uint64 (dict, "snap-max-hard-limit",
-                                       &hard_limit);
-                ret = dict_get_uint64 (dict, "snap-max-soft-limit",
-                                       &soft_limit);
-                ret = dict_get_str (dict, "auto-delete",
-                                    &auto_delete);
-                ret = dict_get_str (dict, "snap-activate-on-create",
-                                    &snap_activate);
-
-                if (!hard_limit && !soft_limit &&
-                    !auto_delete && !snap_activate) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "At least one option from "
-                                "snap-max-hard-limit, snap-max-soft-limit, "
-                                "auto-delete and snap-activate-on-create "
-                                "should be set");
-                        goto out;
-                }
+            volname = NULL;
+            ret = dict_get_str_sizen(dict, "volname", &volname);
 
-                volname = NULL;
-                ret = dict_get_str (dict, "volname", &volname);
-
-                if (hard_limit || soft_limit) {
-                        snprintf (option, sizeof(option), "%s=%"PRIu64,
-                                  hard_limit ? "hard_limit" : "soft_limit",
-                                  hard_limit ? hard_limit:soft_limit);
-                } else if (auto_delete || snap_activate) {
-                        snprintf (option, sizeof(option), "%s=%s",
-                                  auto_delete ? "auto-delete" : "snap-activate",
-                                  auto_delete ? auto_delete:snap_activate);
-                }
-
-                snprintf (msg, sizeof(msg), "config_type=%s;%s",
-                          volname?"volume_config":"system_config", option);
+            snprintf(msg, sizeof(msg), "config_type=%s;%s",
+                     volname ? "volume_config" : "system_config", option);
 
-                gf_event (EVENT_SNAPSHOT_CONFIG_UPDATED, "%s", msg);
+            gf_event(EVENT_SNAPSHOT_CONFIG_UPDATED, "%s", msg);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         default:
-                gf_log ("cli", GF_LOG_WARNING,
-                        "Cannot generate event for unknown type.");
-                ret = 0;
-                goto out;
-        }
+            gf_log("cli", GF_LOG_WARNING,
+                   "Cannot generate event for unknown type.");
+            ret = 0;
+            goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
  * Fetch necessary data from dict at one place instead of *
  * repeating the same code again and again.               *
  */
-int
-gf_cli_snapshot_get_data_from_dict (dict_t *dict, char **snap_name,
-                                    char **volname, char **snap_uuid,
-                                    int8_t *soft_limit_flag,
-                                    char **clone_name)
+static int
+gf_cli_snapshot_get_data_from_dict(dict_t *dict, char **snap_name,
+                                   char **volname, char **snap_uuid,
+                                   int8_t *soft_limit_flag, char **clone_name)
 {
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("cli", dict, out);
-
-        if (snap_name)
-                ret = dict_get_str (dict, "snapname", snap_name);
-
-        if (volname)
-                ret = dict_get_str (dict, "volname1", volname);
-
-        if (snap_uuid)
-                ret = dict_get_str (dict, "snapuuid", snap_uuid);
+    int ret = -1;
 
-        if (soft_limit_flag)
-                ret = dict_get_int8 (dict, "soft-limit-reach",
-                                     soft_limit_flag);
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
 
-        if (clone_name)
-                ret = dict_get_str (dict, "clonename", clone_name);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-gf_cli_snapshot_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        int                   ret                      = -1;
-        gf_cli_rsp            rsp                      = {0, };
-        dict_t               *dict                     = NULL;
-        char                 *snap_name                = NULL;
-        char                 *clone_name               = NULL;
-        int32_t               type                     =  0;
-        call_frame_t         *frame                    = NULL;
-        gf_boolean_t         snap_driven               = _gf_false;
-        int8_t               soft_limit_flag           = -1;
-        char                 *volname                  = NULL;
-        char                 *snap_uuid                = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (req->rpc_status == -1) {
-                goto out;
+    if (snap_name) {
+        ret = dict_get_str_sizen(dict, "snapname", snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapname from dict");
         }
+    }
 
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+    if (volname) {
+        ret = dict_get_str_sizen(dict, "volname1", volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get volname1 from dict");
         }
+    }
 
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
+    if (snap_uuid) {
+        ret = dict_get_str_sizen(dict, "snapuuid", snap_uuid);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapuuid from dict");
         }
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "type", &type);
+    if (soft_limit_flag) {
+        ret = dict_get_int8(dict, "soft-limit-reach", soft_limit_flag);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
-                goto out;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "failed to get soft-limit-reach from dict");
         }
+    }
 
-        ret = gf_cli_snapshot_get_data_from_dict (dict, &snap_name, &volname,
-                                                  &snap_uuid, &soft_limit_flag,
-                                                  &clone_name);
+    if (clone_name) {
+        ret = dict_get_str_sizen(dict, "clonename", clone_name);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch data from dict.");
-                goto out;
+            gf_log("cli", GF_LOG_DEBUG, "failed to get clonename from dict");
         }
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+gf_cli_snapshot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *snap_name = NULL;
+    char *clone_name = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    gf_boolean_t snap_driven = _gf_false;
+    int8_t soft_limit_flag = -1;
+    char *volname = NULL;
+    char *snap_uuid = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get type");
+        goto out;
+    }
+
+    ret = gf_cli_snapshot_get_data_from_dict(
+        dict, &snap_name, &volname, &snap_uuid, &soft_limit_flag, &clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch data from dict.");
+        goto out;
+    }
 
 #if (USE_EVENTS)
-        ret = gf_cli_generate_snapshot_event (&rsp, dict, type, snap_name,
-                                              volname, snap_uuid, clone_name);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to generate snapshot event");
-                goto out;
-        }
+    ret = gf_cli_generate_snapshot_event(&rsp, dict, type, snap_name, volname,
+                                         snap_uuid, clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate snapshot event");
+        goto out;
+    }
 #endif
 
-        /* Snapshot status and delete command is handled separately */
-        if (global_state->mode & GLUSTER_MODE_XML &&
-            GF_SNAP_OPTION_TYPE_STATUS != type &&
-            GF_SNAP_OPTION_TYPE_DELETE != type) {
-                ret = cli_xml_output_snapshot (type, dict, rsp.op_ret,
-                                               rsp.op_errno, rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-                }
-
-                goto out;
+    /* Snapshot status and delete command is handled separately */
+    if (global_state->mode & GLUSTER_MODE_XML &&
+        GF_SNAP_OPTION_TYPE_STATUS != type &&
+        GF_SNAP_OPTION_TYPE_DELETE != type) {
+        ret = cli_xml_output_snapshot(type, dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
 
-        switch (type) {
+        goto out;
+    }
+
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot create: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot create: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                if (!volname) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get volume name");
-                        goto out;
-                }
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+            }
 
-                cli_out ("snapshot create: success: Snap %s created "
-                                        "successfully", snap_name);
+            cli_out("snapshot create: success: Snap %s created successfully",
+                    snap_name);
 
-                if (soft_limit_flag == 1) {
-                        cli_out ("Warning: Soft-limit of volume (%s) is "
-                                "reached. Snapshot creation is not possible "
-                                "once hard-limit is reached.", volname);
-                }
-                ret = 0;
-                break;
+            if (soft_limit_flag == 1) {
+                cli_out(
+                    "Warning: Soft-limit of volume (%s) is "
+                    "reached. Snapshot creation is not possible "
+                    "once hard-limit is reached.",
+                    volname);
+            }
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot clone: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot clone: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                if (!clone_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get clone name");
-                        goto out;
-                }
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snapname name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
+                goto out;
+            }
 
-                cli_out ("snapshot clone: success: Clone %s created "
-                                        "successfully", clone_name);
+            cli_out("snapshot clone: success: Clone %s created successfully",
+                    clone_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot restore: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot restore: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot restore: %s: Snap restored "
-                                 "successfully", snap_name);
+            cli_out("Snapshot restore: %s: Snap restored successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot activate: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot activate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot activate: %s: Snap activated "
-                                 "successfully", snap_name);
+            cli_out("Snapshot activate: %s: Snap activated successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot deactivate: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot deactivate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                if (!snap_name) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot deactivate: %s: Snap deactivated "
-                                 "successfully", snap_name);
+            cli_out("Snapshot deactivate: %s: Snap deactivated successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                if (rsp.op_ret) {
-                        cli_err ("Snapshot info : failed: %s",
-                                  rsp.op_errstr ? rsp.op_errstr :
-                                  "Please check log file for details");
-                        ret = rsp.op_ret;
-                        goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("Snapshot info : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                snap_driven = dict_get_str_boolean (dict, "snap-driven",
-                                                    _gf_false);
-                if (snap_driven == _gf_true) {
-                        ret = cli_call_snapshot_info  (dict, snap_driven);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Snapshot info failed");
-                                goto out;
-                        }
-                } else if (snap_driven == _gf_false) {
-                        ret = cli_get_snaps_in_volume (dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Snapshot info failed");
-                                goto out;
-                        }
+            snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+            if (snap_driven == _gf_true) {
+                ret = cli_call_snapshot_info(dict, snap_driven);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
                 }
-                break;
-
-        case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = cli_snapshot_config_display (dict, &rsp);
+            } else if (snap_driven == _gf_false) {
+                ret = cli_get_snaps_in_volume(dict);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot config output.");
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
                 }
-                break;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = cli_snapshot_config_display(dict, &rsp);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot config output.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_LIST:
-                if (rsp.op_ret) {
-                        cli_err ("Snapshot list : failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                "Please check log file for details");
-                        ret = rsp.op_ret;
-                        goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("Snapshot list : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = cli_snapshot_list (dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot list");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_list(dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to display snapshot list");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = cli_snapshot_remove_reply (&rsp, dict, frame);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to delete snap");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_remove_reply(&rsp, dict, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to delete snap");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = cli_snapshot_status (dict, &rsp, frame);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot status output.");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_status(dict, &rsp, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot status output.");
+                goto out;
+            }
+            break;
 
         default:
-                cli_err ("Unknown command executed");
-                ret = -1;
-                goto out;
-        }
+            cli_err("Unknown command executed");
+            ret = -1;
+            goto out;
+    }
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
 
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    free(rsp.dict.dict_val);
+    free(rsp.op_errstr);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-gf_cli_snapshot_for_delete (call_frame_t *frame, xlator_t *this,
-                            void *data)
-{
-        gf_cli_req      req                     = {{0,}};
-        int32_t         ret                     = -1;
-        int32_t         cmd                     = -1;
-        cli_local_t     *local                  = NULL;
-        dict_t          *snap_dict              = NULL;
-        int32_t         snapcount               = 0;
-        int             i                       = 0;
-        char            question[PATH_MAX]      = "";
-        char            *volname                = NULL;
-        gf_answer_t     answer                  = GF_ANSWER_NO;
-
-        GF_VALIDATE_OR_GOTO ("cli", frame,  out);
-        GF_VALIDATE_OR_GOTO ("cli", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("cli", this, out);
-        GF_VALIDATE_OR_GOTO ("cli", data, out);
-
-        local = frame->local;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                goto out;
-        }
+gf_cli_snapshot_for_delete(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int32_t ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int32_t snapcount = 0;
+    int i = 0;
+    char question[PATH_MAX] = "";
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* No need multiple RPCs for individual snapshot delete*/
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
 
-        /* No need multiple RPCs for individual snapshot delete*/
-        if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_int32_sizen(local->dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
 
-        ret = dict_get_int32 (local->dict, "snapcount",
-                              &snapcount);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+#ifdef HAVE_LIB_XML
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapCount", "%d", snapcount);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get "
-                        "snapcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to write xml element \"snapCount\"");
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-#ifdef HAVE_LIB_XML
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                (xmlChar *)"snapCount",
-                                                "%d", snapcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to write "
-                                "xml element \"snapCount\"");
-                        goto out;
-                }
 #endif /* HAVE_LIB_XML */
-        } else if (snapcount == 0) {
-                cli_out ("No snapshots present");
-                goto out;
+    } else if (snapcount == 0) {
+        cli_out("No snapshots present");
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_DELETE_TYPE_ALL) {
+        snprintf(question, sizeof(question),
+                 "System contains %d snapshot(s).\nDo you still "
+                 "want to continue and delete them? ",
+                 snapcount);
+    } else {
+        ret = dict_get_str_sizen(local->dict, "volname", &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to fetch volname from local dictionary");
+            goto out;
         }
 
-        if (cmd == GF_SNAP_DELETE_TYPE_ALL) {
-                snprintf (question, sizeof (question), "System contains %d "
-                          "snapshot(s).\nDo you still "
-                          "want to continue and delete them? ",
-                           snapcount);
-        } else {
-                ret = dict_get_str (local->dict, "volname", &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to fetch "
-                                "volname from local dictionary");
-                        goto out;
-                }
-
-                snprintf (question, sizeof (question), "Volume (%s) contains "
-                          "%d snapshot(s).\nDo you still want to "
-                           "continue and delete them? ", volname,
-                           snapcount);
-        }
+        snprintf(question, sizeof(question),
+                 "Volume (%s) contains %d snapshot(s).\nDo you still want to "
+                 "continue and delete them? ",
+                 volname, snapcount);
+    }
 
-        answer = cli_cmd_get_confirmation (global_state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot delete operation for snap delete");
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(global_state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled snapshot delete operation for snap delete");
+        goto out;
+    }
 
-        for (i = 1 ; i <= snapcount ; i++) {
-                ret = -1;
+    for (i = 1; i <= snapcount; i++) {
+        ret = -1;
 
-                snap_dict = dict_new();
-                if (!snap_dict)
-                        goto out;
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
 
-                ret = cli_populate_req_dict_for_delete (snap_dict,
-                                                       local->dict, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                "populate snap request dictionary");
-                        goto out;
-                }
+        ret = cli_populate_req_dict_for_delete(snap_dict, local->dict, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
+        }
 
-                ret = cli_to_glusterd (&req, frame,
-                                gf_cli_snapshot_cbk,
-                                (xdrproc_t) xdr_gf_cli_req, snap_dict,
-                                GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                                NULL);
-                if (ret) {
-                        /* Fail the operation if deleting one of the
-                         * snapshots is failed
-                         */
-                        gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd "
-                                "for snapshot delete failed");
-                        goto out;
-                }
-                dict_unref (snap_dict);
-                snap_dict = NULL;
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
+        if (ret) {
+            /* Fail the operation if deleting one of the
+             * snapshots is failed
+             */
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot delete failed");
+            goto out;
         }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
 
 out:
-        if (snap_dict)
-                dict_unref (snap_dict);
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_snapshot_for_status (call_frame_t *frame, xlator_t *this,
-                            void *data)
-{
+static int32_t
+gf_cli_snapshot_for_status(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int snapcount = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* Snapshot status of single snap (i.e. GF_SNAP_STATUS_TYPE_SNAP)
+     * is already handled. Therefore we can return from here.
+     * If want to get status of all snaps in the system or volume then
+     * we should get them one by one.*/
+    if ((cmd == GF_SNAP_STATUS_TYPE_SNAP) ||
+        (cmd == GF_SNAP_STATUS_TYPE_ITER)) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_cli_req        req           = {{0,}};
-        int               ret           = -1;
-        int32_t           cmd           = -1;
-        cli_local_t      *local         = NULL;
-        dict_t           *snap_dict     = NULL;
-        int              snapcount      =  0;
-        int              i              =  0;
+    ret = dict_get_int32_sizen(local->dict, "status.snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("cli", frame, out);
-        GF_VALIDATE_OR_GOTO ("cli", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("cli", this, out);
-        GF_VALIDATE_OR_GOTO ("cli", data, out);
+    if (snapcount == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_out("No snapshots present");
+    }
 
-        local = frame->local;
+    for (i = 0; i < snapcount; i++) {
+        ret = -1;
 
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
-                goto out;
-        }
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
 
-        /* Snapshot status of single snap (i.e. GF_SNAP_STATUS_TYPE_SNAP)
-         * is already handled. Therefore we can return from here.
-         * If want to get status of all snaps in the system or volume then
-         * we should get them one by one.*/
-        if ((cmd == GF_SNAP_STATUS_TYPE_SNAP) ||
-            (cmd == GF_SNAP_STATUS_TYPE_ITER)) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "status.snapcount", &snapcount);
+        ret = cli_populate_req_dict_for_status(snap_dict, local->dict, i);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
         }
 
-        if (snapcount == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
-                cli_out ("No snapshots present");
-        }
-
-        for (i = 0 ; i < snapcount; i++) {
-                ret = -1;
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
 
-                snap_dict = dict_new();
-                if (!snap_dict)
-                        goto out;
-
-                ret = cli_populate_req_dict_for_status (snap_dict,
-                                local->dict, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                        "populate snap request dictionary");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&req, frame,
-                                gf_cli_snapshot_cbk,
-                                (xdrproc_t) xdr_gf_cli_req, snap_dict,
-                                GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                                NULL);
-
-                /* Ignore the return value and error for snapshot
-                 * status of type "ALL" or "VOL"
-                 *
-                 * Scenario : There might be case where status command
-                 * and delete command might be issued at the same time.
-                 * In that case when status tried to fetch detail of
-                 * snap which has been deleted by concurrent command,
-                 * then it will show snapshot not present. Which will
-                 * not be appropriate.
-                 */
-                dict_unref (snap_dict);
-                snap_dict = NULL;
+        /* Ignore the return value and error for snapshot
+         * status of type "ALL" or "VOL"
+         *
+         * Scenario : There might be case where status command
+         * and delete command might be issued at the same time.
+         * In that case when status tried to fetch detail of
+         * snap which has been deleted by concurrent command,
+         * then it will show snapshot not present. Which will
+         * not be appropriate.
+         */
+        if (ret && (cmd != GF_SNAP_STATUS_TYPE_ALL &&
+                    cmd != GF_SNAP_STATUS_TYPE_VOL)) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot status failed");
+            goto out;
         }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (snap_dict)
-                dict_unref (snap_dict);
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_snapshot (call_frame_t *frame, xlator_t *this,
-                 void *data)
-{
-        gf_cli_req        req           = {{0,}};
-        dict_t           *options       = NULL;
-        int               ret           = -1;
-        int               tmp_ret       = -1;
-        cli_local_t      *local         = NULL;
-        char             *err_str       = NULL;
-        int              type           =  -1;
-
-        if (!frame || !this || !data)
-                goto out;
-
-        if (!frame->local)
-                goto out;
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+    int tmp_ret = -1;
+    cli_local_t *local = NULL;
+    char *err_str = NULL;
+    int type = -1;
 
-        local = frame->local;
+    if (!frame || !frame->local || !this || !data)
+        goto out;
 
-        options = data;
+    local = frame->local;
 
-        ret = dict_get_int32 (local->dict, "type", &type);
+    options = data;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_begin_composite_op (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to begin "
-                                "snapshot xml composite op");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_begin_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to begin snapshot xml composite op");
+            goto out;
         }
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options, GLUSTER_CLI_SNAP,
+                          this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for snapshot failed");
+        goto xmlend;
+    }
 
+    ret = dict_get_int32_sizen(local->dict, "type", &type);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_snapshot_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                               NULL);
+    if (GF_SNAP_OPTION_TYPE_STATUS == type) {
+        ret = gf_cli_snapshot_for_status(frame, this, data);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd for "
-                        "snapshot failed");
-                goto xmlend;
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot status command failed");
         }
 
-        if (GF_SNAP_OPTION_TYPE_STATUS == type) {
-                ret = gf_cli_snapshot_for_status (frame, this, data);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "cli to glusterd "
-                                "for snapshot status command failed");
-                }
+        goto xmlend;
+    }
 
-                goto xmlend;
+    if (GF_SNAP_OPTION_TYPE_DELETE == type) {
+        ret = gf_cli_snapshot_for_delete(frame, this, data);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot delete command failed");
         }
 
-        if (GF_SNAP_OPTION_TYPE_DELETE == type) {
-                ret = gf_cli_snapshot_for_delete (frame, this, data);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "cli to glusterd "
-                                "for snapshot delete command failed");
-                }
+        goto xmlend;
+    }
 
-                goto xmlend;
-        }
-
-        ret = 0;
+    ret = 0;
 
 xmlend:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_end_composite_op (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to end "
-                                "snapshot xml composite op");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_end_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to end snapshot xml composite op");
+            goto out;
         }
+    }
 out:
-        if (ret && local && GF_SNAP_OPTION_TYPE_STATUS == type) {
-                tmp_ret = dict_get_str (local->dict, "op_err_str", &err_str);
-                if (tmp_ret || !err_str) {
-                        cli_err ("Snapshot Status : failed: %s", "Please "
-                                 "check log file for details");
-                } else {
-                        cli_err ("Snapshot Status : failed: %s", err_str);
-                        dict_del (local->dict, "op_err_str");
-                }
+    if (ret && local && GF_SNAP_OPTION_TYPE_STATUS == type) {
+        tmp_ret = dict_get_str_sizen(local->dict, "op_err_str", &err_str);
+        if (tmp_ret || !err_str) {
+            cli_err("Snapshot Status : failed: %s",
+                    "Please check log file for details");
+        } else {
+            cli_err("Snapshot Status : failed: %s", err_str);
+            dict_del_sizen(local->dict, "op_err_str");
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                /* XML mode handles its own error */
-                ret = 0;
-        }
-        return ret;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* XML mode handles its own error */
+        ret = 0;
+    }
+    return ret;
 }
 
-int32_t
-gf_cli_barrier_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                  int count, void *myframe)
-{
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        dict_t                          *dict = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to barrier");
-
-        if (rsp.op_ret) {
-                if (rsp.op_errstr && (strlen (rsp.op_errstr) > 1)) {
-                        cli_err ("volume barrier: command unsuccessful : %s",
-                                 rsp.op_errstr);
-                } else {
-                        cli_err ("volume barrier: command unsuccessful");
-                }
+static int32_t
+gf_cli_barrier_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to barrier");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr && (strlen(rsp.op_errstr) > 1)) {
+            cli_err("volume barrier: command unsuccessful : %s", rsp.op_errstr);
         } else {
-                cli_out ("volume barrier: command successful");
+            cli_err("volume barrier: command unsuccessful");
         }
-        ret = rsp.op_ret;
+    } else {
+        cli_out("volume barrier: command successful");
+    }
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_barrier_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int
+gf_cli_barrier_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
+    options = data;
 
-        options = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_barrier_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BARRIER_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_barrier_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_BARRIER_VOLUME, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_get_vol_opt_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
-{
-        gf_cli_rsp   rsp  = {0,};
-        int          ret  = -1;
-        dict_t      *dict = NULL;
-        char        *key  = NULL;
-        char        *value = NULL;
-        char         msg[1024] = {0,};
-        int          i     = 0;
-        char         dict_key[50] = {0,};
-
-        GF_ASSERT (myframe);
-
-        if (-1 == req->rpc_status)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to get volume option");
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume get option: "
-                                  "failed: %s", rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume get option: "
-                                  "failed");
-
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_str ("volGetopts", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        }
-                } else {
-                        cli_err ("%s", msg);
-                }
-                ret = rsp.op_ret;
-                goto out_nolog;
-        }
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed rsp_dict unserialization");
-                goto out;
-        }
+static int32_t
+gf_cli_get_vol_opt_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int i = 0;
+    char dict_key[50] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to get volume option");
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume get option: failed: %s",
+                     rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume get option: failed");
 
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_getopts (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "xml output generation "
-                                "failed");
-                        ret = 0;
-                }
-                goto out;
+            ret = cli_xml_output_str("volGetopts", msg, rsp.op_ret,
+                                     rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        } else {
+            cli_err("%s", msg);
         }
-
-        ret = dict_get_int32 (dict, "count", &count);
+        ret = rsp.op_ret;
+        goto out_nolog;
+    }
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_getopts(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to retrieve count "
-                        "from the dictionary");
-                goto out;
-        }
-        if (count <= 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Value of count :%d is "
-                        "invalid", count);
-                ret = -1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            ret = 0;
+        }
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "warning", &value);
+    if (!ret) {
+        cli_out("%s", value);
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count from the dictionary");
+        goto out;
+    }
+
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR, "Value of count :%d is invalid", count);
+        ret = -1;
+        goto out;
+    }
+
+    cli_out("%-40s%-40s", "Option", "Value");
+    cli_out("%-40s%-40s", "------", "-----");
+    for (i = 1; i <= count; i++) {
+        ret = snprintf(dict_key, sizeof dict_key, "key%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve %s from the dictionary", dict_key);
+            goto out;
         }
-
-        cli_out ("%-40s%-40s", "Option", "Value");
-        cli_out ("%-40s%-40s", "------", "-----");
-        for (i=1; i<=count; i++) {
-                snprintf (dict_key, sizeof dict_key, "key%d", i);
-                ret = dict_get_str (dict, dict_key, &key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to"
-                                " retrieve %s from the "
-                                "dictionary", dict_key);
-                        goto out;
-                }
-                snprintf (dict_key, sizeof dict_key, "value%d", i);
-                ret = dict_get_str (dict, dict_key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "retrieve key value for %s from"
-                                "the dictionary", dict_key);
-                        goto out;
-                }
-                cli_out ("%-40s%-40s", key, value);
+        ret = snprintf(dict_key, sizeof dict_key, "value%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve key value for %s from the dictionary",
+                   dict_key);
+            goto out;
         }
+        cli_out("%-40s%-40s", key, value);
+    }
 
 out:
-        if (ret) {
-                cli_out ("volume get option failed. Check the cli/glusterd log "
-                         "file for more details");
-        }
+    if (ret) {
+        cli_out(
+            "volume get option failed. Check the cli/glusterd log "
+            "file for more details");
+    }
 
 out_nolog:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_get_vol_opt (call_frame_t *frame, xlator_t *this, void *data)
+static int
+gf_cli_get_vol_opt(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req  req     = {{0,}};
-        dict_t     *options = NULL;
-        int         ret     = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
+    options = data;
 
-        options = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_vol_opt_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_GET_VOL_OPT, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_get_vol_opt_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, options,
-                               GLUSTER_CLI_GET_VOL_OPT, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int
-cli_to_glusterd (gf_cli_req *req, call_frame_t *frame,
-                 fop_cbk_fn_t cbkfn, xdrproc_t xdrproc, dict_t *dict,
-                 int procnum, xlator_t *this, rpc_clnt_prog_t *prog,
-                 struct iobref *iobref)
-{
-        int                ret = 0;
-        size_t             len = 0;
-        char               *cmd = NULL;
-        int                i = 0;
-        const char         **words = NULL;
-        cli_local_t        *local = NULL;
-
-        if (!this || !frame || !dict) {
-                ret = -1;
-                goto out;
-        }
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict)
+{
+    int ret = 0;
 
-        if (!frame->local) {
-                ret = -1;
-                goto out;
+    if (cli_default_conn_timeout > 120) {
+        ret = dict_set_uint32(dict, "timeout", cli_default_conn_timeout);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Failed to save timeout to dict");
         }
+    }
+    return ret;
+}
 
-        local = frame->local;
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref)
+{
+    int ret = 0;
+    size_t len = 0;
+    char *cmd = NULL;
+    int i = 0;
+    const char **words = NULL;
+    cli_local_t *local = NULL;
 
-        if (!local->words) {
-                ret = -1;
-                goto out;
-        }
+    if (!this || !frame || !frame->local || !dict) {
+        ret = -1;
+        goto out;
+    }
 
-        words = local->words;
+    local = frame->local;
 
-        while (words[i])
-                len += strlen (words[i++]) + 1;
+    if (!local->words) {
+        ret = -1;
+        goto out;
+    }
 
-        cmd = GF_CALLOC (1, len, gf_common_mt_char);
+    words = local->words;
 
-        if (!cmd) {
-                ret = -1;
-                goto out;
-        }
+    while (words[i])
+        len += strlen(words[i++]) + 1;
 
-        for (i = 0; words[i]; i++) {
-                strncat (cmd, words[i], strlen (words[i]));
-                if (words[i+1] != NULL)
-                        strncat (cmd, " ", strlen (" "));
-        }
+    cmd = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!cmd) {
+        ret = -1;
+        goto out;
+    }
+    cmd[0] = '\0';
 
-        cmd [len - 1] = '\0';
+    for (i = 0; words[i]; i++) {
+        strncat(cmd, words[i], len - 1);
+        if (words[i + 1] != NULL)
+            strncat(cmd, " ", len - 1);
+    }
 
-        ret = dict_set_dynstr (dict, "cmd-str", cmd);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr_sizen(dict, "cmd-str", cmd);
+    if (ret)
+        goto out;
 
-        ret = dict_allocate_and_serialize (dict, &(req->dict).dict_val,
-                                           &(req->dict).dict_len);
+    ret = add_cli_cmd_timeout_to_dict(dict);
 
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &(req->dict).dict_val,
+                                      &(req->dict).dict_len);
 
-        ret = cli_cmd_submit (NULL, req, frame, prog, procnum, iobref, this,
-                              cbkfn, (xdrproc_t) xdrproc);
-out:
-        return ret;
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "failed to get serialized length of dict");
+        goto out;
+    }
 
+    ret = cli_cmd_submit(NULL, req, frame, prog, procnum, iobref, this, cbkfn,
+                         (xdrproc_t)xdrproc);
+out:
+    return ret;
 }
 
-int
-gf_cli_print_bitrot_scrub_status (dict_t *dict)
-{
-        int            i                = 1;
-        int            j                = 0;
-        int            ret              = -1;
-        int            count            = 0;
-        char           key[256]         = {0,};
-        char           *volname         = NULL;
-        char           *node_name       = NULL;
-        char           *scrub_freq      = NULL;
-        char           *state_scrub     = NULL;
-        char           *scrub_impact    = NULL;
-        char           *bad_file_str    = NULL;
-        char           *scrub_log_file  = NULL;
-        char           *bitrot_log_file = NULL;
-        uint64_t       scrub_files      = 0;
-        uint64_t       unsigned_files   = 0;
-        uint64_t       scrub_time       = 0;
-        uint64_t       days             = 0;
-        uint64_t       hours            = 0;
-        uint64_t       minutes          = 0;
-        uint64_t       seconds          = 0;
-        char          *last_scrub       = NULL;
-        uint64_t       error_count      = 0;
-        int8_t         scrub_running    = 0;
-        char          *scrub_state_op   = NULL;
-
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get volume name");
-
-        ret = dict_get_str (dict, "features.scrub", &state_scrub);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub state value");
-
-        ret = dict_get_str (dict, "features.scrub-throttle", &scrub_impact);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub impact "
-                        "value");
-
-        ret = dict_get_str (dict, "features.scrub-freq", &scrub_freq);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub -freq value");
-
-        ret = dict_get_str (dict, "bitrot_log_file", &bitrot_log_file);
+static int
+gf_cli_print_bitrot_scrub_status(dict_t *dict)
+{
+    int i = 1;
+    int j = 0;
+    int ret = -1;
+    int count = 0;
+    char key[64] = {
+        0,
+    };
+    char *volname = NULL;
+    char *node_name = NULL;
+    char *scrub_freq = NULL;
+    char *state_scrub = NULL;
+    char *scrub_impact = NULL;
+    char *bad_file_str = NULL;
+    char *scrub_log_file = NULL;
+    char *bitrot_log_file = NULL;
+    uint64_t scrub_files = 0;
+    uint64_t unsigned_files = 0;
+    uint64_t scrub_time = 0;
+    uint64_t days = 0;
+    uint64_t hours = 0;
+    uint64_t minutes = 0;
+    uint64_t seconds = 0;
+    char *last_scrub = NULL;
+    uint64_t error_count = 0;
+    int8_t scrub_running = 0;
+    char *scrub_state_op = NULL;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "failed to get count value from dictionary");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "features.scrub", &state_scrub);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub state value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-throttle", &scrub_impact);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub impact value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-freq", &scrub_freq);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub -freq value");
+
+    ret = dict_get_str_sizen(dict, "bitrot_log_file", &bitrot_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get bitrot log file location");
+
+    ret = dict_get_str_sizen(dict, "scrub_log_file", &scrub_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrubber log file location");
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "scrub-running-%d", i);
+        ret = dict_get_int8(dict, key, &scrub_running);
         if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get bitrot log file "
-                        "location");
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
+        if (scrub_running)
+            break;
+    }
 
-        ret = dict_get_str (dict, "scrub_log_file", &scrub_log_file);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrubber log file "
-                        "location");
+    if (scrub_running)
+        gf_asprintf(&scrub_state_op, "%s (In Progress)", state_scrub);
+    else
+        gf_asprintf(&scrub_state_op, "%s (Idle)", state_scrub);
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not get count value from"
-                        " dictionary");
-                goto out;
-        }
+    cli_out("\n%s: %s\n", "Volume name ", volname);
 
-        for (i = 1; i <= count; i++) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "scrub-running-%d", i);
-                ret = dict_get_int8 (dict, key, &scrub_running);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get scrubbed "
-                                "files");
-                if (scrub_running)
-                        break;
-        }
+    cli_out("%s: %s\n", "State of scrub", scrub_state_op);
 
-        if (scrub_running)
-                gf_asprintf (&scrub_state_op, "%s (In Progress)", state_scrub);
-        else
-                gf_asprintf (&scrub_state_op, "%s (Idle)", state_scrub);
+    cli_out("%s: %s\n", "Scrub impact", scrub_impact);
 
-        cli_out ("\n%s: %s\n", "Volume name ", volname);
+    cli_out("%s: %s\n", "Scrub frequency", scrub_freq);
 
-        cli_out ("%s: %s\n", "State of scrub", scrub_state_op);
+    cli_out("%s: %s\n", "Bitrot error log location", bitrot_log_file);
 
-        cli_out ("%s: %s\n", "Scrub impact", scrub_impact);
+    cli_out("%s: %s\n", "Scrubber error log location", scrub_log_file);
 
-        cli_out ("%s: %s\n", "Scrub frequency", scrub_freq);
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        last_scrub = NULL;
+        scrub_time = 0;
+        error_count = 0;
+        scrub_files = 0;
+        unsigned_files = 0;
 
-        cli_out ("%s: %s\n", "Bitrot error log location", bitrot_log_file);
-
-        cli_out ("%s: %s\n", "Scrubber error log location", scrub_log_file);
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
 
+        snprintf(key, sizeof(key), "scrubbed-files-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
 
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name       = NULL;
-                last_scrub      = NULL;
-                scrub_time      = 0;
-                days            = 0;
-                hours           = 0;
-                minutes         = 0;
-                seconds         = 0;
-                error_count     = 0;
-                scrub_files     = 0;
-                unsigned_files  = 0;
+        snprintf(key, sizeof(key), "unsigned-files-%d", i);
+        ret = dict_get_uint64(dict, key, &unsigned_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get unsigned files");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+        snprintf(key, sizeof(key), "scrub-duration-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_time);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub duration");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "scrubbed-files-%d", i);
-                ret = dict_get_uint64 (dict, key, &scrub_files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get scrubbed "
-                                "files");
+        snprintf(key, sizeof(key), "last-scrub-time-%d", i);
+        ret = dict_get_str(dict, key, &last_scrub);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub time");
+        snprintf(key, sizeof(key), "error-count-%d", i);
+        ret = dict_get_uint64(dict, key, &error_count);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get error count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "unsigned-files-%d", i);
-                ret = dict_get_uint64 (dict, key, &unsigned_files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get unsigned "
-                                "files");
+        cli_out("\n%s\n",
+                "=========================================================");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "scrub-duration-%d", i);
-                ret = dict_get_uint64 (dict, key, &scrub_time);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get last scrub "
-                                "duration");
+        cli_out("%s: %s\n", "Node", node_name);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "last-scrub-time-%d", i);
-                ret = dict_get_str (dict, key, &last_scrub);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get last scrub"
-                                " time");
-                memset (key, 0, 256);
-                snprintf (key, 256, "error-count-%d", i);
-                ret = dict_get_uint64 (dict, key, &error_count);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get error "
-                                "count");
+        cli_out("%s: %" PRIu64 "\n", "Number of Scrubbed files", scrub_files);
 
-                cli_out ("\n%s\n", "=========================================="
-                         "===============");
+        cli_out("%s: %" PRIu64 "\n", "Number of Skipped files", unsigned_files);
 
-                cli_out ("%s: %s\n", "Node", node_name);
+        if ((!last_scrub) || !strcmp(last_scrub, ""))
+            cli_out("%s: %s\n", "Last completed scrub time",
+                    "Scrubber pending to complete.");
+        else
+            cli_out("%s: %s\n", "Last completed scrub time", last_scrub);
 
-                cli_out ("%s: %"PRIu64 "\n", "Number of Scrubbed files",
-                          scrub_files);
+        /* Printing last scrub duration time in human readable form*/
+        seconds = scrub_time % 60;
+        minutes = (scrub_time / 60) % 60;
+        hours = (scrub_time / 3600) % 24;
+        days = scrub_time / 86400;
+        cli_out("%s: %" PRIu64 ":%" PRIu64 ":%" PRIu64 ":%" PRIu64 "\n",
+                "Duration of last scrub (D:M:H:M:S)", days, hours, minutes,
+                seconds);
 
-                cli_out ("%s: %"PRIu64 "\n", "Number of Skipped files",
-                          unsigned_files);
+        cli_out("%s: %" PRIu64 "\n", "Error count", error_count);
 
-                if ((!last_scrub) || !strcmp (last_scrub, ""))
-                        cli_out ("%s: %s\n", "Last completed scrub time",
-                                 "Scrubber pending to complete.");
-                else
-                        cli_out ("%s: %s\n", "Last completed scrub time",
-                                 last_scrub);
-
-                /* Printing last scrub duration time in human readable form*/
-                seconds    = scrub_time%60;
-                minutes    = (scrub_time/60)%60;
-                hours      = (scrub_time/3600)%24;
-                days       = scrub_time/86400;
-                cli_out ("%s: %"PRIu64 ":%"PRIu64 ":%"PRIu64 ":%"PRIu64 "\n",
-                         "Duration of last scrub (D:M:H:M:S)",
-                         days, hours, minutes, seconds);
-
-                cli_out ("%s: %"PRIu64 "\n", "Error count", error_count);
-
-                if (error_count) {
-                        cli_out ("%s:\n", "Corrupted object's [GFID]");
-                        /* Printing list of bad file's (Corrupted object's)*/
-                        for (j = 0; j < error_count; j++) {
-                                memset (key, 0, 256);
-                                snprintf (key, 256, "quarantine-%d-%d", j, i);
-                                ret = dict_get_str (dict, key, &bad_file_str);
-                                if (!ret) {
-                                        cli_out ("%s\n", bad_file_str);
-                                }
-                        }
+        if (error_count) {
+            cli_out("%s:\n", "Corrupted object's [GFID]");
+            /* Printing list of bad file's (Corrupted object's)*/
+            for (j = 0; j < error_count; j++) {
+                snprintf(key, sizeof(key), "quarantine-%d-%d", j, i);
+                ret = dict_get_str(dict, key, &bad_file_str);
+                if (!ret) {
+                    cli_out("%s\n", bad_file_str);
                 }
+            }
         }
-        cli_out ("%s\n", "=========================================="
-                 "===============");
+    }
+    cli_out("%s\n",
+            "=========================================================");
 
 out:
-        GF_FREE (scrub_state_op);
-        return 0;
+    GF_FREE(scrub_state_op);
+    return 0;
 }
 
-int
-gf_cli_bitrot_cbk (struct rpc_req *req, struct iovec *iov,
-                   int count, void *myframe)
-{
-        int                  ret                       = -1;
-        int                  type                      = 0;
-        gf_cli_rsp           rsp                       = {0, };
-        dict_t               *dict                     = NULL;
-
-        GF_ASSERT (myframe);
-
-        if (req->rpc_status == -1) {
-                goto out;
-        }
+static int
+gf_cli_bitrot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int type = 0;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *scrub_cmd = NULL;
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    char *cmd_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Bitrot command failed : %s", rsp.op_errstr);
+        else
+            cli_err("Bitrot command : failed");
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+        goto out;
+    }
 
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("Bitrot command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("Bitrot command : failed");
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-                goto out;
+        if (!dict) {
+            ret = -1;
+            goto out;
         }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict = dict_new ();
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "failed to unserialize "
-                                "req-buffer to dictionary");
-                        goto out;
-                }
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to bit rot command");
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to bit rot command");
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get command type");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command type");
+        goto out;
+    }
 
-        if ((type == GF_BITROT_CMD_SCRUB_STATUS) &&
-             !(global_state->mode & GLUSTER_MODE_XML)) {
-                ret = gf_cli_print_bitrot_scrub_status (dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to print bitrot "
-                                "scrub status");
-                }
-                goto out;
-        }
+    if ((type == GF_BITROT_CMD_SCRUB_STATUS) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = gf_cli_print_bitrot_scrub_status(dict);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to print bitrot scrub status");
+        }
+        goto out;
+    }
+
+    /* Ignoring the error, as using dict val for cli output only */
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "scrub-value", &scrub_cmd);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "Failed to get scrub command");
+
+    ret = dict_get_str_sizen(dict, "cmd-str", &cmd_str);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get command string");
+
+    if (cmd_str)
+        cmd_op = strrchr(cmd_str, ' ') + 1;
+
+    switch (type) {
+        case GF_BITROT_OPTION_TYPE_ENABLE:
+            cli_out("volume bitrot: success bitrot enabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_DISABLE:
+            cli_out("volume bitrot: success bitrot disabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_CMD_SCRUB_ONDEMAND:
+            cli_out("volume bitrot: scrubber started ondemand for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB:
+            if (!strncmp("pause", scrub_cmd, sizeof("pause")))
+                cli_out("volume bitrot: scrubber paused for volume %s",
+                        volname);
+            if (!strncmp("resume", scrub_cmd, sizeof("resume")))
+                cli_out("volume bitrot: scrubber resumed for volume %s",
+                        volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+            cli_out(
+                "volume bitrot: scrub-frequency is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+            cli_out(
+                "volume bitrot: scrub-throttle is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+    }
 
 xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("volume bitrot: success");
+    if (!rsp.op_ret)
+        cli_out("volume bitrot: success");
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-            dict_unref (dict);
-
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    if (dict)
+        dict_unref(dict);
 
-        cli_cmd_broadcast_response (ret);
-
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int32_t
-gf_cli_bitrot (call_frame_t *frame, xlator_t *this, void *data)
-{
-        gf_cli_req        req           = { {0,} };
-        dict_t           *options       = NULL;
-        int               ret           = -1;
-
-        if (!frame || !this || !data)
-                goto out;
-
-        options = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_bitrot_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_BITROT, this, cli_rpc_prog,
-                               NULL);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd for "
-                        "bitrot failed");
-                goto out;
-        }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
-}
-
-struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_NULL]             = {"NULL", NULL },
-        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli_probe},
-        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli_deprobe},
-        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli_list_friends},
-        [GLUSTER_CLI_UUID_RESET]       = {"UUID_RESET", gf_cli3_1_uuid_reset},
-        [GLUSTER_CLI_UUID_GET]         = {"UUID_GET", gf_cli3_1_uuid_get},
-        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli_create_volume},
-        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli_delete_volume},
-        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli_start_volume},
-        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli_stop_volume},
-        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli_rename_volume},
-        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
-        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli_get_volume},
-        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
-        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli_set_volume},
-        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli_add_brick},
-        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli_remove_brick},
-        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli_replace_brick},
-        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli_log_rotate},
-        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli_getspec},
-        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
-        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli_sync_volume},
-        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli_reset_volume},
-        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli_fsm_log},
-        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli_gsync_set},
-        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli_profile_volume},
-        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli_quota},
-        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli_top_volume},
-        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli_getwd},
-        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli_status_volume},
-        [GLUSTER_CLI_STATUS_ALL]       = {"STATUS_ALL", gf_cli_status_volume_all},
-        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli_mount},
-        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli_umount},
-        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli_heal_volume},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli_statedump_volume},
-        [GLUSTER_CLI_LIST_VOLUME]      = {"LIST_VOLUME", gf_cli_list_volume},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME]  = {"CLEARLOCKS_VOLUME", gf_cli_clearlocks_volume},
-        [GLUSTER_CLI_COPY_FILE]        = {"COPY_FILE", gf_cli_copy_file},
-        [GLUSTER_CLI_SYS_EXEC]         = {"SYS_EXEC", gf_cli_sys_exec},
-        [GLUSTER_CLI_SNAP]             = {"SNAP", gf_cli_snapshot},
-        [GLUSTER_CLI_BARRIER_VOLUME]   = {"BARRIER VOLUME", gf_cli_barrier_volume},
-        [GLUSTER_CLI_GANESHA]          = {"GANESHA", gf_cli_ganesha},
-        [GLUSTER_CLI_GET_VOL_OPT]      = {"GET_VOL_OPT", gf_cli_get_vol_opt},
-        [GLUSTER_CLI_BITROT]           = {"BITROT", gf_cli_bitrot},
-        [GLUSTER_CLI_ATTACH_TIER]      = {"ATTACH_TIER", gf_cli_attach_tier},
-        [GLUSTER_CLI_DETACH_TIER]      = {"DETACH_TIER", gf_cli_detach_tier},
-        [GLUSTER_CLI_TIER]             = {"TIER", gf_cli_tier},
-        [GLUSTER_CLI_GET_STATE]        = {"GET_STATE", gf_cli_get_state},
-        [GLUSTER_CLI_RESET_BRICK]      = {"RESET_BRICK", gf_cli_reset_brick}
+static int32_t
+gf_cli_bitrot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_bitrot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BITROT, this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for bitrot failed");
+        goto out;
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_NULL] = {"NULL", NULL},
+    [GLUSTER_CLI_PROBE] = {"PROBE_QUERY", gf_cli_probe},
+    [GLUSTER_CLI_DEPROBE] = {"DEPROBE_QUERY", gf_cli_deprobe},
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS", gf_cli_list_friends},
+    [GLUSTER_CLI_UUID_RESET] = {"UUID_RESET", gf_cli3_1_uuid_reset},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", gf_cli3_1_uuid_get},
+    [GLUSTER_CLI_CREATE_VOLUME] = {"CREATE_VOLUME", gf_cli_create_volume},
+    [GLUSTER_CLI_DELETE_VOLUME] = {"DELETE_VOLUME", gf_cli_delete_volume},
+    [GLUSTER_CLI_START_VOLUME] = {"START_VOLUME", gf_cli_start_volume},
+    [GLUSTER_CLI_STOP_VOLUME] = {"STOP_VOLUME", gf_cli_stop_volume},
+    [GLUSTER_CLI_RENAME_VOLUME] = {"RENAME_VOLUME", gf_cli_rename_volume},
+    [GLUSTER_CLI_DEFRAG_VOLUME] = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", gf_cli_get_volume},
+    [GLUSTER_CLI_GET_NEXT_VOLUME] = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
+    [GLUSTER_CLI_SET_VOLUME] = {"SET_VOLUME", gf_cli_set_volume},
+    [GLUSTER_CLI_ADD_BRICK] = {"ADD_BRICK", gf_cli_add_brick},
+    [GLUSTER_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", gf_cli_remove_brick},
+    [GLUSTER_CLI_REPLACE_BRICK] = {"REPLACE_BRICK", gf_cli_replace_brick},
+    [GLUSTER_CLI_LOG_ROTATE] = {"LOG ROTATE", gf_cli_log_rotate},
+    [GLUSTER_CLI_GETSPEC] = {"GETSPEC", gf_cli_getspec},
+    [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
+    [GLUSTER_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", gf_cli_sync_volume},
+    [GLUSTER_CLI_RESET_VOLUME] = {"RESET_VOLUME", gf_cli_reset_volume},
+    [GLUSTER_CLI_FSM_LOG] = {"FSM_LOG", gf_cli_fsm_log},
+    [GLUSTER_CLI_GSYNC_SET] = {"GSYNC_SET", gf_cli_gsync_set},
+    [GLUSTER_CLI_PROFILE_VOLUME] = {"PROFILE_VOLUME", gf_cli_profile_volume},
+    [GLUSTER_CLI_QUOTA] = {"QUOTA", gf_cli_quota},
+    [GLUSTER_CLI_TOP_VOLUME] = {"TOP_VOLUME", gf_cli_top_volume},
+    [GLUSTER_CLI_GETWD] = {"GETWD", gf_cli_getwd},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME", gf_cli_status_volume},
+    [GLUSTER_CLI_STATUS_ALL] = {"STATUS_ALL", gf_cli_status_volume_all},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", gf_cli_mount},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", gf_cli_umount},
+    [GLUSTER_CLI_HEAL_VOLUME] = {"HEAL_VOLUME", gf_cli_heal_volume},
+    [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME",
+                                      gf_cli_statedump_volume},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", gf_cli_list_volume},
+    [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME",
+                                     gf_cli_clearlocks_volume},
+    [GLUSTER_CLI_COPY_FILE] = {"COPY_FILE", gf_cli_copy_file},
+    [GLUSTER_CLI_SYS_EXEC] = {"SYS_EXEC", gf_cli_sys_exec},
+    [GLUSTER_CLI_SNAP] = {"SNAP", gf_cli_snapshot},
+    [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER VOLUME", gf_cli_barrier_volume},
+    [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", gf_cli_get_vol_opt},
+    [GLUSTER_CLI_BITROT] = {"BITROT", gf_cli_bitrot},
+    [GLUSTER_CLI_GET_STATE] = {"GET_STATE", gf_cli_get_state},
+    [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", gf_cli_reset_brick},
+    [GLUSTER_CLI_GANESHA] = {"GANESHA", gf_cli_ganesha},
 };
 
 struct rpc_clnt_program cli_prog = {
-        .progname  = "Gluster CLI",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numproc   = GLUSTER_CLI_MAXVALUE,
-        .proctable = gluster_cli_actors,
+    .progname = "Gluster CLI",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numproc = GLUSTER_CLI_MAXVALUE,
+    .proctable = gluster_cli_actors,
 };
 
-struct rpc_clnt_procedure cli_quotad_procs[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", NULL},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", NULL},
-        [GF_AGGREGATOR_GETLIMIT]   = {"GETLIMIT", cli_quotad_getlimit},
+static struct rpc_clnt_procedure cli_quotad_procs[GF_AGGREGATOR_MAXVALUE] = {
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL},
+    [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", cli_quotad_getlimit},
 };
 
 struct rpc_clnt_program cli_quotad_clnt = {
-        .progname  = "CLI Quotad client",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numproc   = GF_AGGREGATOR_MAXVALUE,
-        .proctable = cli_quotad_procs,
+    .progname = "CLI Quotad client",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numproc = GF_AGGREGATOR_MAXVALUE,
+    .proctable = cli_quotad_procs,
 };
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index af4f020bbbd..069de75801c 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -10,15 +10,12 @@
 #include <stdlib.h>
 #include "cli.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "compat.h"
-#include "syscall.h"
-#include "upcall-utils.h"
-
-enum gf_task_types {
-    GF_TASK_TYPE_REBALANCE,
-    GF_TASK_TYPE_REMOVE_BRICK
-};
+#include <glusterfs/run.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/upcall-utils.h>
+
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
 
 /*
  * IMPORTANT NOTE:
@@ -42,4467 +39,3796 @@ enum gf_task_types {
  *      }
  *      #endif
  *
- *  Following the above formate ensures that all xml related code is compliled
+ *  Following the above format ensures that all xml related code is compiled
  *  only when libxml2 is present, and also keeps the rest of the codebase free
  *  of #if (HAVE_LIB_XML)
  */
 
-
 #if (HAVE_LIB_XML)
 
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 
-#define XML_RET_CHECK_AND_GOTO(ret, label)      do {            \
-                if (ret < 0) {                                  \
-                        ret = -1;                               \
-                        goto label;                             \
-                }                                               \
-                else                                            \
-                        ret = 0;                                \
-        }while (0)                                              \
+#define XML_RET_CHECK_AND_GOTO(ret, label)                                     \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        } else                                                                 \
+            ret = 0;                                                           \
+    } while (0)
 
 int
-cli_begin_xml_output (xmlTextWriterPtr *writer, xmlDocPtr *doc)
+cli_begin_xml_output(xmlTextWriterPtr *writer, xmlDocPtr *doc)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        *writer = xmlNewTextWriterDoc (doc, 0);
-        if (writer == NULL) {
-                ret = -1;
-                goto out;
-        }
+    *writer = xmlNewTextWriterDoc(doc, 0);
+    if (*writer == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = xmlTextWriterStartDocument (*writer, "1.0", "UTF-8", "yes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <cliOutput> */
-        ret = xmlTextWriterStartElement (*writer, (xmlChar *)"cliOutput");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <cliOutput> */
+    ret = xmlTextWriterStartElement(*writer, (xmlChar *)"cliOutput");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_end_xml_output (xmlTextWriterPtr writer, xmlDocPtr doc)
+cli_end_xml_output(xmlTextWriterPtr writer, xmlDocPtr doc)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        /* </cliOutput> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </cliOutput> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndDocument (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndDocument(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    /* Dump xml document to stdout and pretty format it */
+    xmlSaveFormatFileEnc("-", doc, "UTF-8", 1);
 
-        /* Dump xml document to stdout and pretty format it */
-        xmlSaveFormatFileEnc ("-", doc, "UTF-8", 1);
-
-        xmlFreeTextWriter (writer);
-        xmlFreeDoc (doc);
+    xmlFreeTextWriter(writer);
+    xmlFreeDoc(doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
-                       char *op_errstr)
+cli_xml_output_common(xmlTextWriterPtr writer, int op_ret, int op_errno,
+                      char *op_errstr)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opRet",
-                                               "%d", op_ret);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opRet", "%d",
+                                          op_ret);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opErrno",
-                                               "%d", op_errno);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrno", "%d",
+                                          op_errno);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (op_errstr)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"opErrstr",
-                                                       "%s", op_errstr);
-        else
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"opErrstr",
-                                                       "%s", "");
+    if (op_errstr)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", op_errstr);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", "");
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
-                    char *op_errstr)
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        if (op) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"cliOp",
-                                                       "%s", op);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (op) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"cliOp", "%s",
+                                              op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (str) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"output",
-                                                       "%s", str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (str) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"output", "%s",
+                                              str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
-                          void *data)
+cli_xml_output_data_pair(dict_t *this, char *key, data_t *value, void *data)
 {
-        int                     ret = -1;
-        xmlTextWriterPtr        *writer = NULL;
+    int ret = -1;
+    xmlTextWriterPtr *writer = NULL;
 
-        writer = (xmlTextWriterPtr *)data;
+    writer = (xmlTextWriterPtr *)data;
 
-        ret = xmlTextWriterWriteFormatElement (*writer, (xmlChar *)key,
-                                               "%s", value->data);
+    ret = xmlTextWriterWriteFormatElement(*writer, (xmlChar *)key, "%s",
+                                          value->data);
 
-        return ret;
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_dict ( char *op, dict_t *dict, int op_ret, int op_errno,
-                      char *op_errstr)
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        /* <"op"> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <"op"> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (dict)
-                dict_foreach (dict, cli_xml_output_data_pair, &writer);
+    if (dict)
+        dict_foreach(dict, cli_xml_output_data_pair, &writer);
 
-        /* </"op"> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </"op"> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
-                                  int   brick_index, int *online,
-                                  gf_boolean_t *node_present)
+cli_xml_output_vol_status_common(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int *online,
+                                 gf_boolean_t *node_present)
 {
-        int             ret             = -1;
-        char            *hostname       = NULL;
-        char            *path           = NULL;
-        char            *uuid           = NULL;
-        int             port            = 0;
-        int             rdma_port       = 0;
-        int             status          = 0;
-        int             pid             = 0;
-        char            key[1024]       = {0,};
-
-        snprintf (key, sizeof (key), "brick%d.hostname", brick_index);
-        ret = dict_get_str (dict, key, &hostname);
-        if (ret) {
-                *node_present = _gf_false;
-                goto out;
+    int ret = -1;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *uuid = NULL;
+    int port = 0;
+    int rdma_port = 0;
+    int status = 0;
+    int pid = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.hostname", brick_index);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret) {
+        *node_present = _gf_false;
+        goto out;
+    }
+    *node_present = _gf_true;
+
+    /* <node>
+     * will be closed in the calling function cli_xml_output_vol_status()*/
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname", "%s",
+                                          hostname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.path", brick_index);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                          path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.peerid", brick_index);
+    ret = dict_get_str(dict, key, &uuid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"peerid", "%s",
+                                          uuid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.status", brick_index);
+    ret = dict_get_int32(dict, key, &status);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    *online = status;
+
+    snprintf(key, sizeof(key), "brick%d.port", brick_index);
+    ret = dict_get_int32(dict, key, &port);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "brick%d.rdma_port", brick_index);
+    ret = dict_get_int32(dict, key, &rdma_port);
+
+    /* If the process is either offline or doesn't provide a port (shd)
+     * port = "N/A"
+     * else print the port number of the process.
+     */
+
+    /*
+     * Tag 'port' can be removed once console management is started
+     * to support new tag ports.
+     */
+
+    if (*online == 1 && port != 0)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%d",
+                                              port);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%s",
+                                              "N/A");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"ports");
+    if (*online == 1 && (port != 0 || rdma_port != 0)) {
+        if (port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%d", port);
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%s", "N/A");
         }
-        *node_present = _gf_true;
-
-        /* <node>
-         * will be closed in the calling function cli_xml_output_vol_status()*/
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hostname",
-                                               "%s", hostname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.path", brick_index);
-        ret = dict_get_str (dict, key, &path);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"path",
-                                               "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.peerid", brick_index);
-        ret = dict_get_str (dict, key, &uuid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"peerid",
-                                               "%s", uuid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.status", brick_index);
-        ret = dict_get_int32 (dict, key, &status);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"status",
-                                               "%d", status);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        *online = status;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.port", brick_index);
-        ret = dict_get_int32 (dict, key, &port);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.rdma_port", brick_index);
-        ret = dict_get_int32 (dict, key, &rdma_port);
-
-        /* If the process is either offline or doesn't provide a port (shd)
-         * port = "N/A"
-         * else print the port number of the process.
-         */
-
-        /*
-         * Tag 'port' can be removed once console management is started
-         * to support new tag ports.
-         */
-
-        if (*online == 1 && port != 0)
-                 ret = xmlTextWriterWriteFormatElement (writer,
-                                                    (xmlChar *)"port",
-                                                        "%d", port);
-         else
-                 ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)"port",
-                                                        "%s", "N/A");
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"ports");
-        if (*online == 1 && (port != 0 || rdma_port != 0)) {
-
-                if (port) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%d", port);
-                } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%s", "N/A");
-                }
-
-                if (rdma_port) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%d", rdma_port);
-                } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%s", "N/A");
-                }
-
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        if (rdma_port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%d", rdma_port);
         } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%s", "N/A");
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%s", "N/A");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%s", "N/A");
         }
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.pid", brick_index);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
-                                               "%d", pid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-cli_xml_output_vol_status_detail (xmlTextWriterPtr writer, dict_t *dict,
-                                  int brick_index)
-{
-        int             ret = -1;
-        uint64_t        size_total = 0;
-        uint64_t        size_free = 0;
-        char            *device = NULL;
-        uint64_t        block_size = 0;
-        char            *mnt_options = NULL;
-        char            *fs_name = NULL;
-        char            *inode_size = NULL;
-        uint64_t        inodes_total = 0;
-        uint64_t        inodes_free = 0;
-        char            key[1024] = {0,};
-
-        snprintf (key, sizeof (key), "brick%d.total", brick_index);
-        ret = dict_get_uint64 (dict, key, &size_total);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"sizeTotal",
-                                                       "%"PRIu64, size_total);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", brick_index);
-        ret = dict_get_uint64 (dict, key, &size_free);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"sizeFree",
-                                                       "%"PRIu64, size_free);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", brick_index);
-        ret = dict_get_str (dict, key, &device);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"device",
-                                                       "%s", device);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", brick_index);
-        ret = dict_get_uint64 (dict, key, &block_size);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"blockSize",
-                                                       "%"PRIu64, block_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", brick_index);
-        ret = dict_get_str (dict, key, &mnt_options);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"mntOptions",
-                                                       "%s", mnt_options);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", brick_index);
-        ret = dict_get_str (dict, key, &fs_name);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"fsName",
-                                                       "%s", fs_name);
-
-        /* inode details are only available for ext 2/3/4 & xfs */
-        if (!fs_name || !IS_EXT_FS(fs_name) || strcmp (fs_name, "xfs")) {
-                        ret = 0;
-                        goto out;
-        }
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", brick_index);
-        ret = dict_get_str (dict, key, &inode_size);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodeSize",
-                                                       "%s", fs_name);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", brick_index);
-        ret = dict_get_uint64 (dict, key, &inodes_total);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodesTotal",
-                                                       "%"PRIu64, inodes_total);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", brick_index);
-        ret = dict_get_uint64 (dict, key, &inodes_free);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodesFree",
-                                                       "%"PRIu64, inodes_free);
+    snprintf(key, sizeof(key), "brick%d.pid", brick_index);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_mempool (xmlTextWriterPtr writer, dict_t *dict,
-                                   char *prefix)
+cli_xml_output_vol_status_detail(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index)
 {
-        int             ret = -1;
-        int             mempool_count = 0;
-        char            *name = NULL;
-        int             hotcount = 0;
-        int             coldcount = 0;
-        uint64_t        paddedsizeof = 0;
-        uint64_t        alloccount = 0;
-        int             maxalloc = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <mempool> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mempool");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.mempool-count", prefix);
-        ret = dict_get_int32 (dict, key, &mempool_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", mempool_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < mempool_count; i++) {
-                /* <pool> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pool");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
-                ret = dict_get_str (dict, key, &name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &hotcount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hotCount",
-                                                       "%d", hotcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &coldcount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"coldCount",
-                                                       "%d", coldcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
-                          prefix, i);
-                ret = dict_get_uint64 (dict, key, &paddedsizeof);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"padddedSizeOf", "%"PRIu64,
-                         paddedsizeof);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"allocCount",
-                                                       "%"PRIu64, alloccount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"maxAlloc",
-                                                       "%d", maxalloc);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"poolMisses",
-                                                       "%"PRIu64, alloccount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"maxStdAlloc",
-                                                       "%d", maxalloc);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-                /* </pool> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        /* </mempool> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t size_total = 0;
+    uint64_t size_free = 0;
+    char *device = NULL;
+    uint64_t block_size = 0;
+    char *mnt_options = NULL;
+    char *fs_name = NULL;
+    char *inode_size = NULL;
+    uint64_t inodes_total = 0;
+    uint64_t inodes_free = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.total", brick_index);
+    ret = dict_get_uint64(dict, key, &size_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeTotal",
+                                              "%" PRIu64, size_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free", brick_index);
+    ret = dict_get_uint64(dict, key, &size_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeFree",
+                                              "%" PRIu64, size_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.device", brick_index);
+    ret = dict_get_str(dict, key, &device);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"device", "%s",
+                                              device);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.block_size", brick_index);
+    ret = dict_get_uint64(dict, key, &block_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"blockSize",
+                                              "%" PRIu64, block_size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", brick_index);
+    ret = dict_get_str(dict, key, &mnt_options);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"mntOptions",
+                                              "%s", mnt_options);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.fs_name", brick_index);
+    ret = dict_get_str(dict, key, &fs_name);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsName", "%s",
+                                              fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.inode_size", brick_index);
+    ret = dict_get_str(dict, key, &inode_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodeSize",
+                                              "%s", fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.total_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesTotal",
+                                              "%" PRIu64, inodes_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesFree",
+                                              "%" PRIu64, inodes_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = 0;
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_mem (xmlTextWriterPtr writer, dict_t *dict,
-                               int brick_index)
+cli_xml_output_vol_status_mempool(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
 {
-        int             ret = -1;
-        int             arena = 0;
-        int             ordblks = 0;
-        int             smblks = 0;
-        int             hblks = 0;
-        int             hblkhd = 0;
-        int             usmblks = 0;
-        int             fsmblks = 0;
-        int             uordblks = 0;
-        int             fordblks = 0;
-        int             keepcost = 0;
-        char            key[1024] = {0,};
-
-        /* <memStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"memStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <mallinfo> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mallinfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.mallinfo.arena", brick_index);
-        ret = dict_get_int32 (dict, key, &arena);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"arena",
-                                               "%d", arena);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &ordblks);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ordblks",
-                                               "%d", ordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int mempool_count = 0;
+    char *name = NULL;
+    int hotcount = 0;
+    int coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int maxalloc = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <mempool> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mempool");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32(dict, key, &mempool_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          mempool_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < mempool_count; i++) {
+        /* <pool> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pool");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", brick_index);
-        ret = dict_get_int32 (dict, key, &smblks);
+        snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_str(dict, key, &name);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"smblks",
-                                               "%d", smblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", brick_index);
-        ret = dict_get_int32 (dict, key, &hblks);
+        snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32(dict, key, &hotcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblks",
-                                               "%d", hblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hotCount",
+                                              "%d", hotcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", brick_index);
-        ret = dict_get_int32 (dict, key, &hblkhd);
+        snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32(dict, key, &coldcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblkhd",
-                                               "%d", hblkhd);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"coldCount",
+                                              "%d", coldcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", brick_index);
-        ret = dict_get_int32 (dict, key, &usmblks);
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"usmblks",
-                                               "%d", usmblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"padddedSizeOf", "%" PRIu64, paddedsizeof);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", brick_index);
-        ret = dict_get_int32 (dict, key, &fsmblks);
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fsmblks",
-                                               "%d", fsmblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"allocCount",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &uordblks);
+        snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uordblks",
-                                               "%d", uordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &fordblks);
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fordblks",
-                                               "%d", fordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"poolMisses",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", brick_index);
-        ret = dict_get_int32 (dict, key, &keepcost);
+        snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"keepcost",
-                                               "%d", keepcost);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </mallinfo> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxStdAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d", brick_index);
-        ret = cli_xml_output_vol_status_mempool (writer, dict, key);
-        if (ret)
-                goto out;
+        /* </pool> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </memStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </mempool> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_clients (xmlTextWriterPtr writer, dict_t *dict,
-                                   int brick_index)
+cli_xml_output_vol_status_mem(xmlTextWriterPtr writer, dict_t *dict,
+                              int brick_index)
 {
-        int             ret = -1;
-        int             client_count = 0;
-        char            *hostname = NULL;
-        uint64_t        bytes_read = 0;
-        uint64_t        bytes_write = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <clientsStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"clientsStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.clientcount", brick_index);
-        ret = dict_get_int32 (dict, key, &client_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"clientCount",
-                                               "%d", client_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < client_count; i++) {
-                /* <client> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"client");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.hostname",
-                          brick_index, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hostname",
-                                                       "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.bytesread",
-                          brick_index, i);
-                ret = dict_get_uint64 (dict, key, &bytes_read);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"bytesRead",
-                                                       "%"PRIu64, bytes_read);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.byteswrite",
-                          brick_index, i);
-                ret = dict_get_uint64 (dict, key, &bytes_write);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"bytesWrite",
-                                                       "%"PRIu64, bytes_write);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </client> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    int ret = -1;
+    int arena = 0;
+    int ordblks = 0;
+    int smblks = 0;
+    int hblks = 0;
+    int hblkhd = 0;
+    int usmblks = 0;
+    int fsmblks = 0;
+    int uordblks = 0;
+    int fordblks = 0;
+    int keepcost = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <memStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"memStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <mallinfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mallinfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.arena", brick_index);
+    ret = dict_get_int32(dict, key, &arena);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"arena", "%d",
+                                          arena);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", brick_index);
+    ret = dict_get_int32(dict, key, &ordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ordblks", "%d",
+                                          ordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", brick_index);
+    ret = dict_get_int32(dict, key, &smblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"smblks", "%d",
+                                          smblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", brick_index);
+    ret = dict_get_int32(dict, key, &hblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblks", "%d",
+                                          hblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", brick_index);
+    ret = dict_get_int32(dict, key, &hblkhd);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblkhd", "%d",
+                                          hblkhd);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", brick_index);
+    ret = dict_get_int32(dict, key, &usmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"usmblks", "%d",
+                                          usmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", brick_index);
+    ret = dict_get_int32(dict, key, &fsmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsmblks", "%d",
+                                          fsmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", brick_index);
+    ret = dict_get_int32(dict, key, &uordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uordblks", "%d",
+                                          uordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", brick_index);
+    ret = dict_get_int32(dict, key, &fordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fordblks", "%d",
+                                          fordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", brick_index);
+    ret = dict_get_int32(dict, key, &keepcost);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"keepcost", "%d",
+                                          keepcost);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </mallinfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d", brick_index);
+    ret = cli_xml_output_vol_status_mempool(writer, dict, key);
+    if (ret)
+        goto out;
+
+    /* </memStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </clientsStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_inode_entry (xmlTextWriterPtr writer, dict_t *dict,
-                                       char *prefix)
+cli_xml_output_vol_status_clients(xmlTextWriterPtr writer, dict_t *dict,
+                                  int brick_index)
 {
-        int             ret = -1;
-        char            *gfid = NULL;
-        uint64_t        nlookup = 0;
-        uint32_t        ref = 0;
-        int             ia_type = 0;
-        char            key[1024] = {0,};
-
-        /* <inode> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inode");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_get_str (dict, key, &gfid);
+    int ret = -1;
+    int client_count = 0;
+    char *hostname = NULL;
+    uint64_t bytes_read = 0;
+    uint64_t bytes_write = 0;
+    uint32_t opversion = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <clientsStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"clientsStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.clientcount", brick_index);
+    ret = dict_get_int32(dict, key, &client_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"clientCount",
+                                          "%d", client_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < client_count; i++) {
+        /* <client> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"client");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.hostname", brick_index, i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gfid",
-                                               "%s", gfid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.nlookup", prefix);
-        ret = dict_get_uint64 (dict, key, &nlookup);
+        snprintf(key, sizeof(key), "brick%d.client%d.bytesread", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_read);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nLookup",
-                                               "%"PRIu64, nlookup);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesRead",
+                                              "%" PRIu64, bytes_read);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ref", prefix);
-        ret = dict_get_uint32 (dict, key, &ref);
+        snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_write);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ref",
-                                               "%"PRIu32, ref);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesWrite",
+                                              "%" PRIu64, bytes_write);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ia_type", prefix);
-        ret = dict_get_int32 (dict, key, &ia_type);
+        snprintf(key, sizeof(key), "brick%d.client%d.opversion", brick_index,
+                 i);
+        ret = dict_get_uint32(dict, key, &opversion);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"iaType",
-                                               "%d", ia_type);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opVersion",
+                                              "%" PRIu32, opversion);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </inode> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </client> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
+    /* </clientsStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_itable (xmlTextWriterPtr writer, dict_t *dict,
-                                  char *prefix)
+cli_xml_output_vol_status_inode_entry(xmlTextWriterPtr writer, dict_t *dict,
+                                      char *prefix)
 {
-        int             ret = -1;
-        uint32_t        active_size = 0;
-        uint32_t        lru_size = 0;
-        uint32_t        purge_size = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        snprintf (key, sizeof (key), "%s.active_size", prefix);
-        ret = dict_get_uint32 (dict, key, &active_size);
-        if (ret)
+    int ret = -1;
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <inode> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inode");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gfid", "%s",
+                                          gfid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nLookup",
+                                          "%" PRIu64, nlookup);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ref", "%" PRIu32,
+                                          ref);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"iaType", "%d",
+                                          ia_type);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </inode> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_itable(xmlTextWriterPtr writer, dict_t *dict,
+                                 char *prefix)
+{
+    int ret = -1;
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activeSize",
+                                          "%" PRIu32, active_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (active_size != 0) {
+        /* <active> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"active");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (i = 0; i < active_size; i++) {
+            snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"activeSize",
-                                               "%"PRIu32, active_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (active_size != 0) {
-                /* <active> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"active");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < active_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.active%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </active> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </active> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lruSize",
+                                          "%" PRIu32, lru_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (lru_size != 0) {
+        /* <lru> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"lru");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.lru_size", prefix);
-        ret = dict_get_uint32 (dict, key, &lru_size);
-        if (ret)
+        for (i = 0; i < lru_size; i++) {
+            snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"lruSize",
-                                               "%"PRIu32, lru_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (lru_size != 0) {
-                /* <lru> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"lru");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < lru_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </lru> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </lru> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"purgeSize",
+                                          "%" PRIu32, purge_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (purge_size != 0) {
+        /* <purge> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"purge");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.purge_size", prefix);
-        ret = dict_get_uint32 (dict, key, &purge_size);
-        if (ret)
+        for (i = 0; i < purge_size; i++) {
+            snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"purgeSize",
-                                               "%"PRIu32, purge_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (purge_size != 0) {
-                /* <purge> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"purge");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < purge_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </purge> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </purge> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_inode (xmlTextWriterPtr writer, dict_t *dict,
-                                 int brick_index)
+cli_xml_output_vol_status_inode(xmlTextWriterPtr writer, dict_t *dict,
+                                int brick_index)
 {
-        int             ret = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <inodeStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inodeStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <inodeStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inodeStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
-        ret = dict_get_int32 (dict, key, &conn_count);
+        snprintf(key, sizeof(key), "brick%d.conn%d.itable", brick_index, i);
+        ret = cli_xml_output_vol_status_itable(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
-                                               "%d", conn_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < conn_count; i++) {
-                /* <connection> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"connection");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conn%d.itable",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_itable (writer, dict, key);
-                if (ret)
-                        goto out;
+            goto out;
 
-                /* </connection> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </inodeStatus> */
-        ret= xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </inodeStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_fdtable (xmlTextWriterPtr writer, dict_t *dict,
-                                   char *prefix)
+cli_xml_output_vol_status_fdtable(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
 {
-        int             ret = -1;
-        int             refcount = 0;
-        uint32_t        maxfds = 0;
-        int             firstfree = 0;
-        int             openfds = 0;
-        int             fd_pid = 0;
-        int             fd_refcount = 0;
-        int             fd_flags = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <fdTable> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdTable");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &refcount);
+    int ret = -1;
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdTable> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdTable");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          refcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxFds",
+                                          "%" PRIu32, maxfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"firstFree", "%d",
+                                          firstfree);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"openFds", "%d",
+                                          openfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
-                                               "%d", refcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.maxfds", prefix);
-        ret = dict_get_uint32 (dict, key, &maxfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"maxFds",
-                                               "%"PRIu32, maxfds);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.firstfree", prefix);
-        ret = dict_get_int32 (dict, key, &firstfree);
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"firstFree",
-                                               "%d", firstfree);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.openfds", prefix);
-        ret = dict_get_int32 (dict, key, &openfds);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"openFds",
-                                               "%d", openfds);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < maxfds; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_pid);
-                if (ret)
-                        continue;
+        /* <fd> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fd");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
-                          prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_refcount);
-                if (ret)
-                        continue;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"entry", "%d",
+                                              i + 1);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_flags);
-                if (ret)
-                        continue;
-
-                /* <fd> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fd");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"entry",
-                                                       "%d", i+1);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"pid",
-                                                       "%d", fd_pid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"refCount",
-                                                       "%d", fd_refcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"flags",
-                                                       "%d", fd_flags);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </fd> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d",
+                                              fd_pid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </fdTable> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount",
+                                              "%d", fd_refcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"flags", "%d",
+                                              fd_flags);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fd> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fdTable> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_fd (xmlTextWriterPtr writer, dict_t *dict,
-                              int brick_index)
+cli_xml_output_vol_status_fd(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index)
 {
-        int             ret = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <fdStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
-        ret = dict_get_int32 (dict, key, &conn_count);
+        snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", brick_index, i);
+        ret = cli_xml_output_vol_status_fdtable(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
-                                               "%d", conn_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < conn_count; i++) {
-                /* <connection> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"connection");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_fdtable (writer, dict, key);
-                if (ret)
-                        goto out;
+            goto out;
 
-                /* </connection> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </fdStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </fdStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callframe (xmlTextWriterPtr writer, dict_t *dict,
-                                     char *prefix)
+cli_xml_output_vol_status_callframe(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
 {
-        int             ret = -1;
-        int             ref_count = 0;
-        char            *translator = NULL;
-        int             complete = 0;
-        char            *parent = NULL;
-        char            *wind_from = NULL;
-        char            *wind_to = NULL;
-        char            *unwind_from = NULL;
-        char            *unwind_to = NULL;
-        char            key[1024] = {0,};
-
-        /* <callFrame> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callFrame");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &ref_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
-                                               "%d", ref_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_get_str (dict, key, &translator);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"translator",
-                                               "%s", translator);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.complete", prefix);
-        ret = dict_get_int32 (dict, key, &complete);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"complete",
-                                               "%d", complete);
-        XML_RET_CHECK_AND_GOTO (ret ,out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.parent", prefix);
-        ret = dict_get_str (dict, key, &parent);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"parent",
-                                                       "%s", parent);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    int ret = -1;
+    int ref_count = 0;
+    char *translator = NULL;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <callFrame> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callFrame");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          ref_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"translator", "%s",
+                                          translator);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"complete", "%d",
+                                          complete);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"parent", "%s",
+                                              parent);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windfrom", prefix);
-        ret = dict_get_str (dict, key, &wind_from);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"windFrom",
-                                                       "%s", wind_from);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windFrom",
+                                              "%s", wind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windto", prefix);
-        ret = dict_get_str (dict, key, &wind_to);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"windTo",
-                                                       "%s", wind_to);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windTo", "%s",
+                                              wind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-        ret = dict_get_str (dict, key, &unwind_from);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"unwindFrom",
-                                                       "%s", unwind_from);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindFrom",
+                                              "%s", unwind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindto", prefix);
-        ret = dict_get_str (dict, key, &unwind_to);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"unwindTo",
-                                                       "%s", unwind_to);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindTo",
+                                              "%s", unwind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </callFrame> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callFrame> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callstack (xmlTextWriterPtr writer, dict_t *dict,
-                                     char *prefix)
+cli_xml_output_vol_status_callstack(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
 {
-        int             ret = -1;
-        int             uid = 0;
-        int             gid = 0;
-        int             pid = 0;
-        uint64_t        unique = 0;
-        int             frame_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <callStack> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callStack");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.uid", prefix);
-        ret = dict_get_int32 (dict, key, &uid);
+    int ret = -1;
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    int frame_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callStack> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callStack");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uid", "%d", uid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gid", "%d", gid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unique",
+                                          "%" PRIu64, unique);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &frame_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"frameCount", "%d",
+                                          frame_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < frame_count; i++) {
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        ret = cli_xml_output_vol_status_callframe(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uid",
-                                               "%d", uid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gid", prefix);
-        ret = dict_get_int32 (dict, key, &gid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gid",
-                                               "%d", gid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", prefix);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
-                                               "%d", pid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unique", prefix);
-        ret = dict_get_uint64 (dict, key, &unique);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"unique",
-                                               "%"PRIu64, unique);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &frame_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"frameCount",
-                                               "%d", frame_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < frame_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
-                ret = cli_xml_output_vol_status_callframe (writer, dict,
-                                                           key);
-                if (ret)
-                        goto out;
-        }
-
-        /* </callStack> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callStack> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callpool (xmlTextWriterPtr writer, dict_t *dict,
-                                    int brick_index)
+cli_xml_output_vol_status_callpool(xmlTextWriterPtr writer, dict_t *dict,
+                                   int brick_index)
 {
-        int             ret = -1;
-        int             call_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <callpoolStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callpoolStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.callpool.count", brick_index);
-        ret = dict_get_int32 (dict, key, &call_count);
+    int ret = -1;
+    int call_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callpoolStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callpoolStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.callpool.count", brick_index);
+    ret = dict_get_int32(dict, key, &call_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          call_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < call_count; i++) {
+        snprintf(key, sizeof(key), "brick%d.callpool.stack%d", brick_index, i);
+        ret = cli_xml_output_vol_status_callstack(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", call_count);
-
-        for (i = 0; i < call_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.callpool.stack%d",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_callstack (writer, dict,
-                                                           key);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /* </callpoolStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callpoolStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                 char *op_errstr)
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "volStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volumes> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_status_end (cli_local_t *local)
+cli_xml_output_vol_status_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        /* </volumes> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </volStatus> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </volStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_remove_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
-                                         char *prefix)
+cli_xml_output_remove_brick_task_params(xmlTextWriterPtr writer, dict_t *dict,
+                                        char *prefix)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             count = 0;
-        int             i = 0;
-        char            *brick = NULL;
-
-        /* <params> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int count = 0;
+    int i = 0;
+    char *brick = NULL;
+
+    /* <params> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"params");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        goto out;
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "%s.brick%d", prefix, i);
+        ret = dict_get_str(dict, key, &brick);
         if (ret)
-                goto out;
-
-        for (i = 1; i <= count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.brick%d", prefix, i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"brick",
-                                                       "%s", brick);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                brick = NULL;
-        }
-
-        /* </param> */
-        ret = xmlTextWriterEndElement (writer);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brick", "%s",
+                                              brick);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        brick = NULL;
+    }
 
+    /* </param> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_tasks (cli_local_t *local, dict_t *dict) {
-        int                     ret = -1;
-        char                    *task_type = NULL;
-        char                    *task_id_str = NULL;
-        int                     status = 0;
-        int                     tasks = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-
-        /* <tasks> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"tasks");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "tasks", &tasks);
+cli_xml_output_vol_status_tasks(cli_local_t *local, dict_t *dict)
+{
+    int ret = -1;
+    char *task_type = NULL;
+    char *task_id_str = NULL;
+    int status = 0;
+    int tasks = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <tasks> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"tasks");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "tasks", &tasks);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < tasks; i++) {
+        /* <task> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"task");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &task_type);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%s", task_type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < tasks; i++) {
-                /* <task> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"task");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str (dict, key, &task_type);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"type",
-                                                       "%s", task_type);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (dict, key, &task_id_str);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s",
-                                             cli_vol_task_status_str[status]);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d", i);
-                if (!strcmp (task_type, "Remove brick")) {
-                        ret = cli_xml_output_remove_brick_task_params
-                                (local->writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_task_status_str[status]);
 
-                /* </task> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d", i);
+        if (!strcmp(task_type, "Remove brick")) {
+            ret = cli_xml_output_remove_brick_task_params(local->writer, dict,
+                                                          key);
+            if (ret)
+                goto out;
         }
 
-        /* </tasks> */
-        ret = xmlTextWriterEndElement (local->writer);
+        /* </task> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
+    /* </tasks> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 #endif
 
 int
-cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int    ret     = -1;
-        char  *volname = NULL;
+    int ret = -1;
+    char *volname = NULL;
 
-        /*<volume>*/
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /*<volume>*/
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"volName", "%s",
-                                               volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_vol_status_tasks (local, dict);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_vol_status_tasks(local, dict);
+    if (ret)
+        goto out;
 
-        /* </volume> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_status (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        char                    *volname = NULL;
-        int                     brick_count = 0;
-        int                     brick_index_max = -1;
-        int                     other_count = 0;
-        int                     index_max = 0;
-        uint32_t                cmd = GF_CLI_STATUS_NONE;
-        int                     online = 0;
-        gf_boolean_t            node_present = _gf_true;
-        int                     i;
-        int                     type            = -1;
-        int                     hot_brick_count = -1;
-
-        /* <volume> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"volName", "%s",
-                                               volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"nodeCount", "%d",
-                                               brick_count);
-        if (ret)
-                goto out;
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret)
+    int ret = -1;
+    char *volname = NULL;
+    int brick_count = 0;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    int online = 0;
+    gf_boolean_t node_present = _gf_true;
+    int i;
+    int type = -1;
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"nodeCount",
+                                          "%d", brick_count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret)
+        goto out;
+
+    for (i = 0; i <= index_max; i++) {
+        ret = cli_xml_output_vol_status_common(local->writer, dict, i, &online,
+                                               &node_present);
+        if (ret) {
+            if (node_present)
                 goto out;
-
-        if (type == GF_CLUSTER_TYPE_TIER) {
-                ret = dict_get_int32 (dict, "hot_brick_count",
-                                      &hot_brick_count);
-                if (ret)
-                        goto out;
-
-                ret = xmlTextWriterStartElement
-                        (local->writer, (xmlChar *)"hotBricks");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
+            else
+                continue;
         }
-        for (i = 0; i <= index_max; i++) {
 
-                if (type == GF_CLUSTER_TYPE_TIER && i == hot_brick_count) {
+        switch (cmd & GF_CLI_STATUS_MASK) {
+            case GF_CLI_STATUS_DETAIL:
+                ret = cli_xml_output_vol_status_detail(local->writer, dict, i);
+                if (ret)
+                    goto out;
+                break;
 
-                        /* </hotBricks>*/
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        ret = xmlTextWriterStartElement (local->writer,
-                                        (xmlChar *)"coldBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
-                ret = cli_xml_output_vol_status_common (local->writer, dict, i,
-                                                        &online, &node_present);
-                if (ret) {
-                        if (node_present)
-                                goto out;
-                        else
-                                continue;
+            case GF_CLI_STATUS_MEM:
+                if (online) {
+                    ret = cli_xml_output_vol_status_mem(local->writer, dict, i);
+                    if (ret)
+                        goto out;
                 }
+                break;
 
-                switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_DETAIL:
-                        ret = cli_xml_output_vol_status_detail (local->writer,
-                                                                dict, i);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_CLI_STATUS_MEM:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_mem
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_CLIENTS:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_clients
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_INODE:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_inode
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_FD:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_fd
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_CALLPOOL:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_callpool
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-                default:
-                        break;
-
+            case GF_CLI_STATUS_CLIENTS:
+                if (online) {
+                    ret = cli_xml_output_vol_status_clients(local->writer, dict,
+                                                            i);
+                    if (ret)
+                        goto out;
                 }
+                break;
 
-                /* </node>  was opened in cli_xml_output_vol_status_common()*/
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            case GF_CLI_STATUS_INODE:
+                if (online) {
+                    ret = cli_xml_output_vol_status_inode(local->writer, dict,
+                                                          i);
+                    if (ret)
+                        goto out;
+                }
+                break;
 
-                /* </coldBricks>*/
-                if (type == GF_CLUSTER_TYPE_TIER && i == brick_index_max) {
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+            case GF_CLI_STATUS_FD:
+                if (online) {
+                    ret = cli_xml_output_vol_status_fd(local->writer, dict, i);
+                    if (ret)
+                        goto out;
                 }
-        }
+                break;
 
-        /* Tasks are only present when a normal volume status call is done on a
-         * single volume or on all volumes
-         */
-        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
-            (cmd & (GF_CLI_STATUS_VOL|GF_CLI_STATUS_ALL))) {
-                ret = cli_xml_output_vol_status_tasks (local, dict);
-                if (ret)
+            case GF_CLI_STATUS_CALLPOOL:
+                if (online) {
+                    ret = cli_xml_output_vol_status_callpool(local->writer,
+                                                             dict, i);
+                    if (ret)
                         goto out;
+                }
+                break;
+            default:
+                break;
         }
 
-        /* </volume> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </node>  was opened in cli_xml_output_vol_status_common()*/
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* Tasks are only present when a normal volume status call is done on a
+     * single volume or on all volumes
+     */
+    if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+        (cmd & (GF_CLI_STATUS_VOL | GF_CLI_STATUS_ALL))) {
+        ret = cli_xml_output_vol_status_tasks(local, dict);
+        if (ret)
+            goto out;
+    }
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_top_rw_perf (xmlTextWriterPtr writer, dict_t *dict,
-                                int brick_index, int member_index)
+cli_xml_output_vol_top_rw_perf(xmlTextWriterPtr writer, dict_t *dict,
+                               int brick_index, int member_index)
 {
-        int        ret = -1;
-        char      *filename = NULL;
-        uint64_t   throughput = 0;
-        long int   time_sec = 0;
-        long int   time_usec = 0;
-        char       timestr[256] = {0,};
-        char       key[1024] = {0,};
-
-        /* <file> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
-                  member_index);
-        ret = dict_get_str (dict, key, &filename);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
-                                               "%s", filename);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
-        ret = dict_get_uint64 (dict, key, &throughput);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%"PRIu64, throughput);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-time-sec-%d", brick_index,
-                  member_index);
-        ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-time-usec-%d", brick_index,
-                  member_index);
-        ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
-        if (ret)
-                goto out;
-
-        gf_time_fmt (timestr, sizeof timestr, time_sec, gf_timefmt_FT);
-        snprintf (timestr + strlen (timestr),
-                  sizeof timestr - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, time_usec);
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"time",
-                                               "%s", timestr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </file> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t throughput = 0;
+    struct timeval tv = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &throughput);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, throughput);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-time-sec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_sec);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%d-time-usec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_usec);
+    if (ret)
+        goto out;
+
+    gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"time", "%s",
+                                          timestr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_top_other (xmlTextWriterPtr writer, dict_t *dict,
-                                int brick_index, int member_index)
+cli_xml_output_vol_top_other(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index, int member_index)
 {
-        int             ret = -1;
-        char            *filename = NULL;
-        uint64_t        count = 0;
-        char            key[1024] = {0,};
-
-        /* <file> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
-                  member_index);
-        ret = dict_get_str (dict, key, &filename);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
-                                               "%s", filename);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
-        ret = dict_get_uint64 (dict, key, &count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%"PRIu64, count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </file> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t count = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
-                        char *op_errstr)
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     brick_count = 0;
-        int                     top_op = GF_CLI_TOP_NONE;
-        char                    *brick_name = NULL;
-        int                     members = 0;
-        uint64_t                current_open = 0;
-        uint64_t                max_open = 0;
-        char                    *max_open_time = NULL;
-        double                  throughput = 0.0;
-        double                  time_taken = 0.0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     j = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int brick_count = 0;
+    int top_op = GF_CLI_TOP_NONE;
+    char *brick_name = NULL;
+    int members = 0;
+    uint64_t current_open = 0;
+    uint64_t max_open = 0;
+    char *max_open_time = NULL;
+    double throughput = 0.0;
+    double time_taken = 0.0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volTop> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volTop");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "1-top-op", &top_op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"topOp", "%d",
+                                          top_op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32(dict, key, &members);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"members",
+                                              "%d", members);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volTop> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volTop");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &current_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"currentOpen", "%" PRIu64, current_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
-                                               "%d", brick_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpen", "%" PRIu64, max_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "1-top-op", &top_op);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"topOp",
-                                               "%d", top_op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+                snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_str(dict, key, &max_open_time);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpenTime", "%s", max_open_time);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        while (i < brick_count) {
-                i++;
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
 
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+                break;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", brick_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key , sizeof (key), "%d-members", i);
-                ret = dict_get_int32 (dict, key, &members);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"members",
-                                                       "%d", members);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                switch (top_op) {
-                case GF_CLI_TOP_OPEN:
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-current-open", i);
-                        ret = dict_get_uint64 (dict, key, &current_open);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"currentOpen", "%"PRIu64,
-                                 current_open);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-max-open", i);
-                        ret = dict_get_uint64 (dict, key, &max_open);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"maxOpen", "%"PRIu64,
-                                 max_open);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
-                        ret = dict_get_str (dict, key, &max_open_time);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"maxOpenTime", "%s",
-                                 max_open_time);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                case GF_CLI_TOP_READ:
-                case GF_CLI_TOP_WRITE:
-                case GF_CLI_TOP_OPENDIR:
-                case GF_CLI_TOP_READDIR:
-
-                        break;
-
-                case GF_CLI_TOP_READ_PERF:
-                case GF_CLI_TOP_WRITE_PERF:
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-throughput", i);
-                        ret = dict_get_double (dict, key, &throughput);
-                        if (!ret) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "%d-time", i);
-                                ret = dict_get_double (dict, key, &time_taken);
-                        }
-
-                        if (!ret) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (writer, (xmlChar *)"throughput",
-                                         "%f", throughput);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (writer, (xmlChar *)"timeTaken",
-                                         "%f", time_taken);
-                        }
-
-                        break;
-
-                default:
-                        ret = -1;
-                        goto out;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time_taken);
                 }
 
-                for (j = 1; j <= members; j++) {
-                        if (top_op == GF_CLI_TOP_READ_PERF ||
-                            top_op == GF_CLI_TOP_WRITE_PERF) {
-                                ret = cli_xml_output_vol_top_rw_perf
-                                        (writer, dict, i, j);
-                        } else {
-                                ret = cli_xml_output_vol_top_other
-                                        (writer, dict, i, j);
-                        }
-                        if (ret)
-                                goto out;
+                if (!ret) {
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"throughput", "%f", throughput);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
+
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"timeTaken", "%f", time_taken);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
                 }
 
+                break;
 
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            default:
+                ret = -1;
+                goto out;
         }
 
-        /* </volTop> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        ret = cli_end_xml_output (writer, doc);
+        for (j = 1; j <= members; j++) {
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                ret = cli_xml_output_vol_top_rw_perf(writer, dict, i, j);
+            } else {
+                ret = cli_xml_output_vol_top_other(writer, dict, i, j);
+            }
+            if (ret)
+                goto out;
+        }
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volTop> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_profile_stats (xmlTextWriterPtr writer, dict_t *dict,
-                                  int brick_index, int interval)
+cli_xml_output_vol_profile_stats(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int interval)
 {
-        int                     ret = -1;
-        uint64_t                read_count = 0;
-        uint64_t                write_count = 0;
-        uint64_t                hits = 0;
-        double                  avg_latency = 0.0;
-        double                  max_latency = 0.0;
-        double                  min_latency = 0.0;
-        uint64_t                duration = 0;
-        uint64_t                total_read = 0;
-        uint64_t                total_write = 0;
-        char                    key[1024] = {0};
-        int                     i = 0;
-
-        /* <cumulativeStats> || <intervalStats> */
-        if (interval == -1)
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"cumulativeStats");
-        else
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"intervalStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <blockStats> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"blockStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < 32; i++) {
-                /* <block> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"block");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"size", "%"PRIu32, (1 << i));
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-read-%d", brick_index,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &read_count);
-                if (ret)
-                        read_count = 0;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"reads", "%"PRIu64, read_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-write-%d", brick_index,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &write_count);
-                if (ret)
-                        write_count = 0;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"writes", "%"PRIu64, write_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </block> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        /* </blockStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <fopStats> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fopStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-hits", brick_index,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &hits);
-                if (ret)
-                        goto cont;
+    int ret = -1;
+    uint64_t read_count = 0;
+    uint64_t write_count = 0;
+    uint64_t hits = 0;
+    double avg_latency = 0.0;
+    double max_latency = 0.0;
+    double min_latency = 0.0;
+    uint64_t duration = 0;
+    uint64_t total_read = 0;
+    uint64_t total_write = 0;
+    char key[1024] = {0};
+    int i = 0;
+
+    /* <cumulativeStats> || <intervalStats> */
+    if (interval == -1)
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"cumulativeStats");
+    else
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"intervalStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <blockStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"blockStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < 32; i++) {
+        /* <block> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"block");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &avg_latency);
-                if (ret)
-                        goto cont;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu32, (1U << i));
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &min_latency);
-                if (ret)
-                        goto cont;
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, brick_index, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &read_count);
+        if (ret)
+            read_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"reads",
+                                              "%" PRIu64, read_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &max_latency);
-                if (ret)
-                        goto cont;
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, brick_index,
+                 interval, (1U << i));
+        ret = dict_get_uint64(dict, key, &write_count);
+        if (ret)
+            write_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"writes",
+                                              "%" PRIu64, write_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* <fop> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fop");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </block> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"name","%s", gf_fop_list[i]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </blockStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"hits", "%"PRIu64, hits);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <fopStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fopStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"avgLatency", "%f", avg_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", brick_index, interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
+        if (ret)
+            goto cont;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"minLatency", "%f", min_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &avg_latency);
+        if (ret)
+            goto cont;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"maxLatency", "%f", max_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &min_latency);
+        if (ret)
+            goto cont;
 
-                /* </fop> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &max_latency);
+        if (ret)
+            goto cont;
 
-cont:
-                hits = 0;
-                avg_latency = 0.0;
-                min_latency = 0.0;
-                max_latency = 0.0;
-        }
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                hits = 0;
-                avg_latency = 0.0;
-                min_latency = 0.0;
-                max_latency = 0.0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-upcall-hits", brick_index,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &hits);
-                if (ret)
-                        continue;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* <fop> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fop");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"name", "%s", gf_fop_list[i]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"hits", "%"PRIu64, hits);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"avgLatency", "%f", avg_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"minLatency", "%f", min_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    cont:
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", brick_index,
+                 interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
+        if (ret)
+            continue;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"maxLatency", "%f", max_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </fop> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </fopStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-duration", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &duration);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"duration",
-                                               "%"PRIu64, duration);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-read", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &total_read);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalRead",
-                                               "%"PRIu64, total_read);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-write", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &total_write);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalWrite",
-                                               "%"PRIu64, total_write);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </cumulativeStats> || </intervalStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fopStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-duration", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &duration);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"duration",
+                                          "%" PRIu64, duration);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_read);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalRead",
+                                          "%" PRIu64, total_read);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_write);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalWrite",
+                                          "%" PRIu64, total_write);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </cumulativeStats> || </intervalStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        int                     op = GF_CLI_STATS_NONE;
-        int                     info_op = GF_CLI_INFO_NONE;
-        int                     brick_count = 0;
-        char                    *brick_name = NULL;
-        int                     interval = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     stats_cleared = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <volProfile> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volProfile");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"volname",
-                                               "%s", volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "op", &op);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"profileOp",
-                                               "%d", op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (GF_CLI_STATS_INFO != op)
-                goto cont;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    int op = GF_CLI_STATS_NONE;
+    int info_op = GF_CLI_INFO_NONE;
+    int brick_count = 0;
+    char *brick_name = NULL;
+    int interval = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int stats_cleared = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volProfile> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volProfile");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volname", "%s",
+                                          volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "op", &op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"profileOp", "%d",
+                                          op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (GF_CLI_STATS_INFO != op)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "info-op", &info_op);
+    if (ret)
+        goto out;
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
-                                               "%d", brick_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickName",
+                                              "%s", brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "info-op", &info_op);
-        if (ret)
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32(dict, key, &stats_cleared);
+            if (ret)
                 goto out;
 
-        while (i < brick_count) {
-                i++;
-
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick_name);
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"clearStats", "%s",
+                stats_cleared ? "Cleared stats." : "Failed to clear stats.");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        } else {
+            snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
                 if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"brickName", "%s", brick_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                if (GF_CLI_INFO_CLEAR == info_op) {
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"clearStats", "%s",
-                                stats_cleared ? "Cleared stats." :
-                                                 "Failed to clear stats.");
-                        if (ret)
-                                goto out;
-                } else {
-                        snprintf (key, sizeof (key), "%d-cumulative", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0) {
-                                ret = cli_xml_output_vol_profile_stats
-                                        (writer, dict, i, interval);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-interval", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0) {
-                                ret = cli_xml_output_vol_profile_stats
-                                        (writer, dict, i, interval);
-                                if (ret)
-                                        goto out;
-                        }
-                }
-
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+                    goto out;
+            }
+
+            snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
+                if (ret)
+                    goto out;
+            }
         }
 
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
 cont:
-        /* </volProfile> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volProfile> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
-                         char *op_errstr)
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     count = 0;
-        char                    *volname = NULL;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = dict_get_str(dict, key, &volname);
         if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <volList> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volList");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"volume",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volume", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </volList> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_info_option (xmlTextWriterPtr writer, char *substr,
-                                char *optstr, char *valstr)
+cli_xml_output_vol_info_option(xmlTextWriterPtr writer, char *substr,
+                               char *optstr, char *valstr)
 {
-        int             ret = -1;
-        char            *ptr1 = NULL;
-        char            *ptr2 = NULL;
-
-        ptr1 = substr;
-        ptr2 = optstr;
-
-        while (ptr1) {
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
-                if (!ptr1)
-                        goto out;
-                if (!ptr2)
-                        goto out;
-        }
-        if (*ptr2 == '\0')
-                goto out;
-
-        /* <option> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"option");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"name",
-                                               "%s", ptr2);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"value",
-                                               "%s", valstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </option> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
+
+    ptr1 = substr;
+    ptr2 = optstr;
+
+    while (ptr1) {
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+        if (!*ptr1)
+            break;
+        if (!*ptr2)
+            break;
+    }
+    if (*ptr2 == '\0')
+        goto out;
+
+    /* <option> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"option");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          ptr2);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"value", "%s",
+                                          valstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </option> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 struct tmp_xml_option_logger {
-        char *key;
-        xmlTextWriterPtr writer;
+    char *key;
+    xmlTextWriterPtr writer;
 };
 
 static int
-_output_vol_info_option (dict_t *d, char *k, data_t *v,
-                         void *data)
+_output_vol_info_option(dict_t *d, char *k, data_t *v, void *data)
 {
-        int                           ret   = 0;
-        char                         *ptr   = NULL;
-        struct tmp_xml_option_logger *tmp   = NULL;
+    int ret = 0;
+    char *ptr = NULL;
+    struct tmp_xml_option_logger *tmp = NULL;
 
-        tmp = data;
+    tmp = data;
 
-        ptr = strstr (k, "option.");
-        if (!ptr)
-                goto out;
+    ptr = strstr(k, "option.");
+    if (!ptr)
+        goto out;
 
-        if (!v) {
-                ret = -1;
-                goto out;
-        }
-        ret = cli_xml_output_vol_info_option (tmp->writer, tmp->key, k,
-                                              v->data);
+    if (!v) {
+        ret = -1;
+        goto out;
+    }
+    ret = cli_xml_output_vol_info_option(tmp->writer, tmp->key, k, v->data);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
-                                 char *prefix)
+cli_xml_output_vol_info_options(xmlTextWriterPtr writer, dict_t *dict,
+                                char *prefix)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            key[1024] = {0,};
-        struct tmp_xml_option_logger tmp = {0,};
-
-        snprintf (key, sizeof (key), "%s.opt_count", prefix);
-        ret = dict_get_int32 (dict, key, &opt_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"optCount",
-                                               "%d", opt_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <options> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"options");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        snprintf (key, sizeof (key), "%s.option.", prefix);
-
-        tmp.key = key;
-        tmp.writer = writer;
-        ret = dict_foreach (dict, _output_vol_info_option, &tmp);
-        if (ret)
-                goto out;
-
-        /* </options> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int opt_count = 0;
+    char key[1024] = {
+        0,
+    };
+    struct tmp_xml_option_logger tmp = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "%s.opt_count", prefix);
+    ret = dict_get_int32(dict, key, &opt_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"optCount", "%d",
+                                          opt_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <options> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"options");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    snprintf(key, sizeof(key), "%s.option.", prefix);
+
+    tmp.key = key;
+    tmp.writer = writer;
+    ret = dict_foreach(dict, _output_vol_info_option, &tmp);
+    if (ret)
+        goto out;
+
+    /* </options> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = 0;
-        int                     count = 0;
-        char                    *volname = NULL;
-        char                    *volume_id = NULL;
-        char                    *uuid = NULL;
-        int                     type = 0;
-        int                     status = 0;
-        int                     brick_count = 0;
-        int                     dist_count = 0;
-        int                     stripe_count = 0;
-        int                     replica_count = 0;
-        int                     arbiter_count = 0;
-        int                     snap_count    = 0;
-        int                     isArbiter = 0;
-        int                     disperse_count = 0;
-        int                     redundancy_count = 0;
-        int                     transport = 0;
-        char                    *brick = NULL;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     j = 1;
-        char                    *caps = NULL;
-        int                     k __attribute__((unused)) = 0;
-        int                     index = 1;
-        int                     tier_vol_type          = 0;
-        /* hot dist count is always zero so need for it to be
-         * included in the array.*/
-        int                     hot_dist_count         = 0;
-        values                  c                      = 0;
-        char                    *keys[MAX]              = {
-                [COLD_BRICK_COUNT]      = "volume%d.cold_brick_count",
-                [COLD_TYPE]             = "volume%d.cold_type",
-                [COLD_DIST_COUNT]       = "volume%d.cold_dist_count",
-                [COLD_REPLICA_COUNT]    = "volume%d.cold_replica_count",
-                [COLD_ARBITER_COUNT]    = "volume%d.cold_arbiter_count",
-                [COLD_DISPERSE_COUNT]   = "volume%d.cold_disperse_count",
-                [COLD_REDUNDANCY_COUNT] = "volume%d.cold_redundancy_count",
-                [HOT_BRICK_COUNT]       = "volume%d.hot_brick_count",
-                [HOT_TYPE]              = "volume%d.hot_type",
-                [HOT_REPLICA_COUNT]     = "volume%d.hot_replica_count"};
-        int                     value[MAX]             = {};
-
-
-        ret = dict_get_int32 (dict, "count", &count);
+    int ret = 0;
+    int count = 0;
+    char *volname = NULL;
+    char *volume_id = NULL;
+    char *uuid = NULL;
+    int type = 0;
+    int status = 0;
+    int brick_count = 0;
+    int dist_count = 0;
+    int stripe_count = 0;
+    int replica_count = 0;
+    int arbiter_count = 0;
+    int snap_count = 0;
+    int isArbiter = 0;
+    int disperse_count = 0;
+    int redundancy_count = 0;
+    int transport = 0;
+    char *brick = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 1;
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"name",
+                                              "%s", volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_str(dict, key, &volume_id);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", volume_id);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_status_str[status]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32(dict, key, &snap_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapshotCount", "%d", snap_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32(dict, key, &brick_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"brickCount", "%d", brick_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32(dict, key, &dist_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"distCount", "%d",
+                                              (brick_count / dist_count));
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32(dict, key, &stripe_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"stripeCount", "%d", stripe_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32(dict, key, &replica_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"replicaCount", "%d", replica_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32(dict, key, &arbiter_count);
         if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"arbiterCount", "%d", arbiter_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32(dict, key, &disperse_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"disperseCount", "%d", disperse_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32(dict, key, &redundancy_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"redundancyCount",
+                                              "%d", redundancy_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32(dict, key, &type);
+        if (ret)
+            goto out;
+        /* For Distributed-(stripe,replicate,stipe-replicate,disperse)
+           types
+         */
+        type = get_vol_type(type, dist_count, brick_count);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%d", type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"typeStr", "%s", vol_type_str[type]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32(dict, key, &transport);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"transport", "%d", transport);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        j = 1;
+
+        /* <bricks> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"bricks");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        while (j <= brick_count) {
+            ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"brick");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d.uuid", i, j);
+            ret = dict_get_str(dict, key, &uuid);
+            if (ret)
                 goto out;
+            ret = xmlTextWriterWriteFormatAttribute(
+                local->writer, (xmlChar *)"uuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < count; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            snprintf(key, sizeof(key), "volume%d.brick%d", i, j);
+            ret = dict_get_str(dict, key, &brick);
+            if (ret)
+                goto out;
+            ret = xmlTextWriterWriteFormatString(local->writer, "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.name", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.volume_id", i);
-                ret = dict_get_str (dict, key, &volume_id);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", volume_id);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret =xmlTextWriterWriteFormatElement
-                        (local->writer, (xmlChar *)"statusStr", "%s",
-                         cli_vol_status_str[status]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.snap_count", i);
-                ret = dict_get_int32 (dict, key, &snap_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                     (xmlChar *)"snapshotCount",
-                                                     "%d", snap_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"name", "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"hostUuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.brick_count", i);
-                ret = dict_get_int32 (dict, key, &brick_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"brickCount",
-                                                       "%d", brick_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.dist_count", i);
-                ret = dict_get_int32 (dict, key, &dist_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"distCount",
-                                                       "%d", dist_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.stripe_count", i);
-                ret = dict_get_int32 (dict, key, &stripe_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"stripeCount",
-                                                       "%d", stripe_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.replica_count", i);
-                ret = dict_get_int32 (dict, key, &replica_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"replicaCount",
-                                                       "%d", replica_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.arbiter_count", i);
-                ret = dict_get_int32 (dict, key, &arbiter_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                      (xmlChar *)"arbiterCount",
-                                                      "%d", arbiter_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.disperse_count", i);
-                ret = dict_get_int32 (dict, key, &disperse_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"disperseCount",
-                                                       "%d", disperse_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.redundancy_count", i);
-                ret = dict_get_int32 (dict, key, &redundancy_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"redundancyCount",
-                                                       "%d", redundancy_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
-                if (ret)
-                        goto out;
-                /* For Distributed-(stripe,replicate,stipe-replicate,disperse)
-                   types
-                 */
-                type = get_vol_type (type, dist_count, brick_count);
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"type",
-                                                       "%d", type);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"typeStr",
-                                                       "%s",
-                                                       vol_type_str[type]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.transport", i);
-                ret = dict_get_int32 (dict, key, &transport);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"transport",
-                                                       "%d", transport);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-#ifdef HAVE_BD_XLATOR
-                /* <xlators> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"xlators");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (k = 0; ; k++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),"volume%d.xlator%d", i, k);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (ret)
-                                break;
-
-                        /* <xlator> */
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)"xlator");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"name", "%s", caps);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        /* <capabilities> */
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "capabilities");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        j = 0;
-                        for (j = 0; ;j++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.xlator%d.caps%d", i, k, j);
-                                ret = dict_get_str (dict, key, &caps);
-                                if (ret)
-                                        break;
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"capability",
-                                         "%s", caps);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        }
-                        /* </capabilities> */
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        /* </xlator> */
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
-                ret = xmlTextWriterFullEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                /* </xlators> */
-#else
-                caps = 0; /* Avoid compiler warnings when BD not enabled */
-#endif
-                j = 1;
-
-                /* <bricks> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"bricks");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                if (type == GF_CLUSTER_TYPE_TIER) {
-                        /*the values for hot stripe, disperse and redundancy
-                         * should not be looped in here as they are zero
-                         * always */
-                        for (c = COLD_BRICK_COUNT; c < MAX; c++) {
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, 256, keys[c], i);
-                                ret = dict_get_int32 (dict, key, &value[c]);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        hot_dist_count = (value[HOT_REPLICA_COUNT] ?
-                                          value[HOT_REPLICA_COUNT] : 1);
-
-                        tier_vol_type = get_vol_type (value[HOT_TYPE],
-                                                      hot_dist_count,
-                                                      value[HOT_BRICK_COUNT]);
-
-                        if ((value[HOT_TYPE] != GF_CLUSTER_TYPE_TIER) &&
-                            (value[HOT_TYPE] > 0) &&
-                            (hot_dist_count < value[HOT_BRICK_COUNT]))
-                                tier_vol_type = value[HOT_TYPE] +
-                                        GF_CLUSTER_TYPE_MAX - 1;
-
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "hotBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"hotBrickType",
-                                 "%s", vol_type_str[tier_vol_type]);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"hotreplicaCount",
-                                                    "%d",
-                                                    value[HOT_REPLICA_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"hotbrickCount",
-                                                    "%d",
-                                                    value[HOT_BRICK_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        if (value[HOT_TYPE] == GF_CLUSTER_TYPE_NONE ||
-                                        value[HOT_TYPE] ==
-                                        GF_CLUSTER_TYPE_TIER) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d", value[HOT_BRICK_COUNT]);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        } else {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d x %d = %d",
-                                         (value[HOT_BRICK_COUNT] /
-                                          hot_dist_count),
-                                         hot_dist_count,
-                                         value[HOT_BRICK_COUNT]);
-                        }
-
-                        while (index <= value[HOT_BRICK_COUNT]) {
-                                snprintf (key, 1024, "volume%d.brick%d", i,
-                                          index);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                index++;
-                        }
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        tier_vol_type = get_vol_type (value[COLD_TYPE],
-                                                      value[COLD_DIST_COUNT],
-                                                      value[COLD_BRICK_COUNT]);
-
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "coldBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"coldBrickType",
-                                 "%s", vol_type_str[tier_vol_type]);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                        (xmlChar *)"coldreplicaCount",
-                                        "%d", value[COLD_REPLICA_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                        (xmlChar *)"coldarbiterCount",
-                                        "%d", value[COLD_ARBITER_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"coldbrickCount",
-                                                    "%d",
-                                                    value[COLD_BRICK_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"colddisperseCount",
-                                              "%d", value[COLD_DISPERSE_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        if (value[COLD_TYPE] == GF_CLUSTER_TYPE_NONE ||
-                                        value[COLD_TYPE] ==
-                                        GF_CLUSTER_TYPE_TIER) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d", value[COLD_BRICK_COUNT]);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        } else if (value[COLD_TYPE] ==
-                                        GF_CLUSTER_TYPE_DISPERSE) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         " %d x (%d + %d) = %d",
-                                         (value[COLD_BRICK_COUNT] /
-                                          value[COLD_DIST_COUNT]),
-                                         value[COLD_DISPERSE_COUNT] -
-                                         value[COLD_REDUNDANCY_COUNT],
-                                         value[COLD_REDUNDANCY_COUNT],
-                                         value[COLD_BRICK_COUNT]);
-                        } else {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d x %d = %d",
-                                         (value[COLD_BRICK_COUNT] /
-                                          value[COLD_DIST_COUNT]),
-                                         value[COLD_DIST_COUNT],
-                                         value[COLD_BRICK_COUNT]);
-                        }
-
-                        index = value[HOT_BRICK_COUNT] + 1;
-
-                        while (index <= brick_count) {
-                                snprintf (key, 1024, "volume%d.brick%d", i,
-                                          index);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.isArbiter", i,
-                                          index);
-                                if (dict_get (dict, key))
-                                        isArbiter = 1;
-                                else
-                                        isArbiter = 0;
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"isArbiter",
-                                         "%d", isArbiter);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                index++;
-                        }
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                } else {
-                        while (j <= brick_count) {
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d", i, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.isArbiter", i, j);
-                                if (dict_get (dict, key))
-                                        isArbiter = 1;
-                                else
-                                        isArbiter = 0;
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"isArbiter",
-                                         "%d", isArbiter);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                /* </brick> */
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                j++;
-                        }
-                }
-                /* </bricks> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", i);
-                ret = cli_xml_output_vol_info_options (local->writer, dict,
-                                                       key);
-                if (ret)
-                        goto out;
+            snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter", i, j);
+            if (dict_get(dict, key))
+                isArbiter = 1;
+            else
+                isArbiter = 0;
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"isArbiter", "%d", isArbiter);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+            /* </brick> */
+            ret = xmlTextWriterEndElement(local->writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (volname) {
-                GF_FREE (local->get_vol.volname);
-                local->get_vol.volname = gf_strdup (volname);
-                local->vol_count += count;
+            j++;
         }
+        /* </bricks> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = cli_xml_output_vol_info_options(local->writer, dict, key);
+        if (ret)
+            goto out;
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (volname) {
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
+        local->vol_count += count;
+    }
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
-                               char *op_errstr)
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        /* <volInfo> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volInfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volInfo> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volumes> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* Init vol count */
-        local->vol_count = 0;
+    /* Init vol count */
+    local->vol_count = 0;
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_info_end (cli_local_t *local)
+cli_xml_output_vol_info_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int             ret = -1;
-
-        GF_ASSERT (local);
+    int ret = -1;
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"count",
-                                               "%d", local->vol_count);
+    GF_ASSERT(local);
 
-        /* </volumes> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"count",
+                                          "%d", local->vol_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </volInfo> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volInfo> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_quota_limit_list_end (cli_local_t *local)
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xmlTextWriterEndElement (local->writer);
-        if (ret) {
-                goto out;
-        }
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_quota_limit_list_begin (cli_local_t *local, int op_ret,
-                                           int op_errno, char *op_errstr)
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        if (ret)
-                goto out;
-
-        /* <volQuota> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volQuota");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
+    /* <volQuota> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volQuota");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 static int
-cli_xml_output_peer_hostnames (xmlTextWriterPtr writer, dict_t *dict,
-                               const char *prefix, int count)
+cli_xml_output_peer_hostnames(xmlTextWriterPtr writer, dict_t *dict,
+                              const char *prefix, int count)
 {
-        int   ret       = -1;
-        int   i         = 0;
-        char *hostname  = NULL;
-        char  key[1024] = {0,};
-
-        /* <hostnames> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"hostnames");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"hostname", "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                hostname = NULL;
-        }
-
-        /* </hostnames> */
-        ret = xmlTextWriterEndElement (writer);
+    int ret = -1;
+    int i = 0;
+    char *hostname = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <hostnames> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"hostnames");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        if (ret < 0)
+            goto out;
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        hostname = NULL;
+    }
 
+    /* </hostnames> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     count = 0;
-        char                    *uuid = NULL;
-        char                    *hostname = NULL;
-        int                     connected = 0;
-        int                     state_id = 0;
-        char                    *state_str = NULL;
-        int                     hostname_count = 0;
-        int                     i = 1;
-        char                    key[1024] = {0,};
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *uuid = NULL;
+    char *hostname = NULL;
+    int connected = 0;
+    int state_id = 0;
+    char *state_str = NULL;
+    int hostname_count = 0;
+    int i = 1;
+    char key[1024] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <peerStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"peerStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (!dict)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    while (i <= count) {
+        /* <peer> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"peer");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_str(dict, key, &uuid);
         if (ret)
-                goto out;
+            goto out;
 
-        /* <peerStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"peerStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (!dict)
-                goto cont;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                              uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &count);
+        snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-
-        while (i <= count) {
-                /* <peer> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"peer");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid);
-                if (ret)
-                        goto out;
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"uuid",
-                                                       "%s", uuid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hostname",
-                                                       "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname_count", i);
-                ret = dict_get_int32 (dict, key, &hostname_count);
-                if ((ret == 0) && (hostname_count > 0)) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "friend%d", i);
-                        ret = cli_xml_output_peer_hostnames (writer, dict, key,
-                                                             hostname_count);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+        snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32(dict, key, &hostname_count);
+        if ((ret == 0) && (hostname_count > 0)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = cli_xml_output_peer_hostnames(writer, dict, key,
+                                                hostname_count);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32(dict, key, &connected);
+        if (ret)
+            goto out;
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"connected",
-                                                       "%d", connected);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connected",
+                                              "%d", connected);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.stateId", i);
-                ret = dict_get_int32 (dict, key, &state_id);
-                if (!ret) {
-                        /* ignore */
+        snprintf(key, sizeof(key), "friend%d.stateId", i);
+        ret = dict_get_int32(dict, key, &state_id);
+        if (!ret) {
+            /* ignore */
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                           (xmlChar *)"state", "%d", state_id);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"state",
+                                                  "%d", state_id);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.state", i);
-                ret = dict_get_str (dict, key, &state_str);
-                if (!ret) {
-                        /* ignore */
+        snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_str(dict, key, &state_str);
+        if (!ret) {
+            /* ignore */
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                       (xmlChar *)"stateStr", "%s", state_str);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"stateStr",
+                                                  "%s", state_str);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                /* </peer> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </peer> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                i++;
-        }
+        i++;
+    }
 
 cont:
-        /* </peerStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </peerStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 /* Used for rebalance stop/status, remove-brick status */
 int
-cli_xml_output_vol_rebalance_status (xmlTextWriterPtr writer, dict_t *dict,
-                                     enum gf_task_types task_type)
+cli_xml_output_vol_rebalance_status(xmlTextWriterPtr writer, dict_t *dict,
+                                    enum gf_task_types task_type)
 {
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *node_name = NULL;
-        char                    *node_uuid = NULL;
-        uint64_t                files = 0;
-        uint64_t                size = 0;
-        uint64_t                lookups = 0;
-        int                     status_rcd = 0;
-        uint64_t                failures = 0;
-        uint64_t                skipped = 0;
-        uint64_t                total_files = 0;
-        uint64_t                total_size = 0;
-        uint64_t                total_lookups = 0;
-        uint64_t                total_failures = 0;
-        uint64_t                total_skipped = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     overall_status = -1;
-        double                  elapsed = 0;
-        double                  overall_elapsed = 0;
-
-        if (!dict) {
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    int count = 0;
+    char *node_name = NULL;
+    char *node_uuid = NULL;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookups = 0;
+    int status_rcd = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    uint64_t total_files = 0;
+    uint64_t total_size = 0;
+    uint64_t total_lookups = 0;
+    uint64_t total_failures = 0;
+    uint64_t total_skipped = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int overall_status = -1;
+    double elapsed = 0;
+    double overall_elapsed = 0;
+
+    if (!dict) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeCount", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < count) {
+        i++;
+        /* Getting status early, to skip nodes that don't have the
+         * rebalance process started
+         */
+        snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32(dict, key, &status_rcd);
 
-        ret = dict_get_int32 (dict, "count", &count);
+        /* If glusterd is down it fails to get the status, try
+         getting status from other nodes */
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        while (i < count) {
-                i++;
-                /* Getting status early, to skip nodes that don't have the
-                 * rebalance process started
-                 */
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "status-%d", i);
-                ret = dict_get_int32 (dict, key, &status_rcd);
-
-                /* If glusterd is down it fails to get the status, try
-                 getting status from other nodes */
-                if (ret)
-                    continue;
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
-
-                /* <node> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"nodeName",
-                                                       "%s", node_name);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-uuid-%d", i);
-                ret = dict_get_str (dict, key, &node_uuid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", node_uuid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "files-%d", i);
-                ret = dict_get_uint64 (dict, key, &files);
-                if (ret)
-                        goto out;
-                total_files += files;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"files",
-                                                       "%"PRIu64, files);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "size-%d", i);
-                ret = dict_get_uint64 (dict, key, &size);
-                if (ret)
-                        goto out;
-                total_size += size;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"size",
-                                                       "%"PRIu64,size);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "lookups-%d", i);
-                ret = dict_get_uint64 (dict, key, &lookups);
-                if (ret)
-                        goto out;
-                total_lookups += lookups;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"lookups",
-                                                       "%"PRIu64, lookups);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "failures-%d", i);
-                ret = dict_get_uint64 (dict, key, &failures);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "skipped-%d", i);
-
-                ret = dict_get_uint64 (dict, key, &skipped);
-                if (ret)
-                        goto out;
+        /* <node> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                if (task_type == GF_TASK_TYPE_REMOVE_BRICK) {
-                        failures += skipped;
-                        skipped = 0;
-                }
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeName",
+                                              "%s", node_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                total_failures += failures;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"failures",
-                                               "%"PRIu64, failures);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "node-uuid-%d", i);
+        ret = dict_get_str(dict, key, &node_uuid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              node_uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                total_skipped += skipped;
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            goto out;
+        total_files += files;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                              "%" PRIu64, files);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"skipped",
-                                               "%"PRIu64, skipped);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            goto out;
+        total_size += size;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu64, size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status_rcd);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookups);
+        if (ret)
+            goto out;
+        total_lookups += lookups;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                              "%" PRIu64, lookups);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s",
-                                         cli_vol_task_status_str[status_rcd]);
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            goto out;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", i);
-                ret = dict_get_double (dict, key, &elapsed);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"runtime",
-                                                       "%.2f", elapsed);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "skipped-%d", i);
 
-                if (elapsed > overall_elapsed) {
-                    overall_elapsed = elapsed;
-                }
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            goto out;
 
-                /* Rebalance has 5 states,
-                 * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
-                 * The precedence used to determine the aggregate status is as
-                 * below,
-                 * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
-                 */
-                /* TODO: Move this to a common place utilities that both CLI and
-                 * glusterd need.
-                 * Till then if the below algorithm is changed, change it in
-                 * glusterd_volume_status_aggregate_tasks_status in
-                 * glusterd-utils.c
-                 */
-
-                if (-1 == overall_status)
-                        overall_status = status_rcd;
-                int rank[] = {
-                        [GF_DEFRAG_STATUS_STARTED] = 1,
-                        [GF_DEFRAG_STATUS_FAILED] = 2,
-                        [GF_DEFRAG_STATUS_STOPPED] = 3,
-                        [GF_DEFRAG_STATUS_COMPLETE] = 4,
-                        [GF_DEFRAG_STATUS_NOT_STARTED] = 5
-                };
-                if (rank[status_rcd] <= rank[overall_status])
-                        overall_status = status_rcd;
-
-                /* </node> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        if (task_type == GF_TASK_TYPE_REMOVE_BRICK) {
+            failures += skipped;
+            skipped = 0;
         }
 
-        /* Aggregate status */
-        /* <aggregate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"aggregate");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"files",
-                                               "%"PRIu64, total_files);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"size",
-                                               "%"PRIu64, total_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"lookups",
-                                               "%"PRIu64, total_lookups);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"failures",
-                                               "%"PRIu64, total_failures);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"skipped",
-                                               "%"PRIu64, total_skipped);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"status",
-                                               "%d", overall_status);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        total_failures += failures;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                              "%" PRIu64, failures);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"statusStr",
-                                               "%s",
-                                      cli_vol_task_status_str[overall_status]);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        total_skipped += skipped;
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"runtime",
-                                               "%.2f", overall_elapsed);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                              "%" PRIu64, skipped);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </aggregate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                              status_rcd);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-#endif
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"statusStr", "%s",
+            cli_vol_task_status_str[status_rcd]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-int
-cli_xml_output_vol_tier_status (xmlTextWriterPtr writer, dict_t *dict,
-                enum gf_task_types task_type)
-{
-#if (HAVE_LIB_XML)
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime",
+                                              "%.2f", elapsed);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *node_name = NULL;
-        char                    *status_str  = NULL;
-        uint64_t                promoted     = 0;
-        uint64_t                demoted      = 0;
-        int                     i = 1;
-        char                    key[1024] = {0,};
-        gf_defrag_status_t      status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
+        if (elapsed > overall_elapsed) {
+            overall_elapsed = elapsed;
+        }
 
+        /* Rebalance has 5 states,
+         * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
+         * The precedence used to determine the aggregate status is as
+         * below,
+         * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
+         */
+        /* TODO: Move this to a common place utilities that both CLI and
+         * glusterd need.
+         * Till then if the below algorithm is changed, change it in
+         * glusterd_volume_status_aggregate_tasks_status in
+         * glusterd-utils.c
+         */
 
-        GF_VALIDATE_OR_GOTO ("cli", dict, out);
+        if (-1 == overall_status)
+            overall_status = status_rcd;
+        int rank[] = {[GF_DEFRAG_STATUS_STARTED] = 1,
+                      [GF_DEFRAG_STATUS_FAILED] = 2,
+                      [GF_DEFRAG_STATUS_STOPPED] = 3,
+                      [GF_DEFRAG_STATUS_COMPLETE] = 4,
+                      [GF_DEFRAG_STATUS_NOT_STARTED] = 5};
+        if (rank[status_rcd] <= rank[overall_status])
+            overall_status = status_rcd;
+
+        /* </node> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
+    /* Aggregate status */
+    /* <aggregate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"aggregate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                          "%" PRIu64, total_files);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        while (i <= count) {
-                promoted = 0;
-                node_name = NULL;
-                demoted = 0;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size", "%" PRIu64,
+                                          total_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                          "%" PRIu64, total_lookups);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"nodeName",
-                                                       "%s", node_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "promoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &promoted);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"promoted"
-                                                       "Files", "%"PRIu64,
-                                                       promoted);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "demoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &demoted);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"demoted"
-                                                       "Files", "%"PRIu64,
-                                                       demoted);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                          "%" PRIu64, total_failures);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                          "%" PRIu64, total_skipped);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                status_str = cli_vol_task_status_str[status_rcd];
+    if (overall_status == -1) {
+        overall_status = status_rcd;
+    }
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s", status_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          overall_status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(
+        writer, (xmlChar *)"statusStr", "%s",
+        cli_vol_task_status_str[overall_status]);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime", "%.2f",
+                                          overall_elapsed);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                i++;
-        }
+    /* </aggregate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-
-#else
-        return 0;
-
-#endif
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
-
-
-
+#endif
 
 int
-cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
-                              int op_errno, char *op_errstr)
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *task_id_str = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volRebalance> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volRebalance");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"op", "%d", op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+    if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REBALANCE);
         if (ret)
-                goto out;
-
-        /* <volRebalance> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRebalance");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-        if (ret == 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"task-id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
-                                               "%d", op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (GF_DEFRAG_CMD_STATUS_TIER == op) {
-                ret = cli_xml_output_vol_tier_status (writer,
-                                dict, GF_TASK_TYPE_REBALANCE);
-                if (ret)
-                        goto out;
-                }
-        if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
-
-                ret = cli_xml_output_vol_rebalance_status (writer, dict,
-                                                      GF_TASK_TYPE_REBALANCE);
-                if (ret)
-                        goto out;
-        }
-
-        /* </volRebalance> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+    }
 
+    /* </volRebalance> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_remove_brick_detach_tier (gf_boolean_t status_op,
-                                             dict_t *dict, int op_ret,
-                                             int op_errno, char *op_errstr,
-                                             const char *op)
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *task_id_str = NULL;
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+    if (status_op) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REMOVE_BRICK);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = xmlTextWriterStartElement (writer, (xmlChar *) op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-        if (ret == 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"task-id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        if (status_op) {
-                ret = cli_xml_output_vol_rebalance_status (writer, dict,
-                                                   GF_TASK_TYPE_REMOVE_BRICK);
-                if (ret)
-                        goto out;
-        }
-
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_replace_brick (dict_t *dict,
-                                  int op_ret, int op_errno, char *op_errstr)
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr)
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        char                    *volid = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    if (dict) {
+        /* <volCreate> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volCreate");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        ret = dict_get_str(dict, "volname", &volname);
         if (ret)
-                goto out;
-
-        if (dict) {
-                /* <volCreate> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"volCreate");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "volume-id", &volid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
-                                                       "%s", volid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "volume-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volCreate> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </volCreate> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
-                               char *op_errstr)
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        char                    *volid = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
 
-        GF_ASSERT (op);
+    GF_ASSERT(op);
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        if (dict) {
-                /* <"op"> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    if (dict) {
+        /* <"op"> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "volname", &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "vol-id", &volid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
-                                                       "%s", volid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "vol-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </"op"> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </"op"> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-_output_gsync_config (FILE *fp, xmlTextWriterPtr writer, char *op_name)
+_output_gsync_config(FILE *fp, xmlTextWriterPtr writer, char *op_name)
 {
-        char  resbuf[256 + PATH_MAX] = {0,};
-        char *ptr                    = NULL;
-        char *v                      = NULL;
-        int   blen                   = sizeof(resbuf);
-        int   ret                    = 0;
-
-        for (;;) {
-                ptr = fgets (resbuf, blen, fp);
-                if (!ptr)
-                        break;
-
-                v = resbuf + strlen (resbuf) - 1;
-                while (isspace (*v)) {
-                        /* strip trailing space */
-                        *v-- = '\0';
-                }
-                if (v == resbuf) {
-                        /* skip empty line */
-                        continue;
-                }
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    char *ptr = NULL;
+    char *v = NULL;
+    int blen = sizeof(resbuf);
+    int ret = 0;
+
+    for (;;) {
+        ptr = fgets(resbuf, blen, fp);
+        if (!ptr)
+            break;
 
-                if (op_name!= NULL){
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)op_name,
-                                                        "%s", resbuf);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        goto out;
-                }
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v)) {
+            /* strip trailing space */
+            *v-- = '\0';
+        }
+        if (v == resbuf) {
+            /* skip empty line */
+            continue;
+        }
 
-                v = strchr (resbuf, ':');
-                if (!v) {
-                        ret = -1;
-                        goto out;
-                }
-                *v++ = '\0';
-                while (isspace (*v))
-                        v++;
-                v = gf_strdup (v);
-                if (!v) {
-                        ret = -1;
-                        goto out;
-                }
+        if (op_name != NULL) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)op_name,
+                                                  "%s", resbuf);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            goto out;
+        }
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)resbuf,
-                                                       "%s", v);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        v = strchr(resbuf, ':');
+        if (!v) {
+            ret = -1;
+            goto out;
+        }
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v) {
+            ret = -1;
+            goto out;
         }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)resbuf, "%s",
+                                              v);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-get_gsync_config (runner_t *runner,
-                  int (*op_conf)(FILE *fp,
-                              xmlTextWriterPtr writer,
-                              char *op_name),
-                  xmlTextWriterPtr writer, char *op_name)
+get_gsync_config(runner_t *runner,
+                 int (*op_conf)(FILE *fp, xmlTextWriterPtr writer,
+                                char *op_name),
+                 xmlTextWriterPtr writer, char *op_name)
 {
-        int ret = 0;
+    int ret = 0;
 
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_log ("cli", GF_LOG_ERROR, "spawning child failed");
-                return -1;
-        }
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_log("cli", GF_LOG_ERROR, "spawning child failed");
+        return -1;
+    }
 
-        ret = op_conf (runner_chio (runner, STDOUT_FILENO), writer, op_name);
+    ret = op_conf(runner_chio(runner, STDOUT_FILENO), writer, op_name);
 
-        ret |= runner_end (runner);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "reading data from child failed");
+    ret |= runner_end(runner);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "reading data from child failed");
 
-        return ret ? -1 : 0;
+    return ret ? -1 : 0;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_generate_gsync_config (dict_t *dict, xmlTextWriterPtr writer)
+cli_xml_generate_gsync_config(dict_t *dict, xmlTextWriterPtr writer)
 {
-        runner_t runner           = {0,};
-        char *subop               = NULL;
-        char *gwd                 = NULL;
-        char *slave               = NULL;
-        char *confpath            = NULL;
-        char *master              = NULL;
-        char *op_name             = NULL;
-        int   ret                 = -1;
-        char  conf_path[PATH_MAX] = "";
-
-        if (dict_get_str (dict, "subop", &subop) != 0) {
-                ret = -1;
-                goto out;
-        }
-
-        if (strcmp (subop, "get") != 0 && strcmp (subop, "get-all") != 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                 (xmlChar *)"message",
-                                 "%s",GEOREP" config updated successfully" );
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                ret = 0;
-                goto out;
-        }
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str(dict, "subop", &subop) != 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"message", "%s",
+            GEOREP " config updated successfully");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = 0;
+        goto out;
+    }
 
-        if (dict_get_str (dict, "glusterd_workdir", &gwd) != 0 ||
-            dict_get_str (dict, "slave", &slave) != 0) {
-                ret = -1;
-                goto out;
-        }
+    if (dict_get_str(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str(dict, "slave", &slave) != 0) {
+        ret = -1;
+        goto out;
+    }
 
-        if (dict_get_str (dict, "master", &master) != 0)
-                master = NULL;
+    if (dict_get_str(dict, "master", &master) != 0)
+        master = NULL;
 
-        if (dict_get_str (dict, "op_name", &op_name) != 0)
-                op_name = NULL;
+    if (dict_get_str(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
 
-        ret = dict_get_str (dict, "conf_path", &confpath);
-        if (!confpath) {
-                ret = snprintf (conf_path, sizeof (conf_path) - 1,
-                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
-                conf_path[ret] = '\0';
-                confpath = conf_path;
-        }
+    ret = dict_get_str(dict, "conf_path", &confpath);
+    if (!confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
 
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", confpath);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
 
-        if (master)
-                runner_argprintf (&runner, ":%s", master);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
 
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
 
-        if (op_name)
-                runner_add_arg (&runner, op_name);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
 
-        ret =  get_gsync_config (&runner, _output_gsync_config,
-                                 writer, op_name);
+    ret = get_gsync_config(&runner, _output_gsync_config, writer, op_name);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_gsync_status (dict_t *dict,
-                                 xmlTextWriterPtr writer)
+cli_xml_output_vol_gsync_status(dict_t *dict, xmlTextWriterPtr writer)
 {
-        int                  ret                         = -1;
-        int                  i                           = 1;
-        int                  j                           = 0;
-        int                  count                       = 0;
-        const int            number_of_fields            = 20;
-        int                  closed                      = 1;
-        int                  session_closed              = 1;
-        gf_gsync_status_t  **status_values               = NULL;
-        char                 status_value_name[PATH_MAX] = "";
-        char                *tmp                         = NULL;
-        char                *volume                      = NULL;
-        char                *volume_next                 = NULL;
-        char                *slave                       = NULL;
-        char                *slave_next                  = NULL;
-        char                *title_values[]              = {"master_node",
-                                                            "",
-                                                            "master_brick",
-                                                            "slave_user",
-                                                            "slave",
-                                                            "slave_node",
-                                                            "status",
-                                                            "crawl_status",
-                                                            /* last_synced */
-                                                            "",
-                                                            "entry",
-                                                            "data",
-                                                            "meta",
-                                                            "failures",
-                                                           /* checkpoint_time */
-                                                            "",
-                                                         "checkpoint_completed",
-                                               /* checkpoint_completion_time */
-                                                            "",
-                                                            "master_node_uuid",
-                                                           /* last_synced_utc */
-                                                            "last_synced",
-                                                       /* checkpoint_time_utc */
-                                                            "checkpoint_time",
-                                            /* checkpoint_completion_time_utc */
-                                                 "checkpoint_completion_time"};
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_int32 (dict, "gsync-count", &count);
-        if (ret)
-                goto out;
-
-        status_values = GF_CALLOC (count, sizeof (gf_gsync_status_t *),
+    int ret = -1;
+    int i = 1;
+    int j = 0;
+    int count = 0;
+    const int number_of_fields = 20;
+    int closed = 1;
+    int session_closed = 1;
+    gf_gsync_status_t **status_values = NULL;
+    char status_value_name[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *volume = NULL;
+    char *volume_next = NULL;
+    char *slave = NULL;
+    char *slave_next = NULL;
+    static const char *title_values[] = {
+        "master_node", "", "master_brick", "slave_user", "slave", "slave_node",
+        "status", "crawl_status",
+        /* last_synced */
+        "", "entry", "data", "meta", "failures",
+        /* checkpoint_time */
+        "", "checkpoint_completed",
+        /* checkpoint_completion_time */
+        "", "master_node_uuid",
+        /* last_synced_utc */
+        "last_synced",
+        /* checkpoint_time_utc */
+        "checkpoint_time",
+        /* checkpoint_completion_time_utc */
+        "checkpoint_completion_time"};
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_int32(dict, "gsync-count", &count);
+    if (ret)
+        goto out;
+
+    status_values = GF_MALLOC(count * sizeof(gf_gsync_status_t *),
                               gf_common_mt_char);
-        if (!status_values) {
-                ret = -1;
-                goto out;
-        }
+    if (!status_values) {
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0; i < count; i++) {
-                status_values[i] = GF_CALLOC (1, sizeof (gf_gsync_status_t),
-                                         gf_common_mt_char);
-                if (!status_values[i]) {
-                        ret = -1;
-                        goto out;
-                }
+    for (i = 0; i < count; i++) {
+        status_values[i] = GF_CALLOC(1, sizeof(gf_gsync_status_t),
+                                     gf_common_mt_char);
+        if (!status_values[i]) {
+            ret = -1;
+            goto out;
+        }
 
-                snprintf (status_value_name, sizeof (status_value_name),
-                          "status_value%d", i);
+        snprintf(status_value_name, sizeof(status_value_name), "status_value%d",
+                 i);
 
-                ret = dict_get_bin (dict, status_value_name,
-                                    (void **)&(status_values[i]));
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "struct member empty.");
-                        goto out;
-                }
+        ret = dict_get_bin(dict, status_value_name,
+                           (void **)&(status_values[i]));
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+            goto out;
         }
+    }
 
-        qsort(status_values, count, sizeof (gf_gsync_status_t *),
-              gf_gsync_status_t_comparator);
-
-        for (i = 0; i < count; i++) {
-                if (closed) {
-                        ret = xmlTextWriterStartElement (writer,
-                                                         (xmlChar *)"volume");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        tmp = get_struct_variable (1, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)"name",
-                                                        "%s",tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterStartElement (writer,
-                                                    (xmlChar *)"sessions");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        closed = 0;
-                }
+    qsort(status_values, count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
 
-                if (session_closed) {
-                        ret = xmlTextWriterStartElement (writer,
-                                                         (xmlChar *)"session");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+    for (i = 0; i < count; i++) {
+        if (closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        session_closed = 0;
+            tmp = get_struct_variable(1, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                        tmp = get_struct_variable (21, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name",
+                                                  "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"session_slave", "%s", tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"sessions");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pair");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            closed = 0;
+        }
 
-                for (j = 0; j < number_of_fields; j++) {
-                        /* XML ignore fields */
-                        if (strcmp(title_values[j], "") == 0)
-                                continue;
+        if (session_closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"session");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        tmp = get_struct_variable (j, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+            session_closed = 0;
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                  (xmlChar *)title_values[j],
-                                                  "%s", tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            tmp = get_struct_variable(21, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"session_slave", "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                if (i+1 < count) {
-                        slave = get_struct_variable (20, status_values[i]);
-                        slave_next = get_struct_variable (20,
-                                                          status_values[i+1]);
-                        volume = get_struct_variable (1, status_values[i]);
-                        volume_next = get_struct_variable (1,
-                                                           status_values[i+1]);
-                        if (!slave || !slave_next || !volume || !volume_next) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pair");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        if (strcmp (volume, volume_next)!=0) {
-                                closed = 1;
-                                session_closed = 1;
+        for (j = 0; j < number_of_fields; j++) {
+            /* XML ignore fields */
+            if (strcmp(title_values[j], "") == 0)
+                continue;
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
+            tmp = get_struct_variable(j, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)title_values[j], "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        } else if (strcmp (slave, slave_next)!=0) {
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                                session_closed = 1;
+        if (i + 1 < count) {
+            slave = get_struct_variable(20, status_values[i]);
+            slave_next = get_struct_variable(20, status_values[i + 1]);
+            volume = get_struct_variable(1, status_values[i]);
+            volume_next = get_struct_variable(1, status_values[i + 1]);
+            if (!slave || !slave_next || !volume || !volume_next) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        }
-                } else {
+            if (strcmp(volume, volume_next) != 0) {
+                closed = 1;
+                session_closed = 1;
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            } else if (strcmp(slave, slave_next) != 0) {
+                session_closed = 1;
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
+        } else {
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
         }
+    }
 out:
-        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (status_values)
+        GF_FREE(status_values);
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
-                          char *op_errstr)
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *master = NULL;
-        char                    *slave = NULL;
-        int                     type = 0;
-
-        GF_ASSERT (dict);
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <geoRep> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"geoRep");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
-                goto out;
-        }
-
-        switch (type) {
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int type = 0;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <geoRep> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"geoRep");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    switch (type) {
         case GF_GSYNC_OPTION_TYPE_START:
         case GF_GSYNC_OPTION_TYPE_STOP:
         case GF_GSYNC_OPTION_TYPE_PAUSE:
         case GF_GSYNC_OPTION_TYPE_RESUME:
         case GF_GSYNC_OPTION_TYPE_CREATE:
         case GF_GSYNC_OPTION_TYPE_DELETE:
-                if (dict_get_str (dict, "master", &master) != 0)
-                        master = "???";
-                if (dict_get_str (dict, "slave", &slave) != 0)
-                        slave = "???";
+            if (dict_get_str(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str(dict, "slave", &slave) != 0)
+                slave = "???";
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"master",
-                                                       "%s", master);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"master",
+                                                  "%s", master);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"slave",
-                                                       "%s", slave);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"slave",
+                                                  "%s", slave);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                break;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_CONFIG:
-                if (op_ret == 0) {
-                        ret = xmlTextWriterStartElement (writer, (xmlChar *)"config");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+            if (op_ret == 0) {
+                ret = xmlTextWriterStartElement(writer, (xmlChar *)"config");
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = cli_xml_generate_gsync_config (dict, writer);
-                        if (ret)
-                                goto out;
+                ret = cli_xml_generate_gsync_config(dict, writer);
+                if (ret)
+                    goto out;
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
 
-                break;
+            break;
         case GF_GSYNC_OPTION_TYPE_STATUS:
-                ret = cli_xml_output_vol_gsync_status (dict, writer);
-                break;
+            ret = cli_xml_output_vol_gsync_status(dict, writer);
+            if (ret) {
+                gf_log("cli", GF_LOG_DEBUG, "Failed to get gsync status");
+                goto out;
+            }
+            break;
         default:
-                ret = 0;
-                break;
-        }
+            ret = 0;
+            break;
+    }
 
-        /* </geoRep> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </geoRep> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
@@ -4516,54 +3842,54 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_create (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_create(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapCreate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapCreate");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapCreate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot clone output in xml format.
@@ -4575,57 +3901,55 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_clone (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_clone(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        char   *str_value       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("cli", writer, out);
-        GF_VALIDATE_OR_GOTO ("cli", doc, out);
-        GF_VALIDATE_OR_GOTO ("cli", dict, out);
-
-        /* <CloneCreate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"CloneCreate");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <volume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "clonename", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get clone name");
-                goto out;
-        }
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get clone uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </volume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </CloneCreate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", writer, out);
+    GF_VALIDATE_OR_GOTO("cli", doc, out);
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+
+    /* <CloneCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"CloneCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "clonename", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </CloneCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /* This function will generate snapshot restore output in xml format.
  *
  * @param writer        xmlTextWriterPtr
@@ -4635,84 +3959,83 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_restore (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_restore(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapRestore> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapRestore");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <volume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get vol name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume id");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </volume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapRestore> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapRestore> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapRestore");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get vol name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume id");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapRestore> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot list output in xml format.
@@ -4724,57 +4047,57 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_list (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_list(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        int     i               = 0;
-        int     snapcount       = 0;
-        char   *str_value       = NULL;
-        char    key[PATH_MAX]   = "";
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char *str_value = NULL;
+    char key[PATH_MAX] = "";
 
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
 
-        /* <snapList> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapList");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapcount");
-                goto out;
-        }
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "count",
-                                               "%d", snapcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 1; i <= snapcount; ++i) {
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
+    for (i = 1; i <= snapcount; ++i) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
+        }
 
-                ret = dict_get_str (dict, key, &str_value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                } else {
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *)"snapshot", "%s", str_value);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+        ret = dict_get_str(dict, key, &str_value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapshot",
+                                                  "%s", str_value);
+            XML_RET_CHECK_AND_GOTO(ret, out);
         }
+    }
 
-        /* </snapList> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate xml output for origin volume
@@ -4788,66 +4111,66 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_orig_vol (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix)
+cli_xml_snapshot_info_orig_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix)
 {
-        int     ret             = -1;
-        int     value           = 0;
-        char   *buffer          = NULL;
-        char    key [PATH_MAX]  = "";
+    int ret = -1;
+    int value = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
 
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
 
-        /* <originVolume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"originVolume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <originVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"originVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%sorigin-volname", keyprefix);
+    snprintf(key, sizeof(key), "%sorigin-volname", keyprefix);
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%ssnapcount", keyprefix);
+    snprintf(key, sizeof(key), "%ssnapcount", keyprefix);
 
-        ret = dict_get_int32 (dict, key, &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "snapCount",
-                                               "%d", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapCount", "%d",
+                                          value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%ssnaps-available", keyprefix);
+    snprintf(key, sizeof(key), "%ssnaps-available", keyprefix);
 
-        ret = dict_get_int32 (dict, key, &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "snapRemaining", "%d", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapRemaining",
+                                          "%d", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </originVolume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </originVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate xml output of snapshot volume info.
@@ -4862,66 +4185,74 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_snap_vol (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix,
-                                gf_boolean_t snap_driven)
+cli_xml_snapshot_info_snap_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
 {
-        char            key [PATH_MAX]          = "";
-        char           *buffer                  = NULL;
-        int             ret                     = -1;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-
-        /* <snapVolume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapVolume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.volname", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* If the command is snap_driven then we need to show origin volume
+     * info. Else this is shown in the start of info display.*/
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.", keyprefix);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+    /* </snapVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "status",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* If the command is snap_driven then we need to show origin volume
-         * info. Else this is shown in the start of info display.*/
-        if (snap_driven) {
-                snprintf (key, sizeof (key), "%s.", keyprefix);
-                ret = cli_xml_snapshot_info_orig_vol (writer, doc, dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot's origin volume");
-                        goto out;
-                }
-        }
-
-        /* </snapVolume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot info of individual snapshot
@@ -4937,112 +4268,118 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_per_snap (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix,
-                                gf_boolean_t snap_driven)
+cli_xml_snapshot_info_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
 {
-        char            key_buffer[PATH_MAX]    = "";
-        char           *buffer                  = NULL;
-        int             volcount                = 0;
-        int             ret                     = -1;
-        int             i                       = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
-                  keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
-                        key_buffer);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
+    char key_buffer[PATH_MAX] = "";
+    char *buffer = NULL;
+    int volcount = 0;
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", buffer);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", "");
+    }
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ", keyprefix);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"createTime", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol-count", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Fail to get snap vol count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    /* Display info of each snapshot volume */
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol%d", keyprefix,
+                       i);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_snap_vol(writer, doc, dict, key_buffer,
+                                             snap_driven);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
-                                key_buffer);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list "
+                   "details of volume in a snap");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "description",
-                                                "%s", buffer);
-        } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "description",
-                                                "%s", "");
-        }
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
-                                keyprefix);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *) "createTime",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.vol-count", keyprefix);
-        ret = dict_get_int32 (dict, key_buffer, &volcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Fail to get snap vol count");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *) "volCount",
-                                               "%d", volcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, key_buffer, &volcount);
-        /* Display info of each snapshot volume */
-        for (i = 1 ; i <= volcount ; i++) {
-                snprintf (key_buffer, sizeof (key_buffer), "%s.vol%d",
-                          keyprefix, i);
-
-                ret = cli_xml_snapshot_info_snap_vol (writer, doc, dict,
-                                                      key_buffer, snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
-                                        "details of volume in a snap");
-                        goto out;
-                }
-        }
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot info output in xml format.
@@ -5054,73 +4391,74 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_info(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int             ret             = -1;
-        int             i               = 0;
-        int             snapcount       = 0;
-        char            key [PATH_MAX]  = "";
-        gf_boolean_t    snap_driven     = _gf_false;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapInfo> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapInfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snap_driven = dict_get_str_boolean (dict, "snap-driven", _gf_false);
-
-        /* If the approach is volume based then we should display orgin volume
-         * information first followed by per snap info*/
-        if (!snap_driven) {
-                ret = cli_xml_snapshot_info_orig_vol (writer, doc, dict, "");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot's origin volume");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char key[PATH_MAX] = "";
+    gf_boolean_t snap_driven = _gf_false;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapInfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+
+    /* If the approach is volume based then we should display origin volume
+     * information first followed by per snap info*/
+    if (!snap_driven) {
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, "");
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "count",
-                                               "%d", snapcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* Get snapshot info of individual snapshots */
-        for (i = 1; i <= snapcount; ++i) {
-                snprintf (key, sizeof (key), "snap%d", i);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = cli_xml_snapshot_info_per_snap (writer, doc, dict,
-                                                      key, snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                }
+    /* Get snapshot info of individual snapshots */
+    for (i = 1; i <= snapcount; ++i) {
+        snprintf(key, sizeof(key), "snap%d", i);
+
+        ret = cli_xml_snapshot_info_per_snap(writer, doc, dict, key,
+                                             snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
         }
+    }
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapInfo> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapInfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status of individual
@@ -5134,139 +4472,148 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_volume_status (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, const char *keyprefix)
+cli_xml_snapshot_volume_status(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, const char *keyprefix)
 {
-        int     ret             = -1;
-        int     brickcount      = 0;
-        int     i               = 0;
-        int     pid             = 0;
-        char   *buffer          = NULL;
-        char    key[PATH_MAX]   = "";
+    int ret = -1;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brickcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get status of every brick belonging to the snapshot volume */
+    for (i = 0; i < brickcount; i++) {
+        /* <snapInfo> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-        snprintf (key, sizeof (key), "%s.brickcount", keyprefix);
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Brick Path");
+            /*
+             * If path itself is not present, then end *
+             * this brick's status and continue to the *
+             * brick                                   *
+             */
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            continue;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                              buffer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, key, &brickcount);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
-                goto out;
-        }
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Volume Group");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "%s", buffer);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "brickCount",
-                                               "%d", brickcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* Get status of every brick belonging to the snapshot volume */
-        for (i = 0 ; i < brickcount ; i++) {
-                /* <snapInfo> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix, i);
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to get Brick Path");
-                        /*
-                         * If path itself is not present, then end *
-                         * this brick's status and continue to the *
-                         * brick                                   *
-                         */
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        continue;
-                }
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "path", "%s", buffer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                          keyprefix, i);
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to get Volume Group");
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                              (xmlChar *) "volumeGroup", "N/A");
-                } else
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                       (xmlChar *) "volumeGroup", "%s", buffer);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.status", keyprefix, i);
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Brick Running");
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                            (xmlChar *) "brick_running", "N/A");
-                } else
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                     (xmlChar *) "brick_running", "%s", buffer);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.pid", keyprefix, i);
-
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO, "Unable to get pid");
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                      (xmlChar *) "pid", "N/A");
-                } else
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                  (xmlChar *) "pid", "%d", pid);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.data", keyprefix, i);
-
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                        "Unable to get Data Percent");
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                          (xmlChar *) "data_percentage", "N/A");
-                } else
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                   (xmlChar *) "data_percentage", "%s", buffer);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.brick%d.lvsize",
-                          keyprefix, i);
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get LV Size");
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                 (xmlChar *) "lvSize", "N/A");
-                } else {
-                        /* Truncate any newline character */
-                        buffer = strtok (buffer, "\n");
-
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                          (xmlChar *) "lvSize", "%s", buffer);
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "%s", buffer);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "%d", pid);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Data Percent");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "%s", buffer);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get LV Size");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "N/A");
+        } else {
+            /* Truncate any newline character */
+            buffer = strtok(buffer, "\n");
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "%s", buffer);
         }
 
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status of individual
@@ -5279,87 +4626,85 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_status_per_snap (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                  dict_t *dict, const char *keyprefix)
+cli_xml_snapshot_status_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                 dict_t *dict, const char *keyprefix)
 {
-        int     ret             = -1;
-        int     volcount        = 0;
-        int     i               = 0;
-        char   *buffer          = NULL;
-        char    key [PATH_MAX]  = "";
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.uuid", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int volcount = 0;
+    int i = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get snapshot status of individual snapshot volume */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
 
-        ret = dict_get_int32 (dict, key, &volcount);
+        ret = cli_xml_snapshot_volume_status(writer, doc, dict, key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "volCount",
-                                               "%d", volcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* Get snapshot status of individual snapshot volume */
-        for (i = 0 ; i < volcount ; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
-
-                ret = cli_xml_snapshot_volume_status (writer, doc,
-                                                             dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                        "Could not get snap volume status");
-                        goto out;
-                }
-
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status output in xml format.
@@ -5371,64 +4716,63 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_status (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_status(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        int     snapcount       = 0;
-        int     i               = 0;
-        int     status_cmd      = 0;
-        char    key [PATH_MAX]  = "";
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "sub-cmd", &status_cmd);
+    int ret = -1;
+    int snapcount = 0;
+    int i = 0;
+    int status_cmd = 0;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((GF_SNAP_STATUS_TYPE_SNAP == status_cmd) ||
+        (GF_SNAP_STATUS_TYPE_ITER == status_cmd)) {
+        snapcount = 1;
+    } else {
+        ret = dict_get_int32(dict, "status.snapcount", &snapcount);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+            goto out;
         }
 
-        if ((GF_SNAP_STATUS_TYPE_SNAP == status_cmd) ||
-            (GF_SNAP_STATUS_TYPE_ITER == status_cmd)) {
-                snapcount = 1;
-        } else {
-                ret = dict_get_int32 (dict, "status.snapcount", &snapcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "count",
-                                                       "%d", snapcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                              snapcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        for (i = 0 ; i < snapcount; i++) {
-                snprintf (key, sizeof (key), "status.snap%d", i);
+    for (i = 0; i < snapcount; i++) {
+        snprintf(key, sizeof(key), "status.snap%d", i);
 
-                ret = cli_xml_snapshot_status_per_snap (writer, doc,
-                                                               dict, key);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "failed to create xml "
-                                "output for snapshot status");
-                        goto out;
-                }
+        ret = cli_xml_snapshot_status_per_snap(writer, doc, dict, key);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "failed to create xml "
+                   "output for snapshot status");
+            goto out;
         }
+    }
 
-        /* </snapStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config show output in xml format.
@@ -5440,153 +4784,149 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config_show (xmlTextWriterPtr writer,
-                              xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_config_show(xmlTextWriterPtr writer, xmlDocPtr doc,
+                             dict_t *dict)
 {
-        int             ret             = -1;
-        uint64_t        i               = 0;
-        uint64_t        value           = 0;
-        uint64_t        volcount        = 0;
-        char            buf[PATH_MAX]   = "";
-        char           *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <systemConfig> */
-        ret = xmlTextWriterStartElement (writer,
-                        (xmlChar *)"systemConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "snap-max-hard-limit");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "hardLimit", "%"PRIu64, value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t i = 0;
+    uint64_t value = 0;
+    uint64_t volcount = 0;
+    char buf[PATH_MAX] = "";
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <systemConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-hard-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                          "%" PRIu64, value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-soft-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                          "%" PRIu64 "%%", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "auto-delete", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch auto-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snap-activate-on-create", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch snap-activate-on-create-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activateOnCreate",
+                                          "%s", str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volumeConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "voldisplaycount", &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch volcount");
+        goto out;
+    }
+
+    /* Get config of all the volumes */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname", i);
+        ret = dict_get_str(dict, buf, &str_value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "snap-max-soft-limit");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "softLimit",
-                        "%"PRIu64"%%", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              str_value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "auto-delete", &str_value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch auto-delete");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "autoDelete", "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "snap-activate-on-create", &str_value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-active-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not fetch snap-activate-on-create-delete");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not fetch"
+                   " effective snap_max_hard_limit for "
+                   "%s",
+                   str_value);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "activateOnCreate", "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </systemConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <volumeConfig> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volumeConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"effectiveHardLimit", "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_uint64 (dict, "voldisplaycount", &volcount);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-soft-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch volcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        /* Get config of all the volumes */
-        for (i = 0; i < volcount; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf), "volume%"PRIu64"-volname", i);
-                ret = dict_get_str (dict, buf, &str_value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "name", "%s", str_value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-snap-max-hard-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "hardLimit", "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-active-hard-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
-                                " effective snap_max_hard_limit for "
-                                "%s", str_value);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "effectiveHardLimit",
-                                "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-snap-max-soft-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "softLimit",
-                                        "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
         /* </volume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = 0;
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config set output in xml format.
@@ -5598,87 +4938,84 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config_set (xmlTextWriterPtr writer, xmlDocPtr doc,
-                             dict_t *dict)
+cli_xml_snapshot_config_set(xmlTextWriterPtr writer, xmlDocPtr doc,
+                            dict_t *dict)
 {
-        int             ret             = -1;
-        uint64_t        hard_limit      = 0;
-        uint64_t        soft_limit      = 0;
-        char           *volname         = NULL;
-        char           *auto_delete     = NULL;
-        char           *snap_activate   = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* This is optional parameter therefore ignore the error */
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
-        /* This is optional parameter therefore ignore the error */
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
-        ret = dict_get_str (dict, "auto-delete", &auto_delete);
-        ret = dict_get_str (dict, "snap-activate-on-create", &snap_activate);
-
-        if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "At least one option from "
-                        "snap-max-hard-limit, snap-max-soft-limit, auto-delete"
-                        " and snap-activate-on-create should be set");
-                goto out;
-        }
-
-        /* Ignore the error, as volname is optional */
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (NULL == volname) {
-                /* <systemConfig> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"systemConfig");
-        } else {
-                /* <volumeConfig> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"volumeConfig");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *volname = NULL;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+    ret = dict_get_str(dict, "auto-delete", &auto_delete);
+    ret = dict_get_str(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "At least one option from "
+               "snap-max-hard-limit, snap-max-soft-limit, auto-delete"
+               " and snap-activate-on-create should be set");
+        goto out;
+    }
+
+    /* Ignore the error, as volname is optional */
+    ret = dict_get_str(dict, "volname", &volname);
+
+    if (NULL == volname) {
+        /* <systemConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    } else {
+        /* <volumeConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "name", "%s", volname);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+    }
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (hard_limit) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "newHardLimit",
-                                                "%"PRIu64, hard_limit);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (hard_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newHardLimit",
+                                              "%" PRIu64, hard_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (soft_limit) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "newSoftLimit",
-                                "%"PRIu64, soft_limit);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (soft_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newSoftLimit",
+                                              "%" PRIu64, soft_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (auto_delete) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "autoDelete", "%s", auto_delete);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (auto_delete) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete",
+                                              "%s", auto_delete);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (snap_activate) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                           (xmlChar *) "activateOnCreate", "%s", snap_activate);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (snap_activate) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"activateOnCreate", "%s", snap_activate);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </volumeConfig> or </systemConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volumeConfig> or </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config output in xml format.
@@ -5690,58 +5027,60 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_config(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int             ret             = -1;
-        int             config_command  = 0;
+    int ret = -1;
+    int config_command = 0;
 
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
 
-        /* <snapConfig> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
-                goto out;
-        }
+    ret = dict_get_int32(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
 
-        switch (config_command) {
+    switch (config_command) {
         case GF_SNAP_CONFIG_TYPE_SET:
-                ret = cli_xml_snapshot_config_set (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create xml "
-                                "output for snapshot config set command");
-                        goto out;
-                }
+            ret = cli_xml_snapshot_config_set(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config set command");
+                goto out;
+            }
 
-                break;
+            break;
         case GF_SNAP_CONFIG_DISPLAY:
-                ret = cli_xml_snapshot_config_show (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create xml "
-                                "output for snapshot config show command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_config_show(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config show command");
+                goto out;
+            }
+            break;
         default:
-                gf_log ("cli", GF_LOG_ERROR, "Unknown config command :%d",
-                        config_command);
-                ret = -1;
-                goto out;
-        }
+            gf_log("cli", GF_LOG_ERROR, "Unknown config command :%d",
+                   config_command);
+            ret = -1;
+            goto out;
+    }
 
-        /* </snapConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot activate or
@@ -5754,67 +5093,66 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_activate_deactivate (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                      dict_t *dict, int cmd)
+cli_xml_snapshot_activate_deactivate(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                     dict_t *dict, int cmd)
 {
-        int     ret     = -1;
-        char   *buffer  = NULL;
-        char   *tag     = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        if (GF_SNAP_OPTION_TYPE_ACTIVATE == cmd) {
-                tag = "snapActivate";
-        } else if (GF_SNAP_OPTION_TYPE_DEACTIVATE == cmd) {
-                tag = "snapDeactivate";
-        } else {
-                gf_log ("cli", GF_LOG_ERROR, "invalid command %d", cmd);
-                goto out;
-        }
-
-        /* <snapActivate> or <snapDeactivate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)tag);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        /* </snapActivate> or </snapDeactivate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *buffer = NULL;
+    char *tag = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    if (GF_SNAP_OPTION_TYPE_ACTIVATE == cmd) {
+        tag = "snapActivate";
+    } else if (GF_SNAP_OPTION_TYPE_DEACTIVATE == cmd) {
+        tag = "snapDeactivate";
+    } else {
+        gf_log("cli", GF_LOG_ERROR, "invalid command %d", cmd);
+        goto out;
+    }
+
+    /* <snapActivate> or <snapDeactivate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)tag);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapActivate> or </snapDeactivate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 #endif /* HAVE_LIB_XML */
 
@@ -5828,191 +5166,183 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-cli_xml_snapshot_delete (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict,
-                         gf_cli_rsp *rsp)
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp)
 {
-        int     ret             = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto xmlend;
-        }
-
-        if (!rsp->op_ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "status",
-                                                       "Success");
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-        } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "status",
-                                                       "Failure");
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-
-                ret = cli_xml_output_common (writer, rsp->op_ret,
-                                             rsp->op_errno,
-                                             rsp->op_errstr);
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-        }
-
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, xmlend);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto xmlend;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    char *str_value = NULL;
+    xmlTextWriterPtr writer = local->writer;
+    xmlDocPtr doc = local->doc;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto xmlend;
+    }
+
+    if (!rsp->op_ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Success");
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Failure");
+        XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+        ret = cli_xml_output_common(writer, rsp->op_ret, rsp->op_errno,
+                                    rsp->op_errstr);
+    }
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto xmlend;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 xmlend:
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 #endif /* HAVE_LIB_XML */
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_output_snap_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr)
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "snapStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_status_end (cli_local_t *local)
+cli_xml_output_snap_status_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapStatus> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_delete_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr)
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int ret = -1;
-        int delete_cmd = -1;
+    int ret = -1;
+    int delete_cmd = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (local->dict, "sub-cmd", &delete_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
-                goto out;
-        }
+    ret = dict_get_int32(local->dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                             op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "snapDelete");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapDelete");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_delete_end (cli_local_t *local)
+cli_xml_output_snap_delete_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapDelete> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </snapDelete> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 /* This function will generate xml output for all the snapshot commands
@@ -6026,493 +5356,484 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-cli_xml_output_snapshot (int cmd_type, dict_t *dict, int op_ret,
-                         int op_errno, char *op_errstr)
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret     = -1;
-        xmlTextWriterPtr        writer  = NULL;
-        xmlDocPtr               doc     = NULL;
-
-        GF_ASSERT (dict);
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml begin block");
-                goto out;
-        }
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml common block");
-                goto out;
-        }
-
-        /* In case of command failure just printing the error message is good
-         * enough */
-        if (0 != op_ret) {
-                goto end;
-        }
-
-        switch (cmd_type) {
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml begin block");
+        goto out;
+    }
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml common block");
+        goto out;
+    }
+
+    /* In case of command failure just printing the error message is good
+     * enough */
+    if (0 != op_ret) {
+        goto end;
+    }
+
+    switch (cmd_type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = cli_xml_snapshot_create (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot create command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_create(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot create command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = cli_xml_snapshot_clone (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot clone command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_clone(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot clone command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = cli_xml_snapshot_restore (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot restore command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_restore(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot restore command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_LIST:
-                ret = cli_xml_snapshot_list (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot list command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_list(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot list command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = cli_xml_snapshot_status (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create"
-                                "xml output for snapshot status command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_status(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create"
+                       "xml output for snapshot status command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                ret = cli_xml_snapshot_info (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot info command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_info(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot info command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = cli_xml_snapshot_activate_deactivate (writer, doc,
-                                                            dict, cmd_type);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot config command");
-                }
-                break;
+            ret = cli_xml_snapshot_activate_deactivate(writer, doc, dict,
+                                                       cmd_type);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = cli_xml_snapshot_config (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot config command");
-                }
-                break;
+            ret = cli_xml_snapshot_config(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
         default:
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unexpected snapshot command: %d", cmd_type);
-                goto out;
-        }
+            gf_log("cli", GF_LOG_ERROR, "Unexpected snapshot command: %d",
+                   cmd_type);
+            goto out;
+    }
 
 end:
-        ret = cli_end_xml_output (writer, doc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml end block");
-                goto out;
-        }
-
-        ret = 0;
+    ret = cli_end_xml_output(writer, doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml end block");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_xml_snapshot_begin_composite_op (cli_local_t *local)
+cli_xml_snapshot_begin_composite_op(cli_local_t *local)
 {
-        int ret         = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        int cmd         = -1;
-        int type        =  -1;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                ret = 0;
-                goto out;
-        }
-
-        if (cmd == GF_SNAP_STATUS_TYPE_ITER ||
-            cmd == GF_SNAP_DELETE_TYPE_SNAP){
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapshot "
-                        "command type from dictionary");
-                goto out;
-        }
-
-        if (GF_SNAP_OPTION_TYPE_STATUS == type)
-                ret = cli_xml_output_snap_status_begin (local, 0, 0, NULL);
-        else if (GF_SNAP_OPTION_TYPE_DELETE == type)
-                ret = cli_xml_output_snap_delete_begin (local, 0, 0, NULL);
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Error creating xml output");
-                goto out;
-        }
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_begin(local, 0, 0, NULL);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_begin(local, 0, 0, NULL);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Error creating xml output");
+        goto out;
+    }
 
 #endif /* HAVE_LIB_XML */
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_snapshot_end_composite_op (cli_local_t *local)
+cli_xml_snapshot_end_composite_op(cli_local_t *local)
 {
-        int ret         = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        int cmd         = -1;
-        int type        = -1;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                ret = 0;
-                goto out;
-        }
-
-        if (cmd == GF_SNAP_STATUS_TYPE_ITER ||
-            cmd == GF_SNAP_DELETE_TYPE_SNAP){
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapshot "
-                        "command type from dictionary");
-                goto out;
-        }
-
-        if (GF_SNAP_OPTION_TYPE_STATUS == type)
-                ret = cli_xml_output_snap_status_end (local);
-        else if (GF_SNAP_OPTION_TYPE_DELETE == type)
-                ret = cli_xml_output_snap_delete_end (local);
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Error creating xml "
-                                "output");
-                goto out;
-        }
-#endif /* HAVE_LIB_XML */
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
         ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_end(local);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_end(local);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Error creating xml "
+               "output");
+        goto out;
+    }
+#endif /* HAVE_LIB_XML */
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_snapshot_status_single_snap (cli_local_t *local, dict_t *dict,
-                                     char *key)
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict, char *key)
 {
 #if (HAVE_LIB_XML)
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("cli", (local != NULL), out);
-        GF_VALIDATE_OR_GOTO ("cli", (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO ("cli", (key != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (local != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (key != NULL), out);
 
-        ret = cli_xml_snapshot_status_per_snap (local->writer, local->doc, dict,
-                                                key);
+    ret = cli_xml_snapshot_status_per_snap(local->writer, local->doc, dict,
+                                           key);
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_xml_output_vol_getopts (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     i = 0;
-        int                     ret = -1;
-        int                     count = 0;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        char                    dict_key[50] = {0,};
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volGetopts");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "count", &count);
+    int i = 0;
+    int ret = -1;
+    int count = 0;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char dict_key[50] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volGetopts");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count "
+               "from the dictionary");
+        goto out;
+    }
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Value of count :%d is "
+               "invalid",
+               count);
+        ret = -1;
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 1; i <= count; i++) {
+        sprintf(dict_key, "key%d", i);
+        ret = dict_get_str(dict, dict_key, &key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to retrieve count "
-                        "from the dictionary");
-                goto out;
-        }
-        if (count <= 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Value of count :%d is "
-                        "invalid", count);
-                ret = -1;
-                goto out;
-        }
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", count);
-
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i=1; i<=count; i++) {
-                sprintf (dict_key, "key%d", i);
-                ret = dict_get_str (dict, dict_key, &key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to"
-                                " retrieve %s from the "
-                                "dictionary", dict_key);
-                        goto out;
-                }
-                sprintf (dict_key, "value%d", i);
-                ret = dict_get_str (dict, dict_key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "retrieve key value for %s from"
-                                "the dictionary", dict_key);
-                        goto out;
-                }
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"Opt");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to"
+                   " retrieve %s from the "
+                   "dictionary",
+                   dict_key);
+            goto out;
+        }
+        sprintf(dict_key, "value%d", i);
+        ret = dict_get_str(dict, dict_key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to "
+                   "retrieve key value for %s from"
+                   "the dictionary",
+                   dict_key);
+            goto out;
+        }
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"Opt");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"Option",
-                                                       "%s", key);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Option", "%s",
+                                              key);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"Value",
-                                                       "%s", value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Value", "%s",
+                                              value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-        ret = cli_end_xml_output (writer, doc);
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_quota_list_xml_error (cli_local_t *local, char *path,
-                          char *errstr)
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr)
 {
 #if (HAVE_LIB_XML)
-        int     ret     =       -1;
+    int ret = -1;
 
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"path",
-                                               "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"errstr",
-                                               "%s", errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"errstr",
+                                          "%s", errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_quota_xml_output (cli_local_t *local, char *path, int64_t hl_str,
-                      char *sl_final, int64_t sl_num, int64_t used,
-                      int64_t avail, char *sl, char *hl,
-                      gf_boolean_t limit_set)
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set)
 {
 #if (HAVE_LIB_XML)
-        int     ret             = -1;
-
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"path",
-                                               "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hard_limit",
-                                               !limit_set ? "N/A" :
-                                               "%"PRId64, hl_str);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"soft_limit_percent",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl_final);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"soft_limit_value",
-                                               !limit_set ? "N/A" :
-                                               "%"PRId64, sl_num);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"used_space",
-                                               "%"PRId64, used);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"avail_space",
-                                               !limit_set ? "N/A" :
-                                               "%"PRId64, avail);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"sl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        hl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_final);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRId64, sl_num);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"used_space", "%" PRId64, used);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"avail_space",
+        !limit_set ? "N/A" : "%" PRId64, avail);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_quota_object_xml_output (cli_local_t *local, char *path, char *sl_str,
-                             int64_t sl_val, quota_limits_t *limits,
-                             quota_meta_t *used_space, int64_t avail,
-                             char *sl, char *hl, gf_boolean_t limit_set)
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set)
 {
 #if (HAVE_LIB_XML)
-        int     ret             = -1;
-
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"path",
-                                              "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hard_limit",
-                                               !limit_set ? "N/A" :
-                                               "%"PRId64, limits->hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"soft_limit_percent",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl_str);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"soft_limit_value",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, sl_val);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        limits->hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"file_count",
-                                               "%"PRId64,
-                                               used_space->file_count);
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRIu64, sl_val);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"dir_count",
-                                               "%"PRIu64,
-                                               used_space->dir_count);
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"file_count", "%" PRId64,
+                                          used_space->file_count);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"dir_count",
+                                          "%" PRIu64, used_space->dir_count);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"available",
-                                               !limit_set ? "N/A" :
-                                               "%"PRId64, avail);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"available",
+                                          !limit_set ? "N/A" : "%" PRId64,
+                                          avail);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"sl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
diff --git a/cli/src/cli.c b/cli/src/cli.c
index 8e4caaa944e..a52b39c5fb8 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2010-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -33,769 +33,880 @@
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
 #include "cli.h"
 #include "cli-quotad-client.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "list.h"
-#include "timer.h"
-#include "stack.h"
-#include "revision.h"
-#include "common-utils.h"
-#include "event.h"
-#include "globals.h"
-#include "syscall.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/revision.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/call-stub.h>
 #include <fnmatch.h>
 
 #include "xdr-generic.h"
 
-extern int connected;
 /* using argp for command line parsing */
 
-const char *argp_program_version = ""                                 \
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__ \
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"  \
-        "Copyright (c) 2006-2011 Gluster Inc. "                       \
-        "<http://www.gluster.com>\n"                                  \
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"              \
-        "You may redistribute copies of GlusterFS under the terms of "\
-        "the GNU General Public License.";
-
+const char *argp_program_version =
+    "" PACKAGE_NAME " " PACKAGE_VERSION
+    "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION
+    "\n"
+    "Copyright (c) 2006-2016 Red Hat, Inc. "
+    "<https://www.gluster.org/>\n"
+    "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+    "It is licensed to you under your choice of the GNU Lesser\n"
+    "General Public License, version 3 or any later version (LGPLv3\n"
+    "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+    "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
-
-
 struct rpc_clnt *global_quotad_rpc;
+
 struct rpc_clnt *global_rpc;
 
 rpc_clnt_prog_t *cli_rpc_prog;
 
-
 extern struct rpc_clnt_program cli_prog;
 
+int cli_default_conn_timeout = 120;
+int cli_ten_minutes_timeout = 600;
+
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-        int            ret         = -1;
-
-        ret = xlator_mem_acct_init (THIS, cli_mt_end);
-        if (ret != 0) {
-                return ret;
-        }
-
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid)
-                return -1;
-
-        ctx->page_size  = 128 * GF_UNIT_KB;
-
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool)
-                return -1;
-
-        ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE,
-                                          STARTING_EVENT_THREADS);
-        if (!ctx->event_pool)
-                return -1;
-
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          cli_mt_call_pool_t);
-        if (!pool)
-                return -1;
-
-        /* frame_mem_pool size 112 * 64 */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
-        if (!pool->frame_mem_pool)
-                return -1;
-
-        /* stack_mem_pool size 256 * 128 */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
-
-        if (!pool->stack_mem_pool)
-                return -1;
-
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
-        if (!ctx->stub_mem_pool)
-                return -1;
-
-        ctx->dict_pool = mem_pool_new (dict_t, 32);
-        if (!ctx->dict_pool)
-                return -1;
-
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
-        if (!ctx->dict_pair_pool)
-                return -1;
-
-        ctx->dict_data_pool = mem_pool_new (data_t, 512);
-        if (!ctx->dict_data_pool)
-                return -1;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t, 256);
-        if (!ctx->logbuf_pool)
-                return -1;
-
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
-
-        LOCK_INIT (&ctx->lock);
-
-        cmd_args = &ctx->cmd_args;
-
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    if (!ctx)
+        return ret;
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    ret = xlator_mem_acct_init(THIS, cli_mt_end);
+    if (ret != 0) {
+        gf_log("cli", GF_LOG_ERROR, "Memory accounting init failed.");
+        return ret;
+    }
+
+    /* Resetting ret to -1 to so in case of failure
+     * we can relese allocated resource.
+     */
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate uuid.");
+        goto out;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create iobuf pool.");
+        goto out;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create event pool.");
+        goto out;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), cli_mt_call_pool_t);
+    if (!pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create call pool.");
+        goto out;
+    }
+
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create frame mem pool.");
+        goto out;
+    }
+
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
+
+    if (!pool->stack_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create stack mem pool.");
+        goto out;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to stub mem pool.");
+        goto out;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pool.");
+        goto out;
+    }
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pair pool.");
+        goto out;
+    }
+
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict data pool.");
+        goto out;
+    }
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create logbuf pool.");
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    cmd_args = &ctx->cmd_args;
+
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
+
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
+
+    ret = 0;
 
-        return 0;
+out:
+    if (ret != 0) {
+        if (pool) {
+            mem_pool_destroy(pool->frame_mem_pool);
+            mem_pool_destroy(pool->stack_mem_pool);
+        }
+        GF_FREE(pool);
+        pool = NULL;
+        GF_FREE(ctx->process_uuid);
+        mem_pool_destroy(ctx->stub_mem_pool);
+        mem_pool_destroy(ctx->dict_pool);
+        mem_pool_destroy(ctx->dict_pair_pool);
+        mem_pool_destroy(ctx->dict_data_pool);
+        mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-logging_init (glusterfs_ctx_t *ctx, struct cli_state *state)
+logging_init(glusterfs_ctx_t *ctx, struct cli_state *state)
 {
-        char *log_file = state->log_file ? state->log_file :
-                         DEFAULT_CLI_LOG_FILE_DIRECTORY "/cli.log";
-
-        /* passing ident as NULL means to use default ident for syslog */
-        if (gf_log_init (ctx, log_file, NULL) == -1) {
-                fprintf (stderr, "ERROR: failed to open logfile %s\n",
-                         log_file);
-                return -1;
-        }
-
-        /* CLI should not have something to DEBUG after the release,
-           hence defaulting to INFO loglevel */
-        gf_log_set_loglevel ((state->log_level == GF_LOG_NONE) ? GF_LOG_INFO :
-                             state->log_level);
-
-        return 0;
+    char *log_file = state->log_file ? state->log_file
+                                     : DEFAULT_CLI_LOG_FILE_DIRECTORY
+                         "/cli.log";
+
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(ctx, log_file, NULL) == -1) {
+        fprintf(stderr, "ERROR: failed to open logfile %s\n", log_file);
+    }
+
+    /* CLI should not have something to DEBUG after the release,
+       hence defaulting to INFO loglevel */
+    gf_log_set_loglevel(ctx, (state->log_level == GF_LOG_NONE)
+                                 ? GF_LOG_INFO
+                                 : state->log_level);
+
+    return 0;
 }
 
 int
-cli_submit_request (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int                     ret         = -1;
-        int                     count      = 0;
-        struct iovec            iov         = {0, };
-        struct iobuf            *iobuf = NULL;
-        char                    new_iobref = 0;
-        ssize_t                 xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        if (!rpc)
-                rpc = global_rpc;
-        /* Send the msg */
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    if (!rpc)
+        rpc = global_rpc;
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data)
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        this = mydata;
+    this = mydata;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-
-                cli_cmd_broadcast_connected ();
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-               break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            cli_cmd_broadcast_connected(_gf_true);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                connected = 0;
-                if (!global_state->prompt && global_state->await_connected) {
-                        ret = 1;
-                        cli_out ("Connection failed. Please check if gluster "
-                                  "daemon is operational.");
-                        exit (ret);
-                }
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            cli_cmd_broadcast_connected(_gf_false);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            if (!global_state->prompt && global_state->await_connected) {
+                ret = 1;
+                cli_out(
+                    "Connection failed. Please check if gluster "
+                    "daemon is operational.");
+                exit(ret);
+            }
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
+static gf_boolean_t
+is_valid_int(char *str)
+{
+    if (*str == '-')
+        ++str;
+
+    /* Handle empty string or just "-".*/
+    if (!*str)
+        return _gf_false;
+
+    /* Check for non-digit chars in the rest of the string */
+    while (*str) {
+        if (!isdigit(*str))
+            return _gf_false;
+        else
+            ++str;
+    }
+    return _gf_true;
+}
 
 /*
  * ret: 0: option successfully processed
  *      1: signalling end of option list
- *     -1: unknown option or other issue
+ *     -1: unknown option
+ *     -2: parsing issue (avoid unknown option error)
  */
 int
-cli_opt_parse (char *opt, struct cli_state *state)
+cli_opt_parse(char *opt, struct cli_state *state)
 {
-        char            *oarg           = NULL;
-        gf_boolean_t    secure_mgmt_tmp = 0;
-
-        if (strcmp (opt, "") == 0)
-                return 1;
-
-        if (strcmp (opt, "version") == 0) {
-                cli_out ("%s", argp_program_version);
-                exit (0);
-        }
-
-        if (strcmp (opt, "print-logdir") == 0) {
-                cli_out ("%s", DEFAULT_LOG_FILE_DIRECTORY);
-                exit (0);
-        }
-
-        if (strcmp (opt, "print-statedumpdir") == 0) {
-                cli_out ("%s", DEFAULT_VAR_RUN_DIRECTORY);
-                exit (0);
-        }
-
-        if (strcmp (opt, "xml") == 0) {
+    char *oarg = NULL;
+    gf_boolean_t secure_mgmt_tmp = 0;
+
+    if (strcmp(opt, "") == 0)
+        return 1;
+    if (strcmp(opt, "help") == 0) {
+        cli_out(
+            " peer help                - display help for peer commands\n"
+            " volume help              - display help for volume commands\n"
+            " volume bitrot help       - display help for volume"
+            " bitrot commands\n"
+            " volume quota help        - display help for volume"
+            " quota commands\n"
+            " snapshot help            - display help for snapshot commands\n"
+            " global help              - list global commands\n");
+        exit(0);
+    }
+
+    if (strcmp(opt, "version") == 0) {
+        cli_out("%s", argp_program_version);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-logdir") == 0) {
+        cli_out("%s", DEFAULT_LOG_FILE_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-statedumpdir") == 0) {
+        cli_out("%s", DEFAULT_VAR_RUN_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "xml") == 0) {
 #if (HAVE_LIB_XML)
-                state->mode |= GLUSTER_MODE_XML;
+        state->mode |= GLUSTER_MODE_XML;
 #else
-                cli_err ("XML output not supported. Ignoring '--xml' option");
+        cli_err("XML output not supported. Ignoring '--xml' option");
 #endif
-                return 0;
-        }
+        return 0;
+    }
 
-        if (strcmp (opt, "wignore") == 0) {
-                state->mode |= GLUSTER_MODE_WIGNORE;
-                return 0;
-        }
+    if (strcmp(opt, "nolog") == 0) {
+        state->mode |= GLUSTER_MODE_GLFSHEAL_NOLOG;
+        return 0;
+    }
 
-        oarg = strtail (opt, "mode=");
-        if (oarg) {
-                if (strcmp (oarg, "script") == 0) {
-                        state->mode |= GLUSTER_MODE_SCRIPT;
-                        return 0;
-                }
+    if (strcmp(opt, "wignore-partition") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE_PARTITION;
+        return 0;
+    }
 
-                if (strcmp (oarg, "interactive") == 0)
-                        return 0;
+    if (strcmp(opt, "wignore") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE;
+        return 0;
+    }
 
-                return -1;
+    oarg = strtail(opt, "mode=");
+    if (oarg) {
+        if (strcmp(oarg, "script") == 0) {
+            state->mode |= GLUSTER_MODE_SCRIPT;
+            return 0;
         }
 
-        oarg = strtail (opt, "remote-host=");
-        if (oarg) {
-                state->remote_host = oarg;
-                return 0;
-        }
+        if (strcmp(oarg, "interactive") == 0)
+            return 0;
 
-        oarg = strtail (opt, "log-file=");
-        if (oarg) {
-                state->log_file = oarg;
-                return 0;
-        }
-        oarg = strtail (opt, "timeout=");
-        if (oarg) {
-                cli_default_conn_timeout = atoi(oarg);
-                return 0;
-        }
+        return -1;
+    }
 
-        oarg = strtail (opt, "log-level=");
-        if (oarg) {
-                int log_level = glusterd_check_log_level(oarg);
-                if (log_level == -1)
-                        return -1;
-                state->log_level = (gf_loglevel_t) log_level;
-                return 0;
-        }
+    oarg = strtail(opt, "remote-host=");
+    if (oarg) {
+        state->remote_host = oarg;
+        return 0;
+    }
 
-        oarg = strtail (opt, "glusterd-sock=");
-        if (oarg) {
-                state->glusterd_sock = oarg;
-                return 0;
+    oarg = strtail(opt, "inet6");
+    if (oarg) {
+        state->address_family = "inet6";
+        return 0;
+    }
+
+    oarg = strtail(opt, "log-file=");
+    if (oarg) {
+        state->log_file = oarg;
+        return 0;
+    }
+    oarg = strtail(opt, "timeout=");
+    if (oarg) {
+        if (!is_valid_int(oarg) || atoi(oarg) <= 0) {
+            cli_err("timeout value should be a positive integer");
+            return -2; /* -2 instead of -1 to avoid unknown option
+                          error */
         }
+        cli_default_conn_timeout = atoi(oarg);
+        return 0;
+    }
+
+    oarg = strtail(opt, "log-level=");
+    if (oarg) {
+        int log_level = glusterd_check_log_level(oarg);
+        if (log_level == -1)
+            return -1;
+        state->log_level = (gf_loglevel_t)log_level;
+        return 0;
+    }
+
+    oarg = strtail(opt, "glusterd-sock=");
+    if (oarg) {
+        state->glusterd_sock = oarg;
+        return 0;
+    }
 
-        oarg = strtail (opt, "secure-mgmt=");
-        if (oarg) {
-                if (gf_string2boolean(oarg,&secure_mgmt_tmp) == 0) {
-                        if (secure_mgmt_tmp) {
-                                /* See declaration for why this is an int. */
-                                state->ctx->secure_mgmt = 1;
-                        }
-                }
-                else {
-                        cli_err ("invalide secure-mgmt value (ignored)");
-                }
-                return 0;
+    oarg = strtail(opt, "secure-mgmt=");
+    if (oarg) {
+        if (gf_string2boolean(oarg, &secure_mgmt_tmp) == 0) {
+            if (secure_mgmt_tmp) {
+                /* See declaration for why this is an int. */
+                state->ctx->secure_mgmt = 1;
+            }
+        } else {
+            cli_err("invalid secure-mgmt value (ignored)");
         }
+        return 0;
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-parse_cmdline (int argc, char *argv[], struct cli_state *state)
+parse_cmdline(int argc, char *argv[], struct cli_state *state)
 {
-        int                 ret            = 0;
-        int                 i              = 0;
-        int                 j              = 0;
-        char               *opt            = NULL;
-        gf_boolean_t       geo_rep_config  = _gf_false;
-
-        state->argc=argc-1;
-        state->argv=&argv[1];
-
-        /* Do this first so that an option can override. */
-        if (sys_access (SECURE_ACCESS_FILE, F_OK) == 0) {
-                state->ctx->secure_mgmt = 1;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    char *opt = NULL;
+
+    state->argc = argc - 1;
+    state->argv = &argv[1];
+
+    /* Do this first so that an option can override. */
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        state->ctx->secure_mgmt = 1;
+        state->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    if (state->argc > GEO_REP_CMD_CONFIG_INDEX &&
+        strtail(state->argv[GEO_REP_CMD_INDEX], "geo") &&
+        strtail(state->argv[GEO_REP_CMD_CONFIG_INDEX], "co"))
+        goto done;
+
+    for (i = 0; i < state->argc; i++) {
+        opt = strtail(state->argv[i], "--");
+        if (!opt)
+            continue;
+        ret = cli_opt_parse(opt, state);
+        if (ret == -1) {
+            cli_out("unrecognized option --%s\n", opt);
+            usage();
+            return ret;
+        } else if (ret == -2) {
+            return ret;
         }
-
-        if (state->argc > GEO_REP_CMD_CONFIG_INDEX &&
-            strtail (state->argv[GEO_REP_CMD_INDEX], "geo") &&
-            strtail (state->argv[GEO_REP_CMD_CONFIG_INDEX], "co"))
-                geo_rep_config = _gf_true;
-
-        for (i = 0; i < state->argc; i++) {
-                opt = strtail (state->argv[i], "--");
-                if (opt && !geo_rep_config) {
-                        ret = cli_opt_parse (opt, state);
-                        if (ret == -1) {
-                                cli_out ("unrecognized option --%s", opt);
-                                return ret;
-                        }
-                        for (j = i; j < state->argc - 1; j++)
-                                state->argv[j] = state->argv[j + 1];
-                        state->argc--;
-                        /* argv shifted, next check should be at i again */
-                        i--;
-                        if (ret == 1) {
-                                /* end of cli options */
-                                ret = 0;
-                                break;
-                        }
-                }
+        for (j = i; j < state->argc - 1; j++)
+            state->argv[j] = state->argv[j + 1];
+        state->argc--;
+        /* argv shifted, next check should be at i again */
+        i--;
+        if (ret == 1) {
+            /* end of cli options */
+            ret = 0;
+            break;
         }
+    }
 
-        state->argv[state->argc] = NULL;
+done:
+    state->argv[state->argc] = NULL;
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_tree_init (struct cli_cmd_tree *tree)
+cli_cmd_tree_init(struct cli_cmd_tree *tree)
 {
-        struct cli_cmd_word  *root = NULL;
-        int                   ret = 0;
+    struct cli_cmd_word *root = NULL;
+    int ret = 0;
 
-        root = &tree->root;
-        root->tree = tree;
+    root = &tree->root;
+    root->tree = tree;
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_state_init (struct cli_state *state)
+cli_state_init(struct cli_state *state)
 {
-        struct cli_cmd_tree  *tree = NULL;
-        int                   ret = 0;
-
+    struct cli_cmd_tree *tree = NULL;
+    int ret = 0;
 
-        state->log_level = GF_LOG_NONE;
+    state->log_level = GF_LOG_NONE;
 
-        tree = &state->tree;
-        tree->state = state;
+    tree = &state->tree;
+    tree->state = state;
 
-        ret = cli_cmd_tree_init (tree);
+    ret = cli_cmd_tree_init(tree);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_usage_out (const char *usage)
+cli_usage_out(const char *usage)
 {
-        GF_ASSERT (usage);
-        GF_ASSERT (usage[0] != '\0');
+    GF_ASSERT(usage);
 
-        if (!usage || usage[0] == '\0')
-                return -1;
+    if (!usage || usage[0] == '\0')
+        return -1;
 
-        cli_err ("Usage: %s", usage);
-        return 0;
+    cli_err("\nUsage:\n%s\n", usage);
+    return 0;
 }
 
 int
-_cli_err (const char *fmt, ...)
+_cli_err(const char *fmt, ...)
 {
-        struct cli_state *state = NULL;
-        va_list           ap;
-        int               ret = 0;
-
-        state = global_state;
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
-        va_start (ap, fmt);
+    va_start(ap, fmt);
 
 #ifdef HAVE_READLINE
-        if (state->rl_enabled && !state->rl_processing) {
-                va_end (ap);
-                return cli_rl_err (state, fmt, ap);
-        }
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_err(state, fmt, ap);
+        va_end(ap);
+        return ret;
+    }
 #endif
 
-        ret = vfprintf (stderr, fmt, ap);
-        fprintf (stderr, "\n");
-        va_end (ap);
+    ret = vfprintf(stderr, fmt, ap);
+    fprintf(stderr, "\n");
+    va_end(ap);
 
-        return ret;
+    return ret;
 }
 
-
 int
-_cli_out (const char *fmt, ...)
+_cli_out(const char *fmt, ...)
 {
-        struct cli_state *state = NULL;
-        va_list           ap;
-        int               ret = 0;
-
-        state = global_state;
-
-        va_start (ap, fmt);
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
+    va_start(ap, fmt);
 #ifdef HAVE_READLINE
-        if (state->rl_enabled && !state->rl_processing) {
-                va_end (ap);
-                return cli_rl_out (state, fmt, ap);
-        }
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_out(state, fmt, ap);
+        va_end(ap);
+        return ret;
+    }
 #endif
 
-        ret = vprintf (fmt, ap);
-        printf ("\n");
-        va_end (ap);
+    ret = vprintf(fmt, ap);
+    printf("\n");
+    va_end(ap);
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_rpc_init (void)
+cli_quotad_clnt_rpc_init(void)
 {
-        struct rpc_clnt *rpc = NULL;
-        dict_t          *rpc_opts = NULL;
-        int             ret = -1;
-
-        rpc_opts = dict_new ();
-        if (!rpc_opts) {
-                        ret = -1;
-                        goto out;
-                }
+    struct rpc_clnt *rpc = NULL;
+    dict_t *rpc_opts = NULL;
+    int ret = -1;
+
+    rpc_opts = dict_new();
+    if (!rpc_opts) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(rpc_opts, "transport.address-family", "unix");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport-type", "socket");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport.socket.connect-path",
+                       "/var/run/gluster/quotad.socket");
+    if (ret)
+        goto out;
+
+    rpc = cli_quotad_clnt_init(THIS, rpc_opts);
+    if (!rpc)
+        goto out;
+
+    global_quotad_rpc = rpc;
+out:
+    if (rpc_opts) {
+        dict_unref(rpc_opts);
+    }
+    return rpc;
+}
 
-        ret = dict_set_str (rpc_opts, "transport.address-family", "unix");
-        if (ret)
-                goto out;
+struct rpc_clnt *
+cli_rpc_init(struct cli_state *state)
+{
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = CLI_GLUSTERD_PORT;
+    xlator_t *this = NULL;
+#ifdef IPV6_DEFAULT
+    char *addr_family = "inet6";
+#else
+    char *addr_family = "inet";
+#endif
 
-        ret = dict_set_str (rpc_opts, "transport-type", "socket");
+    this = THIS;
+    cli_rpc_prog = &cli_prog;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    /* If address family specified in CLI */
+    if (state->address_family) {
+        addr_family = state->address_family;
+    }
+
+    /* Connect to glusterd using the specified method, giving preference
+     * to a unix socket connection.  If nothing is specified, connect to
+     * the default glusterd socket.
+     */
+    if (state->glusterd_sock) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to glusterd using "
+               "sockfile %s",
+               state->glusterd_sock);
+        ret = rpc_transport_unix_options_build(options, state->glusterd_sock,
+                                               0);
         if (ret)
-                goto out;
-
-        ret = dict_set_str (rpc_opts, "transport.socket.connect-path",
-                                            "/var/run/gluster/quotad.socket");
+            goto out;
+    } else if (state->remote_host) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to remote glusterd at "
+               "%s",
+               state->remote_host);
+
+        ret = dict_set_str(options, "remote-host", state->remote_host);
         if (ret)
-                goto out;
-
-        rpc = cli_quotad_clnt_init (THIS, rpc_opts);
-        if (!rpc)
-                goto out;
+            goto out;
 
-        global_quotad_rpc = rpc;
-out:
-        if (ret) {
-                if (rpc_opts)
-                        dict_unref(rpc_opts);
-        }
-        return rpc;
-}
+        if (state->remote_port)
+            port = state->remote_port;
 
-struct rpc_clnt *
-cli_rpc_init (struct cli_state *state)
-{
-        struct rpc_clnt         *rpc = NULL;
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-        int                     port = CLI_GLUSTERD_PORT;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        cli_rpc_prog = &cli_prog;
-        options = dict_new ();
-        if (!options)
-                goto out;
+        ret = dict_set_int32(options, "remote-port", port);
+        if (ret)
+            goto out;
 
-        /* Connect to glusterd using the specified method, giving preference
-         * to a unix socket connection.  If nothing is specified, connect to
-         * the default glusterd socket.
-         */
-        if (state->glusterd_sock) {
-                gf_log ("cli", GF_LOG_INFO, "Connecting to glusterd using "
-                        "sockfile %s", state->glusterd_sock);
-                ret = rpc_transport_unix_options_build (&options,
-                                                        state->glusterd_sock,
-                                                        0);
-                if (ret)
-                        goto out;
-        }
-        else if (state->remote_host) {
-                gf_log ("cli", GF_LOG_INFO, "Connecting to remote glusterd at "
-                        "%s", state->remote_host);
-                ret = dict_set_str (options, "remote-host", state->remote_host);
-                if (ret)
-                        goto out;
-
-                if (state->remote_port)
-                        port = state->remote_port;
-
-                ret = dict_set_int32 (options, "remote-port", port);
-                if (ret)
-                        goto out;
-
-                ret = dict_set_str (options, "transport.address-family",
-                                    "inet");
-                if (ret)
-                        goto out;
-        }
-        else {
-                gf_log ("cli", GF_LOG_DEBUG, "Connecting to glusterd using "
-                        "default socket");
-                ret = rpc_transport_unix_options_build
-                        (&options, DEFAULT_GLUSTERD_SOCKFILE, 0);
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(options, "transport.address-family", addr_family);
+        if (ret)
+            goto out;
+    } else {
+        gf_log("cli", GF_LOG_DEBUG,
+               "Connecting to glusterd using "
+               "default socket");
+        ret = rpc_transport_unix_options_build(options,
+                                               DEFAULT_GLUSTERD_SOCKFILE, 0);
+        if (ret)
+            goto out;
+    }
 
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto out;
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
 
-        ret = rpc_clnt_register_notify (rpc, cli_rpc_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
+    ret = rpc_clnt_register_notify(rpc, cli_rpc_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
 
-        ret = rpc_clnt_start (rpc);
+    ret = rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
-        return rpc;
+    if (options)
+        dict_unref(options);
+
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
+    return rpc;
 }
 
 cli_local_t *
-cli_local_get ()
+cli_local_get()
 {
-        cli_local_t     *local = NULL;
+    cli_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), cli_mt_cli_local_t);
-        LOCK_INIT (&local->lock);
-        INIT_LIST_HEAD (&local->dict_list);
+    local = GF_CALLOC(1, sizeof(*local), cli_mt_cli_local_t);
+    LOCK_INIT(&local->lock);
+    INIT_LIST_HEAD(&local->dict_list);
 
-        return local;
+    return local;
 }
 
 void
-cli_local_wipe (cli_local_t *local)
+cli_local_wipe(cli_local_t *local)
 {
-        if (local) {
-                GF_FREE (local->get_vol.volname);
-                if (local->dict)
-                        dict_unref (local->dict);
-                GF_FREE (local);
-        }
-
-        return;
+    if (local) {
+        GF_FREE(local->get_vol.volname);
+        if (local->dict)
+            dict_unref(local->dict);
+        GF_FREE(local);
+    }
+
+    return;
 }
 
 struct cli_state *global_state;
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        struct cli_state   state = {0, };
-        int                ret = -1;
-        glusterfs_ctx_t   *ctx = NULL;
+    struct cli_state state = {
+        0,
+    };
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                return ENOMEM;
+    mem_pools_init();
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        return ENOMEM;
 
 #ifdef DEBUG
-        gf_mem_acct_enable_set (ctx);
+    gf_mem_acct_enable_set(ctx);
 #endif
 
-        ret = glusterfs_globals_init (ctx);
-        if (ret)
-                return ret;
-
-	THIS->ctx = ctx;
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        return ret;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto out;
+    THIS->ctx = ctx;
 
-        cli_default_conn_timeout = 120;
-        cli_ten_minutes_timeout = 600;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = cli_state_init (&state);
-        if (ret)
-                goto out;
+    cli_default_conn_timeout = 120;
+    cli_ten_minutes_timeout = 600;
 
-        state.ctx = ctx;
-        global_state = &state;
+    ret = cli_state_init(&state);
+    if (ret)
+        goto out;
 
-        ret = parse_cmdline (argc, argv, &state);
-        if (ret)
-                goto out;
+    state.ctx = ctx;
+    global_state = &state;
 
-        ret = logging_init (ctx, &state);
-        if (ret)
-                goto out;
+    ret = parse_cmdline(argc, argv, &state);
+    if (ret)
+        goto out;
 
-        gf_log ("cli", GF_LOG_INFO, "Started running %s with version %s",
-                argv[0], PACKAGE_VERSION);
+    ret = logging_init(ctx, &state);
+    if (ret)
+        goto out;
 
-        global_rpc = cli_rpc_init (&state);
-        if (!global_rpc)
-                goto out;
+    gf_log("cli", GF_LOG_INFO, "Started running %s with version %s", argv[0],
+           PACKAGE_VERSION);
 
-        global_quotad_rpc = cli_quotad_clnt_rpc_init ();
-        if (!global_quotad_rpc)
-                goto out;
+    global_rpc = cli_rpc_init(&state);
+    if (!global_rpc)
+        goto out;
 
-        ret = cli_cmds_register (&state);
-        if (ret)
-                goto out;
+    global_quotad_rpc = cli_quotad_clnt_rpc_init();
+    if (!global_quotad_rpc)
+        goto out;
 
-        ret = cli_cmd_cond_init ();
-        if (ret)
-                goto out;
+    ret = cli_cmds_register(&state);
+    if (ret)
+        goto out;
 
-        ret = cli_input_init (&state);
-        if (ret)
-                goto out;
+    ret = cli_input_init(&state);
+    if (ret)
+        goto out;
 
-        ret = event_dispatch (ctx->event_pool);
+    ret = gf_event_dispatch(ctx->event_pool);
 
 out:
-//        glusterfs_ctx_destroy (ctx);
+    //        glusterfs_ctx_destroy (ctx);
 
-        return ret;
+    mem_pools_fini();
+
+    return ret;
 }
 
 void
-cli_print_line (int len)
+cli_print_line(int len)
 {
-        GF_ASSERT (len > 0);
+    GF_ASSERT(len > 0);
 
-        while (len--)
-                printf ("-");
+    while (len--)
+        printf("-");
 
-        printf ("\n");
+    printf("\n");
 }
 
 void
-print_quota_list_header (int type)
+print_quota_list_header(int type)
 {
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                cli_out ("                  Path                   Hard-limit "
-                         " Soft-limit      Used  Available  Soft-limit "
-                         "exceeded? Hard-limit exceeded?");
-                cli_out ("-----------------------------------------------------"
-                         "-----------------------------------------------------"
-                         "---------------------");
-        } else {
-                cli_out ("                  Path                   Hard-limit  "
-                         " Soft-limit      Files       Dirs     Available  "
-                         "Soft-limit exceeded? Hard-limit exceeded?");
-                cli_out ("-----------------------------------------------------"
-                         "-----------------------------------------------------"
-                         "-------------------------------------");
-        }
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        cli_out(
+            "                  Path                   Hard-limit "
+            " Soft-limit      Used  Available  Soft-limit "
+            "exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "---------------------");
+    } else {
+        cli_out(
+            "                  Path                   Hard-limit  "
+            " Soft-limit      Files       Dirs     Available  "
+            "Soft-limit exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "-------------------------------------");
+    }
 }
 
 void
-print_quota_list_empty (char *path, int type)
+print_quota_list_empty(char *path, int type)
 {
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                cli_out ("%-40s %7s %9s %10s %7s %15s %20s", path,
-                         "N/A", "N/A", "N/A", "N/A", "N/A", "N/A");
-        else
-                cli_out ("%-40s %9s %9s %12s %10s %10s %15s %20s", path,
-                         "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "N/A");
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        cli_out("%-40s %7s %9s %10s %7s %15s %20s", path, "N/A", "N/A", "N/A",
+                "N/A", "N/A", "N/A");
+    else
+        cli_out("%-40s %9s %9s %12s %10s %10s %15s %20s", path, "N/A", "N/A",
+                "N/A", "N/A", "N/A", "N/A", "N/A");
 }
-
diff --git a/cli/src/cli.h b/cli/src/cli.h
index ba0d845381a..c0d933e8f8a 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -11,60 +11,64 @@
 #define __CLI_H__
 
 #include "rpc-clnt.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "protocol-common.h"
-#include "logging.h"
-#include "quota-common-utils.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include "cli1-xdr.h"
+#include "gd-common-utils.h"
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 #endif
 
-#define DEFAULT_EVENT_POOL_SIZE            16384
-#define CLI_GLUSTERD_PORT                  24007
-#define DEFAULT_CLI_LOG_FILE_DIRECTORY     DATADIR "/log/glusterfs"
-#define CLI_VOL_STATUS_BRICK_LEN              43
-#define CLI_TAB_LENGTH                         8
-#define CLI_BRICK_STATUS_LINE_LEN             78
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define CLI_GLUSTERD_PORT 24007
+#define DEFAULT_CLI_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
+#define CLI_VOL_STATUS_BRICK_LEN 43
+#define CLI_TAB_LENGTH 8
+#define CLI_BRICK_STATUS_LINE_LEN 78
 
 /* Geo-rep command positional arguments' index  */
-#define GEO_REP_CMD_INDEX                      1
-#define GEO_REP_CMD_CONFIG_INDEX               4
+#define GEO_REP_CMD_INDEX 1
+#define GEO_REP_CMD_CONFIG_INDEX 4
 
 enum argp_option_keys {
-	ARGP_DEBUG_KEY = 133,
-	ARGP_PORT_KEY = 'p',
+    ARGP_DEBUG_KEY = 133,
+    ARGP_PORT_KEY = 'p',
 };
 
-int cli_default_conn_timeout;
-int cli_ten_minutes_timeout;
+extern int cli_default_conn_timeout;
+extern int cli_ten_minutes_timeout;
 
 typedef enum {
-        COLD_BRICK_COUNT,
-        COLD_TYPE,
-        COLD_DIST_COUNT,
-        COLD_REPLICA_COUNT,
-        COLD_ARBITER_COUNT,
-        COLD_DISPERSE_COUNT,
-        COLD_REDUNDANCY_COUNT,
-        HOT_BRICK_COUNT,
-        HOT_TYPE,
-        HOT_REPLICA_COUNT,
-        MAX
+    COLD_BRICK_COUNT,
+    COLD_TYPE,
+    COLD_DIST_COUNT,
+    COLD_REPLICA_COUNT,
+    COLD_ARBITER_COUNT,
+    COLD_DISPERSE_COUNT,
+    COLD_REDUNDANCY_COUNT,
+    HOT_BRICK_COUNT,
+    HOT_TYPE,
+    HOT_REPLICA_COUNT,
+    MAX
 } values;
 
-#define GLUSTER_MODE_SCRIPT    (1 << 0)
+#define GLUSTER_MODE_SCRIPT (1 << 0)
 #define GLUSTER_MODE_ERR_FATAL (1 << 1)
-#define GLUSTER_MODE_XML       (1 << 2)
-#define GLUSTER_MODE_WIGNORE   (1 << 3)
+#define GLUSTER_MODE_XML (1 << 2)
+#define GLUSTER_MODE_WIGNORE (1 << 3)
+#define GLUSTER_MODE_WIGNORE_PARTITION (1 << 4)
+#define GLUSTER_MODE_GLFSHEAL_NOLOG (1 << 5)
 
-
-#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
-        snprintf (abspath, sizeof (abspath)-1,                          \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path)             \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list%s", volname, path); \
+    } while (0)
 
 struct cli_state;
 struct cli_cmd_word;
@@ -74,423 +78,439 @@ struct cli_cmd;
 extern char *cli_vol_status_str[];
 extern char *cli_vol_task_status_str[];
 
-typedef int (cli_cmd_cbk_t)(struct cli_state *state,
-                            struct cli_cmd_word *word,
-                            const char **words,
-                            int wordcount);
-typedef void (cli_cmd_reg_cbk_t)( struct cli_cmd *this);
+typedef int(cli_cmd_cbk_t)(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount);
+typedef void(cli_cmd_reg_cbk_t)(struct cli_cmd *this);
 
-typedef int (cli_cmd_match_t)(struct cli_cmd_word *word);
-typedef int (cli_cmd_filler_t)(struct cli_cmd_word *word);
+typedef int(cli_cmd_match_t)(struct cli_cmd_word *word);
+typedef int(cli_cmd_filler_t)(struct cli_cmd_word *word);
 
 struct cli_cmd_word {
-        struct cli_cmd_tree   *tree;
-        const char            *word;
-        cli_cmd_filler_t      *filler;
-        cli_cmd_match_t       *match;
-        cli_cmd_cbk_t         *cbkfn;
-        const char            *desc;
-        const char            *pattern;
-        int                    nextwords_cnt;
-        struct cli_cmd_word  **nextwords;
+    struct cli_cmd_tree *tree;
+    const char *word;
+    cli_cmd_filler_t *filler;
+    cli_cmd_match_t *match;
+    cli_cmd_cbk_t *cbkfn;
+    const char *desc;
+    const char *pattern;
+    int nextwords_cnt;
+    struct cli_cmd_word **nextwords;
 };
 
-
 struct cli_cmd_tree {
-        struct cli_state      *state;
-        struct cli_cmd_word    root;
+    struct cli_state *state;
+    struct cli_cmd_word root;
 };
 
-
 struct cli_state {
-        int                   argc;
-        char                **argv;
+    int argc;
+    char **argv;
 
-        char                  debug;
+    char debug;
 
-        /* for events dispatching */
-        glusterfs_ctx_t      *ctx;
+    /* for events dispatching */
+    glusterfs_ctx_t *ctx;
 
-        /* registry of known commands */
-        struct cli_cmd_tree   tree;
+    /* registry of known commands */
+    struct cli_cmd_tree tree;
 
-        /* the thread which "executes" the command in non-interactive mode */
-        /* also the thread which reads from stdin in non-readline mode */
-        pthread_t             input;
+    /* the thread which "executes" the command in non-interactive mode */
+    /* also the thread which reads from stdin in non-readline mode */
+    pthread_t input;
 
-        /* terminal I/O */
-        const char           *prompt;
-        int                   rl_enabled;
-        int                   rl_async;
-        int                   rl_processing;
+    /* terminal I/O */
+    const char *prompt;
+    int rl_enabled;
+    int rl_async;
+    int rl_processing;
 
-        /* autocompletion state */
-        char                **matches;
-        char                **matchesp;
+    /* autocompletion state */
+    char **matches;
+    char **matchesp;
 
-        char                 *remote_host;
-        int                   remote_port;
-        int                   mode;
-        int                   await_connected;
+    char *remote_host;
+    int remote_port;
+    int mode;
+    int await_connected;
 
-        char                 *log_file;
-        gf_loglevel_t         log_level;
+    char *log_file;
+    gf_loglevel_t log_level;
 
-        char                 *glusterd_sock;
+    char *glusterd_sock;
+    char *address_family;
 };
 
 struct cli_local {
-        struct {
-                char    *volname;
-                int     flags;
-        } get_vol;
-
-        dict_t          *dict;
-        const char      **words;
-        /* Marker for volume status all */
-        gf_boolean_t    all;
+    struct {
+        char *volname;
+        int flags;
+    } get_vol;
+
+    dict_t *dict;
+    const char **words;
+    /* Marker for volume status all */
+    gf_boolean_t all;
 #if (HAVE_LIB_XML)
-        xmlTextWriterPtr        writer;
-        xmlDocPtr               doc;
-        int                     vol_count;
+    xmlTextWriterPtr writer;
+    xmlDocPtr doc;
+    int vol_count;
 #endif
-        gf_lock_t               lock;
-        struct list_head        dict_list;
+    gf_lock_t lock;
+    struct list_head dict_list;
 };
 
 struct cli_volume_status {
-        int            port;
-        int            rdma_port;
-        int            online;
-        uint64_t       block_size;
-        uint64_t       total_inodes;
-        uint64_t       free_inodes;
-        char          *brick;
-        char          *pid_str;
-        char          *free;
-        char          *total;
-#ifdef GF_LINUX_HOST_OS
-        char          *fs_name;
-        char          *mount_options;
-        char          *device;
-        char          *inode_size;
-#endif
+    int port;
+    int rdma_port;
+    int online;
+    uint64_t block_size;
+    uint64_t total_inodes;
+    uint64_t free_inodes;
+    char *brick;
+    char *pid_str;
+    char *free;
+    char *total;
+    char *fs_name;
+    char *mount_options;
+    char *device;
+    char *inode_size;
 };
 
 struct snap_config_opt_vals_ {
-        char           *op_name;
-        char           *question;
+    char *op_name;
+    char *question;
 };
 
 typedef struct cli_volume_status cli_volume_status_t;
 
 typedef struct cli_local cli_local_t;
 
-typedef ssize_t (*cli_serialize_t) (struct iovec outmsg, void *args);
+typedef ssize_t (*cli_serialize_t)(struct iovec outmsg, void *args);
 
 extern struct cli_state *global_state; /* use only in readline callback */
 
-typedef const char *(*cli_selector_t) (void *wcon);
+extern struct rpc_clnt *global_quotad_rpc;
 
-char *get_struct_variable (int mem_num, gf_gsync_status_t *sts_val);
+extern struct rpc_clnt *global_rpc;
 
-void *cli_getunamb (const char *tok, void **choices, cli_selector_t sel);
+extern rpc_clnt_prog_t *cli_rpc_prog;
 
-int cli_cmd_register (struct cli_cmd_tree *tree, struct cli_cmd *cmd);
-int cli_cmds_register (struct cli_state *state);
+typedef const char *(*cli_selector_t)(void *wcon);
 
-int cli_input_init (struct cli_state *state);
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val);
 
-int cli_cmd_process (struct cli_state *state, int argc, char *argv[]);
-int cli_cmd_process_line (struct cli_state *state, const char *line);
+void *
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel);
 
-int cli_rl_enable (struct cli_state *state);
-int cli_rl_out (struct cli_state *state, const char *fmt, va_list ap);
-int cli_rl_err (struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd);
+int
+cli_cmds_register(struct cli_state *state);
 
-int cli_usage_out (const char *usage);
+int
+cli_input_init(struct cli_state *state);
 
-int _cli_out (const char *fmt, ...);
-int _cli_err (const char *fmt, ...);
+int
+cli_cmd_process(struct cli_state *state, int argc, char *argv[]);
+int
+cli_cmd_process_line(struct cli_state *state, const char *line);
 
-#define cli_out(fmt...) do {                       \
-                FMT_WARN (fmt);                    \
-                                                   \
-                _cli_out(fmt);                     \
-                                                   \
-        } while (0)
+int
+cli_rl_enable(struct cli_state *state);
+int
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap);
 
-#define cli_err(fmt...) do {                       \
-                FMT_WARN (fmt);                    \
-                                                   \
-                _cli_err(fmt);                     \
-                                                   \
-        } while (0)
+int
+cli_usage_out(const char *usage);
 
 int
-cli_submit_request (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+_cli_out(const char *fmt, ...);
+int
+_cli_err(const char *fmt, ...);
 
-int32_t
-cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
-                             int wordcount, dict_t **options);
+#define cli_out(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_out(fmt);                                                         \
+                                                                               \
+    } while (0)
 
-int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **opt);
+#define cli_err(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_err(fmt);                                                         \
+                                                                               \
+    } while (0)
+
+#define usage()                                                                \
+    do {                                                                       \
+        cli_out(                                                               \
+            " Usage: gluster [options] <help> <peer>"                          \
+            " <pool> <volume>\n"                                               \
+            " Options:\n"                                                      \
+            " --help  Shows the help information\n"                            \
+            " --version  Shows the version\n"                                  \
+            " --print-logdir  Shows the log directory\n"                       \
+            " --print-statedumpdir Shows the state dump directory\n");         \
+                                                                               \
+    } while (0)
+
+int
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **bricks);
 
 int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_inode_quota_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **opt, char **errstr);
 
 int32_t
-cli_cmd_bitrot_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_volume_set_parse (struct cli_state *state, const char **words,
-                          int wordcount, dict_t **options, char **op_errstr);
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **opt);
+
 int32_t
-cli_cmd_ganesha_parse (struct cli_state *state, const char **words,
-                       int wordcount, dict_t **options, char **op_errstr);
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_get_state_parse (struct cli_state *state, const char **words,
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
                          int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options, int *type);
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
-                                  dict_t **options, int *question);
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_tier_parse (const char **words, int wordcount,
-                           dict_t **options);
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *type);
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question);
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *command);
 
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
                                    dict_t **options);
 
 int32_t
-cli_cmd_volume_reset_brick_parse (const char **words, int wordcount,
-                                  dict_t **options);
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options);
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options);
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options);
 int32_t
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
-                                  dict_t **options);
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options);
 
-cli_local_t * cli_local_get ();
+cli_local_t *
+cli_local_get();
 
 void
-cli_local_wipe (cli_local_t *local);
+cli_local_wipe(cli_local_t *local);
+
+gf_boolean_t
+cli_cmd_connected();
 
 int32_t
-cli_cmd_await_connected ();
+cli_cmd_await_connected(unsigned timeout);
 
 int32_t
-cli_cmd_broadcast_connected ();
+cli_cmd_broadcast_connected(gf_boolean_t status);
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data);
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data);
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options);
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options);
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options);
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int wordcount,
-                         dict_t **options);
+cli_cmd_log_level_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options);
+cli_cmd_volume_status_parse(const char **words, int wordcount,
+                            dict_t **options);
 
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options);
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options);
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options);
+cli_cmd_volume_defrag_parse(const char **words, int wordcount,
+                            dict_t **options);
 
 int
-cli_print_brick_status (cli_volume_status_t *status);
+cli_print_brick_status(cli_volume_status_t *status);
 
 void
-cli_print_detailed_status (cli_volume_status_t *status);
+cli_print_detailed_status(cli_volume_status_t *status);
 
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status);
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status);
 
 void
-cli_print_line (int len);
+cli_print_line(int len);
 
 int
-cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
-                    char *op_errstr);
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr);
 
 int
-cli_xml_output_dict (char *op, dict_t *dict, int op_ret, int op_errno,
-                     char *op_errstr);
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr);
 
 int
-cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
-                        char *op_errstr);
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr);
 
 int
-cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr);
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 int
-cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                 char *op_errstr);
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr);
 
 int
-cli_xml_output_vol_status_end (cli_local_t *local);
+cli_xml_output_vol_status_end(cli_local_t *local);
 
 int
-cli_xml_output_vol_status (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
-                         char *op_errstr);
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
 
 int
-cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
-                               char *op_errstr);
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr);
 
 int
-cli_xml_output_vol_info_end (cli_local_t *local);
+cli_xml_output_vol_info_end(cli_local_t *local);
 
 int
-cli_xml_output_vol_info (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_vol_quota_limit_list_begin (cli_local_t *local, int op_ret,
-                                           int op_errno, char *op_errstr);
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr);
 int
-cli_xml_output_vol_quota_limit_list_end (cli_local_t *local);
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local);
 
 int
-cli_quota_list_xml_error (cli_local_t *local, char *path,
-                          char *errstr);
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr);
 
 int
-cli_quota_xml_output (cli_local_t *local, char *path, int64_t hl_str,
-                      char *sl_final, int64_t sl_num, int64_t used,
-                      int64_t avail, char *sl, char *hl,
-                      gf_boolean_t limit_set);
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set);
 
 int
-cli_quota_object_xml_output (cli_local_t *local, char *path, char *sl_str,
-                             int64_t sl_val, quota_limits_t *limits,
-                             quota_meta_t *used_space, int64_t avail,
-                             char *sl, char *hl, gf_boolean_t limit_set);
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set);
 
 int
-cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr);
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 int
-cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
-                              int op_errno, char *op_errstr);
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr);
 
 int
-cli_xml_output_vol_remove_brick_detach_tier (gf_boolean_t status_op,
-                                             dict_t *dict, int op_ret,
-                                             int op_errno, char *op_errstr,
-                                             const char *op);
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op);
 
 int
-cli_xml_output_vol_replace_brick (dict_t *dict, int op_ret,
-                                  int op_errno, char *op_errstr);
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr);
 
 int
-cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr);
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr);
 
 int
-cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
-                               char *op_errstr);
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr);
 
 int
-cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
-                          char *op_errstr);
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr);
 int
-cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
-                       char *op_errstr);
-int
-cli_xml_snapshot_delete (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict,
-                        gf_cli_rsp *rsp);
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp);
+
 int
-cli_xml_snapshot_begin_composite_op (cli_local_t *local);
+cli_xml_snapshot_begin_composite_op(cli_local_t *local);
 
 int
-cli_xml_snapshot_end_composite_op (cli_local_t *local);
+cli_xml_snapshot_end_composite_op(cli_local_t *local);
 
 int
-cli_xml_output_snap_delete_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr);
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
 int
-cli_xml_output_snap_delete_end (cli_local_t *local);
+cli_xml_output_snap_delete_end(cli_local_t *local);
 
 int
-cli_xml_output_snap_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr);
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
 int
-cli_xml_output_snap_status_end (cli_local_t *local);
+cli_xml_output_snap_status_end(cli_local_t *local);
 int
-cli_xml_output_snapshot (int cmd_type, dict_t *dict, int op_ret,
-                         int op_errno, char *op_errstr);
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
 int
-cli_xml_snapshot_status_single_snap (cli_local_t *local, dict_t *dict,
-                                     char *key);
-char *
-is_server_debug_xlator (void *myframe);
-
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict,
+                                    char *key);
 int32_t
-cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
-                        struct cli_state *state);
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state);
 
 int
-cli_xml_output_vol_getopts (dict_t *dict, int op_ret, int op_errno,
-                             char *op_errstr);
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 void
-print_quota_list_header (int type);
+print_quota_list_header(int type);
 
 void
-print_quota_list_empty (char *path, int type);
+print_quota_list_empty(char *path, int type);
 
 int
-gf_gsync_status_t_comparator (const void *p, const void *q);
+gf_gsync_status_t_comparator(const void *p, const void *q);
 #endif /* __CLI_H__ */
diff --git a/cli/src/input.c b/cli/src/input.c
index 10fc39cd6b4..5ac1a20edb1 100644
--- a/cli/src/input.c
+++ b/cli/src/input.c
@@ -19,74 +19,75 @@
 #define CMDBUFSIZ 1024
 
 void *
-cli_batch (void *d)
+cli_batch(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = d;
+    state = d;
 
-        ret = cli_cmd_process (state, state->argc, state->argv);
+    ret = cli_cmd_process(state, state->argc, state->argv);
 
-        gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-        exit (-ret);
+    gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+    exit(-ret);
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_input (void *d)
+cli_input(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
-        char              cmdbuf[CMDBUFSIZ];
-        char             *cmd = NULL;
-        size_t            len = 0;
-
-        state = d;
-
-        for (;;) {
-                printf ("%s", state->prompt);
-
-                cmd = fgets (cmdbuf, CMDBUFSIZ, stdin);
-                if (!cmd)
-                        break;
-                len = strlen(cmd);
-                if (len > 0 && cmd[len - 1] == '\n') //strip trailing \n
-                        cmd[len - 1] = '\0';
-                ret = cli_cmd_process_line (state, cmd);
-                if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
-                        break;
-        }
-
-        exit (-ret);
-
-        return NULL;
+    struct cli_state *state = NULL;
+    int ret = 0;
+    char cmdbuf[CMDBUFSIZ];
+    char *cmd = NULL;
+    size_t len = 0;
+
+    state = d;
+
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        printf("%s", state->prompt);
+
+        cmd = fgets(cmdbuf, CMDBUFSIZ, stdin);
+        if (!cmd)
+            break;
+        len = strlen(cmd);
+        if (len > 0 && cmd[len - 1] == '\n')  // strip trailing \n
+            cmd[len - 1] = '\0';
+        ret = cli_cmd_process_line(state, cmd);
+        if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
+            break;
+    }
+
+    exit(-ret);
+
+    return NULL;
 }
 
-
 int
-cli_input_init (struct cli_state *state)
+cli_input_init(struct cli_state *state)
 {
-        int  ret = 0;
+    int ret = 0;
 
-        if (state->argc) {
-                ret = pthread_create (&state->input, NULL, cli_batch, state);
-                return ret;
-        }
+    if (state->argc) {
+        ret = pthread_create(&state->input, NULL, cli_batch, state);
+        return ret;
+    }
 
-        if (isatty (STDIN_FILENO)) {
-                state->prompt = "gluster> ";
+    if (isatty(STDIN_FILENO)) {
+        state->prompt = "gluster> ";
 
-                cli_rl_enable (state);
-        } else {
-                state->prompt = "";
-                state->mode |= GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
-        }
+        cli_rl_enable(state);
+    } else {
+        state->prompt = "";
+        state->mode |= GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
+    }
 
-        if (!state->rl_enabled)
-                ret = pthread_create (&state->input, NULL, cli_input, state);
+    if (!state->rl_enabled)
+        ret = pthread_create(&state->input, NULL, cli_input, state);
 
-        return ret;
+    return ret;
 }
diff --git a/cli/src/registry.c b/cli/src/registry.c
index 1adf7d6e572..85f7686ade1 100644
--- a/cli/src/registry.c
+++ b/cli/src/registry.c
@@ -15,20 +15,18 @@
 #include "cli.h"
 #include "cli-cmd.h"
 
-
 static int
-__is_spc (int ch)
+__is_spc(int ch)
 {
-        if (ch == ' ')
-                return 1;
-        return 0;
+    if (ch == ' ')
+        return 1;
+    return 0;
 }
 
-
 static int
-__is_div (int ch)
+__is_div(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
         case ')':
         case '<':
@@ -38,369 +36,356 @@ __is_div (int ch)
         case '{':
         case '}':
         case '|':
-                return 1;
-        }
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 static int
-__is_word (const char *word)
+__is_word(const char *word)
 {
-        return (!__is_div (*word) && !__is_spc (*word));
+    return (!__is_div(*word) && !__is_spc(*word));
 }
 
-
 int
-counter_char (int ch)
+counter_char(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
-                return ')';
+            return ')';
         case '<':
-                return '>';
+            return '>';
         case '[':
-                return ']';
+            return ']';
         case '{':
-                return '}';
-        }
+            return '}';
+    }
 
-        return -1;
+    return -1;
 }
 
-
 const char *
-__is_template_balanced (const char *template)
+__is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
-        int   ch = 0;
-
-        trav = template;
-
-        while (*trav) {
-                ch = *trav;
-
-                switch (ch) {
-                case '<':
-                case '(':
-                case '[':
-                        trav = __is_template_balanced (trav+1);
-                        if (!trav)
-                                return NULL;
-                        if (*trav != counter_char (ch))
-                                return NULL;
-                        break;
-                case '>':
-                case ')':
-                case ']':
-                        return trav;
-                }
+    const char *trav = NULL;
+    int ch = 0;
 
-                trav++;
+    trav = template;
+
+    while (*trav) {
+        ch = *trav;
+
+        switch (ch) {
+            case '<':
+            case '(':
+            case '[':
+                trav = __is_template_balanced(trav + 1);
+                if (!trav)
+                    return NULL;
+                if (*trav != counter_char(ch))
+                    return NULL;
+                break;
+            case '>':
+            case ')':
+            case ']':
+                return trav;
         }
 
-        return trav;
-}
+        trav++;
+    }
 
+    return trav;
+}
 
 int
-is_template_balanced (const char *template)
+is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
+    const char *trav = NULL;
 
-        trav = __is_template_balanced (template);
-        if (!trav || *trav)
-                return -1;
+    trav = __is_template_balanced(template);
+    if (!trav || *trav)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
-
 int
-cli_cmd_token_count (const char *template)
+cli_cmd_token_count(const char *template)
 {
-        int         count = 0;
-        const char *trav = NULL;
-        int         is_alnum = 0;
-
-        for (trav = template; *trav; trav++) {
-                switch (*trav) {
-                case '<':
-                case '>':
-                case '(':
-                case ')':
-                case '[':
-                case ']':
-                case '{':
-                case '}':
-                case '|':
-                        count++;
-                        /* fall through */
-                case ' ':
-                        is_alnum = 0;
-                        break;
-                default:
-                        if (!is_alnum) {
-                                is_alnum = 1;
-                                count++;
-                        }
+    int count = 0;
+    const char *trav = NULL;
+    int is_alnum = 0;
+
+    for (trav = template; *trav; trav++) {
+        switch (*trav) {
+            case '<':
+            case '>':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '|':
+                count++;
+                /* fall through */
+            case ' ':
+                is_alnum = 0;
+                break;
+            default:
+                if (!is_alnum) {
+                    is_alnum = 1;
+                    count++;
                 }
         }
+    }
 
-        return count + 1;
+    return count + 1;
 }
 
-
 void
-cli_cmd_tokens_destroy (char **tokens)
+cli_cmd_tokens_destroy(char **tokens)
 {
-        char **tokenp = NULL;
+    char **tokenp = NULL;
 
-        if (!tokens)
-                return;
+    if (!tokens)
+        return;
 
-        tokenp = tokens;
-        while (*tokenp) {
-                free (*tokenp);
-                tokenp++;
-        }
+    tokenp = tokens;
+    while (*tokenp) {
+        free(*tokenp);
+        tokenp++;
+    }
 
-        free (tokens);
+    free(tokens);
 }
 
-
 int
-cli_cmd_tokens_fill (char **tokens, const char *template)
+cli_cmd_tokens_fill(char **tokens, const char *template)
 {
-        const char  *trav = NULL;
-        char       **tokenp = NULL;
-        char        *token = NULL;
-        int          ret = 0;
-        int          ch = 0;
+    const char *trav = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    int ret = 0;
+    int ch = 0;
 
-        tokenp = tokens;
+    tokenp = tokens;
 
-        for (trav = template; *trav; trav++) {
-                ch = *trav;
+    for (trav = template; *trav; trav++) {
+        ch = *trav;
 
-                if (__is_spc (ch))
-                        continue;
+        if (__is_spc(ch))
+            continue;
 
-                if (__is_div (ch)) {
-                        token = calloc (2, 1);
-                        if (!token)
-                                return -1;
-                        token[0] = ch;
+        if (__is_div(ch)) {
+            token = calloc(2, 1);
+            if (!token)
+                return -1;
+            token[0] = ch;
 
-                        *tokenp = token;
-                        tokenp++;
+            *tokenp = token;
+            tokenp++;
 
-                        continue;
-                }
+            continue;
+        }
 
-                token = strdup (trav);
-                *tokenp = token;
-                tokenp++;
+        token = strdup(trav);
+        *tokenp = token;
+        tokenp++;
 
-                for (token++; *token; token++) {
-                        if (__is_spc (*token) || __is_div (*token)) {
-                                *token = 0;
-                                break;
-                        }
-                        trav++;
-                }
+        for (token++; *token; token++) {
+            if (__is_spc(*token) || __is_div(*token)) {
+                *token = 0;
+                break;
+            }
+            trav++;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 char **
-cli_cmd_tokenize (const char *template)
+cli_cmd_tokenize(const char *template)
 {
-        char **tokens = NULL;
-        int    ret = 0;
-        int    count = 0;
+    char **tokens = NULL;
+    int ret = 0;
+    int count = 0;
 
-        ret = is_template_balanced (template);
-        if (ret)
-                return NULL;
+    ret = is_template_balanced(template);
+    if (ret)
+        return NULL;
 
-        count = cli_cmd_token_count (template);
-        if (count <= 0)
-                return NULL;
+    count = cli_cmd_token_count(template);
+    if (count <= 0)
+        return NULL;
 
-        tokens = calloc (count + 1, sizeof (char *));
-        if (!tokens)
-                return NULL;
+    tokens = calloc(count + 1, sizeof(char *));
+    if (!tokens)
+        return NULL;
 
-        ret = cli_cmd_tokens_fill (tokens, template);
-        if (ret)
-                goto err;
+    ret = cli_cmd_tokens_fill(tokens, template);
+    if (ret)
+        goto err;
 
-        return tokens;
+    return tokens;
 err:
-        cli_cmd_tokens_destroy (tokens);
-        return NULL;
+    cli_cmd_tokens_destroy(tokens);
+    return NULL;
 }
 
 void *
-cli_getunamb (const char *tok, void **choices, cli_selector_t sel)
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel)
 {
-        void  **wcon = NULL;
-        char      *w = NULL;
-        unsigned  mn = 0;
-        void    *ret = NULL;
-
-        if (!choices || !tok || !*tok)
-                return NULL;
-
-        for (wcon = choices; *wcon; wcon++) {
-                w = strtail ((char *)sel (*wcon), tok);
-                if (!w)
-                        /* no match */
-                        continue;
-                if (!*w)
-                        /* exact match */
-                        return *wcon;
-
-                ret = *wcon;
-                mn++;
-        }
+    void **wcon = NULL;
+    char *w = NULL;
+    unsigned mn = 0;
+    void *ret = NULL;
 
-#ifdef FORCE_MATCH_EXACT
+    if (!choices || !tok || !*tok)
         return NULL;
+
+    for (wcon = choices; *wcon; wcon++) {
+        w = strtail((char *)sel(*wcon), tok);
+        if (!w)
+            /* no match */
+            continue;
+        if (!*w)
+            /* exact match */
+            return *wcon;
+
+        ret = *wcon;
+        mn++;
+    }
+
+#ifdef FORCE_MATCH_EXACT
+    return NULL;
 #else
-        return (mn == 1) ? ret : NULL;
+    return (mn == 1) ? ret : NULL;
 #endif
 }
 
 static const char *
-sel_cmd_word (void *wcon)
+sel_cmd_word(void *wcon)
 {
-        return ((struct cli_cmd_word *)wcon)->word;
+    return ((struct cli_cmd_word *)wcon)->word;
 }
 
 struct cli_cmd_word *
-cli_cmd_nextword (struct cli_cmd_word *word, const char *token)
+cli_cmd_nextword(struct cli_cmd_word *word, const char *token)
 {
-        return (struct cli_cmd_word *)cli_getunamb (token,
-                                                    (void **)word->nextwords,
-                                                    sel_cmd_word);
+    return (struct cli_cmd_word *)cli_getunamb(token, (void **)word->nextwords,
+                                               sel_cmd_word);
 }
 
-
 struct cli_cmd_word *
-cli_cmd_newword (struct cli_cmd_word *word, const char *token)
+cli_cmd_newword(struct cli_cmd_word *word, const char *token)
 {
-        struct cli_cmd_word **nextwords = NULL;
-        struct cli_cmd_word  *nextword = NULL;
+    struct cli_cmd_word **nextwords = NULL;
+    struct cli_cmd_word *nextword = NULL;
 
-        nextwords = realloc (word->nextwords,
-                             (word->nextwords_cnt + 2) * sizeof (*nextwords));
-        if (!nextwords)
-                return NULL;
+    nextwords = realloc(word->nextwords,
+                        (word->nextwords_cnt + 2) * sizeof(*nextwords));
+    if (!nextwords)
+        return NULL;
 
-        word->nextwords = nextwords;
+    word->nextwords = nextwords;
 
-        nextword = calloc (1, sizeof (*nextword));
-        if (!nextword)
-                return NULL;
+    nextword = calloc(1, sizeof(*nextword));
+    if (!nextword)
+        return NULL;
 
-        nextword->word = strdup (token);
-        if (!nextword->word) {
-                free (nextword);
-                return NULL;
-        }
+    nextword->word = strdup(token);
+    if (!nextword->word) {
+        free(nextword);
+        return NULL;
+    }
 
-        nextword->tree = word->tree;
-        nextwords[word->nextwords_cnt++] = nextword;
-        nextwords[word->nextwords_cnt] = NULL;
+    nextword->tree = word->tree;
+    nextwords[word->nextwords_cnt++] = nextword;
+    nextwords[word->nextwords_cnt] = NULL;
 
-        return nextword;
+    return nextword;
 }
 
-
 int
-cli_cmd_ingest (struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
-                const char *desc, const char *pattern)
+cli_cmd_ingest(struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
+               const char *desc, const char *pattern)
 {
-        int                    ret = 0;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-
-        word = &tree->root;
+    int ret = 0;
+    char **tokenp = NULL;
+    char *token = NULL;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
 
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!__is_word (token))
-                        break;
+    word = &tree->root;
 
-                next = cli_cmd_nextword (word, token);
-                if (!next)
-                        next = cli_cmd_newword (word, token);
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!__is_word(token))
+            break;
 
-                word = next;
-                if (!word)
-                        break;
-        }
+        next = cli_cmd_nextword(word, token);
+        if (!next)
+            next = cli_cmd_newword(word, token);
 
+        word = next;
         if (!word)
-                return -1;
+            break;
+    }
 
-        if (word->cbkfn) {
-                /* warning - command already registered */
-        }
+    if (!word)
+        return -1;
 
-        word->cbkfn = cbkfn;
-        word->desc  = desc;
-        word->pattern = pattern;
+    if (word->cbkfn) {
+        /* warning - command already registered */
+    }
 
-        /* end of static strings in command template */
+    word->cbkfn = cbkfn;
+    word->desc = desc;
+    word->pattern = pattern;
 
-        /* TODO: autocompletion beyond this point is just "nice to have" */
+    /* end of static strings in command template */
 
-        return ret;
-}
+    /* TODO: autocompletion beyond this point is just "nice to have" */
 
+    return ret;
+}
 
 int
-cli_cmd_register (struct cli_cmd_tree *tree, struct cli_cmd *cmd)
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd)
 {
-        char **tokens = NULL;
-        int    ret = 0;
+    char **tokens = NULL;
+    int ret = 0;
 
-        GF_ASSERT (cmd);
+    GF_ASSERT(cmd);
 
-        if (cmd->reg_cbk)
-                cmd->reg_cbk (cmd);
+    if (cmd->reg_cbk)
+        cmd->reg_cbk(cmd);
 
-        if (cmd->disable) {
-                ret = 0;
-                goto out;
-        }
+    if (cmd->disable) {
+        ret = 0;
+        goto out;
+    }
 
-        tokens = cli_cmd_tokenize (cmd->pattern);
-        if (!tokens) {
-                ret = -1;
-                goto out;
-        }
+    tokens = cli_cmd_tokenize(cmd->pattern);
+    if (!tokens) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_ingest (tree, tokens, cmd->cbk, cmd->desc, cmd->pattern);
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    ret = cli_cmd_ingest(tree, tokens, cmd->cbk, cmd->desc, cmd->pattern);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
-
diff --git a/configure.ac b/configure.ac
index 8aa07ffbeb9..e2d6fd66cec 100644
--- a/configure.ac
+++ b/configure.ac
@@ -13,7 +13,7 @@ AC_INIT([glusterfs],
 AC_SUBST([PACKAGE_RELEASE],
          [m4_esyscmd([build-aux/pkg-version --release])])
 
-AM_INIT_AUTOMAKE(tar-pax)
+AM_INIT_AUTOMAKE([tar-pax foreign])
 
 # Removes warnings when using automake 1.14 around (...but option 'subdir-objects' is disabled )
 #but libglusterfs fails to build with contrib (Then are not set up that way?)
@@ -21,30 +21,20 @@ AM_INIT_AUTOMAKE(tar-pax)
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
 
-if make --help 2>&1 | grep -q no-print-directory; then
-   AM_MAKEFLAGS="$AM_MAKEFLAGS --no-print-directory";
-fi
-
-if make --help 2>&1 | grep -q quiet; then
-   AM_MAKEFLAGS="$AM_MAKEFLAGS --quiet"
-fi
-
-if libtool --help 2>&1 | grep -q quiet; then
-   AM_LIBTOOLFLAGS="--quiet";
-fi
-
-AC_CONFIG_HEADERS([config.h])
+AC_CONFIG_HEADERS([config.h site.h])
 
 AC_CONFIG_FILES([Makefile
                 libglusterfs/Makefile
                 libglusterfs/src/Makefile
-                libglusterfs/src/gfdb/Makefile
+                libglusterd/Makefile
+                libglusterd/src/Makefile
                 geo-replication/src/peer_gsec_create
                 geo-replication/src/peer_mountbroker
                 geo-replication/src/peer_mountbroker.py
                 geo-replication/src/peer_georep-sshkey.py
                 extras/peer_add_secret_pub
                 geo-replication/syncdaemon/conf.py
+                geo-replication/gsyncd.conf
                 extras/snap_scheduler/conf.py
                 glusterfsd/Makefile
                 glusterfsd/src/Makefile
@@ -54,8 +44,6 @@ AC_CONFIG_FILES([Makefile
                 rpc/rpc-transport/Makefile
                 rpc/rpc-transport/socket/Makefile
                 rpc/rpc-transport/socket/src/Makefile
-                rpc/rpc-transport/rdma/Makefile
-                rpc/rpc-transport/rdma/src/Makefile
                 rpc/xdr/Makefile
                 rpc/xdr/src/Makefile
                 xlators/Makefile
@@ -70,13 +58,9 @@ AC_CONFIG_FILES([Makefile
                 xlators/storage/Makefile
                 xlators/storage/posix/Makefile
                 xlators/storage/posix/src/Makefile
-                xlators/storage/bd/Makefile
-                xlators/storage/bd/src/Makefile
                 xlators/cluster/Makefile
                 xlators/cluster/afr/Makefile
                 xlators/cluster/afr/src/Makefile
-                xlators/cluster/stripe/Makefile
-                xlators/cluster/stripe/src/Makefile
                 xlators/cluster/dht/Makefile
                 xlators/cluster/dht/src/Makefile
                 xlators/cluster/ec/Makefile
@@ -92,21 +76,23 @@ AC_CONFIG_FILES([Makefile
                 xlators/performance/io-threads/src/Makefile
                 xlators/performance/io-cache/Makefile
                 xlators/performance/io-cache/src/Makefile
-                xlators/performance/symlink-cache/Makefile
-                xlators/performance/symlink-cache/src/Makefile
                 xlators/performance/quick-read/Makefile
                 xlators/performance/quick-read/src/Makefile
                 xlators/performance/open-behind/Makefile
                 xlators/performance/open-behind/src/Makefile
                 xlators/performance/md-cache/Makefile
                 xlators/performance/md-cache/src/Makefile
-                xlators/performance/decompounder/Makefile
-                xlators/performance/decompounder/src/Makefile
+                xlators/performance/nl-cache/Makefile
+                xlators/performance/nl-cache/src/Makefile
                 xlators/debug/Makefile
+                xlators/debug/sink/Makefile
+                xlators/debug/sink/src/Makefile
                 xlators/debug/trace/Makefile
                 xlators/debug/trace/src/Makefile
                 xlators/debug/error-gen/Makefile
                 xlators/debug/error-gen/src/Makefile
+                xlators/debug/delay-gen/Makefile
+                xlators/debug/delay-gen/src/Makefile
                 xlators/debug/io-stats/Makefile
                 xlators/debug/io-stats/src/Makefile
                 xlators/protocol/Makefile
@@ -122,40 +108,34 @@ AC_CONFIG_FILES([Makefile
                 xlators/features/Makefile
                 xlators/features/arbiter/Makefile
                 xlators/features/arbiter/src/Makefile
-                xlators/experimental/fdl/Makefile
-                xlators/experimental/fdl/src/Makefile
+                xlators/features/thin-arbiter/Makefile
+                xlators/features/thin-arbiter/src/Makefile
                 xlators/features/changelog/Makefile
                 xlators/features/changelog/src/Makefile
                 xlators/features/changelog/lib/Makefile
                 xlators/features/changelog/lib/src/Makefile
-                xlators/features/changetimerecorder/Makefile
-                xlators/features/changetimerecorder/src/Makefile
-                xlators/features/glupy/Makefile
-                xlators/features/glupy/examples/Makefile
-                xlators/features/glupy/src/Makefile
-                xlators/features/glupy/src/setup.py
-                xlators/features/glupy/src/__init__.py
-                xlators/features/glupy/src/glupy/Makefile
                 xlators/features/locks/Makefile
                 xlators/features/locks/src/Makefile
                 xlators/features/quota/Makefile
                 xlators/features/quota/src/Makefile
                 xlators/features/marker/Makefile
                 xlators/features/marker/src/Makefile
+                xlators/features/selinux/Makefile
+                xlators/features/selinux/src/Makefile
+                xlators/features/sdfs/Makefile
+                xlators/features/sdfs/src/Makefile
                 xlators/features/read-only/Makefile
                 xlators/features/read-only/src/Makefile
                 xlators/features/compress/Makefile
                 xlators/features/compress/src/Makefile
-                xlators/features/mac-compat/Makefile
-                xlators/features/mac-compat/src/Makefile
+                xlators/features/namespace/Makefile
+                xlators/features/namespace/src/Makefile
                 xlators/features/quiesce/Makefile
                 xlators/features/quiesce/src/Makefile
                 xlators/features/barrier/Makefile
                 xlators/features/barrier/src/Makefile
                 xlators/features/index/Makefile
                 xlators/features/index/src/Makefile
-                xlators/features/protect/Makefile
-                xlators/features/protect/src/Makefile
                 xlators/features/gfid-access/Makefile
                 xlators/features/gfid-access/src/Makefile
                 xlators/features/trash/Makefile
@@ -174,14 +154,21 @@ AC_CONFIG_FILES([Makefile
                 xlators/features/bit-rot/src/bitd/Makefile
                 xlators/features/leases/Makefile
                 xlators/features/leases/src/Makefile
+                xlators/features/cloudsync/Makefile
+                xlators/features/cloudsync/src/Makefile
+                xlators/features/utime/Makefile
+                xlators/features/utime/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile
+                xlators/features/metadisp/Makefile
+                xlators/features/metadisp/src/Makefile
                 xlators/playground/Makefile
                 xlators/playground/template/Makefile
                 xlators/playground/template/src/Makefile
-                xlators/encryption/Makefile
-                xlators/encryption/rot-13/Makefile
-                xlators/encryption/rot-13/src/Makefile
-                xlators/encryption/crypt/Makefile
-                xlators/encryption/crypt/src/Makefile
                 xlators/system/Makefile
                 xlators/system/posix-acl/Makefile
                 xlators/system/posix-acl/src/Makefile
@@ -191,23 +178,6 @@ AC_CONFIG_FILES([Makefile
                 xlators/mgmt/Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
-                xlators/experimental/Makefile
-                xlators/experimental/jbr-client/Makefile
-                xlators/experimental/jbr-client/src/Makefile
-                xlators/experimental/jbr-server/Makefile
-                xlators/experimental/jbr-server/src/Makefile
-                xlators/experimental/dht2/Makefile
-                xlators/experimental/dht2/dht2-client/Makefile
-                xlators/experimental/dht2/dht2-client/src/Makefile
-                xlators/experimental/dht2/dht2-server/Makefile
-                xlators/experimental/dht2/dht2-server/src/Makefile
-                xlators/experimental/posix2/Makefile
-                xlators/experimental/posix2/ds/Makefile
-                xlators/experimental/posix2/ds/src/Makefile
-                xlators/experimental/posix2/mds/Makefile
-                xlators/experimental/posix2/mds/src/Makefile
-                xlators/experimental/posix2/common/Makefile
-                xlators/experimental/posix2/common/src/Makefile
                 cli/Makefile
                 cli/src/Makefile
                 doc/Makefile
@@ -220,6 +190,7 @@ AC_CONFIG_FILES([Makefile
                 extras/init.d/glusterd-Redhat
                 extras/init.d/glusterd-FreeBSD
                 extras/init.d/glusterd-SuSE
+                extras/init.d/glustereventsd-Debian
                 extras/init.d/glustereventsd-Redhat
                 extras/init.d/glustereventsd-FreeBSD
                 extras/ganesha/Makefile
@@ -229,6 +200,8 @@ AC_CONFIG_FILES([Makefile
                 extras/systemd/Makefile
                 extras/systemd/glusterd.service
                 extras/systemd/glustereventsd.service
+                extras/systemd/glusterfssharedstorage.service
+                extras/systemd/gluster-ta-volume.service
                 extras/run-gluster.tmpfiles
                 extras/benchmarking/Makefile
                 extras/hook-scripts/Makefile
@@ -242,6 +215,10 @@ AC_CONFIG_FILES([Makefile
                 extras/hook-scripts/add-brick/Makefile
                 extras/hook-scripts/add-brick/pre/Makefile
                 extras/hook-scripts/add-brick/post/Makefile
+                extras/hook-scripts/create/Makefile
+                extras/hook-scripts/create/post/Makefile
+                extras/hook-scripts/delete/Makefile
+                extras/hook-scripts/delete/pre/Makefile
                 extras/hook-scripts/start/Makefile
                 extras/hook-scripts/start/post/Makefile
                 extras/hook-scripts/set/Makefile
@@ -251,6 +228,7 @@ AC_CONFIG_FILES([Makefile
                 extras/hook-scripts/reset/Makefile
                 extras/hook-scripts/reset/post/Makefile
                 extras/hook-scripts/reset/pre/Makefile
+                extras/python/Makefile
                 extras/snap_scheduler/Makefile
                 events/Makefile
                 events/src/Makefile
@@ -258,10 +236,8 @@ AC_CONFIG_FILES([Makefile
                 events/tools/Makefile
                 contrib/fuse-util/Makefile
                 contrib/umountd/Makefile
-                contrib/uuid/uuid_types.h
                 glusterfs-api.pc
                 libgfchangelog.pc
-                libgfdb.pc
                 api/Makefile
                 api/src/Makefile
                 api/examples/Makefile
@@ -273,12 +249,12 @@ AC_CONFIG_FILES([Makefile
                 heal/Makefile
                 heal/src/Makefile
                 glusterfs.spec
-		tools/glusterfind/src/tool.conf
-	        tools/glusterfind/glusterfind
-	        tools/glusterfind/Makefile
+                tools/glusterfind/src/tool.conf
+                tools/glusterfind/glusterfind
+                tools/glusterfind/Makefile
                 tools/glusterfind/src/Makefile
-                tests/basic/fuse/Makefile
-                tests/basic/gfapi/Makefile])
+                tools/setgfid2path/Makefile
+                tools/setgfid2path/src/Makefile])
 
 AC_CANONICAL_HOST
 
@@ -299,10 +275,124 @@ AC_ARG_ENABLE([debug],
                              [Enable debug build options.]))
 if test "x$enable_debug" = "xyes"; then
         BUILD_DEBUG=yes
-        CFLAGS="${CFLAGS} -g -O0 -DDEBUG"
+        GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
 else
         BUILD_DEBUG=no
-        CFLAGS="${CFLAGS} -g -O2"
+fi
+
+SANITIZER=none
+
+AC_ARG_ENABLE([asan],
+              AC_HELP_STRING([--enable-asan],
+                             [Enable Address Sanitizer support]))
+if test "x$enable_asan" = "xyes"; then
+        SANITIZER=asan
+        AC_CHECK_LIB([asan], [__asan_init], ,
+                [AC_MSG_ERROR([--enable-asan requires libasan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=address -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lasan"
+fi
+
+AC_ARG_ENABLE([tsan],
+              AC_HELP_STRING([--enable-tsan],
+                             [Enable Thread Sanitizer support]))
+if test "x$enable_tsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=tsan
+        AC_CHECK_LIB([tsan], [__tsan_init], ,
+                [AC_MSG_ERROR([--enable-tsan requires libtsan.so, exiting])])
+        if test "x$ac_cv_lib_tsan___tsan_init" = xyes; then
+                AC_MSG_CHECKING([whether tsan API can be used])
+                saved_CFLAGS=${CFLAGS}
+                CFLAGS="${CFLAGS} -fsanitize=thread"
+                AC_COMPILE_IFELSE(
+                [AC_LANG_PROGRAM([
+                   [#include <sanitizer/tsan_interface.h>]],
+                   [[__tsan_create_fiber(0)]])],
+                   [TSAN_API=yes], [TSAN_API=no])
+                AC_MSG_RESULT([$TSAN_API])
+                if test x$TSAN_API = "xyes"; then
+                   AC_DEFINE(HAVE_TSAN_API, 1, [Define if tsan API can be used.])
+                fi
+                CFLAGS=${saved_CFLAGS}
+        fi
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=thread -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -ltsan"
+fi
+
+AC_ARG_ENABLE([ubsan],
+              AC_HELP_STRING([--enable-ubsan],
+                             [Enable Undefined Behavior Sanitizer support]))
+if test "x$enable_ubsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=ubsan
+        AC_CHECK_LIB([ubsan], [__ubsan_default_options], ,
+                [AC_MSG_ERROR([--enable-ubsan requires libubsan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=undefined -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lubsan"
+fi
+
+# Initialize CFLAGS before usage
+BUILD_TCMALLOC=no
+AC_ARG_ENABLE([tcmalloc],
+              AC_HELP_STRING([--enable-tcmalloc],
+                             [Enable linking with tcmalloc library.]))
+if test "x$enable_tcmalloc" = "xyes"; then
+    BUILD_TCMALLOC=yes
+    GF_CFLAGS="${GF_CFLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"
+    AC_CHECK_LIB([tcmalloc], [malloc], [],
+                 [AC_MSG_ERROR([when --enable-tcmalloc is used, tcmalloc library needs to be present])])
+    GF_LDFLAGS="-ltcmalloc ${GF_LDFLAGS}"
+fi
+
+
+dnl When possible, prefer libtirpc over glibc rpc.
+dnl
+dnl On newer linux with only libtirpc, use libtirpc. (Specifying
+dnl --without-libtirpc is an error.)
+dnl
+dnl on older linux with glibc rpc and WITH libtirpc, use libtirpc
+dnl by default except when configured with --without-libtirpc.
+dnl
+dnl on old linux with glibc rpc and WITHOUT libtirpc, default to
+dnl use glibc rpc.
+dnl
+AC_ARG_WITH([libtirpc],
+        [AC_HELP_STRING([--without-libtirpc], [Use legacy glibc RPC.])],
+        [with_libtirpc="no"], [with_libtirpc="yes"])
+
+dnl ipv6-default is off by default
+dnl
+dnl ipv6-default requires libtirpc. (glibc rpc does not support IPv6.)
+dnl ipv6-default can only be enabled if libtipc is enabled.
+dnl
+AC_ARG_WITH([ipv6-default],
+        AC_HELP_STRING([--with-ipv6-default], [Set IPv6 as default.]),
+        [with_ipv6_default=${with_libtirpc}], [with_ipv6_default="no"])
+
+AC_CHECK_FILE([/etc/centos-release])
+if test "x$ac_cv_file__etc_centos_release" = "xyes"; then
+        if grep "release 6" /etc/centos-release; then
+                with_ipv6_default="no"
+        fi
+fi
+
+dnl On some distributions '-ldl' isn't automatically added to LIBS
+AC_CHECK_LIB([dl], [dlopen], [LIB_DL=-ldl])
+AC_SUBST(LIB_DL)
+
+AC_ARG_ENABLE([privport_tracking],
+              AC_HELP_STRING([--disable-privport_tracking],
+                             [Disable internal tracking of privileged ports.]))
+TRACK_PRIVPORTS="yes"
+if test x"$enable_privport_tracking" = x"no"; then
+        TRACK_PRIVPORTS="no"
+        AC_DEFINE(GF_DISABLE_PRIVPORT_TRACKING, 1,
+                  [Disable internal tracking of privileged ports.])
 fi
 
 case $host_os in
@@ -312,18 +402,42 @@ case $host_os in
     fi
     # OSX version lesser than 9 has llvm/clang optimization issues which leads to various segfaults
     if test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -lt 9; then
-       CFLAGS="${CFLAGS} -g -O0 -DDEBUG"
+       GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
     fi
     ;;
 esac
 
+# --enable-valgrind prevents calling dlclose(), this leaks memory
+AC_ARG_ENABLE([valgrind],
+              AC_HELP_STRING([--enable-valgrind@<:@=memcheck,drd@:>@],
+                             [Enable valgrind for resource leak (memcheck, which is
+                              the default) or thread synchronization (drd) debugging.]))
+case x$enable_valgrind in
+  xmemcheck|xyes)
+    AC_DEFINE(RUN_WITH_MEMCHECK, 1,
+              [Define if all processes should run under 'valgrind --tool=memcheck'.])
+    VALGRIND_TOOL=memcheck
+    ;;
+  xdrd)
+    AC_DEFINE(RUN_WITH_DRD, 1,
+              [Define if all processes should run under 'valgrind --tool=drd'.])
+    VALGRIND_TOOL=drd
+    ;;
+  x|xno)
+    VALGRIND_TOOL=no
+    ;;
+  *)
+    AC_MSG_ERROR([Please specify --enable-valgrind@<:@=memcheck,drd@:>@])
+    ;;
+esac
+
 AC_ARG_WITH([previous-options],
         [AS_HELP_STRING([--with-previous-options],
                         [read config.status for configure options])
         ],
         [ if test -r ./config.status && \
              args=$(grep 'ac_cs_config=' config.status | \
-                    sed -e 's/.*"\(.*\)".*/\1/'| sed -e "s/'//g") ; then
+                    sed -e 's/.*"\(.*\)".*/\1/' -e "s/'//g" -e "s/--with-previous-options//g") ; then
             echo "###"
             echo "### Rerunning as '$0 $args'"
             echo "###"
@@ -376,6 +490,13 @@ AC_ARG_WITH([ocf],
             )
 AC_SUBST(OCF_SUBDIR)
 
+AC_ARG_WITH([server],
+            [AS_HELP_STRING([--without-server], [do not build server components])],
+            [with_server='no'],
+            [with_server='yes'],
+            )
+AM_CONDITIONAL([WITH_SERVER], [test x$with_server = xyes])
+
 # LEX needs a check
 AC_PROG_LEX
 if test  "x${LEX}" != "xflex" -a "x${FLEX}" != "xlex"; then
@@ -426,30 +547,50 @@ AC_CHECK_HEADERS([openssl/dh.h])
 
 AC_CHECK_HEADERS([openssl/ecdh.h])
 
+AC_CHECK_LIB([ssl], [SSL_CTX_get0_param], [AC_DEFINE([HAVE_SSL_CTX_GET0_PARAM], [1], [define if found OpenSSL SSL_CTX_get0_param])])
+
 dnl Math library
 AC_CHECK_LIB([m], [pow], [MATH_LIB='-lm'], [MATH_LIB=''])
 AC_SUBST(MATH_LIB)
 
-dnl use libuuid.so or fall-back to contrib/uuid
+dnl depend on libuuid.so
 PKG_CHECK_MODULES([UUID], [uuid],
-        [HAVE_LIBUUID=yes
+        [have_uuid=yes
          AC_DEFINE(HAVE_LIBUUID, 1, [have libuuid.so])
          PKGCONFIG_UUID=uuid],
-        [HAVE_LIBUUID=no
-         UUID_CFLAGS='-I$(CONTRIBDIR)/uuid'])
-AM_CONDITIONAL([HAVE_LIBUUID], [test x$HAVE_LIBUUID = xyes])
+        [have_uuid=no])
+AM_CONDITIONAL([HAVE_LIBUUID], [test x$have_uuid = xyes])
 
 dnl older version of libuuid (from e2fsprogs) require including uuid/uuid.h
 saved_CFLAGS=${CFLAGS}
 CFLAGS="${CFLAGS} ${UUID_CFLAGS}"
-AC_CHECK_HEADER([uuid.h], [], [AC_CHECK_HEADER([uuid/uuid.h])])
+AC_CHECK_HEADER([uuid.h], [], [AC_CHECK_HEADER([uuid/uuid.h])],
+                [[#if HAVE_UUID_H
+                  #include <uuid.h>
+                  #endif
+                ]])
 CFLAGS=${saved_CFLAGS}
 if test "x$ac_cv_header_uuid_uuid_h" = "xyes"; then
     UUID_CFLAGS="${UUID_CFLAGS} -I$(pkg-config --variable=includedir uuid)/uuid"
+    have_uuid=yes
+fi
+
+if test "x$have_uuid" != "xyes"; then
+    case $host_os in
+        *freebsd*)
+            AC_MSG_ERROR([e2fsprogs-libuuid is required to build glusterfs])
+            ;;
+        linux*)
+            AC_MSG_ERROR([libuuid is required to build glusterfs])
+            ;;
+        *)
+            AC_MSG_ERROR([a Linux compatible libuuid is required to build glusterfs])
+            ;;
+    esac
 fi
 
 dnl libglusterfs needs uuid.h, practically everything depends on it
-GF_CPPFLAGS="${GF_CPPFLAGS} ${UUID_CFLAGS}"
+GF_CFLAGS="${GF_CFLAGS} ${UUID_CFLAGS}"
 dnl PKGCONFIG_UUID is used for the dependency in *.pc.in files
 AC_SUBST(PKGCONFIG_UUID)
 
@@ -528,48 +669,54 @@ fi
 
 dnl Check Python Availability
 have_python=no
-AM_PATH_PYTHON(,, [:])
-if test "$PYTHON" != ":"; then
+dnl if the user has not specified a python, pick one
+if test -z "${PYTHON}"; then
+  case $host_os in
+    freebsd*)
+      if test -x /usr/local/bin/python3; then
+        PYTHON=/usr/local/bin/python3
+      else
+        PYTHON=/usr/local/bin/python2
+      fi
+      ;;
+    *)
+      if test -x /usr/bin/python3; then
+        PYTHON=/usr/bin/python3
+      else
+        PYTHON=/usr/bin/python2
+      fi
+      ;;
+  esac
+fi
+AM_PATH_PYTHON([2.6],,[:])
+if test -n "${PYTHON}"; then
    have_python=yes
 fi
-
-dnl Check if version matches that we require
-PYTHONDEV_CPPFLAGS=
-PYTHONDEV_CPPFLAGS=
-BUILD_PYTHON_SITE_PACKAGES=
-BUILD_PYTHON_INC=
-BUILD_PYTHON_LIB=
-have_python2=no
-have_Python_h=no
-
-if echo $PYTHON_VERSION | grep ^2; then
-   have_python2=yes
-
-   dnl Use pkg-config to get runtime search patch missing from ${PYTHON}-config
-   dnl Just do "true" on failure so that configure does not bail out
-   PKG_CHECK_MODULES([PYTHON], "python-$PYTHON_VERSION",,true)
-
-   PYTHONDEV_CPPFLAGS="`${PYTHON}-config --cflags`"
-   dnl Edit out the flags that are not required or are conflicting
-   PYTHONDEV_CPPFLAGS=`echo ${PYTHONDEV_CPPFLAGS} | sed -e 's/-Wp,-D_FORTIFY_SOURCE=[[0-9]]//g'`
-
-   dnl Find python libs at user configured libdir and also "lib" under prefix
-   PYTHONDEV_LDFLAGS="${PYTHON_LIBS} -L`${PYTHON}-config --prefix`/lib -L`${PYTHON}-config --prefix`/$libdir `${PYTHON}-config --ldflags`"
-
-   BUILD_PYTHON_SITE_PACKAGES=`$PYTHON -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib(prefix="${exec_prefix}"))'`
-   BUILD_PYTHON_INC=`$PYTHON -c "from distutils import sysconfig; print sysconfig.get_python_inc()"`
-   BUILD_PYTHON_LIB=python$PYTHON_VERSION
-
-   dnl Now check for python header using the include path obtained above
-   AC_CHECK_HEADERS([${BUILD_PYTHON_INC}/Python.h],[have_Python_h=yes],[])
+AM_CONDITIONAL(HAVE_PYTHON, test "x$have_python" = "xyes")
+
+dnl Use pkg-config to get runtime search path missing from ${PYTHON}-config
+dnl Just do "true" on failure so that configure does not bail out
+dnl Note: python 2.6's devel pkg (e.g. in CentOS/RHEL 6) does not have
+dnl pkg-config files, so this work-around instead
+if test "x${PYTHON_VERSION}" = "x2.6"; then
+  PYTHON_CFLAGS=$(python-config --includes)
+  PYTHON_LIBS=$(python-config --libs)
+else
+  PKG_CHECK_MODULES([PYTHON], "python-${PYTHON_VERSION}",,true)
 fi
 
-AC_SUBST(PYTHONDEV_CPPFLAGS)
-AC_SUBST(PYTHONDEV_LDFLAGS)
+PYTHON_CFLAGS=$(echo ${PYTHON_CFLAGS} | sed -e 's|-I|-isystem |')
+
+BUILD_PYTHON_SITE_PACKAGES=${pythondir}
 AC_SUBST(BUILD_PYTHON_SITE_PACKAGES)
-AC_SUBST(BUILD_PYTHON_INC)
-AC_SUBST(BUILD_PYTHON_LIB)
 
+# Eval two times to expand fully. First eval replaces $exec_prefix into $prefix
+# Second eval will expand $prefix
+build_python_site_packages_temp="${pythondir}"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+BUILD_PYTHON_SITE_PACKAGES_EXPANDED=${build_python_site_packages_temp}
+AC_SUBST(BUILD_PYTHON_SITE_PACKAGES_EXPANDED)
 
 # FUSE section
 AC_ARG_ENABLE([fuse-client],
@@ -582,70 +729,25 @@ if test "x$enable_fuse_client" != "xno"; then
    BUILD_FUSE_CLIENT="yes"
 fi
 
-AC_ARG_ENABLE([bd-xlator],
-              AC_HELP_STRING([--enable-bd-xlator], [Build BD xlator]))
-
-if test "x$enable_bd_xlator" != "xno"; then
-  AC_CHECK_LIB([lvm2app],
-               [lvm_init,lvm_lv_from_name],
-               [HAVE_BD_LIB="yes"],
-               [HAVE_BD_LIB="no"])
-
-if test "x$HAVE_BD_LIB" = "xyes"; then
-    # lvm_lv_from_name() has been made public with lvm2-2.02.79
-    AC_CHECK_DECLS(
-                  [lvm_lv_from_name],
-                  [NEED_LVM_LV_FROM_NAME_DECL="no"],
-                  [NEED_LVM_LV_FROM_NAME_DECL="yes"],
-                  [[#include <lvm2app.h>]])
-  fi
-fi
-
-if test "x$enable_bd_xlator" = "xyes" -a "x$HAVE_BD_LIB" = "xno"; then
-   echo "BD xlator requested but required lvm2 development library not found."
-   exit 1
-fi
-
-BUILD_BD_XLATOR=no
-if test "x${enable-bd-xlator}" != "xno" -a "x${HAVE_BD_LIB}" = "xyes"; then
-  BUILD_BD_XLATOR=yes
-  AC_DEFINE(HAVE_BD_XLATOR, 1, [define if lvm2app library found and bd xlator
-                            enabled])
-  if test "x$NEED_LVM_LV_FROM_NAME_DECL" = "xyes"; then
-    AC_DEFINE(NEED_LVM_LV_FROM_NAME_DECL, 1, [defined if lvm_lv_from_name()
-                  was not found in the lvm2app.h header, but can be linked])
-  fi
-fi
-
-AM_CONDITIONAL([ENABLE_BD_XLATOR], [test x$BUILD_BD_XLATOR = xyes])
-
-dnl check for old openssl
-AC_CHECK_LIB([crypto], CRYPTO_THREADID_set_callback, [AC_DEFINE([HAVE_CRYPTO_THREADID], [1], [use new OpenSSL functions])])
-
-AC_CHECK_LIB([ssl], TLSv1_2_method, [AC_DEFINE([HAVE_TLSV1_2_METHOD], [1], [use new OpenSSL functions])])
+AC_SUBST(FUSE_CLIENT_SUBDIR)
 
-# start encryption/crypt section
+AC_ARG_ENABLE([fuse-notifications],
+  AS_HELP_STRING([--disable-fuse-notifications], [Disable FUSE notifications]))
 
-AC_CHECK_HEADERS([openssl/cmac.h], [have_cmac_h=yes], [have_cmac_h=no])
+AS_IF([test "x$enable_fuse_notifications" != "xno"], [
+  AC_DEFINE([HAVE_FUSE_NOTIFICATIONS], [1], [Use FUSE notifications])
+])
 
-AC_ARG_ENABLE([crypt-xlator],
-        AC_HELP_STRING([--enable-crypt-xlator], [Build crypt encryption xlator]))
+# end FUSE section
 
-if test "x$enable_crypt_xlator" = "xyes" -a "x$have_cmac_h" = "xno"; then
-   AC_MSG_ERROR([Encryption xlator requires OpenSSL with cmac.h])
-fi
 
-BUILD_CRYPT_XLATOR=no
-if test "x$enable_crypt_xlator" != "xno" -a "x$have_cmac_h" = "xyes"; then
-   BUILD_CRYPT_XLATOR=yes
-   AC_DEFINE(HAVE_CRYPT_XLATOR, 1, [enable building crypt encryption xlator])
+AC_CHECK_LIB([ssl], TLS_method, [HAVE_OPENSSL_1_1="yes"], [HAVE_OPENSSL_1_1="no"])
+if test "x$HAVE_OPENSSL_1_1" = "xyes"; then
+    AC_DEFINE([HAVE_TLS_METHOD], [1], [Using OpenSSL-1.1 TLS_method])
+else
+    AC_CHECK_LIB([ssl], TLSv1_2_method, [AC_DEFINE([HAVE_TLSV1_2_METHOD], [1], [Using OpenSSL-1.0 TLSv1_2_method])])
 fi
 
-AM_CONDITIONAL([ENABLE_CRYPT_XLATOR], [test x$BUILD_CRYPT_XLATOR = xyes])
-
-AC_SUBST(FUSE_CLIENT_SUBDIR)
-# end FUSE section
-
 
 # FUSERMOUNT section
 AC_ARG_ENABLE([fusermount],
@@ -676,53 +778,6 @@ if test "x$enable_epoll" != "xno"; then
 fi
 # end EPOLL section
 
-
-# IBVERBS section
-AC_ARG_ENABLE([ibverbs],
-              AC_HELP_STRING([--disable-ibverbs],
-                             [Do not build the ibverbs transport]))
-
-if test "x$enable_ibverbs" != "xno"; then
-  AC_CHECK_LIB([ibverbs],
-               [ibv_get_device_list],
-               [HAVE_LIBIBVERBS="yes"],
-               [HAVE_LIBIBVERBS="no"])
-  AC_CHECK_LIB([rdmacm], [rdma_create_id], [HAVE_RDMACM="yes"], [HAVE_RDMACM="no"])
-  if test "x$HAVE_RDMACM" = "xyes" ; then
-    AC_CHECK_DECLS(
-                  [RDMA_OPTION_ID_REUSEADDR],
-                  [],
-                  [AC_ERROR([Need at least version 1.0.15 of librdmacm])],
-                  [[#include <rdma/rdma_cma.h>]])
-  fi
-fi
-
-if test "x$enable_ibverbs" = "xyes"; then
-   if test "x$HAVE_LIBIBVERBS" = "xno"; then
-      echo "ibverbs-transport requested, but libibverbs is not present."
-      exit 1
-   fi
-
-   if test "x$HAVE_RDMACM" = "xno"; then
-      echo "ibverbs-transport requested, but librdmacm is not present."
-      exit 1
-   fi
-fi
-
-BUILD_RDMA=no
-BUILD_IBVERBS=no
-if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes" -a "x$HAVE_RDMACM" = "xyes"; then
-  IBVERBS_SUBDIR=ib-verbs
-  BUILD_IBVERBS=yes
-  RDMA_SUBDIR=rdma
-  BUILD_RDMA=yes
-fi
-
-AC_SUBST(IBVERBS_SUBDIR)
-AC_SUBST(RDMA_SUBDIR)
-# end IBVERBS section
-
-
 # SYNCDAEMON section
 AC_ARG_ENABLE([georeplication],
               AC_HELP_STRING([--disable-georeplication],
@@ -730,6 +785,9 @@ AC_ARG_ENABLE([georeplication],
 
 BUILD_SYNCDAEMON=no
 case $host_os in
+     freebsd*)
+#do nothing
+       ;;
      linux*)
 #do nothing
        ;;
@@ -742,25 +800,20 @@ case $host_os in
         ;;
 esac
 SYNCDAEMON_COMPILE=0
-if test "x$enable_georeplication" != "xno"; then
-  SYNCDAEMON_SUBDIR=geo-replication
-  SYNCDAEMON_COMPILE=1
-
-  BUILD_SYNCDAEMON="yes"
-  AM_PATH_PYTHON([2.4])
-  echo -n "checking if python is python 2.x... "
-  if echo $PYTHON_VERSION | grep ^2; then
-    :
-  else
-    echo no
-    AC_MSG_ERROR([only python 2.x is supported])
-  fi
-  echo -n "checking if python has ctypes support... "
-  if "$PYTHON" -c 'import ctypes' 2>/dev/null; then
-    echo yes
+if test "x${with_server}" = "xyes" -a "x${enable_georeplication}" != "xno"; then
+  if test "x${have_python}" = "xno" ; then
+    AC_MSG_ERROR([only python 2 and 3 are supported])
   else
-    echo no
-    AC_MSG_ERROR([python does not have ctypes support])
+    SYNCDAEMON_SUBDIR=geo-replication
+    SYNCDAEMON_COMPILE=1
+
+    BUILD_SYNCDAEMON="yes"
+    AC_MSG_CHECKING([if python has ctypes support...])
+    if "${PYTHON}" -c 'import ctypes' 2>/dev/null; then
+      AC_MSG_RESULT("yes")
+    else
+      AC_MSG_ERROR([python does not have ctypes support])
+    fi
   fi
 fi
 AC_SUBST(SYNCDAEMON_COMPILE)
@@ -768,12 +821,23 @@ AC_SUBST(SYNCDAEMON_SUBDIR)
 # end SYNCDAEMON section
 
 # only install scripts from extras/geo-rep when enabled
-if test "x$enable_georeplication" != "xno"; then
+if test "x${with_server}" = "xyes" -a "x$enable_georeplication" != "xno"; then
   GEOREP_EXTRAS_SUBDIR=geo-rep
 fi
 AC_SUBST(GEOREP_EXTRAS_SUBDIR)
 AM_CONDITIONAL(USE_GEOREP, test "x$enable_georeplication" != "xno")
 
+# METADISP section
+AC_ARG_ENABLE([metadisp],
+              AC_HELP_STRING([--enable-metadisp],
+                             [Enable the metadata dispersal xlator]))
+BUILD_METADISP=no
+if test "x${enable_metadisp}" = "xyes"; then
+  BUILD_METADISP=yes
+fi
+AM_CONDITIONAL([BUILD_METADISP], [test "x$BUILD_METADISP" = "xyes"])
+# end METADISP section
+
 # Events section
 AC_ARG_ENABLE([events],
               AC_HELP_STRING([--disable-events],
@@ -788,11 +852,11 @@ if test "x$enable_events" != "xno"; then
 
   BUILD_EVENTS="yes"
 
-  if test "x$have_python2" = "xno"; then
-     if test "x$enable_events" = "xyes"; then
-        AC_MSG_ERROR([python 2.x packages required. exiting..])
+  if test "x${have_python}" = "xno"; then
+     if test "x${enable_events}" = "xyes"; then
+        AC_MSG_ERROR([python 2 or 3 required. exiting.])
      fi
-     AC_MSG_WARN([python 2.x not found, disabling events])
+     AC_MSG_WARN([python not found, disabling events])
      EVENTS_SUBDIR=
      EVENTS_ENABLED=0
      BUILD_EVENTS="no"
@@ -802,7 +866,7 @@ if test "x$enable_events" != "xno"; then
 fi
 AC_SUBST(EVENTS_ENABLED)
 AC_SUBST(EVENTS_SUBDIR)
-AM_CONDITIONAL([BUILD_EVENTS], [test x$BUILD_EVENTS = xyes])
+AM_CONDITIONAL([BUILD_EVENTS], [test "x${BUILD_EVENTS}" = "xyes"])
 # end Events section
 
 # CDC xlator - check if libz is present if so enable HAVE_LIB_Z
@@ -828,8 +892,8 @@ AC_ARG_ENABLE([firewalld],
                              [enable installation configuration for firewalld]),
               [BUILD_FIREWALLD="${enableval}"], [BUILD_FIREWALLD="no"])
 
-if test "x${BUILD_FIREWALLD}" = "xyes"; then
-        if !(which firewalld 1>/dev/null 2>&1) ; then
+if test "x${with_server}" = "xyes" -a "x${BUILD_FIREWALLD}" = "xyes"; then
+        if !(test -d /usr/lib/firewalld/services 1>/dev/null 2>&1) ; then
                 BUILD_FIREWALLD="no (firewalld not installed)"
         fi
 fi
@@ -837,41 +901,13 @@ AM_CONDITIONAL([USE_FIREWALLD],test ["x${BUILD_FIREWALLD}" =  "xyes"])
 
 #endof firewald section
 
-# Data tiering requires sqlite
-AC_ARG_ENABLE([tiering],
-               AC_HELP_STRING([--disable-tiering],
-                             [Disable data classification/tiering]),
-              [BUILD_GFDB="${enableval}"], [BUILD_GFDB="yes"])
-
-case $host_os in
-  darwin*)
-    SQLITE_LIBS="-lsqlite3"
-    AC_CHECK_HEADERS([sqlite3.h], AC_DEFINE(USE_GFDB, 1))
-    ;;
-  *)
-    if test "x${BUILD_GFDB}" = "xyes"; then
-      PKG_CHECK_MODULES([SQLITE], [sqlite3],
-                        AC_DEFINE(USE_GFDB, 1),
-                        AC_MSG_ERROR([pass --disable-tiering to build without sqlite]))
-    else
-      AC_DEFINE(USE_GFDB, 0, [no sqlite, gfdb is disabled])
-    fi
-    ;;
-esac
-
-AC_SUBST(SQLITE_CFLAGS)
-AC_SUBST(SQLITE_LIBS)
-AM_CONDITIONAL(BUILD_GFDB, test "x${BUILD_GFDB}" = "xyes")
-AM_CONDITIONAL(USE_GFDB, test "x${BUILD_GFDB}" = "xyes")
-
 # xml-output
 AC_ARG_ENABLE([xml-output],
               AC_HELP_STRING([--disable-xml-output],
                              [Disable the xml output]))
 BUILD_XML_OUTPUT="yes"
 if test "x$enable_xml_output" != "xno"; then
-    #check if libxml is present if so enable HAVE_LIB_XML
-    m4_ifdef([AM_PATH_XML2],[AM_PATH_XML2([2.6.19])], [no_xml=yes])
+    PKG_CHECK_MODULES([XML], [libxml-2.0], [], [no_xml="yes"])
     if test "x${no_xml}" = "x"; then
         AC_DEFINE([HAVE_LIB_XML], [1], [Define to 1 if using libxml2.])
     else
@@ -891,12 +927,74 @@ else
 fi
 # end of xml-output
 
+dnl cloudsync section
+BUILD_CLOUDSYNC="no"
+AC_CHECK_LIB([curl], [curl_easy_setopt], [LIBCURL="-lcurl"])
+if test -n "$LIBCURL";then
+        HAVE_LIBCURL="yes"
+fi
+AC_CHECK_HEADERS([openssl/hmac.h openssl/evp.h openssl/bio.h openssl/buffer.h], [HAVE_OPENSSL="yes"])
+if test "x$HAVE_LIBCURL" = "xyes" -a "x$HAVE_OPENSSL" = "xyes";then
+   HAVE_AMAZONS3="yes"
+fi
+AM_CONDITIONAL([BUILD_AMAZONS3_PLUGIN], [test "x$HAVE_AMAZONS3" = "xyes"])
+if test "x$HAVE_AMAZONS3" = "xyes";then
+   BUILD_CLOUDSYNC="yes"
+fi
+BUILD_CVLT_PLUGIN="no"
+case $host_os in
+#enable cvlt plugin only for linux platforms
+     linux*)
+       BUILD_CVLT_PLUGIN="yes"
+       BUILD_CLOUDSYNC="yes"
+       ;;
+     *)
+       ;;
+esac
+AM_CONDITIONAL([BUILD_CVLT_PLUGIN], [test "x$BUILD_CVLT_PLUGIN" = "xyes"])
+AM_CONDITIONAL([BUILD_CLOUDSYNC], [test "x$BUILD_CLOUDSYNC" = "xyes"])
+dnl end cloudsync section
+
+dnl SELinux feature enablement
+case $host_os in
+  linux*)
+    AC_ARG_ENABLE([selinux],
+                  AC_HELP_STRING([--disable-selinux],
+                                 [Disable SELinux features]),
+                  [USE_SELINUX="${enableval}"], [USE_SELINUX="yes"])
+    ;;
+  *)
+    USE_SELINUX=no
+    ;;
+esac
+AM_CONDITIONAL(USE_SELINUX, test "x${USE_SELINUX}" = "xyes")
+dnl end of SELinux feature enablement
+
 AC_CHECK_HEADERS([execinfo.h], [have_backtrace=yes])
 if test "x${have_backtrace}" = "xyes"; then
    AC_DEFINE(HAVE_BACKTRACE, 1, [define if found backtrace])
 fi
 AC_SUBST(HAVE_BACKTRACE)
 
+dnl Old (before C11) compiler can compile (but not link) this:
+dnl
+dnl int main () {
+dnl     _Static_assert(1, "True");
+dnl     return 0;
+dnl }
+dnl
+dnl assuming that _Static_assert is an implicitly declared function. So
+dnl we're trying to link just to make sure that this is not the case.
+
+AC_MSG_CHECKING([whether $CC supports C11 _Static_assert])
+AC_TRY_LINK([], [_Static_assert(1, "True");],
+            [STATIC_ASSERT=yes], [STATIC_ASSERT=no])
+
+AC_MSG_RESULT([$STATIC_ASSERT])
+if test x$STATIC_ASSERT = "xyes"; then
+   AC_DEFINE(HAVE_STATIC_ASSERT, 1, [Define if C11 _Static_assert is supported.])
+fi
+
 if test "x${have_backtrace}" != "xyes"; then
 AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);],
    [],
@@ -904,11 +1002,11 @@ AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);],
 fi
 
 dnl glusterfs prints memory usage to stderr by sending it SIGUSR1
-AC_CHECK_FUNC([malloc_stats], [have_malloc_stats=yes])
-if test "x${have_malloc_stats}" = "xyes"; then
-   AC_DEFINE(HAVE_MALLOC_STATS, 1, [define if found malloc_stats])
+AC_CHECK_FUNC([mallinfo], [have_mallinfo=yes])
+if test "x${have_mallinfo}" = "xyes"; then
+   AC_DEFINE(HAVE_MALLINFO, 1, [define if found mallinfo])
 fi
-AC_SUBST(HAVE_MALLOC_STATS)
+AC_SUBST(HAVE_MALLINFO)
 
 dnl Linux, Solaris, Cygwin
 AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec])
@@ -916,7 +1014,7 @@ dnl FreeBSD, NetBSD
 AC_CHECK_MEMBERS([struct stat.st_atimespec.tv_nsec])
 case $host_os in
         *netbsd*)
-        CFLAGS="${CFLAGS} -D_INCOMPLETE_XOPEN_C063 -DCONFIG_MACHINE_BSWAP_H"
+        GF_CFLAGS="${GF_CFLAGS} -D_INCOMPLETE_XOPEN_C063 -DCONFIG_MACHINE_BSWAP_H"
         ;;
 esac
 AC_CHECK_FUNC([linkat], [have_linkat=yes])
@@ -929,35 +1027,42 @@ dnl check for Monotonic clock
 AC_CHECK_LIB([rt], [clock_gettime], ,
              AC_MSG_WARN([System doesn't have monotonic clock using contrib]))
 
-dnl Check for argp
+dnl check for argp, FreeBSD has the header in /usr/local/include
+case $host_os in
+    *freebsd*)
+    CFLAGS="${CFLAGS} -isystem /usr/local/include"
+    ARGP_LDADD=-largp
+    ;;
+    *netbsd*)
+    ARGP_LDADD=-largp
+    ;;
+esac
+dnl argp-standalone does not provide a pkg-config file
 AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp]))
-
-BUILD_ARGP_STANDALONE=no
-if test "x${ac_cv_header_argp_h}" = "xno"; then
-   AC_CONFIG_SUBDIRS(contrib/argp-standalone)
-   BUILD_ARGP_STANDALONE=yes
-   ARGP_STANDALONE_CPPFLAGS='-I${top_srcdir}/contrib/argp-standalone'
-   ARGP_STANDALONE_LDADD='${top_builddir}/contrib/argp-standalone/libargp.a'
-   ARGP_STANDALONE_DIR='${top_builddir}/contrib/argp-standalone'
+if test "x$ac_cv_header_argp_h" != "xyes"; then
+    AC_MSG_ERROR([argp.h not found, install libargp or argp-standalone])
 fi
-
-dnl libglusterfs needs argp.h, practically everything depends on it
-GF_CPPFLAGS="${GF_CPPFLAGS} ${ARGP_STANDALONE_CPPFLAGS}"
-
-AC_SUBST(ARGP_STANDALONE_CPPFLAGS)
-AC_SUBST(ARGP_STANDALONE_LDADD)
-AC_SUBST(ARGP_STANDALONE_DIR)
-
-# Check for atomic operation support
-echo -n "checking for atomic operation support... "
-AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { long int a = 4; __sync_fetch_and_add_8 (&a, 1); }]])],
-                   [have_sync_fetch_and_add_8=yes], [have_sync_fetch_and_add_8=no])
-if test "x${have_sync_fetch_and_add_8}" = "xyes"; then
-  echo "yes"
-  AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, [have atomic builtins])
-else
-  echo "no"
+AC_SUBST(ARGP_LDADD)
+
+dnl Check for atomic operation support
+AC_MSG_CHECKING([for gcc __atomic builtins])
+AC_TRY_LINK([], [int v; __atomic_load_n(&v, __ATOMIC_ACQUIRE);],
+            [have_atomic_builtins=yes], [have_atomic_builtins=no])
+if test "x${have_atomic_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, [define if __atomic_*() builtins are available])
 fi
+AC_SUBST(HAVE_ATOMIC_BUILTINS)
+AC_MSG_RESULT([$have_atomic_builtins])
+
+dnl __sync_*() will not be needed if __atomic_*() is available
+AC_MSG_CHECKING([for gcc __sync builtins])
+AC_TRY_LINK([], [__sync_synchronize();],
+            [have_sync_builtins=yes], [have_sync_builtins=no])
+if test "x${have_sync_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNC_BUILTINS, 1, [define if __sync_*() builtins are available])
+fi
+AC_SUBST(HAVE_SYNC_BUILTINS)
+AC_MSG_RESULT([$have_sync_builtins])
 
 AC_CHECK_HEADER([malloc.h], AC_DEFINE(HAVE_MALLOC_H, 1, [have malloc.h]))
 
@@ -981,6 +1086,45 @@ if test "x${have_posix_fallocate}" = "xyes"; then
    AC_DEFINE(HAVE_POSIX_FALLOCATE, 1, [define if posix_fallocate exists])
 fi
 
+# On fedora-29, copy_file_range syscall and the libc API both are present.
+# Whereas, on some machines such as centos-7, RHEL-7, the API is not there.
+# Only the system call is present. So, this change is to determine whether
+# the API is present or not. If not, then check whether the system call is
+# present or not. Accordingly sys_copy_file_range function will first call
+# the API if it is there. Otherwise it will call syscall(SYS_copy_file_range).
+AC_CHECK_FUNC([copy_file_range], [have_copy_file_range=yes])
+if test "x${have_copy_file_range}" = "xyes"; then
+   AC_DEFINE(HAVE_COPY_FILE_RANGE, 1, [define if copy_file_range exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_copy_file_range], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_copy_file_range}" = "xyes"; then
+      AC_DEFINE(HAVE_COPY_FILE_RANGE_SYS, 1, [define if SYS_copy_file_range is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+AC_CHECK_FUNC([syncfs], [have_syncfs=yes])
+if test "x${have_syncfs}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNCFS, 1, [define if syncfs exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_syncfs], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_syncfs}" = "xyes"; then
+      AC_DEFINE(HAVE_SYNCFS_SYS, 1, [define if SYS_syncfs is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+BUILD_NANOSECOND_TIMESTAMPS=no
+AC_CHECK_FUNC([utimensat], [have_utimensat=yes])
+if test "x${have_utimensat}" = "xyes"; then
+   BUILD_NANOSECOND_TIMESTAMPS=yes
+   AC_DEFINE(HAVE_UTIMENSAT, 1, [define if utimensat exists])
+fi
+
 OLD_CFLAGS=${CFLAGS}
 CFLAGS="-D_GNU_SOURCE"
 AC_CHECK_DECL([SEEK_HOLE], , , [#include <unistd.h>])
@@ -989,6 +1133,16 @@ if test "x${ac_cv_have_decl_SEEK_HOLE}" = "xyes"; then
 fi
 CFLAGS=${OLD_CFLAGS}
 
+AC_CHECK_FUNC([accept4], [have_accept4=yes])
+if test "x${have_accept4}" = "xyes"; then
+   AC_DEFINE(HAVE_ACCEPT4, 1, [define if accept4 exists])
+fi
+
+AC_CHECK_FUNC([paccept], [have_paccept=yes])
+if test "x${have_paccept}" = "xyes"; then
+AC_DEFINE(HAVE_PACCEPT, 1, [define if paccept exists])
+fi
+
 # Check the distribution where you are compiling glusterfs on
 
 GF_DISTRIBUTION=
@@ -1009,7 +1163,34 @@ fi
 AC_SUBST(GF_DISTRIBUTION)
 
 GF_HOST_OS=""
-GF_LDFLAGS="-rdynamic"
+GF_LDFLAGS="${GF_LDFLAGS} -rdynamic"
+
+dnl see --with-libtirpc option check above, libtirpc(-devel) is required for
+dnl ipv6-default
+if test "x${with_libtirpc}" = "xyes" || test "x${with_ipv6_default}" = "xyes" ; then
+    PKG_CHECK_MODULES([TIRPC], [libtirpc],
+       [with_libtirpc="yes"; GF_CFLAGS="$GF_CFLAGS $TIRPC_CFLAGS"; GF_LDFLAGS="$GF_LDFLAGS $TIRPC_LIBS";],
+       [with_libtirpc="missing"; with_ipv6_default="no"])
+fi
+
+if test "x${with_libtirpc}" = "xmissing" ; then
+    AC_CHECK_HEADERS([rpc/rpc.h],[
+        AC_MSG_WARN([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed.
+            Disabling libtirpc and ipv6-default and falling back to legacy glibc rpc headers.
+            This is a transitional warning message. Eventually it will be an error message.
+            ---------------------------------------------------------------------------------])],[
+        AC_MSG_ERROR([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed
+            and there were no legacy glibc rpc headers and library to fall back to.
+            ---------------------------------------------------------------------------------])])
+fi
+
+if test "x$with_ipv6_default" = "xyes" ; then
+   GF_CFLAGS="$GF_CFLAGS -DIPV6_DEFAULT"
+fi
 
 dnl check for gcc -Werror=format-security
 saved_CFLAGS=$CFLAGS
@@ -1017,28 +1198,21 @@ CFLAGS="-Wformat -Werror=format-security"
 AC_MSG_CHECKING([whether $CC accepts -Werror=format-security])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_format_security=yes], [cc_werror_format_security=no])
 echo $cc_werror_format_security
-if test "x$cc_werror_format_security" = "xno"; then
-    CFLAGS="$saved_CFLAGS"
-else
-    CFLAGS="$saved_CFLAGS $CFLAGS"
-    GF_CFLAGS="$GF_CFLAGS $CFLAGS"
+if test "x$cc_werror_format_security" = "xyes"; then
+    GF_CFLAGS="$GF_CFLAGS ${CFLAGS}"
 fi
+CFLAGS="$saved_CFLAGS"
 
 dnl check for gcc -Werror=implicit-function-declaration
 saved_CFLAGS=$CFLAGS
-saved_GF_CFLAGS=$GF_CFLAGS
 CFLAGS="-Werror=implicit-function-declaration"
-GF_CFLAGS="-Werror=implicit-function-declaration"
 AC_MSG_CHECKING([whether $CC accepts -Werror=implicit-function-declaration])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_implicit=yes], [cc_werror_implicit=no])
 echo $cc_werror_implicit
-if test "x$cc_werror_implicit" = "xno"; then
-    CFLAGS="$saved_CFLAGS"
-    GF_CFLAGS="$saved_GF_CFLAGS"
-else
-    CFLAGS="$saved_CFLAGS $CFLAGS"
-    GF_CFLAGS="$saved_GF_CFLAGS $GF_CFLAGS"
+if test "x$cc_werror_implicit" = "xyes"; then
+    GF_CFLAGS="${GF_CFLAGS} ${CFLAGS}"
 fi
+CFLAGS="$saved_CFLAGS"
 
 dnl clang is mostly GCC-compatible, but its version is much lower,
 dnl so we have to check for it.
@@ -1075,20 +1249,25 @@ fi
 
 old_prefix=$prefix
 if test "x$prefix" = xNONE; then
-	prefix=$ac_default_prefix
+        prefix=$ac_default_prefix
 fi
-GLUSTERFS_LIBEXECDIR="$(eval echo $prefix)/libexec/glusterfs"
-GLUSTERFSD_MISCDIR="$(eval echo $prefix)/var/lib/misc/glusterfsd"
+old_exec_prefix=$exec_prefix
+if test "x$exec_prefix" = xNONE; then
+        exec_prefix="$(eval echo $prefix)"
+fi
+GLUSTERFS_LIBEXECDIR="$(eval echo $libexecdir)/glusterfs"
 prefix=$old_prefix
+exec_prefix=$old_exec_prefix
 
 ### Dirty hacky stuff to make LOCALSTATEDIR work
 if test "x$prefix" = xNONE; then
    test $localstatedir = '${prefix}/var' && localstatedir=$ac_default_prefix/var
    localstatedir=/var
-   LOCALSTATEDIR=$(eval echo ${localstatedir})
-else
-   LOCALSTATEDIR=$(eval echo ${localstatedir})
 fi
+localstatedir="$(eval echo ${localstatedir})"
+LOCALSTATEDIR=$localstatedir
+
+GLUSTERFSD_MISCDIR="$(eval echo ${localstatedir})/lib/misc/glusterfsd"
 
 old_prefix=$prefix
 if test "x$prefix" = xNONE; then
@@ -1098,6 +1277,7 @@ GLUSTERD_VOLFILE="$(eval echo ${sysconfdir})/glusterfs/glusterd.vol"
 prefix=$old_prefix
 
 
+GFAPI_EXTRA_LDFLAGS='-Wl,--version-script=$(top_srcdir)/api/src/gfapi.map'
 case $host_os in
      linux*)
         GF_HOST_OS="GF_LINUX_HOST_OS"
@@ -1117,9 +1297,9 @@ case $host_os in
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
         GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_LDADD="${ARGP_LDADD}"
         if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-           GF_LDFLAGS="-lexecinfo"
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
         fi
         GF_FUSE_LDADD="-lperfuse"
         BUILD_FUSE_CLIENT=yes
@@ -1130,17 +1310,13 @@ case $host_os in
         ;;
      *freebsd*)
         GF_HOST_OS="GF_BSD_HOST_OS"
-        GF_CFLAGS="${GF_CFLAGS} ${ARGP_STANDALONE_CPPFLAGS} -O0"
-        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
-        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
-        GF_CFLAGS="${GF_CFLAGS} -D_LIBGEN_H_"
         GF_CFLAGS="${GF_CFLAGS} -DO_DSYNC=0"
         GF_CFLAGS="${GF_CFLAGS} -Dxdr_quad_t=xdr_longlong_t"
         GF_CFLAGS="${GF_CFLAGS} -Dxdr_u_quad_t=xdr_u_longlong_t"
         GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_LDADD="${ARGP_LDADD}"
         if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-           GF_LDFLAGS="-lexecinfo"
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
         fi
         BUILD_FUSE_CLIENT=yes
         BUILD_FUSERMOUNT=no
@@ -1150,27 +1326,18 @@ case $host_os in
      darwin*)
         GF_HOST_OS="GF_DARWIN_HOST_OS"
         LIBTOOL=glibtool
-        GF_CFLAGS="${GF_CFLAGS} ${ARGP_STANDALONE_CPPFLAGS} "
         GF_CFLAGS="${GF_CFLAGS} -D_REENTRANT -D_XOPEN_SOURCE "
         GF_CFLAGS="${GF_CFLAGS} -D_DARWIN_USE_64_BIT_INODE "
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
-        GF_LDFLAGS=""
+        GF_LDADD="${ARGP_LDADD}"
+        GF_LDFLAGS="${GF_LDFLAGS}"
         GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
         BUILD_FUSERMOUNT="no"
         FUSERMOUNT_SUBDIR=""
         GLUSTERD_WORKDIR="${LOCALSTATEDIR}/db/glusterd"
-        ;;
-esac
-
-case $host_os in
-     darwin*)
         GFAPI_EXTRA_LDFLAGS='-Wl,-alias_list,$(top_srcdir)/api/src/gfapi.aliases'
         ;;
-     *)
-        GFAPI_EXTRA_LDFLAGS='-Wl,--version-script=$(top_srcdir)/api/src/gfapi.map'
-        ;;
 esac
 
 # Default value for sbindir
@@ -1188,19 +1355,11 @@ sysconfdirtemp="${sysconfdir}"
 eval sysconfdirtemp=\"${sysconfdirtemp}\"
 SYSCONF_DIR=${sysconfdirtemp}
 
-# Eval two times to expand fully. First eval replaces $exec_prefix into $prefix
-# Second eval will expand $prefix
-build_python_site_packages_temp="${BUILD_PYTHON_SITE_PACKAGES}"
-eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
-eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
-BUILD_PYTHON_SITE_PACKAGES_EXPANDED=${build_python_site_packages_temp}
-
 prefix=$prefix_temp
 exec_prefix=$exec_prefix_temp
 
 AC_SUBST(SBIN_DIR)
 AC_SUBST(SYSCONF_DIR)
-AC_SUBST(BUILD_PYTHON_SITE_PACKAGES_EXPANDED)
 
 # lazy umount emulation
 UMOUNTD_SUBDIR=""
@@ -1215,6 +1374,15 @@ AC_ARG_ENABLE([debug],
               AC_HELP_STRING([--enable-debug],
                              [Enable debug build options.]))
 
+AC_ARG_ENABLE([mempool],
+              AC_HELP_STRING([--disable-mempool],
+                             [Disable the Gluster memory pooler.]))
+
+USE_MEMPOOL="yes"
+if test "x$enable_mempool" = "xno"; then
+        USE_MEMPOOL="no"
+        AC_DEFINE(GF_DISABLE_MEMPOOL, 1, [Disable the Gluster memory pooler.])
+fi
 
 # syslog section
 AC_ARG_ENABLE([syslog],
@@ -1235,7 +1403,7 @@ AC_CHECK_LIB([readline -lcurses],[readline],[RLLIBS="-lreadline -lcurses"])
 AC_CHECK_LIB([readline -ltermcap],[readline],[RLLIBS="-lreadline -ltermcap"])
 AC_CHECK_LIB([readline -lncurses],[readline],[RLLIBS="-lreadline -lncurses"])
 
-if test "x$RLLIBS" != "x"; then
+if test -n "$RLLIBS"; then
    if test "x$RL_UNDO" = "xyes"; then
       AC_DEFINE(HAVE_READLINE, 1, [readline enabled CLI])
       BUILD_READLINE=yes
@@ -1248,48 +1416,26 @@ fi
 BUILD_LIBAIO=no
 AC_CHECK_LIB([aio],[io_setup],[LIBAIO="-laio"])
 
-if test "x$LIBAIO" != "x"; then
+if test -n "$LIBAIO"; then
    AC_DEFINE(HAVE_LIBAIO, 1, [libaio based POSIX enabled])
    BUILD_LIBAIO=yes
 fi
 
-dnl glupy section
-BUILD_GLUPY=no
-
-AC_ARG_ENABLE([glupy], AS_HELP_STRING([--enable-glupy], [build glupy]))
-if test "x$enable_glupy" != "xno"; then enable_glupy=yes; fi
-
-if test "x$enable_glupy" = "xyes"; then
-    GLUPY_SUBDIR=glupy
-    GLUPY_SUBDIR_MAKEFILE=xlators/features/glupy/Makefile
-    GLUPY_SUBDIR_SRC_MAKEFILE=xlators/features/glupy/src/Makefile
-
-    if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
-        case $host_os in
-            darwin*)
-                BUILD_GLUPY=no
-                ;;
-            *)
-                BUILD_GLUPY=yes
-                ;;
-        esac
-    else
-        AC_MSG_WARN([
-        ---------------------------------------------------------------------------------
-        cannot build glupy. python 2.x and python-devel/python-dev package are required.
-        ---------------------------------------------------------------------------------])
-    fi
-
-    if test "x$BUILD_GLUPY" = "xyes"; then
-
-        echo "building glupy with -isystem $BUILD_PYTHON_INC -l $BUILD_PYTHON_LIB"
-
-        AC_SUBST(GLUPY_SUBDIR)
-        AC_SUBST(GLUPY_SUBDIR_MAKEFILE)
-        AC_SUBST(GLUPY_SUBDIR_SRC_MAKEFILE)
-    fi
+dnl gnfs section
+BUILD_GNFS="no"
+RPCBIND_SERVICE=""
+AC_ARG_ENABLE([gnfs],
+              AC_HELP_STRING([--enable-gnfs],
+                             [Enable legacy gnfs server xlator.]))
+if test "x${with_server}" = "xyes" -a "x$enable_gnfs" = "xyes"; then
+    BUILD_GNFS="yes"
+    GF_CFLAGS="$GF_CFLAGS -DBUILD_GNFS"
+    RPCBIND_SERVICE="rpcbind.service"
 fi
-dnl end glupy section
+AM_CONDITIONAL([BUILD_GNFS], [test x$BUILD_GNFS = xyes])
+AC_SUBST(BUILD_GNFS)
+AC_SUBST(RPCBIND_SERVICE)
+dnl end gnfs section
 
 dnl Check for userspace-rcu
 PKG_CHECK_MODULES([URCU], [liburcu-bp], [],
@@ -1298,10 +1444,12 @@ PKG_CHECK_MODULES([URCU], [liburcu-bp], [],
      AC_MSG_ERROR([liburcu-bp not found]))])
 PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.8], [],
   [PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.7],
-    [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])],
+    [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
+     USE_CONTRIB_URCU='yes'],
     [AC_CHECK_HEADERS([urcu/cds.h],
       [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
-       URCU_CDS_LIBS='-lurcu-cds'],
+       URCU_CDS_LIBS='-lurcu-cds'
+       USE_CONTRIB_URCU='yes'],
       [AC_MSG_ERROR([liburcu-cds not found])])])])
 
 BUILD_UNITTEST="no"
@@ -1338,18 +1486,6 @@ AC_SUBST(UNITTEST_LDFLAGS)
 AC_SUBST(CFLAGS)
 # end enable debug section
 
-# experimental section
-AC_ARG_ENABLE([experimental],
-              AC_HELP_STRING([--disable-experimental],
-                             [Disable building experimental xlators]))
-
-BUILD_EXPERIMENTAL="yes"
-if test "x$enable_experimental" = "xno"; then
-  BUILD_EXPERIMENTAL="no"
-fi
-AM_CONDITIONAL([ENABLE_EXPERIMENTAL], [test x$BUILD_EXPERIMENTAL = xyes])
-#end experimental section
-
 # EC dynamic code generation section
 
 EC_DYNAMIC_SUPPORT="none"
@@ -1440,7 +1576,23 @@ AC_SUBST(USE_EC_DYNAMIC_NEON)
 dnl libglusterfs.so uses math functions
 GF_LDADD="${GF_LDADD} ${MATH_LIB}"
 
-GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym'
+case $host_os in
+  dnl Can't use libtool's portable "-no-undefined" as it seems to be ignored on Linux
+  linux*)
+    GF_NO_UNDEFINED='-Wl,--no-undefined'
+    ;;
+  darwin*)
+    GF_NO_UNDEFINED='-Wl,-undefined'
+    ;;
+  *)
+  dnl There's an issue on FreeBSD with reference to __progname used in some parts of code
+    GF_NO_UNDEFINED=''
+    ;;
+esac
+dnl GF_XLATOR_DEFAULT_LDFLAGS is for most xlators that expose a common set of symbols
+GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
+dnl GF_XLATOR_LDFLAGS is for xlators that expose extra symbols, e.g. dht
+GF_XLATOR_LDFLAGS='-avoid-version $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
 
 AC_SUBST(GF_HOST_OS)
 AC_SUBST(GF_CFLAGS)
@@ -1452,19 +1604,34 @@ AC_SUBST(RLLIBS)
 AC_SUBST(LIBAIO)
 AC_SUBST(AM_MAKEFLAGS)
 AC_SUBST(AM_LIBTOOLFLAGS)
+AC_SUBST(GF_NO_UNDEFINED)
 AC_SUBST(GF_XLATOR_DEFAULT_LDFLAGS)
+AC_SUBST(GF_XLATOR_LDFLAGS)
+AC_SUBST(GF_XLATOR_MGNT_LIBADD)
+
+case $host_os in
+     *freebsd*)
+        GF_XLATOR_MGNT_LIBADD="-lutil -lprocstat"
+        ;;
+esac
 
 CONTRIBDIR='$(top_srcdir)/contrib'
 AC_SUBST(CONTRIBDIR)
 
 GF_CPPDEFINES='-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)'
-GF_CPPINCLUDES='-include $(top_builddir)/config.h -I$(top_srcdir)/libglusterfs/src -I$(top_builddir)/libglusterfs/src'
+GF_CPPINCLUDES='-include $(top_builddir)/config.h -include $(top_builddir)/site.h -I$(top_srcdir)/libglusterfs/src -I$(top_builddir)/libglusterfs/src'
+if test "x${USE_CONTRIB_URCU}" = "xyes"; then
+    GF_CPPINCLUDES="${GF_CPPINCLUDES} -I\$(CONTRIBDIR)/userspace-rcu"
+fi
 GF_CPPFLAGS="$GF_CPPFLAGS $GF_CPPDEFINES $GF_CPPINCLUDES"
 AC_SUBST([GF_CPPFLAGS])
 
 AM_CONDITIONAL([GF_LINUX_HOST_OS], test "${GF_HOST_OS}" = "GF_LINUX_HOST_OS")
 AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")
 AM_CONDITIONAL([GF_BSD_HOST_OS], test "${GF_HOST_OS}" = "GF_BSD_HOST_OS")
+if  test "${GF_HOST_OS}" = "GF_BSD_HOST_OS"; then
+    AC_DEFINE(GF_BSD_HOST_OS, 1, [This is a BSD compatible OS.])
+fi
 
 AC_SUBST(GLUSTERD_WORKDIR)
 AM_CONDITIONAL([GF_INSTALL_GLUSTERD_WORKDIR], test ! -d ${GLUSTERD_WORKDIR} && test -d ${sysconfdir}/glusterd )
@@ -1486,8 +1653,6 @@ GFAPI_VERSION="7."${PACKAGE_VERSION}
 LIBGFCHANGELOG_VERSION="0.0.1"
 AC_SUBST(GFAPI_VERSION)
 AC_SUBST(LIBGFCHANGELOG_VERSION)
-LIBGFDB_VERSION="0.0.1"
-AC_SUBST(LIBGFDB_VERSION)
 
 dnl libtool versioning
 LIBGFXDR_LT_VERSION="0:1:0"
@@ -1514,24 +1679,39 @@ echo
 echo "GlusterFS configure summary"
 echo "==========================="
 echo "FUSE client          : $BUILD_FUSE_CLIENT"
-echo "Infiniband verbs     : $BUILD_IBVERBS"
 echo "epoll IO multiplex   : $BUILD_EPOLL"
-echo "argp-standalone      : $BUILD_ARGP_STANDALONE"
 echo "fusermount           : $BUILD_FUSERMOUNT"
 echo "readline             : $BUILD_READLINE"
 echo "georeplication       : $BUILD_SYNCDAEMON"
 echo "Linux-AIO            : $BUILD_LIBAIO"
 echo "Enable Debug         : $BUILD_DEBUG"
-echo "Block Device xlator  : $BUILD_BD_XLATOR"
-echo "glupy                : $BUILD_GLUPY"
+echo "Run with Valgrind    : $VALGRIND_TOOL"
+echo "Sanitizer enabled    : $SANITIZER"
 echo "Use syslog           : $USE_SYSLOG"
 echo "XML output           : $BUILD_XML_OUTPUT"
-echo "Encryption xlator    : $BUILD_CRYPT_XLATOR"
 echo "Unit Tests           : $BUILD_UNITTEST"
+echo "Track priv ports     : $TRACK_PRIVPORTS"
 echo "POSIX ACLs           : $BUILD_POSIX_ACLS"
-echo "Data Classification  : $BUILD_GFDB"
+echo "SELinux features     : $USE_SELINUX"
 echo "firewalld-config     : $BUILD_FIREWALLD"
-echo "Experimental xlators : $BUILD_EXPERIMENTAL"
 echo "Events               : $BUILD_EVENTS"
 echo "EC dynamic support   : $EC_DYNAMIC_SUPPORT"
+echo "Use memory pools     : $USE_MEMPOOL"
+echo "Nanosecond m/atimes  : $BUILD_NANOSECOND_TIMESTAMPS"
+echo "Server components    : $with_server"
+echo "Legacy gNFS server   : $BUILD_GNFS"
+echo "IPV6 default         : $with_ipv6_default"
+echo "Use TIRPC            : $with_libtirpc"
+echo "With Python          : ${PYTHON_VERSION}"
+echo "Cloudsync            : $BUILD_CLOUDSYNC"
+echo "Metadata dispersal   : $BUILD_METADISP"
+echo "Link with TCMALLOC   : $BUILD_TCMALLOC"
 echo
+
+# dnl Note: ${X^^} capitalization assumes bash >= 4.x
+if test "x$SANITIZER" != "xnone"; then
+        echo "Note: since glusterfs processes are daemon processes, use"
+        echo "'export ${SANITIZER^^}_OPTIONS=log_path=/path/to/xxx.log' to collect"
+        echo "sanitizer output. Further details and more options can be"
+        echo "found at https://github.com/google/sanitizers."
+fi
diff --git a/contrib/argp-standalone/Makefile.am b/contrib/argp-standalone/Makefile.am
deleted file mode 100644
index 4775d4876aa..00000000000
--- a/contrib/argp-standalone/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-# From glibc
-
-# Copyright (C) 1997, 2003, 2004 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Library General Public License for more details.
-
-# You should have received a copy of the GNU Library General Public
-# License along with the GNU C Library; see the file COPYING.LIB.  If
-# not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = .
-
-LIBOBJS = @LIBOBJS@
-
-noinst_LIBRARIES = libargp.a
-
-noinst_HEADERS = argp.h argp-fmtstream.h argp-namefrob.h
-
-EXTRA_DIST = mempcpy.c strchrnul.c strndup.c strcasecmp.c vsnprintf.c autogen.sh
-
-# Leaves out argp-fs-xinl.c and argp-xinl.c 
-libargp_a_SOURCES = argp-ba.c argp-eexst.c argp-fmtstream.c \
-		    argp-help.c argp-parse.c argp-pv.c \
-		    argp-pvh.c
-
-libargp_a_LIBADD = $(LIBOBJS)
-
-
diff --git a/contrib/argp-standalone/acinclude.m4 b/contrib/argp-standalone/acinclude.m4
deleted file mode 100644
index fb61e957dfa..00000000000
--- a/contrib/argp-standalone/acinclude.m4
+++ /dev/null
@@ -1,1084 +0,0 @@
-dnl Try to detect the type of the third arg to getsockname() et al
-AC_DEFUN([LSH_TYPE_SOCKLEN_T],
-[AH_TEMPLATE([socklen_t], [Length type used by getsockopt])
-AC_CACHE_CHECK([for socklen_t in sys/socket.h], ac_cv_type_socklen_t,
-[AC_EGREP_HEADER(socklen_t, sys/socket.h,
-  [ac_cv_type_socklen_t=yes], [ac_cv_type_socklen_t=no])])
-if test $ac_cv_type_socklen_t = no; then
-        AC_MSG_CHECKING(for AIX)
-        AC_EGREP_CPP(yes, [
-#ifdef _AIX
- yes
-#endif
-],[
-AC_MSG_RESULT(yes)
-AC_DEFINE(socklen_t, size_t)
-],[
-AC_MSG_RESULT(no)
-AC_DEFINE(socklen_t, int)
-])
-fi
-])
-
-dnl Choose cc flags for compiling position independent code
-AC_DEFUN([LSH_CCPIC],
-[AC_MSG_CHECKING(CCPIC)
-AC_CACHE_VAL(lsh_cv_sys_ccpic,[
-  if test -z "$CCPIC" ; then
-    if test "$GCC" = yes ; then
-      case `uname -sr` in
-	BSD/OS*)
-         case `uname -r` in
-           4.*) CCPIC="-fPIC";;
-           *) CCPIC="";;
-         esac
-	;;
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-	SunOS\ 5.*)
-	  # Could also use -fPIC, if there are a large number of symbol reference
-	  CCPIC="-fPIC"
-	;;
-	CYGWIN*)
-	  CCPIC=""
-	;;
-	*)
-	  CCPIC="-fpic"
-	;;
-      esac
-    else
-      case `uname -sr` in
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-        IRIX*)
-          CCPIC="-share"
-        ;;
-	hp*|HP*) CCPIC="+z"; ;;
-	FreeBSD*) CCPIC="-fpic";;
-	SCO_SV*) CCPIC="-KPIC -dy -Bdynamic";;
-        UnixWare*|OpenUNIX*) CCPIC="-KPIC -dy -Bdynamic";;
-	Solaris*) CCPIC="-KPIC -Bdynamic";;
-	Windows_NT*) CCPIC="-shared" ;;
-      esac
-    fi
-  fi
-  OLD_CFLAGS="$CFLAGS"
-  CFLAGS="$CFLAGS $CCPIC"
-  AC_TRY_COMPILE([], [exit(0);],
-    lsh_cv_sys_ccpic="$CCPIC", lsh_cv_sys_ccpic='')
-  CFLAGS="$OLD_CFLAGS"
-])
-CCPIC="$lsh_cv_sys_ccpic"
-AC_MSG_RESULT($CCPIC)
-AC_SUBST([CCPIC])])
-
-dnl LSH_PATH_ADD(path-id, directory)
-AC_DEFUN([LSH_PATH_ADD],
-[AC_MSG_CHECKING($2)
-ac_exists=no
-if test -d "$2/." ; then
-  ac_real_dir=`cd $2 && pwd`
-  if test -n "$ac_real_dir" ; then
-    ac_exists=yes
-    for old in $1_REAL_DIRS ; do
-      ac_found=no
-      if test x$ac_real_dir = x$old ; then
-        ac_found=yes;
-	break;
-      fi
-    done
-    if test $ac_found = yes ; then
-      AC_MSG_RESULT(already added)
-    else
-      AC_MSG_RESULT(added)
-      # LDFLAGS="$LDFLAGS -L $2"
-      $1_REAL_DIRS="$ac_real_dir [$]$1_REAL_DIRS"
-      $1_DIRS="$2 [$]$1_DIRS"
-    fi
-  fi
-fi
-if test $ac_exists = no ; then
-  AC_MSG_RESULT(not found)
-fi
-])
-
-dnl LSH_RPATH_ADD(dir)
-AC_DEFUN([LSH_RPATH_ADD], [LSH_PATH_ADD(RPATH_CANDIDATE, $1)])
-
-dnl LSH_RPATH_INIT(candidates)
-AC_DEFUN([LSH_RPATH_INIT],
-[AC_MSG_CHECKING([for -R flag])
-RPATHFLAG=''
-case `uname -sr` in
-  OSF1\ V4.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 6.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 5.*)
-    RPATHFLAG="-rpath "
-    ;;
-  SunOS\ 5.*)
-    if test "$TCC" = "yes"; then
-      # tcc doesn't know about -R
-      RPATHFLAG="-Wl,-R,"
-    else
-      RPATHFLAG=-R
-    fi
-    ;;
-  Linux\ 2.*)
-    RPATHFLAG="-Wl,-rpath,"
-    ;;
-  *)
-    :
-    ;;
-esac
-
-if test x$RPATHFLAG = x ; then
-  AC_MSG_RESULT(none)
-else
-  AC_MSG_RESULT([using $RPATHFLAG])
-fi
-
-RPATH_CANDIDATE_REAL_DIRS=''
-RPATH_CANDIDATE_DIRS=''
-
-AC_MSG_RESULT([Searching for libraries])
-
-for d in $1 ; do
-  LSH_RPATH_ADD($d)
-done
-])    
-
-dnl Try to execute a main program, and if it fails, try adding some
-dnl -R flag.
-dnl LSH_RPATH_FIX
-AC_DEFUN([LSH_RPATH_FIX],
-[if test $cross_compiling = no -a "x$RPATHFLAG" != x ; then
-  ac_success=no
-  AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-    ac_success=yes, ac_success=no, :)
-  
-  if test $ac_success = no ; then
-    AC_MSG_CHECKING([Running simple test program failed. Trying -R flags])
-dnl echo RPATH_CANDIDATE_DIRS = $RPATH_CANDIDATE_DIRS
-    ac_remaining_dirs=''
-    ac_rpath_save_LDFLAGS="$LDFLAGS"
-    for d in $RPATH_CANDIDATE_DIRS ; do
-      if test $ac_success = yes ; then
-  	ac_remaining_dirs="$ac_remaining_dirs $d"
-      else
-  	LDFLAGS="$RPATHFLAG$d $LDFLAGS"
-dnl echo LDFLAGS = $LDFLAGS
-  	AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-  	  [ac_success=yes
-  	  ac_rpath_save_LDFLAGS="$LDFLAGS"
-  	  AC_MSG_RESULT([adding $RPATHFLAG$d])
-  	  ],
-  	  [ac_remaining_dirs="$ac_remaining_dirs $d"], :)
-  	LDFLAGS="$ac_rpath_save_LDFLAGS"
-      fi
-    done
-    RPATH_CANDIDATE_DIRS=$ac_remaining_dirs
-  fi
-  if test $ac_success = no ; then
-    AC_MSG_RESULT(failed)
-  fi
-fi
-])
-
-dnl Like AC_CHECK_LIB, but uses $KRB_LIBS rather than $LIBS.
-dnl LSH_CHECK_KRB_LIB(LIBRARY, FUNCTION, [, ACTION-IF-FOUND [,
-dnl                  ACTION-IF-NOT-FOUND [, OTHER-LIBRARIES]]])
-
-AC_DEFUN([LSH_CHECK_KRB_LIB],
-[AC_CHECK_LIB([$1], [$2],
-  ifelse([$3], ,
-      [[ac_tr_lib=HAVE_LIB`echo $1 | sed -e 's/[^a-zA-Z0-9_]/_/g' \
-     	    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
-        AC_DEFINE_UNQUOTED($ac_tr_lib)
-        KRB_LIBS="-l$1 $KRB_LIBS"
-      ]], [$3]),
-  ifelse([$4], , , [$4
-])dnl
-, [$5 $KRB_LIBS])
-])
-
-dnl LSH_LIB_ARGP(ACTION-IF-OK, ACTION-IF-BAD)
-AC_DEFUN([LSH_LIB_ARGP],
-[ ac_argp_save_LIBS="$LIBS"
-  ac_argp_save_LDFLAGS="$LDFLAGS"
-  ac_argp_ok=no
-  # First check if we can link with argp.
-  AC_SEARCH_LIBS(argp_parse, argp,
-  [ LSH_RPATH_FIX
-    AC_CACHE_CHECK([for working argp],
-      lsh_cv_lib_argp_works,
-      [ AC_TRY_RUN(
-[#include <argp.h>
-#include <stdlib.h>
-
-static const struct argp_option
-options[] =
-{
-  { NULL, 0, NULL, 0, NULL, 0 }
-};
-
-struct child_state
-{
-  int n;
-};
-
-static error_t
-child_parser(int key, char *arg, struct argp_state *state)
-{
-  struct child_state *input = (struct child_state *) state->input;
-  
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_END:
-      if (!input->n)
-	input->n = 1;
-      break;
-    }
-  return 0;
-}
-
-const struct argp child_argp =
-{
-  options,
-  child_parser,
-  NULL, NULL, NULL, NULL, NULL
-};
-
-struct main_state
-{
-  struct child_state child;
-  int m;
-};
-
-static error_t
-main_parser(int key, char *arg, struct argp_state *state)
-{
-  struct main_state *input = (struct main_state *) state->input;
-
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_INIT:
-      state->child_inputs[0] = &input->child;
-      break;
-    case ARGP_KEY_END:
-      if (!input->m)
-	input->m = input->child.n;
-      
-      break;
-    }
-  return 0;
-}
-
-static const struct argp_child
-main_children[] =
-{
-  { &child_argp, 0, "", 0 },
-  { NULL, 0, NULL, 0}
-};
-
-static const struct argp
-main_argp =
-{ options, main_parser, 
-  NULL,
-  NULL,
-  main_children,
-  NULL, NULL
-};
-
-int main(int argc, char **argv)
-{
-  struct main_state input = { { 0 }, 0 };
-  char *v[2] = { "foo", NULL };
-
-  argp_parse(&main_argp, 1, v, 0, NULL, &input);
-
-  if ( (input.m == 1) && (input.child.n == 1) )
-    return 0;
-  else
-    return 1;
-}
-], lsh_cv_lib_argp_works=yes,
-   lsh_cv_lib_argp_works=no,
-   lsh_cv_lib_argp_works=no)])
-
-  if test x$lsh_cv_lib_argp_works = xyes ; then
-    ac_argp_ok=yes
-  else
-    # Reset link flags
-    LIBS="$ac_argp_save_LIBS"
-    LDFLAGS="$ac_argp_save_LDFLAGS"
-  fi])
-
-  if test x$ac_argp_ok = xyes ; then
-    ifelse([$1],, true, [$1])
-  else
-    ifelse([$2],, true, [$2])
-  fi   
-])
-
-dnl LSH_GCC_ATTRIBUTES
-dnl Check for gcc's __attribute__ construction
-
-AC_DEFUN([LSH_GCC_ATTRIBUTES],
-[AC_CACHE_CHECK(for __attribute__,
-	       lsh_cv_c_attribute,
-[ AC_TRY_COMPILE([
-#include <stdlib.h>
-],
-[
-static void foo(void) __attribute__ ((noreturn));
-
-static void __attribute__ ((noreturn))
-foo(void)
-{
-  exit(1);
-}
-],
-lsh_cv_c_attribute=yes,
-lsh_cv_c_attribute=no)])
-
-AH_TEMPLATE([HAVE_GCC_ATTRIBUTE], [Define if the compiler understands __attribute__])
-if test "x$lsh_cv_c_attribute" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_ATTRIBUTE)
-fi
-
-AH_BOTTOM(
-[#if __GNUC__ || HAVE_GCC_ATTRIBUTE
-# define NORETURN __attribute__ ((__noreturn__))
-# define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# define UNUSED __attribute__ ((__unused__))
-#else
-# define NORETURN
-# define PRINTF_STYLE(f, a)
-# define UNUSED
-#endif
-])])
-
-AC_DEFUN([LSH_GCC_FUNCTION_NAME],
-[# Check for gcc's __FUNCTION__ variable
-AH_TEMPLATE([HAVE_GCC_FUNCTION],
-	    [Define if the compiler understands __FUNCTION__])
-AH_BOTTOM(
-[#if HAVE_GCC_FUNCTION
-# define FUNCTION_NAME __FUNCTION__
-#else
-# define FUNCTION_NAME "Unknown"
-#endif
-])
-
-AC_CACHE_CHECK(for __FUNCTION__,
-	       lsh_cv_c_FUNCTION,
-  [ AC_TRY_COMPILE(,
-      [ #if __GNUC__ == 3
-	#  error __FUNCTION__ is broken in gcc-3
-	#endif
-        void foo(void) { char c = __FUNCTION__[0]; } ],
-      lsh_cv_c_FUNCTION=yes,
-      lsh_cv_c_FUNCTION=no)])
-
-if test "x$lsh_cv_c_FUNCTION" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_FUNCTION)
-fi
-])
-
-# Check for alloca, and include the standard blurb in config.h
-AC_DEFUN([LSH_FUNC_ALLOCA],
-[AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([malloc.h])
-AH_BOTTOM(
-[/* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# else
-#  ifdef _AIX
- #pragma alloca
-#  else
-#   ifndef alloca /* predefined by HP cc +Olibcalls */
-char *alloca ();
-#   endif
-#  endif
-# endif
-#else /* defined __GNUC__ */
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# endif
-#endif
-/* Needed for alloca on windows */
-#if HAVE_MALLOC_H
-# include <malloc.h>
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRERROR],
-[AC_CHECK_FUNCS(strerror)
-AH_BOTTOM(
-[#if HAVE_STRERROR
-#define STRERROR strerror
-#else
-#define STRERROR(x) (sys_errlist[x])
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRSIGNAL],
-[AC_CHECK_FUNCS(strsignal)
-AC_CHECK_DECLS([sys_siglist, _sys_siglist])
-AH_BOTTOM(
-[#if HAVE_STRSIGNAL
-# define STRSIGNAL strsignal
-#else /* !HAVE_STRSIGNAL */
-# if HAVE_DECL_SYS_SIGLIST
-#  define STRSIGNAL(x) (sys_siglist[x])
-# else
-#  if HAVE_DECL__SYS_SIGLIST
-#   define STRSIGNAL(x) (_sys_siglist[x])
-#  else
-#   define STRSIGNAL(x) "Unknown signal"
-#   if __GNUC__
-#    warning Using dummy STRSIGNAL
-#   endif
-#  endif
-# endif
-#endif /* !HAVE_STRSIGNAL */
-])])
-
-dnl LSH_MAKE_CONDITIONAL(symbol, test)
-AC_DEFUN([LSH_MAKE_CONDITIONAL],
-[if $2 ; then
-  IF_$1=''
-  UNLESS_$1='# '
-else
-  IF_$1='# '
-  UNLESS_$1=''
-fi 
-AC_SUBST(IF_$1)
-AC_SUBST(UNLESS_$1)])
-
-dnl LSH_DEPENDENCY_TRACKING
-
-dnl Defines compiler flags DEP_FLAGS to generate dependency
-dnl information, and DEP_PROCESS that is any shell commands needed for
-dnl massaging the dependency information further. Dependencies are
-dnl generated as a side effect of compilation. Dependency files
-dnl themselves are not treated as targets.
-
-AC_DEFUN([LSH_DEPENDENCY_TRACKING],
-[AC_ARG_ENABLE(dependency_tracking,
-  AC_HELP_STRING([--disable-dependency-tracking],
-    [Disable dependency tracking. Dependency tracking doesn't work with BSD make]),,
-  [enable_dependency_tracking=yes])
-
-DEP_FLAGS=''
-DEP_PROCESS='true'
-if test x$enable_dependency_tracking = xyes ; then
-  if test x$GCC = xyes ; then
-    gcc_version=`gcc --version | head -1`
-    case "$gcc_version" in
-      2.*|*[[!0-9.]]2.*)
-        enable_dependency_tracking=no
-        AC_MSG_WARN([Dependency tracking disabled, gcc-3.x is needed])
-      ;;
-      *)
-        DEP_FLAGS='-MT $[]@ -MD -MP -MF $[]@.d'
-        DEP_PROCESS='true'
-      ;;
-    esac
-  else
-    enable_dependency_tracking=no
-    AC_MSG_WARN([Dependency tracking disabled])
-  fi
-fi
-
-if test x$enable_dependency_tracking = xyes ; then
-  DEP_INCLUDE='include '
-else
-  DEP_INCLUDE='# '
-fi
-
-AC_SUBST([DEP_INCLUDE])
-AC_SUBST([DEP_FLAGS])
-AC_SUBST([DEP_PROCESS])])
-
-dnl @synopsis AX_CREATE_STDINT_H [( HEADER-TO-GENERATE [, HEADERS-TO-CHECK])]
-dnl
-dnl the "ISO C9X: 7.18 Integer types <stdint.h>" section requires the
-dnl existence of an include file <stdint.h> that defines a set of 
-dnl typedefs, especially uint8_t,int32_t,uintptr_t.
-dnl Many older installations will not provide this file, but some will
-dnl have the very same definitions in <inttypes.h>. In other enviroments
-dnl we can use the inet-types in <sys/types.h> which would define the
-dnl typedefs int8_t and u_int8_t respectivly.
-dnl
-dnl This macros will create a local "_stdint.h" or the headerfile given as 
-dnl an argument. In many cases that file will just "#include <stdint.h>" 
-dnl or "#include <inttypes.h>", while in other environments it will provide 
-dnl the set of basic 'stdint's definitions/typedefs: 
-dnl   int8_t,uint8_t,int16_t,uint16_t,int32_t,uint32_t,intptr_t,uintptr_t
-dnl   int_least32_t.. int_fast32_t.. intmax_t
-dnl which may or may not rely on the definitions of other files,
-dnl or using the AC_CHECK_SIZEOF macro to determine the actual
-dnl sizeof each type.
-dnl
-dnl if your header files require the stdint-types you will want to create an
-dnl installable file mylib-int.h that all your other installable header
-dnl may include. So if you have a library package named "mylib", just use
-dnl      AX_CREATE_STDINT_H(mylib-int.h) 
-dnl in configure.ac and go to install that very header file in Makefile.am
-dnl along with the other headers (mylib.h) - and the mylib-specific headers
-dnl can simply use "#include <mylib-int.h>" to obtain the stdint-types.
-dnl
-dnl Remember, if the system already had a valid <stdint.h>, the generated
-dnl file will include it directly. No need for fuzzy HAVE_STDINT_H things...
-dnl
-dnl @, (status: used on new platforms) (see http://ac-archive.sf.net/gstdint/)
-dnl @version $Id: acinclude.m4,v 1.27 2004/11/23 21:27:35 nisse Exp $
-dnl @author  Guido Draheim <guidod@gmx.de> 
-
-AC_DEFUN([AX_CREATE_STDINT_H],
-[# ------ AX CREATE STDINT H -------------------------------------
-AC_MSG_CHECKING([for stdint types])
-ac_stdint_h=`echo ifelse($1, , _stdint.h, $1)`
-# try to shortcircuit - if the default include path of the compiler
-# can find a "stdint.h" header then we assume that all compilers can.
-AC_CACHE_VAL([ac_cv_header_stdint_t],[
-old_CXXFLAGS="$CXXFLAGS" ; CXXFLAGS=""
-old_CPPFLAGS="$CPPFLAGS" ; CPPFLAGS=""
-old_CFLAGS="$CFLAGS"     ; CFLAGS=""
-AC_TRY_COMPILE([#include <stdint.h>],[int_least32_t v = 0;],
-[ac_cv_stdint_result="(assuming C99 compatible system)"
- ac_cv_header_stdint_t="stdint.h"; ],
-[ac_cv_header_stdint_t=""])
-CXXFLAGS="$old_CXXFLAGS"
-CPPFLAGS="$old_CPPFLAGS"
-CFLAGS="$old_CFLAGS" ])
-
-v="... $ac_cv_header_stdint_h"
-if test "$ac_stdint_h" = "stdint.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./stdint.h?)])
-elif test "$ac_stdint_h" = "inttypes.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./inttypes.h?)])
-elif test "_$ac_cv_header_stdint_t" = "_" ; then
- AC_MSG_RESULT([(putting them into $ac_stdint_h)$v])
-else
- ac_cv_header_stdint="$ac_cv_header_stdint_t"
- AC_MSG_RESULT([$ac_cv_header_stdint (shortcircuit)])
-fi
-
-if test "_$ac_cv_header_stdint_t" = "_" ; then # can not shortcircuit..
-
-dnl .....intro message done, now do a few system checks.....
-dnl btw, all CHECK_TYPE macros do automatically "DEFINE" a type, therefore
-dnl we use the autoconf implementation detail _AC CHECK_TYPE_NEW instead
-
-inttype_headers=`echo $2 | sed -e 's/,/ /g'`
-
-ac_cv_stdint_result="(no helpful system typedefs seen)"
-AC_CACHE_CHECK([for stdint uintptr_t], [ac_cv_header_stdint_x],[
- ac_cv_header_stdint_x="" # the 1997 typedefs (inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in stdint.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_uintptr_t 
-   unset ac_cv_type_uint64_t
-   _AC_CHECK_TYPE_NEW(uintptr_t,[ac_cv_header_stdint_x=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uintptr_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uintptr_t])
- ])
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-AC_CACHE_CHECK([for stdint uint32_t], [ac_cv_header_stdint_o],[
- ac_cv_header_stdint_o="" # the 1995 typedefs (sys/inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in inttypes.h sys/inttypes.h stdint.h $inttype_headers ; do
-   unset ac_cv_type_uint32_t
-   unset ac_cv_type_uint64_t
-   AC_CHECK_TYPE(uint32_t,[ac_cv_header_stdint_o=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uint32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uint32_t])
- ])
-fi
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-if test "_$ac_cv_header_stdint_o" = "_" ; then
-AC_CACHE_CHECK([for stdint u_int32_t], [ac_cv_header_stdint_u],[
- ac_cv_header_stdint_u="" # the BSD typedefs (sys/types.h)
-  AC_MSG_RESULT([(..)])
-  for i in sys/types.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_u_int32_t
-   unset ac_cv_type_u_int64_t
-   AC_CHECK_TYPE(u_int32_t,[ac_cv_header_stdint_u=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(u_int64_t,[and64="/u_int64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen u_int32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint u_int32_t])
- ])
-fi fi
-
-dnl if there was no good C99 header file, do some typedef checks...
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-   AC_MSG_CHECKING([for stdint datatype model])
-   AC_MSG_RESULT([(..)])
-   AC_CHECK_SIZEOF(char)
-   AC_CHECK_SIZEOF(short)
-   AC_CHECK_SIZEOF(int)
-   AC_CHECK_SIZEOF(long)
-   AC_CHECK_SIZEOF(void*)
-   ac_cv_stdint_char_model=""
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_char"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_short"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model=""
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_long"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_voidp"
-   name="$ac_cv_stdint_long_model"
-   case "$ac_cv_stdint_char_model/$ac_cv_stdint_long_model" in
-    122/242)     name="$name,  IP16 (standard 16bit machine)" ;;
-    122/244)     name="$name,  LP32 (standard 32bit mac/win)" ;;
-    122/*)       name="$name        (unusual int16 model)" ;; 
-    124/444)     name="$name, ILP32 (standard 32bit unixish)" ;;
-    124/488)     name="$name,  LP64 (standard 64bit unixish)" ;;
-    124/448)     name="$name, LLP64 (unusual  64bit unixish)" ;;
-    124/*)       name="$name        (unusual int32 model)" ;; 
-    128/888)     name="$name, ILP64 (unusual  64bit numeric)" ;;
-    128/*)       name="$name        (unusual int64 model)" ;; 
-    222/*|444/*) name="$name        (unusual dsptype)" ;;
-     *)          name="$name        (very unusal model)" ;;
-   esac
-   AC_MSG_RESULT([combined for stdint datatype model...  $name])
-fi
-
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_x"
-elif  test "_$ac_cv_header_stdint_o" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_o"
-elif  test "_$ac_cv_header_stdint_u" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_u"
-else
-   ac_cv_header_stdint="stddef.h"
-fi
-
-AC_MSG_CHECKING([for extra inttypes in chosen header])
-AC_MSG_RESULT([($ac_cv_header_stdint)])
-dnl see if int_least and int_fast types are present in _this_ header.
-unset ac_cv_type_int_least32_t
-unset ac_cv_type_int_fast32_t
-AC_CHECK_TYPE(int_least32_t,,,[#include <$ac_cv_header_stdint>])
-AC_CHECK_TYPE(int_fast32_t,,,[#include<$ac_cv_header_stdint>])
-AC_CHECK_TYPE(intmax_t,,,[#include <$ac_cv_header_stdint>])
-
-fi # shortcircut to system "stdint.h"
-# ------------------ PREPARE VARIABLES ------------------------------
-if test "$GCC" = "yes" ; then
-ac_cv_stdint_message="using gnu compiler "`$CC --version | head -1` 
-else
-ac_cv_stdint_message="using $CC"
-fi
-
-AC_MSG_RESULT([make use of $ac_cv_header_stdint in $ac_stdint_h dnl
-$ac_cv_stdint_result])
-
-# ----------------- DONE inttypes.h checks START header -------------
-AC_CONFIG_COMMANDS([$ac_stdint_h],[
-AC_MSG_NOTICE(creating $ac_stdint_h : $_ac_stdint_h)
-ac_stdint=$tmp/_stdint.h
-
-echo "#ifndef" $_ac_stdint_h >$ac_stdint
-echo "#define" $_ac_stdint_h "1" >>$ac_stdint
-echo "#ifndef" _GENERATED_STDINT_H >>$ac_stdint
-echo "#define" _GENERATED_STDINT_H '"'$PACKAGE $VERSION'"' >>$ac_stdint
-echo "/* generated $ac_cv_stdint_message */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_t" != "_" ; then 
-echo "#define _STDINT_HAVE_STDINT_H" "1" >>$ac_stdint
-fi
-
-cat >>$ac_stdint <<STDINT_EOF
-
-/* ................... shortcircuit part ........................... */
-
-#if defined HAVE_STDINT_H || defined _STDINT_HAVE_STDINT_H
-#include <stdint.h>
-#else
-#include <stddef.h>
-
-/* .................... configured part ............................ */
-
-STDINT_EOF
-
-echo "/* whether we have a C99 compatible stdint header file */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-  ac_header="$ac_cv_header_stdint_x"
-  echo "#define _STDINT_HEADER_INTPTR" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_INTPTR */" >>$ac_stdint
-fi
-
-echo "/* whether we have a C96 compatible inttypes header file */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_o" != "_" ; then
-  ac_header="$ac_cv_header_stdint_o"
-  echo "#define _STDINT_HEADER_UINT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_UINT32 */" >>$ac_stdint
-fi
-
-echo "/* whether we have a BSD compatible inet types header */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_u" != "_" ; then
-  ac_header="$ac_cv_header_stdint_u"
-  echo "#define _STDINT_HEADER_U_INT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_U_INT32 */" >>$ac_stdint
-fi
-
-echo "" >>$ac_stdint
-
-if test "_$ac_header" != "_" ; then if test "$ac_header" != "stddef.h" ; then
-  echo "#include <$ac_header>" >>$ac_stdint
-  echo "" >>$ac_stdint
-fi fi
-
-echo "/* which 64bit typedef has been found */" >>$ac_stdint
-if test "$ac_cv_type_uint64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_UINT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_UINT64_T */" >>$ac_stdint
-fi
-if test "$ac_cv_type_u_int64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_U_INT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_U_INT64_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* which type model has been detected */" >>$ac_stdint
-if test "_$ac_cv_stdint_char_model" != "_" ; then
-echo "#define   _STDINT_CHAR_MODEL" "$ac_cv_stdint_char_model" >>$ac_stdint
-echo "#define   _STDINT_LONG_MODEL" "$ac_cv_stdint_long_model" >>$ac_stdint
-else
-echo "/* #undef _STDINT_CHAR_MODEL // skipped */" >>$ac_stdint
-echo "/* #undef _STDINT_LONG_MODEL // skipped */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* whether int_least types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_least32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_LEAST32_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_LEAST32_T */" >>$ac_stdint
-fi
-echo "/* whether int_fast types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_fast32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_FAST32_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_FAST32_T */" >>$ac_stdint
-fi
-echo "/* whether intmax_t type was detected */" >>$ac_stdint
-if test "$ac_cv_type_intmax_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INTMAX_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INTMAX_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-  cat >>$ac_stdint <<STDINT_EOF
-/* .................... detections part ............................ */
-
-/* whether we need to define bitspecific types from compiler base types */
-#ifndef _STDINT_HEADER_INTPTR
-#ifndef _STDINT_HEADER_UINT32
-#ifndef _STDINT_HEADER_U_INT32
-#define _STDINT_NEED_INT_MODEL_T
-#else
-#define _STDINT_HAVE_U_INT_TYPES
-#endif
-#endif
-#endif
-
-#ifdef _STDINT_HAVE_U_INT_TYPES
-#undef _STDINT_NEED_INT_MODEL_T
-#endif
-
-#ifdef  _STDINT_CHAR_MODEL
-#if     _STDINT_CHAR_MODEL+0 == 122 || _STDINT_CHAR_MODEL+0 == 124
-#ifndef _STDINT_BYTE_MODEL
-#define _STDINT_BYTE_MODEL 12
-#endif
-#endif
-#endif
-
-#ifndef _STDINT_HAVE_INT_LEAST32_T
-#define _STDINT_NEED_INT_LEAST_T
-#endif
-
-#ifndef _STDINT_HAVE_INT_FAST32_T
-#define _STDINT_NEED_INT_FAST_T
-#endif
-
-#ifndef _STDINT_HEADER_INTPTR
-#define _STDINT_NEED_INTPTR_T
-#ifndef _STDINT_HAVE_INTMAX_T
-#define _STDINT_NEED_INTMAX_T
-#endif
-#endif
-
-
-/* .................... definition part ............................ */
-
-/* some system headers have good uint64_t */
-#ifndef _HAVE_UINT64_T
-#if     defined _STDINT_HAVE_UINT64_T  || defined HAVE_UINT64_T
-#define _HAVE_UINT64_T
-#elif   defined _STDINT_HAVE_U_INT64_T || defined HAVE_U_INT64_T
-#define _HAVE_UINT64_T
-typedef u_int64_t uint64_t;
-#endif
-#endif
-
-#ifndef _HAVE_UINT64_T
-/* .. here are some common heuristics using compiler runtime specifics */
-#if defined __STDC_VERSION__ && defined __STDC_VERSION__ >= 199901L
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-#elif !defined __STRICT_ANSI__
-#if defined _MSC_VER || defined __WATCOMC__ || defined __BORLANDC__
-#define _HAVE_UINT64_T
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-
-#elif defined __GNUC__ || defined __MWERKS__ || defined __ELF__
-/* note: all ELF-systems seem to have loff-support which needs 64-bit */
-#if !defined _NO_LONGLONG
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-#endif
-
-#elif defined __alpha || (defined __mips && defined _ABIN32)
-#if !defined _NO_LONGLONG
-typedef long int64_t;
-typedef unsigned long uint64_t;
-#endif
-  /* compiler/cpu type to define int64_t */
-#endif
-#endif
-#endif
-
-#if defined _STDINT_HAVE_U_INT_TYPES
-/* int8_t int16_t int32_t defined by inet code, redeclare the u_intXX types */
-typedef u_int8_t uint8_t;
-typedef u_int16_t uint16_t;
-typedef u_int32_t uint32_t;
-
-/* glibc compatibility */
-#ifndef __int8_t_defined
-#define __int8_t_defined
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INT_MODEL_T
-/* we must guess all the basic types. Apart from byte-adressable system, */
-/* there a few 32-bit-only dsp-systems that we guard with BYTE_MODEL 8-} */
-/* (btw, those nibble-addressable systems are way off, or so we assume) */
-
-dnl   /* have a look at "64bit and data size neutrality" at */
-dnl   /* http://unix.org/version2/whatsnew/login_64bit.html */
-dnl   /* (the shorthand "ILP" types always have a "P" part) */
-
-#if defined _STDINT_BYTE_MODEL
-#if _STDINT_LONG_MODEL+0 == 242
-/* 2:4:2 =  IP16 = a normal 16-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned long   uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          long    int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL == 444
-/* 2:4:4 =  LP32 = a 32-bit system derived from a 16-bit */
-/* 4:4:4 = ILP32 = a normal 32-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 484 || _STDINT_LONG_MODEL+0 == 488
-/* 4:8:4 =  IP32 = a 32-bit system prepared for 64-bit    */
-/* 4:8:8 =  LP64 = a normal 64-bit system                 */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* this system has a "long" of 64bit */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long   uint64_t;
-typedef          long    int64_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 448
-/*      LLP64   a 64-bit system derived from a 32-bit system */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* assuming the system has a "long long" */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-typedef          long long  int64_t;
-#endif
-#else
-#define _STDINT_NO_INT32_T
-#endif
-#else
-#define _STDINT_NO_INT8_T
-#define _STDINT_NO_INT32_T
-#endif
-#endif
-
-/*
- * quote from SunOS-5.8 sys/inttypes.h:
- * Use at your own risk.  As of February 1996, the committee is squarely
- * behind the fixed sized types; the "least" and "fast" types are still being
- * discussed.  The probability that the "fast" types may be removed before
- * the standard is finalized is high enough that they are not currently
- * implemented.
- */
-
-#if defined _STDINT_NEED_INT_LEAST_T
-typedef  int8_t    int_least8_t;
-typedef  int16_t   int_least16_t;
-typedef  int32_t   int_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_least64_t;
-#endif
-
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_least64_t;
-#endif
-  /* least types */
-#endif
-
-#if defined _STDINT_NEED_INT_FAST_T
-typedef  int8_t    int_fast8_t; 
-typedef  int       int_fast16_t;
-typedef  int32_t   int_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_fast64_t;
-#endif
-
-typedef uint8_t   uint_fast8_t; 
-typedef unsigned  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_fast64_t;
-#endif
-  /* fast types */
-#endif
-
-#ifdef _STDINT_NEED_INTMAX_T
-#ifdef _HAVE_UINT64_T
-typedef  int64_t       intmax_t;
-typedef uint64_t      uintmax_t;
-#else
-typedef          long  intmax_t;
-typedef unsigned long uintmax_t;
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INTPTR_T
-#ifndef __intptr_t_defined
-#define __intptr_t_defined
-/* we encourage using "long" to store pointer values, never use "int" ! */
-#if   _STDINT_LONG_MODEL+0 == 242 || _STDINT_LONG_MODEL+0 == 484
-typedef  unsinged int   uintptr_t;
-typedef           int    intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL+0 == 444
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 448 && defined _HAVE_UINT64_T
-typedef        uint64_t uintptr_t;
-typedef         int64_t  intptr_t;
-#else /* matches typical system types ILP32 and LP64 - but not IP16 or LLP64 */
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#endif
-#endif
-#endif
-
-  /* shortcircuit*/
-#endif
-  /* once */
-#endif
-#endif
-STDINT_EOF
-    if cmp -s $ac_stdint_h $ac_stdint 2>/dev/null; then
-      AC_MSG_NOTICE([$ac_stdint_h is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$ac_stdint_h"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f $ac_stdint_h
-      mv $ac_stdint $ac_stdint_h
-    fi
-],[# variables for create stdint.h replacement
-PACKAGE="$PACKAGE"
-VERSION="$VERSION"
-ac_stdint_h="$ac_stdint_h"
-_ac_stdint_h=AS_TR_CPP(_$PACKAGE-$ac_stdint_h)
-ac_cv_stdint_message="$ac_cv_stdint_message"
-ac_cv_header_stdint_t="$ac_cv_header_stdint_t"
-ac_cv_header_stdint_x="$ac_cv_header_stdint_x"
-ac_cv_header_stdint_o="$ac_cv_header_stdint_o"
-ac_cv_header_stdint_u="$ac_cv_header_stdint_u"
-ac_cv_type_uint64_t="$ac_cv_type_uint64_t"
-ac_cv_type_u_int64_t="$ac_cv_type_u_int64_t"
-ac_cv_stdint_char_model="$ac_cv_stdint_char_model"
-ac_cv_stdint_long_model="$ac_cv_stdint_long_model"
-ac_cv_type_int_least32_t="$ac_cv_type_int_least32_t"
-ac_cv_type_int_fast32_t="$ac_cv_type_int_fast32_t"
-ac_cv_type_intmax_t="$ac_cv_type_intmax_t"
-])
-])
diff --git a/contrib/argp-standalone/argp-ba.c b/contrib/argp-standalone/argp-ba.c
deleted file mode 100644
index 0d3958c1151..00000000000
--- a/contrib/argp-standalone/argp-ba.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Default definition for ARGP_PROGRAM_BUG_ADDRESS.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program, it should point to string that is the
-   bug-reporting address for the program.  It will be printed by argp_help if
-   the ARGP_HELP_BUG_ADDR flag is set (as it is by various standard help
-   messages), embedded in a sentence that says something like `Report bugs to
-   ADDR.'.  */
-const char *argp_program_bug_address = 0;
diff --git a/contrib/argp-standalone/argp-eexst.c b/contrib/argp-standalone/argp-eexst.c
deleted file mode 100644
index 46b27847ad4..00000000000
--- a/contrib/argp-standalone/argp-eexst.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Default definition for ARGP_ERR_EXIT_STATUS
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_SYSEXITS_H
-# include <sysexits.h>
-#else
-# define EX_USAGE 64
-#endif
-
-#include "argp.h"
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-error_t argp_err_exit_status = EX_USAGE;
diff --git a/contrib/argp-standalone/argp-fmtstream.c b/contrib/argp-standalone/argp-fmtstream.c
deleted file mode 100644
index 494b6b31d12..00000000000
--- a/contrib/argp-standalone/argp-fmtstream.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/* Word-wrapping and line-truncating streams
-   Copyright (C) 1997, 1998, 1999, 2001 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-#ifndef ARGP_FMTSTREAM_USE_LINEWRAP
-
-#ifndef isblank
-#define isblank(ch) ((ch)==' ' || (ch)=='\t')
-#endif
-
-#if defined _LIBC && defined USE_IN_LIBIO
-# include <libio/libioP.h>
-# define __vsnprintf(s, l, f, a) _IO_vsnprintf (s, l, f, a)
-#endif
-
-#define INIT_BUF_SIZE 200
-#define PRINTF_SIZE_GUESS 150
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-argp_fmtstream_t
-__argp_make_fmtstream (FILE *stream,
-		       size_t lmargin, size_t rmargin, ssize_t wmargin)
-{
-  argp_fmtstream_t fs = malloc (sizeof (struct argp_fmtstream));
-  if (fs)
-    {
-      fs->stream = stream;
-
-      fs->lmargin = lmargin;
-      fs->rmargin = rmargin;
-      fs->wmargin = wmargin;
-      fs->point_col = 0;
-      fs->point_offs = 0;
-
-      fs->buf = malloc (INIT_BUF_SIZE);
-      if (! fs->buf)
-	{
-	  free (fs);
-	  fs = 0;
-	}
-      else
-	{
-	  fs->p = fs->buf;
-	  fs->end = fs->buf + INIT_BUF_SIZE;
-	}
-    }
-
-  return fs;
-}
-#ifdef weak_alias
-weak_alias (__argp_make_fmtstream, argp_make_fmtstream)
-#endif
-
-/* Flush FS to its stream, and free it (but don't close the stream).  */
-void
-__argp_fmtstream_free (argp_fmtstream_t fs)
-{
-  __argp_fmtstream_update (fs);
-  if (fs->p > fs->buf)
-    FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-  free (fs->buf);
-  free (fs);
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_free, argp_fmtstream_free)
-#endif
-
-/* Process FS's buffer so that line wrapping is done from POINT_OFFS to the
-   end of its buffer.  This code is mostly from glibc stdio/linewrap.c.  */
-void
-__argp_fmtstream_update (argp_fmtstream_t fs)
-{
-  char *buf, *nl;
-  size_t len;
-
-  /* Scan the buffer for newlines.  */
-  buf = fs->buf + fs->point_offs;
-  while (buf < fs->p)
-    {
-      size_t r;
-
-      if (fs->point_col == 0 && fs->lmargin != 0)
-	{
-	  /* We are starting a new line.  Print spaces to the left margin.  */
-	  const size_t pad = fs->lmargin;
-	  if (fs->p + pad < fs->end)
-	    {
-	      /* We can fit in them in the buffer by moving the
-		 buffer text up and filling in the beginning.  */
-	      memmove (buf + pad, buf, fs->p - buf);
-	      fs->p += pad; /* Compensate for bigger buffer. */
-	      memset (buf, ' ', pad); /* Fill in the spaces.  */
-	      buf += pad; /* Don't bother searching them.  */
-	    }
-	  else
-	    {
-	      /* No buffer space for spaces.  Must flush.  */
-	      size_t i;
-	      for (i = 0; i < pad; i++)
-		PUTC_UNLOCKED (' ', fs->stream);
-	    }
-	  fs->point_col = pad;
-	}
-
-      len = fs->p - buf;
-      nl = memchr (buf, '\n', len);
-
-      if (fs->point_col < 0)
-	fs->point_col = 0;
-
-      if (!nl)
-	{
-	  /* The buffer ends in a partial line.  */
-
-	  if (fs->point_col + len < fs->rmargin)
-	    {
-	      /* The remaining buffer text is a partial line and fits
-		 within the maximum line width.  Advance point for the
-		 characters to be written and stop scanning.  */
-	      fs->point_col += len;
-	      break;
-	    }
-	  else
-	    /* Set the end-of-line pointer for the code below to
-	       the end of the buffer.  */
-	    nl = fs->p;
-	}
-      else if (fs->point_col + (nl - buf) < (ssize_t) fs->rmargin)
-	{
-	  /* The buffer contains a full line that fits within the maximum
-	     line width.  Reset point and scan the next line.  */
-	  fs->point_col = 0;
-	  buf = nl + 1;
-	  continue;
-	}
-
-      /* This line is too long.  */
-      r = fs->rmargin - 1;
-
-      if (fs->wmargin < 0)
-	{
-	  /* Truncate the line by overwriting the excess with the
-	     newline and anything after it in the buffer.  */
-	  if (nl < fs->p)
-	    {
-	      memmove (buf + (r - fs->point_col), nl, fs->p - nl);
-	      fs->p -= buf + (r - fs->point_col) - nl;
-	      /* Reset point for the next line and start scanning it.  */
-	      fs->point_col = 0;
-	      buf += r + 1; /* Skip full line plus \n. */
-	    }
-	  else
-	    {
-	      /* The buffer ends with a partial line that is beyond the
-		 maximum line width.  Advance point for the characters
-		 written, and discard those past the max from the buffer.  */
-	      fs->point_col += len;
-	      fs->p -= fs->point_col - r;
-	      break;
-	    }
-	}
-      else
-	{
-	  /* Do word wrap.  Go to the column just past the maximum line
-	     width and scan back for the beginning of the word there.
-	     Then insert a line break.  */
-
-	  char *p, *nextline;
-	  int i;
-
-	  p = buf + (r + 1 - fs->point_col);
-	  while (p >= buf && !isblank (*p))
-	    --p;
-	  nextline = p + 1;	/* This will begin the next line.  */
-
-	  if (nextline > buf)
-	    {
-	      /* Swallow separating blanks.  */
-	      if (p >= buf)
-		do
-		  --p;
-		while (p >= buf && isblank (*p));
-	      nl = p + 1;	/* The newline will replace the first blank. */
-	    }
-	  else
-	    {
-	      /* A single word that is greater than the maximum line width.
-		 Oh well.  Put it on an overlong line by itself.  */
-	      p = buf + (r + 1 - fs->point_col);
-	      /* Find the end of the long word.  */
-	      do
-		++p;
-	      while (p < nl && !isblank (*p));
-	      if (p == nl)
-		{
-		  /* It already ends a line.  No fussing required.  */
-		  fs->point_col = 0;
-		  buf = nl + 1;
-		  continue;
-		}
-	      /* We will move the newline to replace the first blank.  */
-	      nl = p;
-	      /* Swallow separating blanks.  */
-	      do
-		++p;
-	      while (isblank (*p));
-	      /* The next line will start here.  */
-	      nextline = p;
-	    }
-
-	  /* Note: There are a bunch of tests below for
-	     NEXTLINE == BUF + LEN + 1; this case is where NL happens to fall
-	     at the end of the buffer, and NEXTLINE is in fact empty (and so
-	     we need not be careful to maintain its contents).  */
-
-	  if (nextline == buf + len + 1
-	      ? fs->end - nl < fs->wmargin + 1
-	      : nextline - (nl + 1) < fs->wmargin)
-	    {
-	      /* The margin needs more blanks than we removed.  */
-	      if (fs->end - fs->p > fs->wmargin + 1)
-		/* Make some space for them.  */
-		{
-		  size_t mv = fs->p - nextline;
-		  memmove (nl + 1 + fs->wmargin, nextline, mv);
-		  nextline = nl + 1 + fs->wmargin;
-		  len = nextline + mv - buf;
-		  *nl++ = '\n';
-		}
-	      else
-		/* Output the first line so we can use the space.  */
-		{
-		  if (nl > fs->buf)
-		    FWRITE_UNLOCKED (fs->buf, 1, nl - fs->buf, fs->stream);
-		  PUTC_UNLOCKED ('\n', fs->stream);
-		  len += buf - fs->buf;
-		  nl = buf = fs->buf;
-		}
-	    }
-	  else
-	    /* We can fit the newline and blanks in before
-	       the next word.  */
-	    *nl++ = '\n';
-
-	  if (nextline - nl >= fs->wmargin
-	      || (nextline == buf + len + 1 && fs->end - nextline >= fs->wmargin))
-	    /* Add blanks up to the wrap margin column.  */
-	    for (i = 0; i < fs->wmargin; ++i)
-	      *nl++ = ' ';
-	  else
-	    for (i = 0; i < fs->wmargin; ++i)
-	      PUTC_UNLOCKED (' ', fs->stream);
-
-	  /* Copy the tail of the original buffer into the current buffer
-	     position.  */
-	  if (nl < nextline)
-	    memmove (nl, nextline, buf + len - nextline);
-	  len -= nextline - buf;
-
-	  /* Continue the scan on the remaining lines in the buffer.  */
-	  buf = nl;
-
-	  /* Restore bufp to include all the remaining text.  */
-	  fs->p = nl + len;
-
-	  /* Reset the counter of what has been output this line.  If wmargin
-	     is 0, we want to avoid the lmargin getting added, so we set
-	     point_col to a magic value of -1 in that case.  */
-	  fs->point_col = fs->wmargin ? fs->wmargin : -1;
-	}
-    }
-
-  /* Remember that we've scanned as far as the end of the buffer.  */
-  fs->point_offs = fs->p - fs->buf;
-}
-
-/* Ensure that FS has space for AMOUNT more bytes in its buffer, either by
-   growing the buffer, or by flushing it.  True is returned iff we succeed. */
-int
-__argp_fmtstream_ensure (struct argp_fmtstream *fs, size_t amount)
-{
-  if ((size_t) (fs->end - fs->p) < amount)
-    {
-      ssize_t wrote;
-
-      /* Flush FS's buffer.  */
-      __argp_fmtstream_update (fs);
-
-      wrote = FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-      if (wrote == fs->p - fs->buf)
-	{
-	  fs->p = fs->buf;
-	  fs->point_offs = 0;
-	}
-      else
-	{
-	  fs->p -= wrote;
-	  fs->point_offs -= wrote;
-	  memmove (fs->buf, fs->buf + wrote, fs->p - fs->buf);
-	  return 0;
-	}
-
-      if ((size_t) (fs->end - fs->buf) < amount)
-	/* Gotta grow the buffer.  */
-	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
-
-	  if (! new_buf)
-	    {
-	      __set_errno (ENOMEM);
-	      return 0;
-	    }
-
-	  fs->buf = new_buf;
-	  fs->end = new_buf + new_size;
-	  fs->p = fs->buf;
-	}
-    }
-
-  return 1;
-}
-
-ssize_t
-__argp_fmtstream_printf (struct argp_fmtstream *fs, const char *fmt, ...)
-{
-  size_t out;
-  size_t avail;
-  size_t size_guess = PRINTF_SIZE_GUESS; /* How much space to reserve. */
-
-  do
-    {
-      va_list args;
-
-      if (! __argp_fmtstream_ensure (fs, size_guess))
-	return -1;
-
-      va_start (args, fmt);
-      avail = fs->end - fs->p;
-      out = __vsnprintf (fs->p, avail, fmt, args);
-      va_end (args);
-      if (out >= avail)
-	size_guess = out + 1;
-    }
-  while (out >= avail);
-
-  fs->p += out;
-
-  return out;
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_printf, argp_fmtstream_printf)
-#endif
-
-#if __STDC_VERSION__ - 199900L < 1
-/* Duplicate the inline definitions in argp-fmtstream.h, for compilers
- * that don't do inlining. */
-size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-#endif /* !ARGP_FMTSTREAM_USE_LINEWRAP */
diff --git a/contrib/argp-standalone/argp-fmtstream.h b/contrib/argp-standalone/argp-fmtstream.h
deleted file mode 100644
index 828f4357d56..00000000000
--- a/contrib/argp-standalone/argp-fmtstream.h
+++ /dev/null
@@ -1,327 +0,0 @@
-/* Word-wrapping and line-truncating streams.
-   Copyright (C) 1997, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  If the system does have it, it is just a wrapper for
-   that.  This header file is only used internally while compiling argp, and
-   shouldn't be installed.  */
-
-#ifndef _ARGP_FMTSTREAM_H
-#define _ARGP_FMTSTREAM_H
-
-#include <stdio.h>
-#include <string.h>
-
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#else
-/* This is a kludge to make the code compile on windows. Perhaps it
-   would be better to just replace ssize_t with int through out the
-   code. */
-# define ssize_t int
-#endif
-
-#if _LIBC || (defined (HAVE_FLOCKFILE) && defined(HAVE_PUTC_UNLOCKED) \
-     && defined (HAVE_FPUTS_UNLOCKED) && defined (HAVE_FWRITE_UNLOCKED) )
-/* Use locking funxtions */
-# define FLOCKFILE(f) flockfile(f)
-# define FUNLOCKFILE(f) funlockfile(f)
-# define PUTC_UNLOCKED(c, f) putc_unlocked((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs_unlocked((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite_unlocked((b), (s), (n), (f))
-#else
-/* Disable stdio locking */
-# define FLOCKFILE(f)
-# define FUNLOCKFILE(f)
-# define PUTC_UNLOCKED(c, f) putc((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite((b), (s), (n), (f))
-#endif /* No thread safe i/o */
-
-#if    (_LIBC - 0 && !defined (USE_IN_LIBIO)) \
-    || (defined (__GNU_LIBRARY__) && defined (HAVE_LINEWRAP_H))
-/* line_wrap_stream is available, so use that.  */
-#define ARGP_FMTSTREAM_USE_LINEWRAP
-#endif
-
-#ifdef ARGP_FMTSTREAM_USE_LINEWRAP
-/* Just be a simple wrapper for line_wrap_stream; the semantics are
-   *slightly* different, as line_wrap_stream doesn't actually make a new
-   object, it just modifies the given stream (reversibly) to do
-   line-wrapping.  Since we control who uses this code, it doesn't matter.  */
-
-#include <linewrap.h>
-
-typedef FILE *argp_fmtstream_t;
-
-#define argp_make_fmtstream line_wrap_stream
-#define __argp_make_fmtstream line_wrap_stream
-#define argp_fmtstream_free line_unwrap_stream
-#define __argp_fmtstream_free line_unwrap_stream
-
-#define __argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define __argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define __argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define __argp_fmtstream_printf fprintf
-#define argp_fmtstream_printf fprintf
-
-#define __argp_fmtstream_lmargin line_wrap_lmargin
-#define argp_fmtstream_lmargin line_wrap_lmargin
-#define __argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define __argp_fmtstream_rmargin line_wrap_rmargin
-#define argp_fmtstream_rmargin line_wrap_rmargin
-#define __argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define __argp_fmtstream_wmargin line_wrap_wmargin
-#define argp_fmtstream_wmargin line_wrap_wmargin
-#define __argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define __argp_fmtstream_point line_wrap_point
-#define argp_fmtstream_point line_wrap_point
-
-#else /* !ARGP_FMTSTREAM_USE_LINEWRAP */
-/* Guess we have to define our own version.  */
-
-#ifndef __const
-#define __const const
-#endif
-
-
-struct argp_fmtstream
-{
-  FILE *stream;			/* The stream we're outputting to.  */
-
-  size_t lmargin, rmargin;	/* Left and right margins.  */
-  ssize_t wmargin;		/* Margin to wrap to, or -1 to truncate.  */
-
-  /* Point in buffer to which we've processed for wrapping, but not output.  */
-  size_t point_offs;
-  /* Output column at POINT_OFFS, or -1 meaning 0 but don't add lmargin.  */
-  ssize_t point_col;
-
-  char *buf;			/* Output buffer.  */
-  char *p;			/* Current end of text in BUF. */
-  char *end;			/* Absolute end of BUF.  */
-};
-
-typedef struct argp_fmtstream *argp_fmtstream_t;
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-extern argp_fmtstream_t __argp_make_fmtstream (FILE *__stream,
-					       size_t __lmargin,
-					       size_t __rmargin,
-					       ssize_t __wmargin);
-extern argp_fmtstream_t argp_make_fmtstream (FILE *__stream,
-					     size_t __lmargin,
-					     size_t __rmargin,
-					     ssize_t __wmargin);
-
-/* Flush __FS to its stream, and free it (but don't close the stream).  */
-extern void __argp_fmtstream_free (argp_fmtstream_t __fs);
-extern void argp_fmtstream_free (argp_fmtstream_t __fs);
-
-extern ssize_t __argp_fmtstream_printf (argp_fmtstream_t __fs,
-				       __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-extern ssize_t argp_fmtstream_printf (argp_fmtstream_t __fs,
-				      __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-
-#if __STDC_VERSION__ - 199900L < 1
-extern int __argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-extern int argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-
-extern int __argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-extern int argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-
-extern size_t __argp_fmtstream_write (argp_fmtstream_t __fs,
-				      __const char *__str, size_t __len);
-extern size_t argp_fmtstream_write (argp_fmtstream_t __fs,
-				    __const char *__str, size_t __len);
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-/* Access macros for various bits of state.  */
-#define argp_fmtstream_lmargin(__fs) ((__fs)->lmargin)
-#define argp_fmtstream_rmargin(__fs) ((__fs)->rmargin)
-#define argp_fmtstream_wmargin(__fs) ((__fs)->wmargin)
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-#if __STDC_VERSION__ - 199900L < 1
-/* Set __FS's left margin to LMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					  size_t __lmargin);
-extern size_t __argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					    size_t __lmargin);
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					  size_t __rmargin);
-extern size_t __argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					    size_t __rmargin);
-
-/* Set __FS's wrap margin to __WMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					  size_t __wmargin);
-extern size_t __argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					    size_t __wmargin);
-
-/* Return the column number of the current output point in __FS.  */
-extern size_t argp_fmtstream_point (argp_fmtstream_t __fs);
-extern size_t __argp_fmtstream_point (argp_fmtstream_t __fs);
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-/* Internal routines.  */
-extern void _argp_fmtstream_update (argp_fmtstream_t __fs);
-extern void __argp_fmtstream_update (argp_fmtstream_t __fs);
-extern int _argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-extern int __argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-
-#ifdef __OPTIMIZE__
-/* Inline versions of above routines.  */
-
-#if !_LIBC
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#define __argp_fmtstream_write argp_fmtstream_write
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#define __argp_fmtstream_point argp_fmtstream_point
-#define __argp_fmtstream_update _argp_fmtstream_update
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#endif
-
-#ifndef ARGP_FS_EI
-#if defined(__GNUC__) && !defined(__GNUC_STDC_INLINE__)
-#define ARGP_FS_EI extern inline
-#else
-#define ARGP_FS_EI inline
-#endif
-#endif
-
-ARGP_FS_EI size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#if !_LIBC
-#undef __argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#undef __argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#undef __argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#endif
-
-#endif /* __OPTIMIZE__ */
-
-#endif /* ARGP_FMTSTREAM_USE_LINEWRAP */
-
-#endif /* argp-fmtstream.h */
diff --git a/contrib/argp-standalone/argp-help.c b/contrib/argp-standalone/argp-help.c
deleted file mode 100644
index ced78c4cb26..00000000000
--- a/contrib/argp-standalone/argp-help.c
+++ /dev/null
@@ -1,1849 +0,0 @@
-/* Hierarchial argument parsing help output
-   Copyright (C) 1995,96,97,98,99,2000, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <ctype.h>
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-# endif
-#endif
-
-#include "argp.h"
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-
-#ifndef _LIBC
-# ifndef __strchrnul
-#  define __strchrnul strchrnul
-# endif
-# ifndef __mempcpy
-#  define __mempcpy mempcpy
-# endif
-/* We need to use a different name, as __strndup is likely a macro. */
-# define STRNDUP strndup
-# if HAVE_STRERROR
-#  define STRERROR strerror
-# else
-#  define STRERROR(x) (sys_errlist[x])
-# endif
-#else /* _LIBC */
-# define FLOCKFILE __flockfile
-# define FUNLOCKFILE __funlockfile
-# define STRNDUP __strndup
-# define STRERROR strerror
-#endif
-
-#if !_LIBC
-# if !HAVE_STRNDUP
-char *strndup (const char *s, size_t size);
-# endif /* !HAVE_STRNDUP */
-
-# if !HAVE_MEMPCPY
-void *mempcpy (void *to, const void *from, size_t size);
-# endif /* !HAVE_MEMPCPY */
-
-# if !HAVE_STRCHRNUL
-char *strchrnul(const char *s, int c);
-# endif /* !HAVE_STRCHRNUL */
-
-# if !HAVE_STRCASECMP
-int strcasecmp(const char *s1, const char *s2);
-#endif
-
-#endif /* !_LIBC */
-
-
-/* User-selectable (using an environment variable) formatting parameters.
-
-   These may be specified in an environment variable called `ARGP_HELP_FMT',
-   with a contents like:  VAR1=VAL1,VAR2=VAL2,BOOLVAR2,no-BOOLVAR2
-   Where VALn must be a positive integer.  The list of variables is in the
-   UPARAM_NAMES vector, below.  */
-
-/* Default parameters.  */
-#define DUP_ARGS      0		/* True if option argument can be duplicated. */
-#define DUP_ARGS_NOTE 1		/* True to print a note about duplicate args. */
-#define SHORT_OPT_COL 2		/* column in which short options start */
-#define LONG_OPT_COL  6		/* column in which long options start */
-#define DOC_OPT_COL   2		/* column in which doc options start */
-#define OPT_DOC_COL  29		/* column in which option text starts */
-#define HEADER_COL    1		/* column in which group headers are printed */
-#define USAGE_INDENT 12		/* indentation of wrapped usage lines */
-#define RMARGIN      79		/* right margin used for wrapping */
-
-/* User-selectable (using an environment variable) formatting parameters.
-   They must all be of type `int' for the parsing code to work.  */
-struct uparams
-{
-  /* If true, arguments for an option are shown with both short and long
-     options, even when a given option has both, e.g. `-x ARG, --longx=ARG'.
-     If false, then if an option has both, the argument is only shown with
-     the long one, e.g., `-x, --longx=ARG', and a message indicating that
-     this really means both is printed below the options.  */
-  int dup_args;
-
-  /* This is true if when DUP_ARGS is false, and some duplicate arguments have
-     been suppressed, an explanatory message should be printed.  */
-  int dup_args_note;
-
-  /* Various output columns.  */
-  int short_opt_col;
-  int long_opt_col;
-  int doc_opt_col;
-  int opt_doc_col;
-  int header_col;
-  int usage_indent;
-  int rmargin;
-
-  int valid;			/* True when the values in here are valid.  */
-};
-
-/* This is a global variable, as user options are only ever read once.  */
-static struct uparams uparams = {
-  DUP_ARGS, DUP_ARGS_NOTE,
-  SHORT_OPT_COL, LONG_OPT_COL, DOC_OPT_COL, OPT_DOC_COL, HEADER_COL,
-  USAGE_INDENT, RMARGIN,
-  0
-};
-
-/* A particular uparam, and what the user name is.  */
-struct uparam_name
-{
-  const char *name;		/* User name.  */
-  int is_bool;			/* Whether it's `boolean'.  */
-  size_t uparams_offs;		/* Location of the (int) field in UPARAMS.  */
-};
-
-/* The name-field mappings we know about.  */
-static const struct uparam_name uparam_names[] =
-{
-  { "dup-args",       1, offsetof (struct uparams, dup_args) },
-  { "dup-args-note",  1, offsetof (struct uparams, dup_args_note) },
-  { "short-opt-col",  0, offsetof (struct uparams, short_opt_col) },
-  { "long-opt-col",   0, offsetof (struct uparams, long_opt_col) },
-  { "doc-opt-col",    0, offsetof (struct uparams, doc_opt_col) },
-  { "opt-doc-col",    0, offsetof (struct uparams, opt_doc_col) },
-  { "header-col",     0, offsetof (struct uparams, header_col) },
-  { "usage-indent",   0, offsetof (struct uparams, usage_indent) },
-  { "rmargin",        0, offsetof (struct uparams, rmargin) },
-  { 0, 0, 0 }
-};
-
-/* Read user options from the environment, and fill in UPARAMS appropiately.  */
-static void
-fill_in_uparams (const struct argp_state *state)
-{
-
-  const char *var = getenv ("ARGP_HELP_FMT");
-
-#define SKIPWS(p) do { while (isspace (*p)) p++; } while (0);
-
-  if (var)
-    /* Parse var. */
-    while (*var)
-      {
-	SKIPWS (var);
-
-	if (isalpha (*var))
-	  {
-	    size_t var_len;
-	    const struct uparam_name *un;
-	    int unspec = 0, val = 0;
-	    const char *arg = var;
-
-	    while (isalnum (*arg) || *arg == '-' || *arg == '_')
-	      arg++;
-	    var_len = arg - var;
-
-	    SKIPWS (arg);
-
-	    if (*arg == '\0' || *arg == ',')
-	      unspec = 1;
-	    else if (*arg == '=')
-	      {
-		arg++;
-		SKIPWS (arg);
-	      }
-
-	    if (unspec)
-	      {
-		if (var[0] == 'n' && var[1] == 'o' && var[2] == '-')
-		  {
-		    val = 0;
-		    var += 3;
-		    var_len -= 3;
-		  }
-		else
-		  val = 1;
-	      }
-	    else if (isdigit (*arg))
-	      {
-		val = atoi (arg);
-		while (isdigit (*arg))
-		  arg++;
-		SKIPWS (arg);
-	      }
-
-	    for (un = uparam_names; un->name; un++)
-	      if (strlen (un->name) == var_len
-		  && strncmp (var, un->name, var_len) == 0)
-		{
-		  if (unspec && !un->is_bool)
-		    __argp_failure (state, 0, 0,
-				    dgettext (state->root_argp->argp_domain, "\
-%.*s: ARGP_HELP_FMT parameter requires a value"),
-				    (int) var_len, var);
-		  else
-		    *(int *)((char *)&uparams + un->uparams_offs) = val;
-		  break;
-		}
-	    if (! un->name)
-	      __argp_failure (state, 0, 0,
-			      dgettext (state->root_argp->argp_domain, "\
-%.*s: Unknown ARGP_HELP_FMT parameter"),
-			      (int) var_len, var);
-
-	    var = arg;
-	    if (*var == ',')
-	      var++;
-	  }
-	else if (*var)
-	  {
-	    __argp_failure (state, 0, 0,
-			    dgettext (state->root_argp->argp_domain,
-				      "Garbage in ARGP_HELP_FMT: %s"), var);
-	    break;
-	  }
-      }
-}
-
-/* Returns true if OPT hasn't been marked invisible.  Visibility only affects
-   whether OPT is displayed or used in sorting, not option shadowing.  */
-#define ovisible(opt) (! ((opt)->flags & OPTION_HIDDEN))
-
-/* Returns true if OPT is an alias for an earlier option.  */
-#define oalias(opt) ((opt)->flags & OPTION_ALIAS)
-
-/* Returns true if OPT is an documentation-only entry.  */
-#define odoc(opt) ((opt)->flags & OPTION_DOC)
-
-/* Returns true if OPT is the end-of-list marker for a list of options.  */
-#define oend(opt) __option_is_end (opt)
-
-/* Returns true if OPT has a short option.  */
-#define oshort(opt) __option_is_short (opt)
-
-/*
-   The help format for a particular option is like:
-
-     -xARG, -yARG, --long1=ARG, --long2=ARG        Documentation...
-
-   Where ARG will be omitted if there's no argument, for this option, or
-   will be surrounded by "[" and "]" appropiately if the argument is
-   optional.  The documentation string is word-wrapped appropiately, and if
-   the list of options is long enough, it will be started on a separate line.
-   If there are no short options for a given option, the first long option is
-   indented slighly in a way that's supposed to make most long options appear
-   to be in a separate column.
-
-   For example, the following output (from ps):
-
-     -p PID, --pid=PID          List the process PID
-	 --pgrp=PGRP            List processes in the process group PGRP
-     -P, -x, --no-parent        Include processes without parents
-     -Q, --all-fields           Don't elide unusable fields (normally if there's
-				some reason ps can't print a field for any
-				process, it's removed from the output entirely)
-     -r, --reverse, --gratuitously-long-reverse-option
-				Reverse the order of any sort
-	 --session[=SID]        Add the processes from the session SID (which
-				defaults to the sid of the current process)
-
-    Here are some more options:
-     -f ZOT, --foonly=ZOT       Glork a foonly
-     -z, --zaza                 Snit a zar
-
-     -?, --help                 Give this help list
-	 --usage                Give a short usage message
-     -V, --version              Print program version
-
-   The struct argp_option array for the above could look like:
-
-   {
-     {"pid",       'p',      "PID",  0, "List the process PID"},
-     {"pgrp",      OPT_PGRP, "PGRP", 0, "List processes in the process group PGRP"},
-     {"no-parent", 'P',	      0,     0, "Include processes without parents"},
-     {0,           'x',       0,     OPTION_ALIAS},
-     {"all-fields",'Q',       0,     0, "Don't elide unusable fields (normally"
-                                        " if there's some reason ps can't"
-					" print a field for any process, it's"
-                                        " removed from the output entirely)" },
-     {"reverse",   'r',       0,     0, "Reverse the order of any sort"},
-     {"gratuitously-long-reverse-option", 0, 0, OPTION_ALIAS},
-     {"session",   OPT_SESS,  "SID", OPTION_ARG_OPTIONAL,
-                                        "Add the processes from the session"
-					" SID (which defaults to the sid of"
-					" the current process)" },
-
-     {0,0,0,0, "Here are some more options:"},
-     {"foonly", 'f', "ZOT", 0, "Glork a foonly"},
-     {"zaza", 'z', 0, 0, "Snit a zar"},
-
-     {0}
-   }
-
-   Note that the last three options are automatically supplied by argp_parse,
-   unless you tell it not to with ARGP_NO_HELP.
-
-*/
-
-/* Returns true if CH occurs between BEG and END.  */
-static int
-find_char (char ch, char *beg, char *end)
-{
-  while (beg < end)
-    if (*beg == ch)
-      return 1;
-    else
-      beg++;
-  return 0;
-}
-
-struct hol_cluster;		/* fwd decl */
-
-struct hol_entry
-{
-  /* First option.  */
-  const struct argp_option *opt;
-  /* Number of options (including aliases).  */
-  unsigned num;
-
-  /* A pointers into the HOL's short_options field, to the first short option
-     letter for this entry.  The order of the characters following this point
-     corresponds to the order of options pointed to by OPT, and there are at
-     most NUM.  A short option recorded in a option following OPT is only
-     valid if it occurs in the right place in SHORT_OPTIONS (otherwise it's
-     probably been shadowed by some other entry).  */
-  char *short_options;
-
-  /* Entries are sorted by their group first, in the order:
-       1, 2, ..., n, 0, -m, ..., -2, -1
-     and then alphabetically within each group.  The default is 0.  */
-  int group;
-
-  /* The cluster of options this entry belongs to, or 0 if none.  */
-  struct hol_cluster *cluster;
-
-  /* The argp from which this option came.  */
-  const struct argp *argp;
-};
-
-/* A cluster of entries to reflect the argp tree structure.  */
-struct hol_cluster
-{
-  /* A descriptive header printed before options in this cluster.  */
-  const char *header;
-
-  /* Used to order clusters within the same group with the same parent,
-     according to the order in which they occurred in the parent argp's child
-     list.  */
-  int index;
-
-  /* How to sort this cluster with respect to options and other clusters at the
-     same depth (clusters always follow options in the same group).  */
-  int group;
-
-  /* The cluster to which this cluster belongs, or 0 if it's at the base
-     level.  */
-  struct hol_cluster *parent;
-
-  /* The argp from which this cluster is (eventually) derived.  */
-  const struct argp *argp;
-
-  /* The distance this cluster is from the root.  */
-  int depth;
-
-  /* Clusters in a given hol are kept in a linked list, to make freeing them
-     possible.  */
-  struct hol_cluster *next;
-};
-
-/* A list of options for help.  */
-struct hol
-{
-  /* An array of hol_entry's.  */
-  struct hol_entry *entries;
-  /* The number of entries in this hol.  If this field is zero, the others
-     are undefined.  */
-  unsigned num_entries;
-
-  /* A string containing all short options in this HOL.  Each entry contains
-     pointers into this string, so the order can't be messed with blindly.  */
-  char *short_options;
-
-  /* Clusters of entries in this hol.  */
-  struct hol_cluster *clusters;
-};
-
-/* Create a struct hol from the options in ARGP.  CLUSTER is the
-   hol_cluster in which these entries occur, or 0, if at the root.  */
-static struct hol *
-make_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  char *so;
-  const struct argp_option *o;
-  const struct argp_option *opts = argp->options;
-  struct hol_entry *entry;
-  unsigned num_short_options = 0;
-  struct hol *hol = malloc (sizeof (struct hol));
-
-  assert (hol);
-
-  hol->num_entries = 0;
-  hol->clusters = 0;
-
-  if (opts)
-    {
-      int cur_group = 0;
-
-      /* The first option must not be an alias.  */
-      assert (! oalias (opts));
-
-      /* Calculate the space needed.  */
-      for (o = opts; ! oend (o); o++)
-	{
-	  if (! oalias (o))
-	    hol->num_entries++;
-	  if (oshort (o))
-	    num_short_options++;	/* This is an upper bound.  */
-	}
-
-      hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
-      hol->short_options = malloc (num_short_options + 1);
-
-      assert (hol->entries && hol->short_options);
-
-      /* Fill in the entries.  */
-      so = hol->short_options;
-      for (o = opts, entry = hol->entries; ! oend (o); entry++)
-	{
-	  entry->opt = o;
-	  entry->num = 0;
-	  entry->short_options = so;
-	  entry->group = cur_group =
-	    o->group
-	    ? o->group
-	    : ((!o->name && !o->key)
-	       ? cur_group + 1
-	       : cur_group);
-	  entry->cluster = cluster;
-	  entry->argp = argp;
-
-	  do
-	    {
-	      entry->num++;
-	      if (oshort (o) && ! find_char (o->key, hol->short_options, so))
-		/* O has a valid short option which hasn't already been used.*/
-		*so++ = o->key;
-	      o++;
-	    }
-	  while (! oend (o) && oalias (o));
-	}
-      *so = '\0';		/* null terminated so we can find the length */
-    }
-
-  return hol;
-}
-
-/* Add a new cluster to HOL, with the given GROUP and HEADER (taken from the
-   associated argp child list entry), INDEX, and PARENT, and return a pointer
-   to it.  ARGP is the argp that this cluster results from.  */
-static struct hol_cluster *
-hol_add_cluster (struct hol *hol, int group, const char *header, int index,
-		 struct hol_cluster *parent, const struct argp *argp)
-{
-  struct hol_cluster *cl = malloc (sizeof (struct hol_cluster));
-  if (cl)
-    {
-      cl->group = group;
-      cl->header = header;
-
-      cl->index = index;
-      cl->parent = parent;
-      cl->argp = argp;
-      cl->depth = parent ? parent->depth + 1 : 0;
-
-      cl->next = hol->clusters;
-      hol->clusters = cl;
-    }
-  return cl;
-}
-
-/* Free HOL and any resources it uses.  */
-static void
-hol_free (struct hol *hol)
-{
-  struct hol_cluster *cl = hol->clusters;
-
-  while (cl)
-    {
-      struct hol_cluster *next = cl->next;
-      free (cl);
-      cl = next;
-    }
-
-  if (hol->num_entries > 0)
-    {
-      free (hol->entries);
-      free (hol->short_options);
-    }
-
-  free (hol);
-}
-
-static inline int
-hol_entry_short_iterate (const struct hol_entry *entry,
-			 int (*func)(const struct argp_option *opt,
-				     const struct argp_option *real,
-				     const char *domain, void *cookie),
-			 const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-  char *so = entry->short_options;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (oshort (opt) && *so == opt->key)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-	so++;
-      }
-
-  return val;
-}
-
-static inline int
-hol_entry_long_iterate (const struct hol_entry *entry,
-			int (*func)(const struct argp_option *opt,
-				    const struct argp_option *real,
-				    const char *domain, void *cookie),
-			const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (opt->name)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-      }
-
-  return val;
-}
-
-/* Iterator that returns true for the first short option.  */
-static inline int
-until_short (const struct argp_option *opt, const struct argp_option *real UNUSED,
-	     const char *domain UNUSED, void *cookie UNUSED)
-{
-  return oshort (opt) ? opt->key : 0;
-}
-
-/* Returns the first valid short option in ENTRY, or 0 if there is none.  */
-static char
-hol_entry_first_short (const struct hol_entry *entry)
-{
-  return hol_entry_short_iterate (entry, until_short,
-				  entry->argp->argp_domain, 0);
-}
-
-/* Returns the first valid long option in ENTRY, or 0 if there is none.  */
-static const char *
-hol_entry_first_long (const struct hol_entry *entry)
-{
-  const struct argp_option *opt;
-  unsigned num;
-  for (opt = entry->opt, num = entry->num; num > 0; opt++, num--)
-    if (opt->name && ovisible (opt))
-      return opt->name;
-  return 0;
-}
-
-/* Returns the entry in HOL with the long option name NAME, or 0 if there is
-   none.  */
-static struct hol_entry *
-hol_find_entry (struct hol *hol, const char *name)
-{
-  struct hol_entry *entry = hol->entries;
-  unsigned num_entries = hol->num_entries;
-
-  while (num_entries-- > 0)
-    {
-      const struct argp_option *opt = entry->opt;
-      unsigned num_opts = entry->num;
-
-      while (num_opts-- > 0)
-	if (opt->name && ovisible (opt) && strcmp (opt->name, name) == 0)
-	  return entry;
-	else
-	  opt++;
-
-      entry++;
-    }
-
-  return 0;
-}
-
-/* If an entry with the long option NAME occurs in HOL, set it's special
-   sort position to GROUP.  */
-static void
-hol_set_group (struct hol *hol, const char *name, int group)
-{
-  struct hol_entry *entry = hol_find_entry (hol, name);
-  if (entry)
-    entry->group = group;
-}
-
-/* Order by group:  0, 1, 2, ..., n, -m, ..., -2, -1.
-   EQ is what to return if GROUP1 and GROUP2 are the same.  */
-static int
-group_cmp (int group1, int group2, int eq)
-{
-  if (group1 == group2)
-    return eq;
-  else if ((group1 < 0 && group2 < 0) || (group1 >= 0 && group2 >= 0))
-    return group1 - group2;
-  else
-    return group2 - group1;
-}
-
-/* Compare clusters CL1 & CL2 by the order that they should appear in
-   output.  */
-static int
-hol_cluster_cmp (const struct hol_cluster *cl1, const struct hol_cluster *cl2)
-{
-  /* If one cluster is deeper than the other, use its ancestor at the same
-     level, so that finding the common ancestor is straightforward.  */
-  while (cl1->depth < cl2->depth)
-    cl1 = cl1->parent;
-  while (cl2->depth < cl1->depth)
-    cl2 = cl2->parent;
-
-  /* Now reduce both clusters to their ancestors at the point where both have
-     a common parent; these can be directly compared.  */
-  while (cl1->parent != cl2->parent)
-    cl1 = cl1->parent, cl2 = cl2->parent;
-
-  return group_cmp (cl1->group, cl2->group, cl2->index - cl1->index);
-}
-
-/* Return the ancestor of CL that's just below the root (i.e., has a parent
-   of 0).  */
-static struct hol_cluster *
-hol_cluster_base (struct hol_cluster *cl)
-{
-  while (cl->parent)
-    cl = cl->parent;
-  return cl;
-}
-
-/* Return true if CL1 is a child of CL2.  */
-static int
-hol_cluster_is_child (const struct hol_cluster *cl1,
-		      const struct hol_cluster *cl2)
-{
-  while (cl1 && cl1 != cl2)
-    cl1 = cl1->parent;
-  return cl1 == cl2;
-}
-
-/* Given the name of a OPTION_DOC option, modifies NAME to start at the tail
-   that should be used for comparisons, and returns true iff it should be
-   treated as a non-option.  */
-
-/* FIXME: Can we use unsigned char * for the argument? */
-static int
-canon_doc_option (const char **name)
-{
-  int non_opt;
-  /* Skip initial whitespace.  */
-  while (isspace ( (unsigned char) **name))
-    (*name)++;
-  /* Decide whether this looks like an option (leading `-') or not.  */
-  non_opt = (**name != '-');
-  /* Skip until part of name used for sorting.  */
-  while (**name && !isalnum ( (unsigned char) **name))
-    (*name)++;
-  return non_opt;
-}
-
-/* Order ENTRY1 & ENTRY2 by the order which they should appear in a help
-   listing.  */
-static int
-hol_entry_cmp (const struct hol_entry *entry1,
-	       const struct hol_entry *entry2)
-{
-  /* The group numbers by which the entries should be ordered; if either is
-     in a cluster, then this is just the group within the cluster.  */
-  int group1 = entry1->group, group2 = entry2->group;
-
-  if (entry1->cluster != entry2->cluster)
-    {
-      /* The entries are not within the same cluster, so we can't compare them
-	 directly, we have to use the appropiate clustering level too.  */
-      if (! entry1->cluster)
-	/* ENTRY1 is at the `base level', not in a cluster, so we have to
-	   compare it's group number with that of the base cluster in which
-	   ENTRY2 resides.  Note that if they're in the same group, the
-	   clustered option always comes laster.  */
-	return group_cmp (group1, hol_cluster_base (entry2->cluster)->group, -1);
-      else if (! entry2->cluster)
-	/* Likewise, but ENTRY2's not in a cluster.  */
-	return group_cmp (hol_cluster_base (entry1->cluster)->group, group2, 1);
-      else
-	/* Both entries are in clusters, we can just compare the clusters.  */
-	return hol_cluster_cmp (entry1->cluster, entry2->cluster);
-    }
-  else if (group1 == group2)
-    /* The entries are both in the same cluster and group, so compare them
-       alphabetically.  */
-    {
-      int short1 = hol_entry_first_short (entry1);
-      int short2 = hol_entry_first_short (entry2);
-      int doc1 = odoc (entry1->opt);
-      int doc2 = odoc (entry2->opt);
-      /* FIXME: Can we use unsigned char * instead? */
-      const char *long1 = hol_entry_first_long (entry1);
-      const char *long2 = hol_entry_first_long (entry2);
-
-      if (doc1)
-	doc1 = canon_doc_option (&long1);
-      if (doc2)
-	doc2 = canon_doc_option (&long2);
-
-      if (doc1 != doc2)
-	/* `documentation' options always follow normal options (or
-	   documentation options that *look* like normal options).  */
-	return doc1 - doc2;
-      else if (!short1 && !short2 && long1 && long2)
-	/* Only long options.  */
-	return __strcasecmp (long1, long2);
-      else
-	/* Compare short/short, long/short, short/long, using the first
-	   character of long options.  Entries without *any* valid
-	   options (such as options with OPTION_HIDDEN set) will be put
-	   first, but as they're not displayed, it doesn't matter where
-	   they are.  */
-	{
-	  unsigned char first1 = short1 ? short1 : long1 ? *long1 : 0;
-	  unsigned char first2 = short2 ? short2 : long2 ? *long2 : 0;
-#ifdef _tolower
-	  int lower_cmp = _tolower (first1) - _tolower (first2);
-#else
-	  int lower_cmp = tolower (first1) - tolower (first2);
-#endif
-	  /* Compare ignoring case, except when the options are both the
-	     same letter, in which case lower-case always comes first.  */
-	  /* NOTE: The subtraction below does the right thing
-	     even with eight-bit chars: first1 and first2 are
-	     converted to int *before* the subtraction. */
-	  return lower_cmp ? lower_cmp : first2 - first1;
-	}
-    }
-  else
-    /* Within the same cluster, but not the same group, so just compare
-       groups.  */
-    return group_cmp (group1, group2, 0);
-}
-
-/* Version of hol_entry_cmp with correct signature for qsort.  */
-static int
-hol_entry_qcmp (const void *entry1_v, const void *entry2_v)
-{
-  return hol_entry_cmp (entry1_v, entry2_v);
-}
-
-/* Sort HOL by group and alphabetically by option name (with short options
-   taking precedence over long).  Since the sorting is for display purposes
-   only, the shadowing of options isn't effected.  */
-static void
-hol_sort (struct hol *hol)
-{
-  if (hol->num_entries > 0)
-    qsort (hol->entries, hol->num_entries, sizeof (struct hol_entry),
-	   hol_entry_qcmp);
-}
-
-/* Append MORE to HOL, destroying MORE in the process.  Options in HOL shadow
-   any in MORE with the same name.  */
-static void
-hol_append (struct hol *hol, struct hol *more)
-{
-  struct hol_cluster **cl_end = &hol->clusters;
-
-  /* Steal MORE's cluster list, and add it to the end of HOL's.  */
-  while (*cl_end)
-    cl_end = &(*cl_end)->next;
-  *cl_end = more->clusters;
-  more->clusters = 0;
-
-  /* Merge entries.  */
-  if (more->num_entries > 0)
-    {
-      if (hol->num_entries == 0)
-	{
-	  hol->num_entries = more->num_entries;
-	  hol->entries = more->entries;
-	  hol->short_options = more->short_options;
-	  more->num_entries = 0;	/* Mark MORE's fields as invalid.  */
-	}
-      else
-	/* Append the entries in MORE to those in HOL, taking care to only add
-	   non-shadowed SHORT_OPTIONS values.  */
-	{
-	  unsigned left;
-	  char *so, *more_so;
-	  struct hol_entry *e;
-	  unsigned num_entries = hol->num_entries + more->num_entries;
-	  struct hol_entry *entries =
-	    malloc (num_entries * sizeof (struct hol_entry));
-	  unsigned hol_so_len = strlen (hol->short_options);
-	  char *short_options =
-	    malloc (hol_so_len + strlen (more->short_options) + 1);
-
-	  __mempcpy (__mempcpy (entries, hol->entries,
-				hol->num_entries * sizeof (struct hol_entry)),
-		     more->entries,
-		     more->num_entries * sizeof (struct hol_entry));
-
-	  __mempcpy (short_options, hol->short_options, hol_so_len);
-
-	  /* Fix up the short options pointers from HOL.  */
-	  for (e = entries, left = hol->num_entries; left > 0; e++, left--)
-	    e->short_options += (short_options - hol->short_options);
-
-	  /* Now add the short options from MORE, fixing up its entries
-	     too.  */
-	  so = short_options + hol_so_len;
-	  more_so = more->short_options;
-	  for (left = more->num_entries; left > 0; e++, left--)
-	    {
-	      int opts_left;
-	      const struct argp_option *opt;
-
-	      e->short_options = so;
-
-	      for (opts_left = e->num, opt = e->opt; opts_left; opt++, opts_left--)
-		{
-		  int ch = *more_so;
-		  if (oshort (opt) && ch == opt->key)
-		    /* The next short option in MORE_SO, CH, is from OPT.  */
-		    {
-		      if (! find_char (ch, short_options,
-				       short_options + hol_so_len))
-			/* The short option CH isn't shadowed by HOL's options,
-			   so add it to the sum.  */
-			*so++ = ch;
-		      more_so++;
-		    }
-		}
-	    }
-
-	  *so = '\0';
-
-	  free (hol->entries);
-	  free (hol->short_options);
-
-	  hol->entries = entries;
-	  hol->num_entries = num_entries;
-	  hol->short_options = short_options;
-	}
-    }
-
-  hol_free (more);
-}
-
-/* Inserts enough spaces to make sure STREAM is at column COL.  */
-static void
-indent_to (argp_fmtstream_t stream, unsigned col)
-{
-  int needed = col - __argp_fmtstream_point (stream);
-  while (needed-- > 0)
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* Output to STREAM either a space, or a newline if there isn't room for at
-   least ENSURE characters before the right margin.  */
-static void
-space (argp_fmtstream_t stream, size_t ensure)
-{
-  if (__argp_fmtstream_point (stream) + ensure
-      >= __argp_fmtstream_rmargin (stream))
-    __argp_fmtstream_putc (stream, '\n');
-  else
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* If the option REAL has an argument, we print it in using the printf
-   format REQ_FMT or OPT_FMT depending on whether it's a required or
-   optional argument.  */
-static void
-arg (const struct argp_option *real, const char *req_fmt, const char *opt_fmt,
-     const char *domain UNUSED, argp_fmtstream_t stream)
-{
-  if (real->arg)
-    {
-      if (real->flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, opt_fmt,
-				 dgettext (domain, real->arg));
-      else
-	__argp_fmtstream_printf (stream, req_fmt,
-				 dgettext (domain, real->arg));
-    }
-}
-
-/* Helper functions for hol_entry_help.  */
-
-/* State used during the execution of hol_help.  */
-struct hol_help_state
-{
-  /* PREV_ENTRY should contain the previous entry printed, or 0.  */
-  struct hol_entry *prev_entry;
-
-  /* If an entry is in a different group from the previous one, and SEP_GROUPS
-     is true, then a blank line will be printed before any output. */
-  int sep_groups;
-
-  /* True if a duplicate option argument was suppressed (only ever set if
-     UPARAMS.dup_args is false).  */
-  int suppressed_dup_arg;
-};
-
-/* Some state used while printing a help entry (used to communicate with
-   helper functions).  See the doc for hol_entry_help for more info, as most
-   of the fields are copied from its arguments.  */
-struct pentry_state
-{
-  const struct hol_entry *entry;
-  argp_fmtstream_t stream;
-  struct hol_help_state *hhstate;
-
-  /* True if nothing's been printed so far.  */
-  int first;
-
-  /* If non-zero, the state that was used to print this help.  */
-  const struct argp_state *state;
-};
-
-/* If a user doc filter should be applied to DOC, do so.  */
-static const char *
-filter_doc (const char *doc, int key, const struct argp *argp,
-	    const struct argp_state *state)
-{
-  if (argp->help_filter)
-    /* We must apply a user filter to this output.  */
-    {
-      void *input = __argp_input (argp, state);
-      return (*argp->help_filter) (key, doc, input);
-    }
-  else
-    /* No filter.  */
-    return doc;
-}
-
-/* Prints STR as a header line, with the margin lines set appropiately, and
-   notes the fact that groups should be separated with a blank line.  ARGP is
-   the argp that should dictate any user doc filtering to take place.  Note
-   that the previous wrap margin isn't restored, but the left margin is reset
-   to 0.  */
-static void
-print_header (const char *str, const struct argp *argp,
-	      struct pentry_state *pest)
-{
-  const char *tstr = dgettext (argp->argp_domain, str);
-  const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_HEADER, argp, pest->state);
-
-  if (fstr)
-    {
-      if (*fstr)
-	{
-	  if (pest->hhstate->prev_entry)
-	    /* Precede with a blank line.  */
-	    __argp_fmtstream_putc (pest->stream, '\n');
-	  indent_to (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_lmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_wmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_puts (pest->stream, fstr);
-	  __argp_fmtstream_set_lmargin (pest->stream, 0);
-	  __argp_fmtstream_putc (pest->stream, '\n');
-	}
-
-      pest->hhstate->sep_groups = 1; /* Separate subsequent groups. */
-    }
-
-  if (fstr != tstr)
-    free ((char *) fstr);
-}
-
-/* Inserts a comma if this isn't the first item on the line, and then makes
-   sure we're at least to column COL.  If this *is* the first item on a line,
-   prints any pending whitespace/headers that should precede this line. Also
-   clears FIRST.  */
-static void
-comma (unsigned col, struct pentry_state *pest)
-{
-  if (pest->first)
-    {
-      const struct hol_entry *pe = pest->hhstate->prev_entry;
-      const struct hol_cluster *cl = pest->entry->cluster;
-
-      if (pest->hhstate->sep_groups && pe && pest->entry->group != pe->group)
-	__argp_fmtstream_putc (pest->stream, '\n');
-
-      if (cl && cl->header && *cl->header
-	  && (!pe
-	      || (pe->cluster != cl
-		  && !hol_cluster_is_child (pe->cluster, cl))))
-	/* If we're changing clusters, then this must be the start of the
-	   ENTRY's cluster unless that is an ancestor of the previous one
-	   (in which case we had just popped into a sub-cluster for a bit).
-	   If so, then print the cluster's header line.  */
-	{
-	  int old_wm = __argp_fmtstream_wmargin (pest->stream);
-	  print_header (cl->header, cl->argp, pest);
-	  __argp_fmtstream_set_wmargin (pest->stream, old_wm);
-	}
-
-      pest->first = 0;
-    }
-  else
-    __argp_fmtstream_puts (pest->stream, ", ");
-
-  indent_to (pest->stream, col);
-}
-
-/* Print help for ENTRY to STREAM.  */
-static void
-hol_entry_help (struct hol_entry *entry, const struct argp_state *state,
-		argp_fmtstream_t stream, struct hol_help_state *hhstate)
-{
-  unsigned num;
-  const struct argp_option *real = entry->opt, *opt;
-  char *so = entry->short_options;
-  int have_long_opt = 0;	/* We have any long options.  */
-  /* Saved margins.  */
-  int old_lm = __argp_fmtstream_set_lmargin (stream, 0);
-  int old_wm = __argp_fmtstream_wmargin (stream);
-  /* PEST is a state block holding some of our variables that we'd like to
-     share with helper functions.  */
-
-  /* Decent initializers are a GNU extension, so don't use it here. */
-  struct pentry_state pest;
-  pest.entry = entry;
-  pest.stream = stream;
-  pest.hhstate = hhstate;
-  pest.first = 1;
-  pest.state = state;
-
-  if (! odoc (real))
-    for (opt = real, num = entry->num; num > 0; opt++, num--)
-      if (opt->name && ovisible (opt))
-	{
-	  have_long_opt = 1;
-	  break;
-	}
-
-  /* First emit short options.  */
-  __argp_fmtstream_set_wmargin (stream, uparams.short_opt_col); /* For truly bizarre cases. */
-  for (opt = real, num = entry->num; num > 0; opt++, num--)
-    if (oshort (opt) && opt->key == *so)
-      /* OPT has a valid (non shadowed) short option.  */
-      {
-	if (ovisible (opt))
-	  {
-	    comma (uparams.short_opt_col, &pest);
-	    __argp_fmtstream_putc (stream, '-');
-	    __argp_fmtstream_putc (stream, *so);
-	    if (!have_long_opt || uparams.dup_args)
-	      arg (real, " %s", "[%s]", state->root_argp->argp_domain, stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-	so++;
-      }
-
-  /* Now, long options.  */
-  if (odoc (real))
-    /* A `documentation' option.  */
-    {
-      __argp_fmtstream_set_wmargin (stream, uparams.doc_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.doc_opt_col, &pest);
-	    /* Calling gettext here isn't quite right, since sorting will
-	       have been done on the original; but documentation options
-	       should be pretty rare anyway...  */
-	    __argp_fmtstream_puts (stream,
-				   dgettext (state->root_argp->argp_domain,
-					     opt->name));
-	  }
-    }
-  else
-    /* A real long option.  */
-    {
-      int first_long_opt = 1;
-
-      __argp_fmtstream_set_wmargin (stream, uparams.long_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.long_opt_col, &pest);
-	    __argp_fmtstream_printf (stream, "--%s", opt->name);
-	    if (first_long_opt || uparams.dup_args)
-	      arg (real, "=%s", "[=%s]", state->root_argp->argp_domain,
-		   stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-    }
-
-  /* Next, documentation strings.  */
-  __argp_fmtstream_set_lmargin (stream, 0);
-
-  if (pest.first)
-    {
-      /* Didn't print any switches, what's up?  */
-      if (!oshort (real) && !real->name)
-	/* This is a group header, print it nicely.  */
-	print_header (real->doc, entry->argp, &pest);
-      else
-	/* Just a totally shadowed option or null header; print nothing.  */
-	goto cleanup;		/* Just return, after cleaning up.  */
-    }
-  else
-    {
-      const char *tstr = real->doc ? dgettext (state->root_argp->argp_domain,
-					       real->doc) : 0;
-      const char *fstr = filter_doc (tstr, real->key, entry->argp, state);
-      if (fstr && *fstr)
-	{
-	  unsigned int col = __argp_fmtstream_point (stream);
-
-	  __argp_fmtstream_set_lmargin (stream, uparams.opt_doc_col);
-	  __argp_fmtstream_set_wmargin (stream, uparams.opt_doc_col);
-
-	  if (col > (unsigned int) (uparams.opt_doc_col + 3))
-	    __argp_fmtstream_putc (stream, '\n');
-	  else if (col >= (unsigned int) uparams.opt_doc_col)
-	    __argp_fmtstream_puts (stream, "   ");
-	  else
-	    indent_to (stream, uparams.opt_doc_col);
-
-	  __argp_fmtstream_puts (stream, fstr);
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-
-      /* Reset the left margin.  */
-      __argp_fmtstream_set_lmargin (stream, 0);
-      __argp_fmtstream_putc (stream, '\n');
-    }
-
-  hhstate->prev_entry = entry;
-
-cleanup:
-  __argp_fmtstream_set_lmargin (stream, old_lm);
-  __argp_fmtstream_set_wmargin (stream, old_wm);
-}
-
-/* Output a long help message about the options in HOL to STREAM.  */
-static void
-hol_help (struct hol *hol, const struct argp_state *state,
-	  argp_fmtstream_t stream)
-{
-  unsigned num;
-  struct hol_entry *entry;
-  struct hol_help_state hhstate = { 0, 0, 0 };
-
-  for (entry = hol->entries, num = hol->num_entries; num > 0; entry++, num--)
-    hol_entry_help (entry, state, stream, &hhstate);
-
-  if (hhstate.suppressed_dup_arg && uparams.dup_args_note)
-    {
-      const char *tstr = dgettext (state->root_argp->argp_domain, "\
-Mandatory or optional arguments to long options are also mandatory or \
-optional for any corresponding short options.");
-      const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_DUP_ARGS_NOTE,
-				     state ? state->root_argp : 0, state);
-      if (fstr && *fstr)
-	{
-	  __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, fstr);
-	  __argp_fmtstream_putc (stream, '\n');
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-    }
-}
-
-/* Helper functions for hol_usage.  */
-
-/* If OPT is a short option without an arg, append its key to the string
-   pointer pointer to by COOKIE, and advance the pointer.  */
-static int
-add_argless_short_opt (const struct argp_option *opt,
-		       const struct argp_option *real,
-		       const char *domain UNUSED, void *cookie)
-{
-  char **snao_end = cookie;
-  if (!(opt->arg || real->arg)
-      && !((opt->flags | real->flags) & OPTION_NO_USAGE))
-    *(*snao_end)++ = opt->key;
-  return 0;
-}
-
-/* If OPT is a short option with an arg, output a usage entry for it to the
-   stream pointed at by COOKIE.  */
-static int
-usage_argful_short_opt (const struct argp_option *opt,
-			const struct argp_option *real,
-			const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (arg && !(flags & OPTION_NO_USAGE))
-    {
-      arg = dgettext (domain, arg);
-
-      if (flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, " [-%c[%s]]", opt->key, arg);
-      else
-	{
-	  /* Manually do line wrapping so that it (probably) won't
-	     get wrapped at the embedded space.  */
-	  space (stream, 6 + strlen (arg));
-	  __argp_fmtstream_printf (stream, "[-%c %s]", opt->key, arg);
-	}
-    }
-
-  return 0;
-}
-
-/* Output a usage entry for the long option opt to the stream pointed at by
-   COOKIE.  */
-static int
-usage_long_opt (const struct argp_option *opt,
-		const struct argp_option *real,
-		const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (! (flags & OPTION_NO_USAGE))
-    {
-      if (arg)
-	{
-	  arg = dgettext (domain, arg);
-	  if (flags & OPTION_ARG_OPTIONAL)
-	    __argp_fmtstream_printf (stream, " [--%s[=%s]]", opt->name, arg);
-	  else
-	    __argp_fmtstream_printf (stream, " [--%s=%s]", opt->name, arg);
-	}
-      else
-	__argp_fmtstream_printf (stream, " [--%s]", opt->name);
-    }
-
-  return 0;
-}
-
-/* Print a short usage description for the arguments in HOL to STREAM.  */
-static void
-hol_usage (struct hol *hol, argp_fmtstream_t stream)
-{
-  if (hol->num_entries > 0)
-    {
-      unsigned nentries;
-      struct hol_entry *entry;
-      char *short_no_arg_opts = alloca (strlen (hol->short_options) + 1);
-      char *snao_end = short_no_arg_opts;
-
-      /* First we put a list of short options without arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, add_argless_short_opt,
-				 entry->argp->argp_domain, &snao_end);
-      if (snao_end > short_no_arg_opts)
-	{
-	  *snao_end++ = 0;
-	  __argp_fmtstream_printf (stream, " [-%s]", short_no_arg_opts);
-	}
-
-      /* Now a list of short options *with* arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, usage_argful_short_opt,
-				 entry->argp->argp_domain, stream);
-
-      /* Finally, a list of long options (whew!).  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_long_iterate (entry, usage_long_opt,
-				entry->argp->argp_domain, stream);
-    }
-}
-
-/* Make a HOL containing all levels of options in ARGP.  CLUSTER is the
-   cluster in which ARGP's entries should be clustered, or 0.  */
-static struct hol *
-argp_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  const struct argp_child *child = argp->children;
-  struct hol *hol = make_hol (argp, cluster);
-  if (child)
-    while (child->argp)
-      {
-	struct hol_cluster *child_cluster =
-	  ((child->group || child->header)
-	   /* Put CHILD->argp within its own cluster.  */
-	   ? hol_add_cluster (hol, child->group, child->header,
-			      child - argp->children, cluster, argp)
-	   /* Just merge it into the parent's cluster.  */
-	   : cluster);
-	hol_append (hol, argp_hol (child->argp, child_cluster)) ;
-	child++;
-      }
-  return hol;
-}
-
-/* Calculate how many different levels with alternative args strings exist in
-   ARGP.  */
-static size_t
-argp_args_levels (const struct argp *argp)
-{
-  size_t levels = 0;
-  const struct argp_child *child = argp->children;
-
-  if (argp->args_doc && strchr (argp->args_doc, '\n'))
-    levels++;
-
-  if (child)
-    while (child->argp)
-      levels += argp_args_levels ((child++)->argp);
-
-  return levels;
-}
-
-/* Print all the non-option args documented in ARGP to STREAM.  Any output is
-   preceded by a space.  LEVELS is a pointer to a byte vector the length
-   returned by argp_args_levels; it should be initialized to zero, and
-   updated by this routine for the next call if ADVANCE is true.  True is
-   returned as long as there are more patterns to output.  */
-static int
-argp_args_usage (const struct argp *argp, const struct argp_state *state,
-		 char **levels, int advance, argp_fmtstream_t stream)
-{
-  char *our_level = *levels;
-  int multiple = 0;
-  const struct argp_child *child = argp->children;
-  const char *tdoc = dgettext (argp->argp_domain, argp->args_doc), *nl = 0;
-  const char *fdoc = filter_doc (tdoc, ARGP_KEY_HELP_ARGS_DOC, argp, state);
-
-  if (fdoc)
-    {
-      const char *cp = fdoc;
-      nl = __strchrnul (cp, '\n');
-      if (*nl != '\0')
-	/* This is a `multi-level' args doc; advance to the correct position
-	   as determined by our state in LEVELS, and update LEVELS.  */
-	{
-	  int i;
-	  multiple = 1;
-	  for (i = 0; i < *our_level; i++)
-	    cp = nl + 1, nl = __strchrnul (cp, '\n');
-	  (*levels)++;
-	}
-
-      /* Manually do line wrapping so that it (probably) won't get wrapped at
-	 any embedded spaces.  */
-      space (stream, 1 + nl - cp);
-
-      __argp_fmtstream_write (stream, cp, nl - cp);
-    }
-  if (fdoc && fdoc != tdoc)
-    free ((char *)fdoc);	/* Free user's modified doc string.  */
-
-  if (child)
-    while (child->argp)
-      advance = !argp_args_usage ((child++)->argp, state, levels, advance, stream);
-
-  if (advance && multiple)
-    {
-      /* Need to increment our level.  */
-      if (*nl)
-	/* There's more we can do here.  */
-	{
-	  (*our_level)++;
-	  advance = 0;		/* Our parent shouldn't advance also. */
-	}
-      else if (*our_level > 0)
-	/* We had multiple levels, but used them up; reset to zero.  */
-	*our_level = 0;
-    }
-
-  return !advance;
-}
-
-/* Print the documentation for ARGP to STREAM; if POST is false, then
-   everything preceeding a `\v' character in the documentation strings (or
-   the whole string, for those with none) is printed, otherwise, everything
-   following the `\v' character (nothing for strings without).  Each separate
-   bit of documentation is separated a blank line, and if PRE_BLANK is true,
-   then the first is as well.  If FIRST_ONLY is true, only the first
-   occurrence is output.  Returns true if anything was output.  */
-static int
-argp_doc (const struct argp *argp, const struct argp_state *state,
-	  int post, int pre_blank, int first_only,
-	  argp_fmtstream_t stream)
-{
-  const char *text;
-  const char *inp_text;
-  void *input = 0;
-  int anything = 0;
-  size_t inp_text_limit = 0;
-  const char *doc = dgettext (argp->argp_domain, argp->doc);
-  const struct argp_child *child = argp->children;
-
-  if (doc)
-    {
-      char *vt = strchr (doc, '\v');
-      inp_text = post ? (vt ? vt + 1 : 0) : doc;
-      inp_text_limit = (!post && vt) ? (vt - doc) : 0;
-    }
-  else
-    inp_text = 0;
-
-  if (argp->help_filter)
-    /* We have to filter the doc strings.  */
-    {
-      if (inp_text_limit)
-	/* Copy INP_TEXT so that it's nul-terminated.  */
-	inp_text = STRNDUP (inp_text, inp_text_limit);
-      input = __argp_input (argp, state);
-      text =
-	(*argp->help_filter) (post
-			      ? ARGP_KEY_HELP_POST_DOC
-			      : ARGP_KEY_HELP_PRE_DOC,
-			      inp_text, input);
-    }
-  else
-    text = (const char *) inp_text;
-
-  if (text)
-    {
-      if (pre_blank)
-	__argp_fmtstream_putc (stream, '\n');
-
-      if (text == inp_text && inp_text_limit)
-	__argp_fmtstream_write (stream, inp_text, inp_text_limit);
-      else
-	__argp_fmtstream_puts (stream, text);
-
-      if (__argp_fmtstream_point (stream) > __argp_fmtstream_lmargin (stream))
-	__argp_fmtstream_putc (stream, '\n');
-
-      anything = 1;
-    }
-
-  if (text && text != inp_text)
-    free ((char *) text);	/* Free TEXT returned from the help filter.  */
-  if (inp_text && inp_text_limit && argp->help_filter)
-    free ((char *) inp_text);	/* We copied INP_TEXT, so free it now.  */
-
-  if (post && argp->help_filter)
-    /* Now see if we have to output a ARGP_KEY_HELP_EXTRA text.  */
-    {
-      text = (*argp->help_filter) (ARGP_KEY_HELP_EXTRA, 0, input);
-      if (text)
-	{
-	  if (anything || pre_blank)
-	    __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, text);
-	  free ((char *) text);
-	  if (__argp_fmtstream_point (stream)
-	      > __argp_fmtstream_lmargin (stream))
-	    __argp_fmtstream_putc (stream, '\n');
-	  anything = 1;
-	}
-    }
-
-  if (child)
-    while (child->argp && !(first_only && anything))
-      anything |=
-	argp_doc ((child++)->argp, state,
-		  post, anything || pre_blank, first_only,
-		  stream);
-
-  return anything;
-}
-
-/* Output a usage message for ARGP to STREAM.  If called from
-   argp_state_help, STATE is the relevent parsing state.  FLAGS are from the
-   set ARGP_HELP_*.  NAME is what to use wherever a `program name' is
-   needed. */
-
-static void
-_help (const struct argp *argp, const struct argp_state *state, FILE *stream,
-       unsigned flags, const char *name)
-{
-  int anything = 0;		/* Whether we've output anything.  */
-  struct hol *hol = 0;
-  argp_fmtstream_t fs;
-
-  if (! stream)
-    return;
-
-  FLOCKFILE (stream);
-
-  if (! uparams.valid)
-    fill_in_uparams (state);
-
-  fs = __argp_make_fmtstream (stream, 0, uparams.rmargin, 0);
-  if (! fs)
-    {
-      FUNLOCKFILE (stream);
-      return;
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG))
-    {
-      hol = argp_hol (argp, 0);
-
-      /* If present, these options always come last.  */
-      hol_set_group (hol, "help", -1);
-      hol_set_group (hol, "version", -1);
-
-      hol_sort (hol);
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE))
-    /* Print a short `Usage:' message.  */
-    {
-      int first_pattern = 1, more_patterns;
-      size_t num_pattern_levels = argp_args_levels (argp);
-      char *pattern_levels = alloca (num_pattern_levels);
-
-      memset (pattern_levels, 0, num_pattern_levels);
-
-      do
-	{
-	  int old_lm;
-	  int old_wm = __argp_fmtstream_set_wmargin (fs, uparams.usage_indent);
-	  char *levels = pattern_levels;
-
-	  if (first_pattern)
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "Usage:"),
-				     name);
-	  else
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "  or: "),
-				     name);
-
-	  /* We set the lmargin as well as the wmargin, because hol_usage
-	     manually wraps options with newline to avoid annoying breaks.  */
-	  old_lm = __argp_fmtstream_set_lmargin (fs, uparams.usage_indent);
-
-	  if (flags & ARGP_HELP_SHORT_USAGE)
-	    /* Just show where the options go.  */
-	    {
-	      if (hol->num_entries > 0)
-		__argp_fmtstream_puts (fs, dgettext (argp->argp_domain,
-						     " [OPTION...]"));
-	    }
-	  else
-	    /* Actually print the options.  */
-	    {
-	      hol_usage (hol, fs);
-	      flags |= ARGP_HELP_SHORT_USAGE; /* But only do so once.  */
-	    }
-
-	  more_patterns = argp_args_usage (argp, state, &levels, 1, fs);
-
-	  __argp_fmtstream_set_wmargin (fs, old_wm);
-	  __argp_fmtstream_set_lmargin (fs, old_lm);
-
-	  __argp_fmtstream_putc (fs, '\n');
-	  anything = 1;
-
-	  first_pattern = 0;
-	}
-      while (more_patterns);
-    }
-
-  if (flags & ARGP_HELP_PRE_DOC)
-    anything |= argp_doc (argp, state, 0, 0, 1, fs);
-
-  if (flags & ARGP_HELP_SEE)
-    {
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain, "\
-Try `%s --help' or `%s --usage' for more information.\n"),
-			       name, name);
-      anything = 1;
-    }
-
-  if (flags & ARGP_HELP_LONG)
-    /* Print a long, detailed help message.  */
-    {
-      /* Print info about all the options.  */
-      if (hol->num_entries > 0)
-	{
-	  if (anything)
-	    __argp_fmtstream_putc (fs, '\n');
-	  hol_help (hol, state, fs);
-	  anything = 1;
-	}
-    }
-
-  if (flags & ARGP_HELP_POST_DOC)
-    /* Print any documentation strings at the end.  */
-    anything |= argp_doc (argp, state, 1, anything, 0, fs);
-
-  if ((flags & ARGP_HELP_BUG_ADDR) && argp_program_bug_address)
-    {
-      if (anything)
-	__argp_fmtstream_putc (fs, '\n');
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain,
-					     "Report bugs to %s.\n"),
- 			       argp_program_bug_address);
-      anything = 1;
-    }
-
-  FUNLOCKFILE (stream);
-
-  if (hol)
-    hol_free (hol);
-
-  __argp_fmtstream_free (fs);
-}
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  NAME is what to use wherever a `program name' is needed. */
-void __argp_help (const struct argp *argp, FILE *stream,
-		  unsigned flags, char *name)
-{
-  _help (argp, 0, stream, flags, name);
-}
-#ifdef weak_alias
-weak_alias (__argp_help, argp_help)
-#endif
-
-char *__argp_basename(char *name)
-{
-  char *short_name = strrchr(name, '/');
-  return short_name ? short_name + 1 : name;
-}
-
-char *
-__argp_short_program_name(const struct argp_state *state)
-{
-  if (state)
-    return state->name;
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-  return program_invocation_short_name;
-#elif HAVE_DECL_PROGRAM_INVOCATION_NAME
-  return __argp_basename(program_invocation_name);
-#else /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-  /* FIXME: What now? Miles suggests that it is better to use NULL,
-     but currently the value is passed on directly to fputs_unlocked,
-     so that requires more changes. */
-# if __GNUC__
-  return "";
-# endif /* __GNUC__ */
-#endif /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-}
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-void
-__argp_state_help (const struct argp_state *state, FILE *stream, unsigned flags)
-{
-  if ((!state || ! (state->flags & ARGP_NO_ERRS)) && stream)
-    {
-      if (state && (state->flags & ARGP_LONG_ONLY))
-	flags |= ARGP_HELP_LONG_ONLY;
-
-      _help (state ? state->root_argp : 0, state, stream, flags,
-	     __argp_short_program_name(state));
-
-      if (!state || ! (state->flags & ARGP_NO_EXIT))
-	{
-	  if (flags & ARGP_HELP_EXIT_ERR)
-	    exit (argp_err_exit_status);
-	  if (flags & ARGP_HELP_EXIT_OK)
-	    exit (0);
-	}
-  }
-}
-#ifdef weak_alias
-weak_alias (__argp_state_help, argp_state_help)
-#endif
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-void
-__argp_error (const struct argp_state *state, const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  va_list ap;
-
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-	  PUTC_UNLOCKED (':', stream);
-	  PUTC_UNLOCKED (' ', stream);
-
-	  va_start (ap, fmt);
-	  vfprintf (stream, fmt, ap);
-	  va_end (ap);
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  __argp_state_help (state, stream, ARGP_HELP_STD_ERR);
-
-	  FUNLOCKFILE (stream);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_error, argp_error)
-#endif
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-void
-__argp_failure (const struct argp_state *state, int status, int errnum,
-		const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-
-	  if (fmt)
-	    {
-	      va_list ap;
-
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-
-	      va_start (ap, fmt);
-	      vfprintf (stream, fmt, ap);
-	      va_end (ap);
-	    }
-
-	  if (errnum)
-	    {
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-	      fputs (STRERROR (errnum), stream);
-	    }
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  FUNLOCKFILE (stream);
-
-	  if (status && (!state || !(state->flags & ARGP_NO_EXIT)))
-	    exit (status);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_failure, argp_failure)
-#endif
diff --git a/contrib/argp-standalone/argp-namefrob.h b/contrib/argp-standalone/argp-namefrob.h
deleted file mode 100644
index 0ce11481a7b..00000000000
--- a/contrib/argp-standalone/argp-namefrob.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Name frobnication for compiling argp outside of glibc
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#if !_LIBC
-/* This code is written for inclusion in gnu-libc, and uses names in the
-   namespace reserved for libc.  If we're not compiling in libc, define those
-   names to be the normal ones instead.  */
-
-/* argp-parse functions */
-#undef __argp_parse
-#define __argp_parse argp_parse
-#undef __option_is_end
-#define __option_is_end _option_is_end
-#undef __option_is_short
-#define __option_is_short _option_is_short
-#undef __argp_input
-#define __argp_input _argp_input
-
-/* argp-help functions */
-#undef __argp_help
-#define __argp_help argp_help
-#undef __argp_error
-#define __argp_error argp_error
-#undef __argp_failure
-#define __argp_failure argp_failure
-#undef __argp_state_help
-#define __argp_state_help argp_state_help
-#undef __argp_usage
-#define __argp_usage argp_usage
-#undef __argp_basename
-#define __argp_basename _argp_basename
-#undef __argp_short_program_name
-#define __argp_short_program_name _argp_short_program_name
-
-/* argp-fmtstream functions */
-#undef __argp_make_fmtstream
-#define __argp_make_fmtstream argp_make_fmtstream
-#undef __argp_fmtstream_free
-#define __argp_fmtstream_free argp_fmtstream_free
-#undef __argp_fmtstream_putc
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#define __argp_fmtstream_write argp_fmtstream_write
-#undef __argp_fmtstream_printf
-#define __argp_fmtstream_printf argp_fmtstream_printf
-#undef __argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#define __argp_fmtstream_point argp_fmtstream_point
-#undef __argp_fmtstream_update
-#define __argp_fmtstream_update _argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#undef __argp_fmtstream_lmargin
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#undef __argp_fmtstream_rmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#undef __argp_fmtstream_wmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* normal libc functions we call */
-#undef __sleep
-#define __sleep sleep
-#undef __strcasecmp
-#define __strcasecmp strcasecmp
-#undef __vsnprintf
-#define __vsnprintf vsnprintf
-
-#endif /* !_LIBC */
-
-#ifndef __set_errno
-#define __set_errno(e) (errno = (e))
-#endif
diff --git a/contrib/argp-standalone/argp-parse.c b/contrib/argp-standalone/argp-parse.c
deleted file mode 100644
index 78f7bf139b6..00000000000
--- a/contrib/argp-standalone/argp-parse.c
+++ /dev/null
@@ -1,1305 +0,0 @@
-/* Hierarchial argument parsing
-   Copyright (C) 1995, 96, 97, 98, 99, 2000,2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <limits.h>
-#include <assert.h>
-
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.
-   When compiling libc, the _ macro is predefined.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-#  define gettext(msgid) (msgid)
-# endif
-#endif
-#ifndef N_
-# define N_(msgid) (msgid)
-#endif
-
-#if _LIBC - 0
-#include <bits/libc-lock.h>
-#else
-#ifdef HAVE_CTHREADS_H
-#include <cthreads.h>
-#endif
-#endif /* _LIBC */
-
-#include "argp.h"
-#include "argp-namefrob.h"
-
-
-/* The meta-argument used to prevent any further arguments being interpreted
-   as options.  */
-#define QUOTE "--"
-
-/* EZ alias for ARGP_ERR_UNKNOWN.  */
-#define EBADKEY ARGP_ERR_UNKNOWN
-
-
-/* Default options.  */
-
-/* When argp is given the --HANG switch, _ARGP_HANG is set and argp will sleep
-   for one second intervals, decrementing _ARGP_HANG until it's zero.  Thus
-   you can force the program to continue by attaching a debugger and setting
-   it to 0 yourself.  */
-volatile int _argp_hang;
-
-#define OPT_PROGNAME	-2
-#define OPT_USAGE	-3
-#if HAVE_SLEEP && HAVE_GETPID
-#define OPT_HANG	-4
-#endif
-
-static const struct argp_option argp_default_options[] =
-{
-  {"help",	  '?',    	0, 0,  N_("Give this help list"), -1},
-  {"usage",	  OPT_USAGE,	0, 0,  N_("Give a short usage message"), 0 },
-  {"program-name",OPT_PROGNAME,"NAME", OPTION_HIDDEN,
-     N_("Set the program name"), 0},
-#if OPT_HANG
-  {"HANG",	  OPT_HANG,    "SECS", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
-     N_("Hang for SECS seconds (default 3600)"), 0 },
-#endif
-  {0, 0, 0, 0, 0, 0}
-};
-
-static error_t
-argp_default_parser (int key, char *arg, struct argp_state *state)
-{
-  switch (key)
-    {
-    case '?':
-      __argp_state_help (state, state->out_stream, ARGP_HELP_STD_HELP);
-      break;
-    case OPT_USAGE:
-      __argp_state_help (state, state->out_stream,
-		       ARGP_HELP_USAGE | ARGP_HELP_EXIT_OK);
-      break;
-
-    case OPT_PROGNAME:		/* Set the program name.  */
-#if HAVE_DECL_PROGRAM_INVOCATION_NAME
-      program_invocation_name = arg;
-#endif
-      /* [Note that some systems only have PROGRAM_INVOCATION_SHORT_NAME (aka
-	 __PROGNAME), in which case, PROGRAM_INVOCATION_NAME is just defined
-	 to be that, so we have to be a bit careful here.]  */
-
-      /* Update what we use for messages.  */
-
-      state->name = __argp_basename(arg);
-      
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-      program_invocation_short_name = state->name;
-#endif
-
-      if ((state->flags & (ARGP_PARSE_ARGV0 | ARGP_NO_ERRS))
-	  == ARGP_PARSE_ARGV0)
-	/* Update what getopt uses too.  */
-	state->argv[0] = arg;
-
-      break;
-      
-#if OPT_HANG
-    case OPT_HANG:
-      _argp_hang = atoi (arg ? arg : "3600");
-      fprintf(state->err_stream, "%s: pid = %ld\n",
-	      state->name, (long) getpid());
-      while (_argp_hang-- > 0)
-	__sleep (1);
-      break;
-#endif
-      
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_default_argp =
-  {argp_default_options, &argp_default_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-static const struct argp_option argp_version_options[] =
-{
-  {"version",	  'V',    	0, 0,  N_("Print program version"), -1},
-  {0, 0, 0, 0, 0, 0 }
-};
-
-static error_t
-argp_version_parser (int key, char *arg UNUSED, struct argp_state *state)
-{
-  switch (key)
-    {
-    case 'V':
-      if (argp_program_version_hook)
-	(*argp_program_version_hook) (state->out_stream, state);
-      else if (argp_program_version)
-	fprintf (state->out_stream, "%s\n", argp_program_version);
-      else
-	__argp_error (state, dgettext (state->root_argp->argp_domain,
-				       "(PROGRAM ERROR) No version known!?"));
-      if (! (state->flags & ARGP_NO_EXIT))
-	exit (0);
-      break;
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_version_argp =
-  {argp_version_options, &argp_version_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-
-/* The state of a `group' during parsing.  Each group corresponds to a
-   particular argp structure from the tree of such descending from the top
-   level argp passed to argp_parse.  */
-struct group
-{
-  /* This group's parsing function.  */
-  argp_parser_t parser;
-
-  /* Which argp this group is from.  */
-  const struct argp *argp;
-
-  /* The number of non-option args sucessfully handled by this parser.  */
-  unsigned args_processed;
-
-  /* This group's parser's parent's group.  */
-  struct group *parent;
-  unsigned parent_index;	/* And the our position in the parent.   */
-
-  /* These fields are swapped into and out of the state structure when
-     calling this group's parser.  */
-  void *input, **child_inputs;
-  void *hook;
-};
-
-/* Call GROUP's parser with KEY and ARG, swapping any group-specific info
-   from STATE before calling, and back into state afterwards.  If GROUP has
-   no parser, EBADKEY is returned.  */
-static error_t
-group_parse (struct group *group, struct argp_state *state, int key, char *arg)
-{
-  if (group->parser)
-    {
-      error_t err;
-      state->hook = group->hook;
-      state->input = group->input;
-      state->child_inputs = group->child_inputs;
-      state->arg_num = group->args_processed;
-      err = (*group->parser)(key, arg, state);
-      group->hook = state->hook;
-      return err;
-    }
-  else
-    return EBADKEY;
-}
-
-struct parser
-{
-  const struct argp *argp;
-
-  const char *posixly_correct;
-  
-  /* True if there are only no-option arguments left, which are just
-     passed verbatim with ARGP_KEY_ARG. This is set if we encounter a
-     quote, or the end of the proper options, but may be cleared again
-     if the user moves the next argument pointer backwards. */
-  int args_only;
-
-  /* Describe how to deal with options that follow non-option ARGV-elements.
-
-     If the caller did not specify anything, the default is
-     REQUIRE_ORDER if the environment variable POSIXLY_CORRECT is
-     defined, PERMUTE otherwise.
-
-     REQUIRE_ORDER means don't recognize them as options; stop option
-     processing when the first non-option is seen. This is what Unix
-     does. This mode of operation is selected by either setting the
-     environment variable POSIXLY_CORRECT, or using `+' as the first
-     character of the list of option characters.
-
-     PERMUTE is the default. We permute the contents of ARGV as we
-     scan, so that eventually all the non-options are at the end. This
-     allows options to be given in any order, even with programs that
-     were not written to expect this.
-
-     RETURN_IN_ORDER is an option available to programs that were
-     written to expect options and other ARGV-elements in any order
-     and that care about the ordering of the two. We describe each
-     non-option ARGV-element as if it were the argument of an option
-     with character code 1. Using `-' as the first character of the
-     list of option characters selects this mode of operation.
-
-  */
-  enum { REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER } ordering;
-
-  /* A segment of non-option arguments that have been skipped for
-     later processing, after all options. `first_nonopt' is the index
-     in ARGV of the first of them; `last_nonopt' is the index after
-     the last of them.
-
-     If quoted or args_only is non-zero, this segment should be empty. */
-
-  /* FIXME: I'd prefer to use unsigned, but it's more consistent to
-     use the same type as for state.next. */
-  int first_nonopt;
-  int last_nonopt;
-  
-  /* String of all recognized short options. Needed for ARGP_LONG_ONLY. */
-  /* FIXME: Perhaps change to a pointer to a suitable bitmap instead? */
-  char *short_opts;
-
-  /* For parsing combined short options. */
-  char *nextchar;
-  
-  /* States of the various parsing groups.  */
-  struct group *groups;
-  /* The end of the GROUPS array.  */
-  struct group *egroup;
-  /* An vector containing storage for the CHILD_INPUTS field in all groups.  */
-  void **child_inputs;
-  
-  /* State block supplied to parsing routines.  */
-  struct argp_state state;
-
-  /* Memory used by this parser.  */
-  void *storage;
-};
-
-/* Search for a group defining a short option. */
-static const struct argp_option *
-find_short_option(struct parser *parser, int key, struct group **p)
-{
-  struct group *group;
-  
-  assert(key >= 0);
-  assert(isascii(key));
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	if (opts->key == key)
-	  {
-	    *p = group;
-	    return opts;
-	  }
-    }
-  return NULL;
-}
-
-enum match_result { MATCH_EXACT, MATCH_PARTIAL, MATCH_NO };
-
-/* If defined, allow complete.el-like abbreviations of long options. */
-#ifndef ARGP_COMPLETE
-#define ARGP_COMPLETE 0
-#endif
-
-/* Matches an encountern long-option argument ARG against an option NAME.
- * ARG is terminated by NUL or '='. */
-static enum match_result
-match_option(const char *arg, const char *name)
-{
-  unsigned i, j;
-  for (i = j = 0;; i++, j++)
-    {
-      switch(arg[i])
-	{
-	case '\0':
-	case '=':
-	  return name[j] ? MATCH_PARTIAL : MATCH_EXACT;
-#if ARGP_COMPLETE
-	case '-':
-	  while (name[j] != '-')
-	    if (!name[j++])
-	      return MATCH_NO;
-	  break;
-#endif
-	default:
-	  if (arg[i] != name[j])
-	    return MATCH_NO;
-	}
-    }
-}
-
-static const struct argp_option *
-find_long_option(struct parser *parser,
-		 const char *arg,
-		 struct group **p)
-{
-  struct group *group;
-
-  /* Partial match found so far. */
-  struct group *matched_group = NULL;
-  const struct argp_option *matched_option = NULL;
-
-  /* Number of partial matches. */
-  int num_partial = 0;
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	{
-	  if (!opts->name)
-	    continue;
-	  switch (match_option(arg, opts->name))
-	    {
-	    case MATCH_NO:
-	      break;
-	    case MATCH_PARTIAL:
-	      num_partial++;
-
-	      matched_group = group;
-	      matched_option = opts;
-
-	      break;
-	    case MATCH_EXACT:
-	      /* Exact match. */
-	      *p = group;
-	      return opts;
-	    }
-	}
-    }
-  if (num_partial == 1)
-    {
-      *p = matched_group;
-      return matched_option;
-    }
-
-  return NULL;
-}
-
-
-/* The next usable entries in the various parser tables being filled in by
-   convert_options.  */
-struct parser_convert_state
-{
-  struct parser *parser;
-  char *short_end;
-  void **child_inputs_end;
-};
-
-/* Initialize GROUP from ARGP. If CVT->SHORT_END is non-NULL, short
-   options are recorded in the short options string. Returns the next
-   unused group entry. CVT holds state used during the conversion. */
-static struct group *
-convert_options (const struct argp *argp,
-		 struct group *parent, unsigned parent_index,
-		 struct group *group, struct parser_convert_state *cvt)
-{
-  const struct argp_option *opt = argp->options;
-  const struct argp_child *children = argp->children;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      if (cvt->short_end)
-	{
-	  /* Record any short options. */
-	  for ( ; !__option_is_end (opt); opt++)
-	    if (__option_is_short(opt))
-	      *cvt->short_end++ = opt->key;
-	}
-      
-      group->parser = argp->parser;
-      group->argp = argp;
-      group->args_processed = 0;
-      group->parent = parent;
-      group->parent_index = parent_index;
-      group->input = 0;
-      group->hook = 0;
-      group->child_inputs = 0;
-      
-      if (children)
-	/* Assign GROUP's CHILD_INPUTS field some space from
-	   CVT->child_inputs_end.*/
-	{
-	  unsigned num_children = 0;
-	  while (children[num_children].argp)
-	    num_children++;
-	  group->child_inputs = cvt->child_inputs_end;
-	  cvt->child_inputs_end += num_children;
-	}
-      parent = group++;
-    }
-  else
-    parent = 0;
-
-  if (children)
-    {
-      unsigned index = 0;
-      while (children->argp)
-	group =
-	  convert_options (children++->argp, parent, index++, group, cvt);
-    }
-
-  return group;
-}
-/* Allocate and initialize the group structures, so that they are
-   ordered as if by traversing the corresponding argp parser tree in
-   pre-order. Also build the list of short options, if that is needed. */
-static void
-parser_convert (struct parser *parser, const struct argp *argp)
-{
-  struct parser_convert_state cvt;
-
-  cvt.parser = parser;
-  cvt.short_end = parser->short_opts;
-  cvt.child_inputs_end = parser->child_inputs;
-
-  parser->argp = argp;
-
-  if (argp)
-    parser->egroup = convert_options (argp, 0, 0, parser->groups, &cvt);
-  else
-    parser->egroup = parser->groups; /* No parsers at all! */
-
-  if (parser->short_opts)
-    *cvt.short_end ='\0';
-}
-
-/* Lengths of various parser fields which we will allocated.  */
-struct parser_sizes
-{
-  /* Needed only ARGP_LONG_ONLY */
-  size_t short_len;		/* Number of short options.  */
-
-  size_t num_groups;		/* Group structures we allocate.  */
-  size_t num_child_inputs;	/* Child input slots.  */
-};
-
-/* For ARGP, increments the NUM_GROUPS field in SZS by the total
-   number of argp structures descended from it, and the SHORT_LEN by
-   the total number of short options. */
-static void
-calc_sizes (const struct argp *argp,  struct parser_sizes *szs)
-{
-  const struct argp_child *child = argp->children;
-  const struct argp_option *opt = argp->options;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      szs->num_groups++;
-      if (opt)
-	{
-	  while (__option_is_short (opt++))
-	    szs->short_len++;
-	}
-    }
-
-  if (child)
-    while (child->argp)
-      {
-	calc_sizes ((child++)->argp, szs);
-	szs->num_child_inputs++;
-      }
-}
-
-/* Initializes PARSER to parse ARGP in a manner described by FLAGS.  */
-static error_t
-parser_init (struct parser *parser, const struct argp *argp,
-	     int argc, char **argv, int flags, void *input)
-{
-  error_t err = 0;
-  struct group *group;
-  struct parser_sizes szs;
-
-  parser->posixly_correct = getenv ("POSIXLY_CORRECT");
-
-  if (flags & ARGP_IN_ORDER)
-    parser->ordering = RETURN_IN_ORDER;
-  else if (flags & ARGP_NO_ARGS)
-    parser->ordering = REQUIRE_ORDER;
-  else if (parser->posixly_correct)
-    parser->ordering = REQUIRE_ORDER;
-  else
-    parser->ordering = PERMUTE;
-  
-  szs.short_len = 0;
-  szs.num_groups = 0;
-  szs.num_child_inputs = 0;
-
-  if (argp)
-    calc_sizes (argp, &szs);
-
-  if (!(flags & ARGP_LONG_ONLY))
-    /* We have no use for the short option array. */
-    szs.short_len = 0;
-  
-  /* Lengths of the various bits of storage used by PARSER.  */
-#define GLEN (szs.num_groups + 1) * sizeof (struct group)
-#define CLEN (szs.num_child_inputs * sizeof (void *))
-#define SLEN (szs.short_len + 1)
-#define STORAGE(offset) ((void *) (((char *) parser->storage) + (offset)))
-
-  parser->storage = malloc (GLEN + CLEN + SLEN);
-  if (! parser->storage)
-    return ENOMEM;
-
-  parser->groups = parser->storage;
-
-  parser->child_inputs = STORAGE(GLEN);
-  memset (parser->child_inputs, 0, szs.num_child_inputs * sizeof (void *));
-
-  if (flags & ARGP_LONG_ONLY)
-    parser->short_opts = STORAGE(GLEN + CLEN);
-  else
-    parser->short_opts = NULL;
-
-  parser_convert (parser, argp);
-
-  memset (&parser->state, 0, sizeof (struct argp_state));
-  
-  parser->state.root_argp = parser->argp;
-  parser->state.argc = argc;
-  parser->state.argv = argv;
-  parser->state.flags = flags;
-  parser->state.err_stream = stderr;
-  parser->state.out_stream = stdout;
-  parser->state.pstate = parser;
-
-  parser->args_only = 0;
-  parser->nextchar = NULL;
-  parser->first_nonopt = parser->last_nonopt = 0;
-    
-  /* Call each parser for the first time, giving it a chance to propagate
-     values to child parsers.  */
-  if (parser->groups < parser->egroup)
-    parser->groups->input = input;
-  for (group = parser->groups;
-       group < parser->egroup && (!err || err == EBADKEY);
-       group++)
-    {
-      if (group->parent)
-	/* If a child parser, get the initial input value from the parent. */
-	group->input = group->parent->child_inputs[group->parent_index];
-
-      if (!group->parser
-	  && group->argp->children && group->argp->children->argp)
-	/* For the special case where no parsing function is supplied for an
-	   argp, propagate its input to its first child, if any (this just
-	   makes very simple wrapper argps more convenient).  */
-	group->child_inputs[0] = group->input;
-
-      err = group_parse (group, &parser->state, ARGP_KEY_INIT, 0);
-    }
-  if (err == EBADKEY)
-    err = 0;			/* Some parser didn't understand.  */
-
-  if (err)
-    return err;
-
-  if (argv[0] && !(parser->state.flags & ARGP_PARSE_ARGV0))
-    /* There's an argv[0]; use it for messages.  */
-    {
-      parser->state.name = __argp_basename(argv[0]);
-
-      /* Don't parse it as an argument. */
-      parser->state.next = 1;
-    }
-  else
-    parser->state.name = __argp_short_program_name(NULL);
-  
-  return 0;
-}
-
-/* Free any storage consumed by PARSER (but not PARSER itself).  */
-static error_t
-parser_finalize (struct parser *parser,
-		 error_t err, int arg_ebadkey, int *end_index)
-{
-  struct group *group;
-
-  if (err == EBADKEY && arg_ebadkey)
-    /* Suppress errors generated by unparsed arguments.  */
-    err = 0;
-
-  if (! err)
-    {
-      if (parser->state.next == parser->state.argc)
-	/* We successfully parsed all arguments!  Call all the parsers again,
-	   just a few more times... */
-	{
-	  for (group = parser->groups;
-	       group < parser->egroup && (!err || err==EBADKEY);
-	       group++)
-	    if (group->args_processed == 0)
-	      err = group_parse (group, &parser->state, ARGP_KEY_NO_ARGS, 0);
-	  for (group = parser->egroup - 1;
-	       group >= parser->groups && (!err || err==EBADKEY);
-	       group--)
-	    err = group_parse (group, &parser->state, ARGP_KEY_END, 0);
-
-	  if (err == EBADKEY)
-	    err = 0;		/* Some parser didn't understand.  */
-
-	  /* Tell the user that all arguments are parsed.  */
-	  if (end_index)
-	    *end_index = parser->state.next;
-	}
-      else if (end_index)
-	/* Return any remaining arguments to the user.  */
-	*end_index = parser->state.next;
-      else
-	/* No way to return the remaining arguments, they must be bogus. */
-	{
-	  if (!(parser->state.flags & ARGP_NO_ERRS)
-	      && parser->state.err_stream)
-	    fprintf (parser->state.err_stream,
-		     dgettext (parser->argp->argp_domain,
-			       "%s: Too many arguments\n"),
-		     parser->state.name);
-	  err = EBADKEY;
-	}
-    }
-
-  /* Okay, we're all done, with either an error or success; call the parsers
-     to indicate which one.  */
-
-  if (err)
-    {
-      /* Maybe print an error message.  */
-      if (err == EBADKEY)
-	/* An appropriate message describing what the error was should have
-	   been printed earlier.  */
-	__argp_state_help (&parser->state, parser->state.err_stream,
-			   ARGP_HELP_STD_ERR);
-
-      /* Since we didn't exit, give each parser an error indication.  */
-      for (group = parser->groups; group < parser->egroup; group++)
-	group_parse (group, &parser->state, ARGP_KEY_ERROR, 0);
-    }
-  else
-    /* Notify parsers of success, and propagate back values from parsers.  */
-    {
-      /* We pass over the groups in reverse order so that child groups are
-	 given a chance to do there processing before passing back a value to
-	 the parent.  */
-      for (group = parser->egroup - 1
-	   ; group >= parser->groups && (!err || err == EBADKEY)
-	   ; group--)
-	err = group_parse (group, &parser->state, ARGP_KEY_SUCCESS, 0);
-      if (err == EBADKEY)
-	err = 0;		/* Some parser didn't understand.  */
-    }
-
-  /* Call parsers once more, to do any final cleanup.  Errors are ignored.  */
-  for (group = parser->egroup - 1; group >= parser->groups; group--)
-    group_parse (group, &parser->state, ARGP_KEY_FINI, 0);
-
-  if (err == EBADKEY)
-    err = EINVAL;
-
-  free (parser->storage);
-
-  return err;
-}
-
-/* Call the user parsers to parse the non-option argument VAL, at the
-   current position, returning any error. The state NEXT pointer
-   should point to the argument; this function will adjust it
-   correctly to reflect however many args actually end up being
-   consumed. */
-static error_t
-parser_parse_arg (struct parser *parser, char *val)
-{
-  /* Save the starting value of NEXT */
-  int index = parser->state.next;
-  error_t err = EBADKEY;
-  struct group *group;
-  int key = 0;			/* Which of ARGP_KEY_ARG[S] we used.  */
-
-  /* Try to parse the argument in each parser.  */
-  for (group = parser->groups
-       ; group < parser->egroup && err == EBADKEY
-       ; group++)
-    {
-      parser->state.next++;	/* For ARGP_KEY_ARG, consume the arg.  */
-      key = ARGP_KEY_ARG;
-      err = group_parse (group, &parser->state, key, val);
-
-      if (err == EBADKEY)
-	/* This parser doesn't like ARGP_KEY_ARG; try ARGP_KEY_ARGS instead. */
-	{
-	  parser->state.next--;	/* For ARGP_KEY_ARGS, put back the arg.  */
-	  key = ARGP_KEY_ARGS;
-	  err = group_parse (group, &parser->state, key, 0);
-	}
-    }
-
-  if (! err)
-    {
-      if (key == ARGP_KEY_ARGS)
-	/* The default for ARGP_KEY_ARGS is to assume that if NEXT isn't
-	   changed by the user, *all* arguments should be considered
-	   consumed.  */
-	parser->state.next = parser->state.argc;
-
-      if (parser->state.next > index)
-	/* Remember that we successfully processed a non-option
-	   argument -- but only if the user hasn't gotten tricky and set
-	   the clock back.  */
-	(--group)->args_processed += (parser->state.next - index);
-      else
-	/* The user wants to reparse some args, so try looking for options again.  */
-	parser->args_only = 0;
-    }
-
-  return err;
-}
-
-/* Exchange two adjacent subsequences of ARGV.
-   One subsequence is elements [first_nonopt,last_nonopt)
-   which contains all the non-options that have been skipped so far.
-   The other is elements [last_nonopt,next), which contains all
-   the options processed since those non-options were skipped.
-
-   `first_nonopt' and `last_nonopt' are relocated so that they describe
-   the new indices of the non-options in ARGV after they are moved.  */
-
-static void
-exchange (struct parser *parser)
-{
-  int bottom = parser->first_nonopt;
-  int middle = parser->last_nonopt;
-  int top = parser->state.next;
-  char **argv = parser->state.argv;
-  
-  char *tem;
-
-  /* Exchange the shorter segment with the far end of the longer segment.
-     That puts the shorter segment into the right place.
-     It leaves the longer segment in the right place overall,
-     but it consists of two parts that need to be swapped next.  */
-
-  while (top > middle && middle > bottom)
-    {
-      if (top - middle > middle - bottom)
-	{
-	  /* Bottom segment is the short one.  */
-	  int len = middle - bottom;
-	  register int i;
-
-	  /* Swap it with the top part of the top segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[top - (middle - bottom) + i];
-	      argv[top - (middle - bottom) + i] = tem;
-	    }
-	  /* Exclude the moved bottom segment from further swapping.  */
-	  top -= len;
-	}
-      else
-	{
-	  /* Top segment is the short one.  */
-	  int len = top - middle;
-	  register int i;
-
-	  /* Swap it with the bottom part of the bottom segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[middle + i];
-	      argv[middle + i] = tem;
-	    }
-	  /* Exclude the moved top segment from further swapping.  */
-	  bottom += len;
-	}
-    }
-
-  /* Update records for the slots the non-options now occupy.  */
-
-  parser->first_nonopt += (parser->state.next - parser->last_nonopt);
-  parser->last_nonopt = parser->state.next;
-}
-
-
-
-enum arg_type { ARG_ARG, ARG_SHORT_OPTION,
-		ARG_LONG_OPTION, ARG_LONG_ONLY_OPTION,
-		ARG_QUOTE };
-
-static enum arg_type
-classify_arg(struct parser *parser, char *arg, char **opt)
-{
-  if (arg[0] == '-')
-    /* Looks like an option... */
-    switch (arg[1])
-      {
-      case '\0':
-	/* "-" is not an option. */
-	return ARG_ARG;
-      case '-':
-	/* Long option, or quote. */
-	if (!arg[2])
-	  return ARG_QUOTE;
-	  
-	/* A long option. */
-	if (opt)
-	  *opt = arg + 2;
-	return ARG_LONG_OPTION;
-
-      default:
-	/* Short option. But if ARGP_LONG_ONLY, it can also be a long option. */
-
-	if (opt)
-	  *opt = arg + 1;
-
-	if (parser->state.flags & ARGP_LONG_ONLY)
-	  {
-	    /* Rules from getopt.c:
-
-	       If long_only and the ARGV-element has the form "-f",
-	       where f is a valid short option, don't consider it an
-	       abbreviated form of a long option that starts with f.
-	       Otherwise there would be no way to give the -f short
-	       option.
-
-	       On the other hand, if there's a long option "fubar" and
-	       the ARGV-element is "-fu", do consider that an
-	       abbreviation of the long option, just like "--fu", and
-	       not "-f" with arg "u".
-
-	       This distinction seems to be the most useful approach. */
-
-	    assert(parser->short_opts);
-	    
-	    if (arg[2] || !strchr(parser->short_opts, arg[1]))
-	      return ARG_LONG_ONLY_OPTION;
-	  }
-
-	return ARG_SHORT_OPTION;
-      }
-  
-  else
-    return ARG_ARG;
-}
-
-/* Parse the next argument in PARSER (as indicated by PARSER->state.next).
-   Any error from the parsers is returned, and *ARGP_EBADKEY indicates
-   whether a value of EBADKEY is due to an unrecognized argument (which is
-   generally not fatal).  */
-static error_t
-parser_parse_next (struct parser *parser, int *arg_ebadkey)
-{
-  if (parser->state.quoted && parser->state.next < parser->state.quoted)
-    /* The next argument pointer has been moved to before the quoted
-       region, so pretend we never saw the quoting `--', and start
-       looking for options again. If the `--' is still there we'll just
-       process it one more time. */
-    parser->state.quoted = parser->args_only = 0;
-
-  /* Give FIRST_NONOPT & LAST_NONOPT rational values if NEXT has been
-     moved back by the user (who may also have changed the arguments).  */
-  if (parser->last_nonopt > parser->state.next)
-    parser->last_nonopt = parser->state.next;
-  if (parser->first_nonopt > parser->state.next)
-    parser->first_nonopt = parser->state.next;
-
-  if (parser->nextchar)
-    /* Deal with short options. */
-    {
-      struct group *group;
-      char c;
-      const struct argp_option *option;
-      char *value = NULL;;
-
-      assert(!parser->args_only);
-
-      c = *parser->nextchar++;
-      
-      option = find_short_option(parser, c, &group);
-      if (!option)
-	{
-	  if (parser->posixly_correct)
-	    /* 1003.2 specifies the format of this message.  */
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: illegal option -- %c\n"),
-		     parser->state.name, c);
-	  else
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: invalid option -- %c\n"),
-		     parser->state.name, c);
-
-	  *arg_ebadkey = 0;
-	  return EBADKEY;
-	}
-
-      if (!*parser->nextchar)
-	parser->nextchar = NULL;
-
-      if (option->arg)
-	{
-	  value = parser->nextchar;
-	  parser->nextchar = NULL;
-	      
-	  if (!value
-	      && !(option->flags & OPTION_ARG_OPTIONAL))
-	    /* We need an mandatory argument. */
-	    {
-	      if (parser->state.next == parser->state.argc)
-		/* Missing argument */
-		{
-		  /* 1003.2 specifies the format of this message.  */
-		  fprintf (parser->state.err_stream,
-			   dgettext(parser->state.root_argp->argp_domain,
-				    "%s: option requires an argument -- %c\n"),
-			   parser->state.name, c);
-
-		  *arg_ebadkey = 0;
-		  return EBADKEY;
-		}
-	      value = parser->state.argv[parser->state.next++];
-	    }
-	}
-      return group_parse(group, &parser->state,
-			 option->key, value);
-    }
-  else
-    /* Advance to the next ARGV-element.  */
-    {
-      if (parser->args_only)
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->state.next >= parser->state.argc)
-	    /* We're done. */
-	    return EBADKEY;
-	  else
-	    return parser_parse_arg(parser,
-				    parser->state.argv[parser->state.next]);
-	}
-      
-      if (parser->state.next >= parser->state.argc)
-	/* Almost done. If there are non-options that we skipped
-	   previously, we should process them now. */
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->first_nonopt != parser->last_nonopt)
-	    {
-	      exchange(parser);
-	      
-	      /* Start processing the arguments we skipped previously. */
-	      parser->state.next = parser->first_nonopt;
-	      
-	      parser->first_nonopt = parser->last_nonopt = 0;
-
-	      parser->args_only = 1;
-	      return 0;
-	    }
-	  else
-	    /* Indicate that we're really done. */
-	    return EBADKEY;
-	}
-      else
-	/* Look for options. */
-	{
-	  char *arg = parser->state.argv[parser->state.next];
-
-	  char *optstart;
-	  enum arg_type token = classify_arg(parser, arg, &optstart);
-	  
-	  switch (token)
-	    {
-	    case ARG_ARG:
-	      switch (parser->ordering)
-		{
-		case PERMUTE:
-		  if (parser->first_nonopt == parser->last_nonopt)
-		    /* Skipped sequence is empty; start a new one. */
-		    parser->first_nonopt = parser->last_nonopt = parser->state.next;
-
-		  else if (parser->last_nonopt != parser->state.next)
-		    /* We have a non-empty skipped sequence, and
-		       we're not at the end-point, so move it. */
-		    exchange(parser);
-
-		  assert(parser->last_nonopt == parser->state.next);
-		  
-		  /* Skip this argument for now. */
-		  parser->state.next++;
-		  parser->last_nonopt = parser->state.next; 
-		  
-		  return 0;
-
-		case REQUIRE_ORDER:
-		  /* Implicit quote before the first argument. */
-		   parser->args_only = 1;
-		   return 0;
-		   
-		case RETURN_IN_ORDER:
-		  *arg_ebadkey = 1;
-		  return parser_parse_arg(parser, arg);
-
-		default:
-		  abort();
-		}
-	    case ARG_QUOTE:
-	      /* Skip it, then exchange with any previous non-options. */
-	      parser->state.next++;
-	      assert (parser->last_nonopt != parser->state.next);
-
-	      if (parser->first_nonopt != parser->last_nonopt)
-		{
-		  exchange(parser);
-		  
-		  /* Start processing the skipped and the quoted
-		     arguments. */
-
-		  parser->state.quoted = parser->state.next = parser->first_nonopt;
-
-		  /* Also empty the skipped-list, to avoid confusion
-		     if the user resets the next pointer. */
-		  parser->first_nonopt = parser->last_nonopt = 0;
-		}
-	      else
-		parser->state.quoted = parser->state.next;
-
-	      parser->args_only = 1;	      
-	      return 0;
-
-	    case ARG_LONG_ONLY_OPTION:
-	    case ARG_LONG_OPTION:
-	      {
-		struct group *group;
-		const struct argp_option *option;
-		char *value;
-
-		parser->state.next++;
-		option = find_long_option(parser, optstart, &group);
-		
-		if (!option)
-		  {
-		    /* NOTE: This includes any "=something" in the output. */
-		    fprintf (parser->state.err_stream,
-			     dgettext(parser->state.root_argp->argp_domain,
-				      "%s: unrecognized option `%s'\n"),
-			     parser->state.name, arg);
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-
-		value = strchr(optstart, '=');
-		if (value)
-		  value++;
-		
-		if (value && !option->arg)
-		  /* Unexpected argument. */
-		  {
-		    if (token == ARG_LONG_OPTION)
-		      /* --option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `--%s' doesn't allow an argument\n"),
-			       parser->state.name, option->name);
-		    else
-		      /* +option or -option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `%c%s' doesn't allow an argument\n"),
-			       parser->state.name, arg[0], option->name);
-
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-		
-		if (option->arg && !value
-		    && !(option->flags & OPTION_ARG_OPTIONAL))
-		  /* We need an mandatory argument. */
-		  {
-		    if (parser->state.next == parser->state.argc)
-		      /* Missing argument */
-		      {
-			if (token == ARG_LONG_OPTION)
-			  /* --option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `--%s' requires an argument\n"),
-				 parser->state.name, option->name);
-			else
-			  /* +option or -option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `%c%s' requires an argument\n"),
-				   parser->state.name, arg[0], option->name);
-
-			*arg_ebadkey = 0;
-			return EBADKEY;
-		      }
-
-		    value = parser->state.argv[parser->state.next++];
-		  }
-		*arg_ebadkey = 0;
-		return group_parse(group, &parser->state,
-				   option->key, value);
-	      }
-	    case ARG_SHORT_OPTION:
-	      parser->state.next++;
-	      parser->nextchar = optstart;
-	      return 0;
-
-	    default:
-	      abort();
-	    }
-	}
-    }
-}
-
-/* Parse the options strings in ARGC & ARGV according to the argp in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If END_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, EINVAL is returned; if some parser routine
-   returned a non-zero value, it is returned; otherwise 0 is returned.  */
-error_t
-__argp_parse (const struct argp *argp, int argc, char **argv, unsigned flags,
-	      int *end_index, void *input)
-{
-  error_t err;
-  struct parser parser;
-
-  /* If true, then err == EBADKEY is a result of a non-option argument failing
-     to be parsed (which in some cases isn't actually an error).  */
-  int arg_ebadkey = 0;
-
-  if (! (flags & ARGP_NO_HELP))
-    /* Add our own options.  */
-    {
-      struct argp_child *child = alloca (4 * sizeof (struct argp_child));
-      struct argp *top_argp = alloca (sizeof (struct argp));
-
-      /* TOP_ARGP has no options, it just serves to group the user & default
-	 argps.  */
-      memset (top_argp, 0, sizeof (*top_argp));
-      top_argp->children = child;
-
-      memset (child, 0, 4 * sizeof (struct argp_child));
-
-      if (argp)
-	(child++)->argp = argp;
-      (child++)->argp = &argp_default_argp;
-      if (argp_program_version || argp_program_version_hook)
-	(child++)->argp = &argp_version_argp;
-      child->argp = 0;
-
-      argp = top_argp;
-    }
-
-  /* Construct a parser for these arguments.  */
-  err = parser_init (&parser, argp, argc, argv, flags, input);
-
-  if (! err)
-    /* Parse! */
-    {
-      while (! err)
-	err = parser_parse_next (&parser, &arg_ebadkey);
-      err = parser_finalize (&parser, err, arg_ebadkey, end_index);
-    }
-
-  return err;
-}
-#ifdef weak_alias
-weak_alias (__argp_parse, argp_parse)
-#endif
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-void *
-__argp_input (const struct argp *argp, const struct argp_state *state)
-{
-  if (state)
-    {
-      struct group *group;
-      struct parser *parser = state->pstate;
-
-      for (group = parser->groups; group < parser->egroup; group++)
-	if (group->argp == argp)
-	  return group->input;
-    }
-
-  return 0;
-}
-#ifdef weak_alias
-weak_alias (__argp_input, _argp_input)
-#endif
-
-/* Defined here, in case a user is not inlining the definitions in
- * argp.h */
-void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      /* FIXME: whether or not a particular key implies a short option
-       * ought not to be locale dependent. */
-      return __key > 0 && isprint (__key);
-    }
-}
-
-int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
diff --git a/contrib/argp-standalone/argp-pv.c b/contrib/argp-standalone/argp-pv.c
deleted file mode 100644
index d7d374a66bd..00000000000
--- a/contrib/argp-standalone/argp-pv.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which will
-   print this this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-const char *argp_program_version = 0;
diff --git a/contrib/argp-standalone/argp-pvh.c b/contrib/argp-standalone/argp-pvh.c
deleted file mode 100644
index 829a1cda80d..00000000000
--- a/contrib/argp-standalone/argp-pvh.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION_HOOK.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "argp.h"
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which calls
-   this function with a stream to print the version to and a pointer to the
-   current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-void (*argp_program_version_hook) (FILE *stream, struct argp_state *state) = 0;
diff --git a/contrib/argp-standalone/argp.h b/contrib/argp-standalone/argp.h
deleted file mode 100644
index 29d3dfe9720..00000000000
--- a/contrib/argp-standalone/argp.h
+++ /dev/null
@@ -1,602 +0,0 @@
-/* Hierarchial argument parsing.
-   Copyright (C) 1995, 96, 97, 98, 99, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _ARGP_H
-#define _ARGP_H
-
-#include <stdio.h>
-#include <ctype.h>
-
-#define __need_error_t
-#include <errno.h>
-
-#ifndef __THROW
-# define __THROW
-#endif
-
-#ifndef __const
-# define __const const
-#endif
-
-#ifndef __error_t_defined
-typedef int error_t;
-# define __error_t_defined
-#endif
-
-/* FIXME: What's the right way to check for __restrict? Sun's cc seems
-   not to have it. Perhaps it's easiest to just delete the use of
-   __restrict from the prototypes. */
-#ifndef __restrict
-# ifndef __GNUC___
-#  define __restrict
-# endif
-#endif
-
-/* NOTE: We can't use the autoconf tests, since this is supposed to be
-   an installed header file and argp's config.h is of course not
-   installed. */
-#ifndef PRINTF_STYLE
-# if __GNUC__ >= 2
-#  define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# else
-#  define PRINTF_STYLE(f, a)
-# endif
-#endif
-
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-/* A description of a particular option.  A pointer to an array of
-   these is passed in the OPTIONS field of an argp structure.  Each option
-   entry can correspond to one long option and/or one short option; more
-   names for the same option can be added by following an entry in an option
-   array with options having the OPTION_ALIAS flag set.  */
-struct argp_option
-{
-  /* The long option name.  For more than one name for the same option, you
-     can use following options with the OPTION_ALIAS flag set.  */
-  __const char *name;
-
-  /* What key is returned for this option.  If > 0 and printable, then it's
-     also accepted as a short option.  */
-  int key;
-
-  /* If non-NULL, this is the name of the argument associated with this
-     option, which is required unless the OPTION_ARG_OPTIONAL flag is set. */
-  __const char *arg;
-
-  /* OPTION_ flags.  */
-  int flags;
-
-  /* The doc string for this option.  If both NAME and KEY are 0, This string
-     will be printed outdented from the normal option column, making it
-     useful as a group header (it will be the first thing printed in its
-     group); in this usage, it's conventional to end the string with a `:'.  */
-  __const char *doc;
-
-  /* The group this option is in.  In a long help message, options are sorted
-     alphabetically within each group, and the groups presented in the order
-     0, 1, 2, ..., n, -m, ..., -2, -1.  Every entry in an options array with
-     if this field 0 will inherit the group number of the previous entry, or
-     zero if it's the first one, unless its a group header (NAME and KEY both
-     0), in which case, the previous entry + 1 is the default.  Automagic
-     options such as --help are put into group -1.  */
-  int group;
-};
-
-/* The argument associated with this option is optional.  */
-#define OPTION_ARG_OPTIONAL	0x1
-
-/* This option isn't displayed in any help messages.  */
-#define OPTION_HIDDEN	       	0x2
-
-/* This option is an alias for the closest previous non-alias option.  This
-   means that it will be displayed in the same help entry, and will inherit
-   fields other than NAME and KEY from the aliased option.  */
-#define OPTION_ALIAS		0x4
-
-/* This option isn't actually an option (and so should be ignored by the
-   actual option parser), but rather an arbitrary piece of documentation that
-   should be displayed in much the same manner as the options.  If this flag
-   is set, then the option NAME field is displayed unmodified (e.g., no `--'
-   prefix is added) at the left-margin (where a *short* option would normally
-   be displayed), and the documentation string in the normal place.  For
-   purposes of sorting, any leading whitespace and puncuation is ignored,
-   except that if the first non-whitespace character is not `-', this entry
-   is displayed after all options (and OPTION_DOC entries with a leading `-')
-   in the same group.  */
-#define OPTION_DOC		0x8
-
-/* This option shouldn't be included in `long' usage messages (but is still
-   included in help messages).  This is mainly intended for options that are
-   completely documented in an argp's ARGS_DOC field, in which case including
-   the option in the generic usage list would be redundant.  For instance,
-   if ARGS_DOC is "FOO BAR\n-x BLAH", and the `-x' option's purpose is to
-   distinguish these two cases, -x should probably be marked
-   OPTION_NO_USAGE.  */
-#define OPTION_NO_USAGE		0x10
-
-struct argp;			/* fwd declare this type */
-struct argp_state;		/* " */
-struct argp_child;		/* " */
-
-/* The type of a pointer to an argp parsing function.  */
-typedef error_t (*argp_parser_t) (int key, char *arg,
-				  struct argp_state *state);
-
-/* What to return for unrecognized keys.  For special ARGP_KEY_ keys, such
-   returns will simply be ignored.  For user keys, this error will be turned
-   into EINVAL (if the call to argp_parse is such that errors are propagated
-   back to the user instead of exiting); returning EINVAL itself would result
-   in an immediate stop to parsing in *all* cases.  */
-#define ARGP_ERR_UNKNOWN	E2BIG /* Hurd should never need E2BIG.  XXX */
-
-/* Special values for the KEY argument to an argument parsing function.
-   ARGP_ERR_UNKNOWN should be returned if they aren't understood.
-
-   The sequence of keys to a parsing function is either (where each
-   uppercased word should be prefixed by `ARGP_KEY_' and opt is a user key):
-
-       INIT opt... NO_ARGS END SUCCESS  -- No non-option arguments at all
-   or  INIT (opt | ARG)... END SUCCESS  -- All non-option args parsed
-   or  INIT (opt | ARG)... SUCCESS      -- Some non-option arg unrecognized
-
-   The third case is where every parser returned ARGP_KEY_UNKNOWN for an
-   argument, in which case parsing stops at that argument (returning the
-   unparsed arguments to the caller of argp_parse if requested, or stopping
-   with an error message if not).
-
-   If an error occurs (either detected by argp, or because the parsing
-   function returned an error value), then the parser is called with
-   ARGP_KEY_ERROR, and no further calls are made.  */
-
-/* This is not an option at all, but rather a command line argument.  If a
-   parser receiving this key returns success, the fact is recorded, and the
-   ARGP_KEY_NO_ARGS case won't be used.  HOWEVER, if while processing the
-   argument, a parser function decrements the NEXT field of the state it's
-   passed, the option won't be considered processed; this is to allow you to
-   actually modify the argument (perhaps into an option), and have it
-   processed again.  */
-#define ARGP_KEY_ARG		0
-/* There are remaining arguments not parsed by any parser, which may be found
-   starting at (STATE->argv + STATE->next).  If success is returned, but
-   STATE->next left untouched, it's assumed that all arguments were consume,
-   otherwise, the parser should adjust STATE->next to reflect any arguments
-   consumed.  */
-#define ARGP_KEY_ARGS		0x1000006
-/* There are no more command line arguments at all.  */
-#define ARGP_KEY_END		0x1000001
-/* Because it's common to want to do some special processing if there aren't
-   any non-option args, user parsers are called with this key if they didn't
-   successfully process any non-option arguments.  Called just before
-   ARGP_KEY_END (where more general validity checks on previously parsed
-   arguments can take place).  */
-#define ARGP_KEY_NO_ARGS	0x1000002
-/* Passed in before any parsing is done.  Afterwards, the values of each
-   element of the CHILD_INPUT field, if any, in the state structure is
-   copied to each child's state to be the initial value of the INPUT field.  */
-#define ARGP_KEY_INIT		0x1000003
-/* Use after all other keys, including SUCCESS & END.  */
-#define ARGP_KEY_FINI		0x1000007
-/* Passed in when parsing has successfully been completed (even if there are
-   still arguments remaining).  */
-#define ARGP_KEY_SUCCESS	0x1000004
-/* Passed in if an error occurs.  */
-#define ARGP_KEY_ERROR		0x1000005
-
-/* An argp structure contains a set of options declarations, a function to
-   deal with parsing one, documentation string, a possible vector of child
-   argp's, and perhaps a function to filter help output.  When actually
-   parsing options, getopt is called with the union of all the argp
-   structures chained together through their CHILD pointers, with conflicts
-   being resolved in favor of the first occurrence in the chain.  */
-struct argp
-{
-  /* An array of argp_option structures, terminated by an entry with both
-     NAME and KEY having a value of 0.  */
-  __const struct argp_option *options;
-
-  /* What to do with an option from this structure.  KEY is the key
-     associated with the option, and ARG is any associated argument (NULL if
-     none was supplied).  If KEY isn't understood, ARGP_ERR_UNKNOWN should be
-     returned.  If a non-zero, non-ARGP_ERR_UNKNOWN value is returned, then
-     parsing is stopped immediately, and that value is returned from
-     argp_parse().  For special (non-user-supplied) values of KEY, see the
-     ARGP_KEY_ definitions below.  */
-  argp_parser_t parser;
-
-  /* A string describing what other arguments are wanted by this program.  It
-     is only used by argp_usage to print the `Usage:' message.  If it
-     contains newlines, the strings separated by them are considered
-     alternative usage patterns, and printed on separate lines (lines after
-     the first are prefix by `  or: ' instead of `Usage:').  */
-  __const char *args_doc;
-
-  /* If non-NULL, a string containing extra text to be printed before and
-     after the options in a long help message (separated by a vertical tab
-     `\v' character).  */
-  __const char *doc;
-
-  /* A vector of argp_children structures, terminated by a member with a 0
-     argp field, pointing to child argps should be parsed with this one.  Any
-     conflicts are resolved in favor of this argp, or early argps in the
-     CHILDREN list.  This field is useful if you use libraries that supply
-     their own argp structure, which you want to use in conjunction with your
-     own.  */
-  __const struct argp_child *children;
-
-  /* If non-zero, this should be a function to filter the output of help
-     messages.  KEY is either a key from an option, in which case TEXT is
-     that option's help text, or a special key from the ARGP_KEY_HELP_
-     defines, below, describing which other help text TEXT is.  The function
-     should return either TEXT, if it should be used as-is, a replacement
-     string, which should be malloced, and will be freed by argp, or NULL,
-     meaning `print nothing'.  The value for TEXT is *after* any translation
-     has been done, so if any of the replacement text also needs translation,
-     that should be done by the filter function.  INPUT is either the input
-     supplied to argp_parse, or NULL, if argp_help was called directly.  */
-  char *(*help_filter) (int __key, __const char *__text, void *__input);
-
-  /* If non-zero the strings used in the argp library are translated using
-     the domain described by this string.  Otherwise the currently installed
-     default domain is used.  */
-  const char *argp_domain;
-};
-
-/* Possible KEY arguments to a help filter function.  */
-#define ARGP_KEY_HELP_PRE_DOC	0x2000001 /* Help text preceeding options. */
-#define ARGP_KEY_HELP_POST_DOC	0x2000002 /* Help text following options. */
-#define ARGP_KEY_HELP_HEADER	0x2000003 /* Option header string. */
-#define ARGP_KEY_HELP_EXTRA	0x2000004 /* After all other documentation;
-					     TEXT is NULL for this key.  */
-/* Explanatory note emitted when duplicate option arguments have been
-   suppressed.  */
-#define ARGP_KEY_HELP_DUP_ARGS_NOTE 0x2000005
-#define ARGP_KEY_HELP_ARGS_DOC	0x2000006 /* Argument doc string.  */
-
-/* When an argp has a non-zero CHILDREN field, it should point to a vector of
-   argp_child structures, each of which describes a subsidiary argp.  */
-struct argp_child
-{
-  /* The child parser.  */
-  __const struct argp *argp;
-
-  /* Flags for this child.  */
-  int flags;
-
-  /* If non-zero, an optional header to be printed in help output before the
-     child options.  As a side-effect, a non-zero value forces the child
-     options to be grouped together; to achieve this effect without actually
-     printing a header string, use a value of "".  */
-  __const char *header;
-
-  /* Where to group the child options relative to the other (`consolidated')
-     options in the parent argp; the values are the same as the GROUP field
-     in argp_option structs, but all child-groupings follow parent options at
-     a particular group level.  If both this field and HEADER are zero, then
-     they aren't grouped at all, but rather merged with the parent options
-     (merging the child's grouping levels with the parents).  */
-  int group;
-};
-
-/* Parsing state.  This is provided to parsing functions called by argp,
-   which may examine and, as noted, modify fields.  */
-struct argp_state
-{
-  /* The top level ARGP being parsed.  */
-  __const struct argp *root_argp;
-
-  /* The argument vector being parsed.  May be modified.  */
-  int argc;
-  char **argv;
-
-  /* The index in ARGV of the next arg that to be parsed.  May be modified. */
-  int next;
-
-  /* The flags supplied to argp_parse.  May be modified.  */
-  unsigned flags;
-
-  /* While calling a parsing function with a key of ARGP_KEY_ARG, this is the
-     number of the current arg, starting at zero, and incremented after each
-     such call returns.  At all other times, this is the number of such
-     arguments that have been processed.  */
-  unsigned arg_num;
-
-  /* If non-zero, the index in ARGV of the first argument following a special
-     `--' argument (which prevents anything following being interpreted as an
-     option).  Only set once argument parsing has proceeded past this point. */
-  int quoted;
-
-  /* An arbitrary pointer passed in from the user.  */
-  void *input;
-  /* Values to pass to child parsers.  This vector will be the same length as
-     the number of children for the current parser.  */
-  void **child_inputs;
-
-  /* For the parser's use.  Initialized to 0.  */
-  void *hook;
-
-  /* The name used when printing messages.  This is initialized to ARGV[0],
-     or PROGRAM_INVOCATION_NAME if that is unavailable.  */
-  char *name;
-
-  /* Streams used when argp prints something.  */
-  FILE *err_stream;		/* For errors; initialized to stderr. */
-  FILE *out_stream;		/* For information; initialized to stdout. */
-
-  void *pstate;			/* Private, for use by argp.  */
-};
-
-/* Flags for argp_parse (note that the defaults are those that are
-   convenient for program command line parsing): */
-
-/* Don't ignore the first element of ARGV.  Normally (and always unless
-   ARGP_NO_ERRS is set) the first element of the argument vector is
-   skipped for option parsing purposes, as it corresponds to the program name
-   in a command line.  */
-#define ARGP_PARSE_ARGV0  0x01
-
-/* Don't print error messages for unknown options to stderr; unless this flag
-   is set, ARGP_PARSE_ARGV0 is ignored, as ARGV[0] is used as the program
-   name in the error messages.  This flag implies ARGP_NO_EXIT (on the
-   assumption that silent exiting upon errors is bad behaviour).  */
-#define ARGP_NO_ERRS	0x02
-
-/* Don't parse any non-option args.  Normally non-option args are parsed by
-   calling the parse functions with a key of ARGP_KEY_ARG, and the actual arg
-   as the value.  Since it's impossible to know which parse function wants to
-   handle it, each one is called in turn, until one returns 0 or an error
-   other than ARGP_ERR_UNKNOWN; if an argument is handled by no one, the
-   argp_parse returns prematurely (but with a return value of 0).  If all
-   args have been parsed without error, all parsing functions are called one
-   last time with a key of ARGP_KEY_END.  This flag needn't normally be set,
-   as the normal behavior is to stop parsing as soon as some argument can't
-   be handled.  */
-#define ARGP_NO_ARGS	0x04
-
-/* Parse options and arguments in the same order they occur on the command
-   line -- normally they're rearranged so that all options come first. */
-#define ARGP_IN_ORDER	0x08
-
-/* Don't provide the standard long option --help, which causes usage and
-      option help information to be output to stdout, and exit (0) called. */
-#define ARGP_NO_HELP	0x10
-
-/* Don't exit on errors (they may still result in error messages).  */
-#define ARGP_NO_EXIT	0x20
-
-/* Use the gnu getopt `long-only' rules for parsing arguments.  */
-#define ARGP_LONG_ONLY	0x40
-
-/* Turns off any message-printing/exiting options.  */
-#define ARGP_SILENT    (ARGP_NO_EXIT | ARGP_NO_ERRS | ARGP_NO_HELP)
-
-/* Parse the options strings in ARGC & ARGV according to the options in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If ARG_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, ARGP_ERR_UNKNOWN is returned; if some parser
-   routine returned a non-zero value, it is returned; otherwise 0 is
-   returned.  This function may also call exit unless the ARGP_NO_HELP flag
-   is set.  INPUT is a pointer to a value to be passed in to the parser.  */
-extern error_t argp_parse (__const struct argp *__restrict argp,
-			   int argc, char **__restrict argv,
-			   unsigned flags, int *__restrict arg_index,
-			   void *__restrict input) __THROW;
-extern error_t __argp_parse (__const struct argp *__restrict argp,
-			     int argc, char **__restrict argv,
-			     unsigned flags, int *__restrict arg_index,
-			     void *__restrict input) __THROW;
-
-/* Global variables.  */
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   will print this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-extern __const char *argp_program_version;
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   calls this function with a stream to print the version to and a pointer to
-   the current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-extern void (*argp_program_version_hook) (FILE *__restrict __stream,
-					  struct argp_state *__restrict
-					  __state);
-
-/* If defined or set by the user program, it should point to string that is
-   the bug-reporting address for the program.  It will be printed by
-   argp_help if the ARGP_HELP_BUG_ADDR flag is set (as it is by various
-   standard help messages), embedded in a sentence that says something like
-   `Report bugs to ADDR.'.  */
-extern __const char *argp_program_bug_address;
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-extern error_t argp_err_exit_status;
-
-/* Flags for argp_help.  */
-#define ARGP_HELP_USAGE		0x01 /* a Usage: message. */
-#define ARGP_HELP_SHORT_USAGE	0x02 /*  " but don't actually print options. */
-#define ARGP_HELP_SEE		0x04 /* a `Try ... for more help' message. */
-#define ARGP_HELP_LONG		0x08 /* a long help message. */
-#define ARGP_HELP_PRE_DOC	0x10 /* doc string preceding long help.  */
-#define ARGP_HELP_POST_DOC	0x20 /* doc string following long help.  */
-#define ARGP_HELP_DOC		(ARGP_HELP_PRE_DOC | ARGP_HELP_POST_DOC)
-#define ARGP_HELP_BUG_ADDR	0x40 /* bug report address */
-#define ARGP_HELP_LONG_ONLY	0x80 /* modify output appropriately to
-					reflect ARGP_LONG_ONLY mode.  */
-
-/* These ARGP_HELP flags are only understood by argp_state_help.  */
-#define ARGP_HELP_EXIT_ERR	0x100 /* Call exit(1) instead of returning.  */
-#define ARGP_HELP_EXIT_OK	0x200 /* Call exit(0) instead of returning.  */
-
-/* The standard thing to do after a program command line parsing error, if an
-   error message has already been printed.  */
-#define ARGP_HELP_STD_ERR \
-  (ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do after a program command line parsing error, if no
-   more specific error message has been printed.  */
-#define ARGP_HELP_STD_USAGE \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do in response to a --help option.  */
-#define ARGP_HELP_STD_HELP \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG | ARGP_HELP_EXIT_OK \
-   | ARGP_HELP_DOC | ARGP_HELP_BUG_ADDR)
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  */
-extern void argp_help (__const struct argp *__restrict __argp,
-		       FILE *__restrict __stream,
-		       unsigned __flags, char *__restrict __name) __THROW;
-extern void __argp_help (__const struct argp *__restrict __argp,
-			 FILE *__restrict __stream, unsigned __flags,
-			 char *__name) __THROW;
-
-/* The following routines are intended to be called from within an argp
-   parsing routine (thus taking an argp_state structure as the first
-   argument).  They may or may not print an error message and exit, depending
-   on the flags in STATE -- in any case, the caller should be prepared for
-   them *not* to exit, and should return an appropiate error after calling
-   them.  [argp_usage & argp_error should probably be called argp_state_...,
-   but they're used often enough that they should be short]  */
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-extern void argp_state_help (__const struct argp_state *__restrict __state,
-			     FILE *__restrict __stream,
-			     unsigned int __flags) __THROW;
-extern void __argp_state_help (__const struct argp_state *__restrict __state,
-			       FILE *__restrict __stream,
-			       unsigned int __flags) __THROW;
-
-/* Possibly output the standard usage message for ARGP to stderr and exit.  */
-extern void argp_usage (__const struct argp_state *__state) __THROW;
-extern void __argp_usage (__const struct argp_state *__state) __THROW;
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-extern void argp_error (__const struct argp_state *__restrict __state,
-			__const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-extern void __argp_error (__const struct argp_state *__restrict __state,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-extern void argp_failure (__const struct argp_state *__restrict __state,
-			  int __status, int __errnum,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-extern void __argp_failure (__const struct argp_state *__restrict __state,
-			    int __status, int __errnum,
-			    __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-
-/* Returns true if the option OPT is a valid short option.  */
-extern int _option_is_short (__const struct argp_option *__opt) __THROW;
-extern int __option_is_short (__const struct argp_option *__opt) __THROW;
-
-/* Returns true if the option OPT is in fact the last (unused) entry in an
-   options array.  */
-extern int _option_is_end (__const struct argp_option *__opt) __THROW;
-extern int __option_is_end (__const struct argp_option *__opt) __THROW;
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-extern void *_argp_input (__const struct argp *__restrict __argp,
-			  __const struct argp_state *__restrict __state)
-     __THROW;
-extern void *__argp_input (__const struct argp *__restrict __argp,
-			   __const struct argp_state *__restrict __state)
-     __THROW;
-
-/* Used for extracting the program name from argv[0] */
-extern char *_argp_basename(char *name) __THROW;
-extern char *__argp_basename(char *name) __THROW;
-
-/* Getting the program name given an argp state */
-extern char *
-_argp_short_program_name(const struct argp_state *state) __THROW;
-extern char *
-__argp_short_program_name(const struct argp_state *state) __THROW;
-
-
-#ifdef __USE_EXTERN_INLINES
-
-# if !_LIBC
-#  define __argp_usage argp_usage
-#  define __argp_state_help argp_state_help
-#  define __option_is_short _option_is_short
-#  define __option_is_end _option_is_end
-# endif
-
-# ifndef ARGP_EI
-#  define ARGP_EI extern __inline__
-# endif
-
-ARGP_EI void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-ARGP_EI int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      return __key > 0 && isprint (__key);
-    }
-}
-
-ARGP_EI int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
-
-# if !_LIBC
-#  undef __argp_usage
-#  undef __argp_state_help
-#  undef __option_is_short
-#  undef __option_is_end
-# endif
-#endif /* Use extern inlines.  */
-
-#ifdef  __cplusplus
-}
-#endif
-
-#endif /* argp.h */
diff --git a/contrib/argp-standalone/autogen.sh b/contrib/argp-standalone/autogen.sh
deleted file mode 100755
index 8337353b5ae..00000000000
--- a/contrib/argp-standalone/autogen.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-aclocal -I .
-autoheader
-autoconf
-automake --add-missing --copy --foreign
diff --git a/contrib/argp-standalone/configure.ac b/contrib/argp-standalone/configure.ac
deleted file mode 100644
index c0867eb5b35..00000000000
--- a/contrib/argp-standalone/configure.ac
+++ /dev/null
@@ -1,105 +0,0 @@
-dnl Process this file with autoconf to produce a configure script.
-
-dnl This configure.ac is only for building a standalone argp library.
-AC_INIT([argp], [standalone-1.3])
-AC_PREREQ(2.54)
-AC_CONFIG_SRCDIR([argp-ba.c])
-# Needed to stop autoconf from looking for files in parent directories.
-AC_CONFIG_AUX_DIR([.])
-
-AM_INIT_AUTOMAKE
-AC_CONFIG_HEADERS(config.h)
-
-m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
-
-# GNU libc defaults to supplying the ISO C library functions only. The
-# _GNU_SOURCE define enables these extensions, in particular we want
-# errno.h to declare program_invocation_name. Enable it on all
-# systems; no problems have been reported with it so far.
-AC_GNU_SOURCE
-
-# Checks for programs.
-AC_PROG_CC
-AC_PROG_MAKE_SET
-AC_PROG_RANLIB
-AC_PROG_CC
-
-if test "x$am_cv_prog_cc_stdc" = xno ; then
-  AC_ERROR([the C compiler doesn't handle ANSI-C])
-fi
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_HEADER_STDC
-AC_CHECK_HEADERS(libintl.h limits.h malloc.h unistd.h sysexits.h stdarg.h)
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_C_CONST
-AC_C_INLINE
-AC_TYPE_SIZE_T
-
-LSH_GCC_ATTRIBUTES
-
-# Checks for library functions.
-AC_FUNC_ALLOCA
-AC_FUNC_VPRINTF
-AC_CHECK_FUNCS(strerror sleep getpid snprintf)
-
-AC_REPLACE_FUNCS(mempcpy strndup strchrnul strcasecmp vsnprintf)
-
-dnl ARGP_CHECK_FUNC(includes, function-call [, if-found [, if-not-found]])
-AC_DEFUN([ARGP_CHECK_FUNC],
-  [AS_VAR_PUSHDEF([ac_func], m4_substr([$2], 0, m4_index([$2], [(])))
-   AS_VAR_PUSHDEF([ac_var], [ac_cv_func_call_]ac_func)
-   AH_TEMPLATE(AS_TR_CPP(HAVE_[]ac_func),
-               [Define to 1 if you have the `]ac_func[' function.])
-   AC_CACHE_CHECK([for $2], ac_var,
-     [AC_TRY_LINK([$1], [$2],
-                  [AS_VAR_SET(ac_var, yes)],
-                  [AS_VAR_SET(ac_var, no)])])
-   if test AS_VAR_GET(ac_var) = yes ; then
-     ifelse([$3],,
-            [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_[]ac_func))],
-            [$3
-])
-   else
-     ifelse([$4],, true, [$4])
-   fi
-   AS_VAR_POPDEF([ac_var])
-   AS_VAR_POPDEF([ac_func])
-   ])
-
-# At least on freebsd, putc_unlocked is a macro, so the standard
-# AC_CHECK_FUNCS doesn't work well.
-ARGP_CHECK_FUNC([#include <stdio.h>], [putc_unlocked('x', stdout)])
-
-AC_CHECK_FUNCS(flockfile)
-AC_CHECK_FUNCS(fputs_unlocked fwrite_unlocked)
-
-# Used only by argp-test.c, so don't use AC_REPLACE_FUNCS.
-AC_CHECK_FUNCS(strdup asprintf)
-
-AC_CHECK_DECLS([program_invocation_name, program_invocation_short_name],
-               [], [], [[#include <errno.h>]])
-
-# Set these flags *last*, or else the test programs won't compile
-if test x$GCC = xyes ; then
-  # Using -ggdb3 makes (some versions of) Redhat's gcc-2.96 dump core
-  if "$CC" --version | grep '^2\.96$' 1>/dev/null 2>&1; then
-    true
-  else
-    CFLAGS="$CFLAGS -ggdb3"
-  fi
-  CFLAGS="$CFLAGS -Wall -W \
- -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes \
- -Waggregate-return \
- -Wpointer-arith -Wbad-function-cast -Wnested-externs"
-fi
-
-CPPFLAGS="$CPPFLAGS -I$srcdir"
-
-dnl Added for C99 standards
-CFLAGS="$CFLAGS -std=gnu89 -static"
-
-AC_OUTPUT(Makefile)
diff --git a/contrib/argp-standalone/mempcpy.c b/contrib/argp-standalone/mempcpy.c
deleted file mode 100644
index 21d8bd2ed94..00000000000
--- a/contrib/argp-standalone/mempcpy.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <string.h>
-
-void *
-mempcpy (void *, const void *, size_t) ;
-
-void *
-mempcpy (void *to, const void *from, size_t size)
-{
-  memcpy(to, from, size);
-  return (char *) to + size;
-}
-
diff --git a/contrib/argp-standalone/strcasecmp.c b/contrib/argp-standalone/strcasecmp.c
deleted file mode 100644
index 9c1637232fd..00000000000
--- a/contrib/argp-standalone/strcasecmp.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* strcasecmp.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <ctype.h>
-int strcasecmp(const char *, const char *);
-
-int strcasecmp(const char *s1, const char *s2)
-{
-  unsigned i;
-  
-  for (i = 0; s1[i] && s2[i]; i++)
-    {
-      unsigned char c1 = tolower( (unsigned char) s1[i]);
-      unsigned char c2 = tolower( (unsigned char) s2[i]);
-
-      if (c1 < c2)
-	return -1;
-      else if (c1 > c2)
-	return 1;
-    }
-
-  return !s2[i] - !s1[i];	
-}
diff --git a/contrib/argp-standalone/strchrnul.c b/contrib/argp-standalone/strchrnul.c
deleted file mode 100644
index ee4145e4eda..00000000000
--- a/contrib/argp-standalone/strchrnul.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* strchrnul.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-/* FIXME: What is this function supposed to do? My guess is that it is
- * like strchr, but returns a pointer to the NUL character, not a NULL
- * pointer, if the character isn't found. */
-
-char *strchrnul(const char *, int );
-
-char *strchrnul(const char *s, int c)
-{
-  const char *p = s;
-  while (*p && (*p != c))
-    p++;
-
-  return (char *) p;
-}
diff --git a/contrib/argp-standalone/strndup.c b/contrib/argp-standalone/strndup.c
deleted file mode 100644
index 4147b7a2051..00000000000
--- a/contrib/argp-standalone/strndup.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <stdlib.h>
-#include <string.h>
-
-char *
-strndup (const char *, size_t);
-
-char *
-strndup (const char *s, size_t size)
-{
-  char *r;
-  char *end = memchr(s, 0, size);
-  
-  if (end)
-    /* Length + 1 */
-    size = end - s + 1;
-  
-  r = malloc(size);
-
-  if (size)
-    {
-      memcpy(r, s, size-1);
-      r[size-1] = '\0';
-    }
-  return r;
-}
diff --git a/contrib/argp-standalone/vsnprintf.c b/contrib/argp-standalone/vsnprintf.c
deleted file mode 100644
index 33c9a5d0042..00000000000
--- a/contrib/argp-standalone/vsnprintf.c
+++ /dev/null
@@ -1,839 +0,0 @@
-/* Copied from http://www.fiction.net/blong/programs/snprintf.c */
-
-/*
- * Copyright Patrick Powell 1995
- * This code is based on code written by Patrick Powell (papowell@astart.com)
- * It may be used for any purpose as long as this notice remains intact
- * on all source code distributions
- */
-
-/**************************************************************
- * Original:
- * Patrick Powell Tue Apr 11 09:48:21 PDT 1995
- * A bombproof version of doprnt (dopr) included.
- * Sigh.  This sort of thing is always nasty do deal with.  Note that
- * the version here does not include floating point...
- *
- * snprintf() is used instead of sprintf() as it does limit checks
- * for string length.  This covers a nasty loophole.
- *
- * The other functions are there to prevent NULL pointers from
- * causing nast effects.
- *
- * More Recently:
- *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43
- *  This was ugly.  It is still ugly.  I opted out of floating point
- *  numbers, but the formatter understands just about everything
- *  from the normal C string format, at least as far as I can tell from
- *  the Solaris 2.5 printf(3S) man page.
- *
- *  Brandon Long <blong@fiction.net> 10/22/97 for mutt 0.87.1
- *    Ok, added some minimal floating point support, which means this
- *    probably requires libm on most operating systems.  Don't yet
- *    support the exponent (e,E) and sigfig (g,G).  Also, fmtint()
- *    was pretty badly broken, it just wasn't being exercised in ways
- *    which showed it, so that's been fixed.  Also, formated the code
- *    to mutt conventions, and removed dead code left over from the
- *    original.  Also, there is now a builtin-test, just compile with:
- *           gcc -DTEST_SNPRINTF -o snprintf snprintf.c -lm
- *    and run snprintf for results.
- * 
- *  Thomas Roessler <roessler@guug.de> 01/27/98 for mutt 0.89i
- *    The PGP code was using unsigned hexadecimal formats. 
- *    Unfortunately, unsigned formats simply didn't work.
- *
- *  Michael Elkins <me@cs.hmc.edu> 03/05/98 for mutt 0.90.8
- *    The original code assumed that both snprintf() and vsnprintf() were
- *    missing.  Some systems only have snprintf() but not vsnprintf(), so
- *    the code is now broken down under HAVE_SNPRINTF and HAVE_VSNPRINTF.
- *
- *  Andrew Tridgell (tridge@samba.org) Oct 1998
- *    fixed handling of %.0f
- *    added test for HAVE_LONG_DOUBLE
- *
- *  Russ Allbery <rra@stanford.edu> 2000-08-26
- *    fixed return value to comply with C99
- *    fixed handling of snprintf(NULL, ...)
- *
- * Niels M�ller <nisse@lysator.liu.se> 2004-03-05
- *    fixed calls to isdigit to use unsigned char.
- *    fixed calls to va_arg; short arguments are always passed as int.
- *
- **************************************************************/
-
-#if HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#if !defined(HAVE_SNPRINTF) || !defined(HAVE_VSNPRINTF)
-
-#include <string.h>
-#include <ctype.h>
-#include <sys/types.h>
-
-/* Define this as a fall through, HAVE_STDARG_H is probably already set */
-
-#define HAVE_VARARGS_H
-
-
-/* varargs declarations: */
-
-#if defined(HAVE_STDARG_H)
-# include <stdarg.h>
-# define HAVE_STDARGS    /* let's hope that works everywhere (mj) */
-# define VA_LOCAL_DECL   va_list ap
-# define VA_START(f)     va_start(ap, f)
-# define VA_SHIFT(v,t)  ;   /* no-op for ANSI */
-# define VA_END          va_end(ap)
-#else
-# if defined(HAVE_VARARGS_H)
-#  include <varargs.h>
-#  undef HAVE_STDARGS
-#  define VA_LOCAL_DECL   va_list ap
-#  define VA_START(f)     va_start(ap)      /* f is ignored! */
-#  define VA_SHIFT(v,t) v = va_arg(ap,t)
-#  define VA_END        va_end(ap)
-# else
-/*XX ** NO VARARGS ** XX*/
-# endif
-#endif
-
-#ifdef HAVE_LONG_DOUBLE
-#define LDOUBLE long double
-#else
-#define LDOUBLE double
-#endif
-
-int snprintf (char *str, size_t count, const char *fmt, ...);
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-
-static int dopr (char *buffer, size_t maxlen, const char *format, 
-                 va_list args);
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-		   char *value, int flags, int min, int max);
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags);
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags);
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c );
-
-/*
- * dopr(): poor man's version of doprintf
- */
-
-/* format read states */
-#define DP_S_DEFAULT 0
-#define DP_S_FLAGS   1
-#define DP_S_MIN     2
-#define DP_S_DOT     3
-#define DP_S_MAX     4
-#define DP_S_MOD     5
-#define DP_S_CONV    6
-#define DP_S_DONE    7
-
-/* format flags - Bits */
-#define DP_F_MINUS 	(1 << 0)
-#define DP_F_PLUS  	(1 << 1)
-#define DP_F_SPACE 	(1 << 2)
-#define DP_F_NUM   	(1 << 3)
-#define DP_F_ZERO  	(1 << 4)
-#define DP_F_UP    	(1 << 5)
-#define DP_F_UNSIGNED 	(1 << 6)
-
-/* Conversion Flags */
-#define DP_C_SHORT   1
-#define DP_C_LONG    2
-#define DP_C_LDOUBLE 3
-
-#define char_to_int(p) (p - '0')
-#define MAX(p,q) ((p >= q) ? p : q)
-#define MIN(p,q) ((p <= q) ? p : q)
-
-static int dopr (char *buffer, size_t maxlen, const char *format, va_list args)
-{
-  unsigned char ch;
-  long value;
-  LDOUBLE fvalue;
-  char *strvalue;
-  int min;
-  int max;
-  int state;
-  int flags;
-  int cflags;
-  int total;
-  size_t currlen;
-  
-  state = DP_S_DEFAULT;
-  currlen = flags = cflags = min = 0;
-  max = -1;
-  ch = *format++;
-  total = 0;
-
-  while (state != DP_S_DONE)
-  {
-    if (ch == '\0')
-      state = DP_S_DONE;
-
-    switch(state) 
-    {
-    case DP_S_DEFAULT:
-      if (ch == '%') 
-	state = DP_S_FLAGS;
-      else 
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-      ch = *format++;
-      break;
-    case DP_S_FLAGS:
-      switch (ch) 
-      {
-      case '-':
-	flags |= DP_F_MINUS;
-        ch = *format++;
-	break;
-      case '+':
-	flags |= DP_F_PLUS;
-        ch = *format++;
-	break;
-      case ' ':
-	flags |= DP_F_SPACE;
-        ch = *format++;
-	break;
-      case '#':
-	flags |= DP_F_NUM;
-        ch = *format++;
-	break;
-      case '0':
-	flags |= DP_F_ZERO;
-        ch = *format++;
-	break;
-      default:
-	state = DP_S_MIN;
-	break;
-      }
-      break;
-    case DP_S_MIN:
-      if (isdigit(ch)) 
-      {
-	min = 10*min + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	min = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_DOT;
-      } 
-      else 
-	state = DP_S_DOT;
-      break;
-    case DP_S_DOT:
-      if (ch == '.') 
-      {
-	state = DP_S_MAX;
-	ch = *format++;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MAX:
-      if (isdigit(ch)) 
-      {
-	if (max < 0)
-	  max = 0;
-	max = 10*max + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	max = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_MOD;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MOD:
-      /* Currently, we don't support Long Long, bummer */
-      switch (ch) 
-      {
-      case 'h':
-	cflags = DP_C_SHORT;
-	ch = *format++;
-	break;
-      case 'l':
-	cflags = DP_C_LONG;
-	ch = *format++;
-	break;
-      case 'L':
-	cflags = DP_C_LDOUBLE;
-	ch = *format++;
-	break;
-      default:
-	break;
-      }
-      state = DP_S_CONV;
-      break;
-    case DP_S_CONV:
-      switch (ch) 
-      {
-      case 'd':
-      case 'i':
-	if (cflags == DP_C_SHORT) 
-	  value = (short) va_arg (args, int);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, long int);
-	else
-	  value = va_arg (args, int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'o':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 8, min, max, flags);
-	break;
-      case 'u':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'X':
-	flags |= DP_F_UP;
-      case 'x':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 16, min, max, flags);
-	break;
-      case 'f':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	/* um, floating point? */
-	total += fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
-	break;
-      case 'E':
-	flags |= DP_F_UP;
-      case 'e':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'G':
-	flags |= DP_F_UP;
-      case 'g':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'c':
-	total += dopr_outch (buffer, &currlen, maxlen, va_arg (args, int));
-	break;
-      case 's':
-	strvalue = va_arg (args, char *);
-	total += fmtstr (buffer, &currlen, maxlen, strvalue, flags, min, max);
-	break;
-      case 'p':
-	strvalue = va_arg (args, void *);
-	total += fmtint (buffer, &currlen, maxlen, (long) strvalue, 16, min,
-                         max, flags);
-	break;
-      case 'n':
-	if (cflags == DP_C_SHORT) 
-	{
-	  short int *num;
-	  num = va_arg (args, short int *);
-	  *num = currlen;
-        } 
-	else if (cflags == DP_C_LONG) 
-	{
-	  long int *num;
-	  num = va_arg (args, long int *);
-	  *num = currlen;
-        } 
-	else 
-	{
-	  int *num;
-	  num = va_arg (args, int *);
-	  *num = currlen;
-        }
-	break;
-      case '%':
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-	break;
-      case 'w':
-	/* not supported yet, treat as next char */
-	ch = *format++;
-	break;
-      default:
-	/* Unknown, skip */
-	break;
-      }
-      ch = *format++;
-      state = DP_S_DEFAULT;
-      flags = cflags = min = 0;
-      max = -1;
-      break;
-    case DP_S_DONE:
-      break;
-    default:
-      /* hmm? */
-      break; /* some picky compilers need this */
-    }
-  }
-  if (buffer != NULL)
-  {
-    if (currlen < maxlen - 1) 
-      buffer[currlen] = '\0';
-    else 
-      buffer[maxlen - 1] = '\0';
-  }
-  return total;
-}
-
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-                   char *value, int flags, int min, int max)
-{
-  int padlen, strln;     /* amount to pad */
-  int cnt = 0;
-  int total = 0;
-  
-  if (value == 0)
-  {
-    value = "<NULL>";
-  }
-
-  for (strln = 0; value[strln]; ++strln); /* strlen */
-  if (max >= 0 && max < strln)
-    strln = max;
-  padlen = min - strln;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justify */
-
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  while (*value && ((max < 0) || (cnt < max)))
-  {
-    total += dopr_outch (buffer, currlen, maxlen, *value++);
-    ++cnt;
-  }
-  while (padlen < 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-  return total;
-}
-
-/* Have to handle DP_F_NUM (ie 0x and 0 alternates) */
-
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags)
-{
-  int signvalue = 0;
-  unsigned long uvalue;
-  char convert[20];
-  int place = 0;
-  int spadlen = 0; /* amount to space pad */
-  int zpadlen = 0; /* amount to zero pad */
-  int caps = 0;
-  int total = 0;
-  
-  if (max < 0)
-    max = 0;
-
-  uvalue = value;
-
-  if(!(flags & DP_F_UNSIGNED))
-  {
-    if( value < 0 ) {
-      signvalue = '-';
-      uvalue = -value;
-    }
-    else
-      if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-	signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-  }
-  
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-
-  do {
-    convert[place++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")
-      [uvalue % (unsigned)base  ];
-    uvalue = (uvalue / (unsigned)base );
-  } while(uvalue && (place < 20));
-  if (place == 20) place--;
-  convert[place] = 0;
-
-  zpadlen = max - place;
-  spadlen = min - MAX (max, place) - (signvalue ? 1 : 0);
-  if (zpadlen < 0) zpadlen = 0;
-  if (spadlen < 0) spadlen = 0;
-  if (flags & DP_F_ZERO)
-  {
-    zpadlen = MAX(zpadlen, spadlen);
-    spadlen = 0;
-  }
-  if (flags & DP_F_MINUS) 
-    spadlen = -spadlen; /* Left Justifty */
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "zpad: %d, spad: %d, min: %d, max: %d, place: %d\n",
-      zpadlen, spadlen, min, max, place));
-#endif
-
-  /* Spaces */
-  while (spadlen > 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --spadlen;
-  }
-
-  /* Sign */
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  /* Zeros */
-  if (zpadlen > 0) 
-  {
-    while (zpadlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --zpadlen;
-    }
-  }
-
-  /* Digits */
-  while (place > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, convert[--place]);
-  
-  /* Left Justified spaces */
-  while (spadlen < 0) {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++spadlen;
-  }
-
-  return total;
-}
-
-static LDOUBLE abs_val (LDOUBLE value)
-{
-  LDOUBLE result = value;
-
-  if (value < 0)
-    result = -value;
-
-  return result;
-}
-
-static LDOUBLE pow10_argp (int exp)
-{
-  LDOUBLE result = 1;
-
-  while (exp)
-  {
-    result *= 10;
-    exp--;
-  }
-  
-  return result;
-}
-
-static long round_argp (LDOUBLE value)
-{
-  long intpart;
-
-  intpart = value;
-  value = value - intpart;
-  if (value >= 0.5)
-    intpart++;
-
-  return intpart;
-}
-
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags)
-{
-  int signvalue = 0;
-  LDOUBLE ufvalue;
-  char iconvert[20];
-  char fconvert[20];
-  int iplace = 0;
-  int fplace = 0;
-  int padlen = 0; /* amount to pad */
-  int zpadlen = 0; 
-  int caps = 0;
-  int total = 0;
-  long intpart;
-  long fracpart;
-  
-  /* 
-   * AIX manpage says the default is 0, but Solaris says the default
-   * is 6, and sprintf on AIX defaults to 6
-   */
-  if (max < 0)
-    max = 6;
-
-  ufvalue = abs_val (fvalue);
-
-  if (fvalue < 0)
-    signvalue = '-';
-  else
-    if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-      signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-
-#if 0
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-#endif
-
-  intpart = ufvalue;
-
-  /* 
-   * Sorry, we only support 9 digits past the decimal because of our 
-   * conversion method
-   */
-  if (max > 9)
-    max = 9;
-
-  /* We "cheat" by converting the fractional part to integer by
-   * multiplying by a factor of 10
-   */
-  fracpart = round_argp ((pow10_argp (max)) * (ufvalue - intpart));
-
-  if (fracpart >= pow10_argp (max))
-  {
-    intpart++;
-    fracpart -= pow10_argp (max);
-  }
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "fmtfp: %f =? %d.%d\n", fvalue, intpart, fracpart));
-#endif
-
-  /* Convert integer part */
-  do {
-    iconvert[iplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[intpart % 10];
-    intpart = (intpart / 10);
-  } while(intpart && (iplace < 20));
-  if (iplace == 20) iplace--;
-  iconvert[iplace] = 0;
-
-  /* Convert fractional part */
-  do {
-    fconvert[fplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[fracpart % 10];
-    fracpart = (fracpart / 10);
-  } while(fracpart && (fplace < 20));
-  if (fplace == 20) fplace--;
-  fconvert[fplace] = 0;
-
-  /* -1 for decimal point, another -1 if we are printing a sign */
-  padlen = min - iplace - max - 1 - ((signvalue) ? 1 : 0); 
-  zpadlen = max - fplace;
-  if (zpadlen < 0)
-    zpadlen = 0;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justifty */
-
-  if ((flags & DP_F_ZERO) && (padlen > 0)) 
-  {
-    if (signvalue) 
-    {
-      total += dopr_outch (buffer, currlen, maxlen, signvalue);
-      --padlen;
-      signvalue = 0;
-    }
-    while (padlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --padlen;
-    }
-  }
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  while (iplace > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, iconvert[--iplace]);
-
-  /*
-   * Decimal point.  This should probably use locale to find the correct
-   * char to print out.
-   */
-  if (max > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '.');
-
-    while (fplace > 0) 
-      total += dopr_outch (buffer, currlen, maxlen, fconvert[--fplace]);
-  }
-
-  while (zpadlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '0');
-    --zpadlen;
-  }
-
-  while (padlen < 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-
-  return total;
-}
-
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c)
-{
-  if (*currlen + 1 < maxlen)
-    buffer[(*currlen)++] = c;
-  return 1;
-}
-
-#ifndef HAVE_VSNPRINTF
-int vsnprintf (char *str, size_t count, const char *fmt, va_list args)
-{
-  if (str != NULL)
-    str[0] = 0;
-  return dopr(str, count, fmt, args);
-}
-#endif /* !HAVE_VSNPRINTF */
-
-#ifndef HAVE_SNPRINTF
-/* VARARGS3 */
-#ifdef HAVE_STDARGS
-int snprintf (char *str,size_t count,const char *fmt,...)
-#else
-int snprintf (va_alist) va_dcl
-#endif
-{
-#ifndef HAVE_STDARGS
-  char *str;
-  size_t count;
-  char *fmt;
-#endif
-  VA_LOCAL_DECL;
-  int total;
-    
-  VA_START (fmt);
-  VA_SHIFT (str, char *);
-  VA_SHIFT (count, size_t );
-  VA_SHIFT (fmt, char *);
-  total = vsnprintf(str, count, fmt, ap);
-  VA_END;
-  return total;
-}
-#endif /* !HAVE_SNPRINTF */
-
-#ifdef TEST_SNPRINTF
-#ifndef LONG_STRING
-#define LONG_STRING 1024
-#endif
-int main (void)
-{
-  char buf1[LONG_STRING];
-  char buf2[LONG_STRING];
-  char *fp_fmt[] = {
-    "%-1.5f",
-    "%1.5f",
-    "%123.9f",
-    "%10.5f",
-    "% 10.5f",
-    "%+22.9f",
-    "%+4.9f",
-    "%01.3f",
-    "%4f",
-    "%3.1f",
-    "%3.2f",
-    "%.0f",
-    "%.1f",
-    NULL
-  };
-  double fp_nums[] = { -1.5, 134.21, 91340.2, 341.1234, 0203.9, 0.96, 0.996, 
-    0.9996, 1.996, 4.136, 0};
-  char *int_fmt[] = {
-    "%-1.5d",
-    "%1.5d",
-    "%123.9d",
-    "%5.5d",
-    "%10.5d",
-    "% 10.5d",
-    "%+22.33d",
-    "%01.3d",
-    "%4d",
-    NULL
-  };
-  long int_nums[] = { -1, 134, 91340, 341, 0203, 0};
-  int x, y;
-  int fail = 0;
-  int num = 0;
-
-  printf ("Testing snprintf format codes against system sprintf...\n");
-
-  for (x = 0; fp_fmt[x] != NULL ; x++)
-    for (y = 0; fp_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), fp_fmt[x], fp_nums[y]);
-      sprintf (buf2, fp_fmt[x], fp_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    fp_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-
-  for (x = 0; int_fmt[x] != NULL ; x++)
-    for (y = 0; int_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), int_fmt[x], int_nums[y]);
-      sprintf (buf2, int_fmt[x], int_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    int_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-  printf ("%d tests failed out of %d.\n", fail, num);
-}
-#endif /* SNPRINTF_TEST */
-
-#endif /* !HAVE_SNPRINTF */
diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index 9358ac810e1..7d28462de47 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -8,6 +8,6 @@
 */
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
-int gf_fuse_mount (const char *mountpoint, char *fsname,
-                   unsigned long mountflags, char *mnt_param,
+int gf_fuse_unmount_daemon (const char *mountpoint, int fd);
+int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
                    pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-lib/misc.c b/contrib/fuse-lib/misc.c
index 0c41b1a1917..1a9b418e511 100644
--- a/contrib/fuse-lib/misc.c
+++ b/contrib/fuse-lib/misc.c
@@ -10,7 +10,7 @@
 #include <string.h>
 #include <limits.h>
 #include <fcntl.h>
-#include "glusterfs.h"
+#include "glusterfs/glusterfs.h"
 #include "fuse_kernel.h"
 #include "fuse-misc.h"
 
@@ -41,7 +41,6 @@ void
 convert_fuse_file_lock (struct fuse_file_lock *fl, struct gf_flock *flock,
                         uint64_t lk_owner)
 {
-        memset (flock, 0, sizeof (struct flock));
         flock->l_type = fl->type;
         flock->l_whence = SEEK_SET;
         flock->l_start = fl->start;
diff --git a/contrib/fuse-lib/mount-common.c b/contrib/fuse-lib/mount-common.c
index e9f80fe8154..cffd4c01ed5 100644
--- a/contrib/fuse-lib/mount-common.c
+++ b/contrib/fuse-lib/mount-common.c
@@ -32,7 +32,7 @@ mtab_needs_update (const char *mnt)
         struct stat stbuf;
 
         /* If mtab is within new mount, don't touch it */
-        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
+        if (strncmp (mnt, _PATH_MOUNTED, sizeof (_PATH_MOUNTED) - 1) == 0 &&
             _PATH_MOUNTED[strlen (mnt)] == '/')
                 return 0;
 
@@ -255,16 +255,16 @@ fuse_mnt_umount (const char *progname, const char *abs_mnt,
                         exit (1);
                 }
 #ifdef GF_LINUX_HOST_OS
-                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
+                execl ("umount", "umount", "-i", rel_mnt,
                        lazy ? "-l" : NULL, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
                                progname, strerror (errno));
 #elif __NetBSD__
                 /* exitting the filesystem causes the umount */
                 exit (0);
 #else
-                execl ("/sbin/umount", "/sbin/umount", "-f", rel_mnt, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /sbin/umount: %s",
+                execl ("umount", "umount", "-f", rel_mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
                                progname, strerror (errno));
 #endif /* GF_LINUX_HOST_OS */
                 exit (1);
diff --git a/contrib/fuse-lib/mount-gluster-compat.h b/contrib/fuse-lib/mount-gluster-compat.h
index 562f089dd1f..d3646d08d8e 100644
--- a/contrib/fuse-lib/mount-gluster-compat.h
+++ b/contrib/fuse-lib/mount-gluster-compat.h
@@ -30,17 +30,59 @@
 #include <sys/wait.h>
 #include <sys/mount.h>
 
+#ifdef GF_LINUX_HOST_OS
+typedef unsigned long mount_flag_t;
+#endif
+
 #if defined(__NetBSD__)
 #include <perfuse.h>
 #define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
 #define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+#define MS_NODEV  MNT_NODEV
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+typedef int mount_flag_t;
 #endif
 
 #if defined(GF_DARWIN_HOST_OS) || defined(__FreeBSD__)
 #include <sys/param.h>
 #include <sys/mount.h>
 #define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#endif
+
+#if defined(__FreeBSD__)
 #define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+/* "nodev"/MNT_NODEV was removed from FreBSD, as it became unneeded because "As
+ * of FreeBSD 6.0 device nodes may be created in regular file systems but such
+ * nodes cannot be used to access devices." (See
+ * https://freebsd.org/cgi/man.cgi?query=mknod&sektion=8 .
+ * Also see:
+ * - https://github.com/freebsd/freebsd/commit/266790a
+ * - https://github.com/freebsd/freebsd/commit/a5e716d
+ * - 700008 in
+ *   https://www.freebsd.org/doc/en/books/porters-handbook/versions-7.html .)
+ */
+#if __FreeBSD_version < 700008
+#define MS_NODEV  MNT_NODEV
+#else
+#define MS_NODEV  0
+#endif
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+#if __FreeBSD_version < 1000715
+typedef int  mount_flag_t;
+#else
+/* __FreeBSD_version was not bumped for this type change. Anyway, see
+ * https://github.com/freebsd/freebsd/commit/e8d76f8
+ * and respective __FreeBSD_version:
+ * https://github.com/freebsd/freebsd/blob/e8d76f8/sys/sys/param.h#L61 .
+ * We use the subsequent value, 1000715, to switch. (Also see:
+ * https://www.freebsd.org/doc/en/books/porters-handbook/versions-10.html .)
+ */
+typedef long long mount_flag_t;
+#endif
 #endif
 
 #ifdef GF_LINUX_HOST_OS
@@ -54,9 +96,9 @@
 #define FREE(ptr) free (ptr)
 #define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
 #else /* FUSE_UTIL */
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 1edde86014a..06ff191f542 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -52,12 +52,16 @@ gf_fuse_unmount (const char *mountpoint, int fd)
         if (geteuid () == 0) {
                 fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
                 return;
+        } else {
+            GFFUSE_LOGERR ("fuse: Effective-uid: %d", geteuid());
         }
 
         res = umount2 (mountpoint, 2);
         if (res == 0)
                 return;
 
+        GFFUSE_LOGERR ("fuse: failed to unmount %s: %s",
+                       mountpoint, strerror (errno));
         pid = fork ();
         if (pid == -1)
                 return;
@@ -67,6 +71,8 @@ gf_fuse_unmount (const char *mountpoint, int fd)
                                        "--", mountpoint, NULL };
 
                 execvp (FUSERMOUNT_PROG, (char **)argv);
+                GFFUSE_LOGERR ("fuse: failed to execute fuserumount: %s",
+                               strerror (errno));
                 _exit (1);
         }
         waitpid (pid, NULL, 0);
@@ -75,6 +81,54 @@ gf_fuse_unmount (const char *mountpoint, int fd)
 
 /* gluster-specific routines */
 
+/* Unmounting in a daemon that lurks 'till main process exits */
+int
+gf_fuse_unmount_daemon (const char *mountpoint, int fd)
+{
+        int   ret = -1;
+        pid_t pid = -1;
+
+        if (fd == -1)
+                return -1;
+
+        int ump[2] = {0,};
+
+        ret = pipe(ump);
+        if (ret == -1) {
+                close (fd);
+                return -1;
+        }
+
+        pid = fork ();
+        switch (pid) {
+        case 0:
+        {
+                char c = 0;
+                sigset_t sigset;
+
+                close_fds_except (ump, 1);
+
+                setsid();
+                (void)chdir("/");
+                sigfillset(&sigset);
+                sigprocmask(SIG_BLOCK, &sigset, NULL);
+
+                read (ump[0], &c, 1);
+
+                gf_fuse_unmount (mountpoint, fd);
+                exit (0);
+        }
+        case -1:
+                close (fd);
+                fd = -1;
+                ret = -1;
+                close (ump[1]);
+        }
+        close (ump[0]);
+
+        return ret;
+}
+
 static char *
 escape (char *s)
 {
@@ -106,8 +160,7 @@ escape (char *s)
 
 static int
 fuse_mount_fusermount (const char *mountpoint, char *fsname,
-                       unsigned long mountflags, char *mnt_param,
-                       int fd)
+                       char *mnt_param, int fd)
 {
         int  pid = -1;
         int  res = 0;
@@ -130,8 +183,7 @@ fuse_mount_fusermount (const char *mountpoint, char *fsname,
                 return -1;
         }
         ret = asprintf (&fm_mnt_params,
-                        "%s%s,fsname=%s,nonempty,subtype=glusterfs",
-                        (mountflags & MS_RDONLY) ? "ro," : "",
+                        "%s,fsname=%s,nonempty,subtype=glusterfs",
                         mnt_param, efsname);
         FREE (efsname);
         if (ret == -1) {
@@ -224,19 +276,101 @@ build_iovec_argf(struct iovec **iov, int *iovlen, const char *name,
 }
 #endif /* __FreeBSD__ */
 
+struct mount_flags {
+        const char *opt;
+        mount_flag_t flag;
+        int on;
+} mount_flags[] = {
+        /* We provide best effort cross platform support for mount flags by
+         * defining the ones which are commonly used in Unix-like OS-es.
+         */
+        {"ro",      MS_RDONLY,      1},
+        {"nosuid",  MS_NOSUID,      1},
+        {"nodev",   MS_NODEV,       1},
+        {"noatime", MS_NOATIME,     1},
+        {"noexec",  MS_NOEXEC,      1},
+#ifdef GF_LINUX_HOST_OS
+        {"rw",      MS_RDONLY,      0},
+        {"suid",    MS_NOSUID,      0},
+        {"dev",     MS_NODEV,       0},
+        {"exec",    MS_NOEXEC,      0},
+        {"async",   MS_SYNCHRONOUS, 0},
+        {"sync",    MS_SYNCHRONOUS, 1},
+        {"atime",   MS_NOATIME,     0},
+        {"dirsync", MS_DIRSYNC,     1},
+#endif
+        {NULL,      0,              0}
+};
+
+static int
+mount_param_to_flag (char *mnt_param, mount_flag_t *mntflags,
+                     char **mnt_param_new)
+{
+        gf_boolean_t found = _gf_false;
+        struct mount_flags *flag = NULL;
+        char *param_tok = NULL;
+        token_iter_t tit = {0,};
+        gf_boolean_t iter_end = _gf_false;
+
+        /* Allocate a buffer that will hold the mount parameters remaining
+         * after the ones corresponding to mount flags are processed and
+         * removed.The length of the original params are a good upper bound
+         * of the size needed.
+         */
+        *mnt_param_new = strdup (mnt_param);
+        if (!*mnt_param_new)
+                return -1;
+
+        for (param_tok = token_iter_init (*mnt_param_new, ',', &tit) ;;) {
+                iter_end = next_token (&param_tok, &tit);
+
+                found = _gf_false;
+                for (flag = mount_flags; flag->opt; flag++) {
+                        /* Compare the mount flag name to the param
+                         * name at hand.
+                         */
+                        if (strcmp (flag->opt, param_tok) == 0) {
+                                /* If there is a match, adjust mntflags
+                                 * accordingly and break.
+                                 */
+                                if (flag->on) {
+                                        *mntflags |= flag->flag;
+                                } else {
+                                        *mntflags &= ~flag->flag;
+                                }
+                                found = _gf_true;
+                                break;
+                        }
+                }
+                /* Exclude flag names from new parameter list. */
+                if (found)
+                        drop_token (param_tok, &tit);
+
+                if (iter_end)
+                        break;
+        }
+
+        return 0;
+}
+
 static int
 fuse_mount_sys (const char *mountpoint, char *fsname,
-                unsigned long mountflags, char *mnt_param, int fd)
+                char *mnt_param, int fd)
 {
         int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
-
-        ret = asprintf (&mnt_param_mnt,
-                        "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
-                        mnt_param, fd, S_IFDIR, getuid (), getgid ());
+        mount_flag_t mountflags = 0;
+        char *mnt_param_new = NULL;
+
+        ret = mount_param_to_flag (mnt_param, &mountflags, &mnt_param_new);
+        if (ret == 0)
+                ret = asprintf (&mnt_param_mnt,
+                                "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
+                                mnt_param_new, fd, S_IFDIR, getuid (),
+                                getgid ());
         if (ret == -1) {
                 GFFUSE_LOGERR ("Out of memory");
 
@@ -256,6 +390,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname,
         build_iovec (&iov, &iovlen, "from", "/dev/fuse", -1);
         build_iovec (&iov, &iovlen, "volname", source, -1);
         build_iovec (&iov, &iovlen, "fd", fdstr, -1);
+        build_iovec (&iov, &iovlen, "allow_other", NULL, -1);
         ret = nmount (iov, iovlen, mountflags);
 #else
         ret = mount (source, mountpoint, fstype, mountflags,
@@ -295,7 +430,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname,
 
                 ret = asprintf (&mnt_param_mtab, "%s%s",
                                 mountflags & MS_RDONLY ? "ro," : "",
-                                mnt_param);
+                                mnt_param_new);
                 if (ret == -1)
                         GFFUSE_LOGERR ("Out of memory");
                 else {
@@ -320,6 +455,7 @@ out:
                         umount2 (mountpoint, 2); /* lazy umount */
         }
         FREE (mnt_param_mnt);
+        FREE (mnt_param_new);
         if (source != fsname)
                 FREE (source);
 
@@ -328,8 +464,7 @@ out:
 
 int
 gf_fuse_mount (const char *mountpoint, char *fsname,
-               unsigned long mountflags, char *mnt_param,
-               pid_t *mnt_pid, int status_fd)
+               char *mnt_param, pid_t *mnt_pid, int status_fd)
 {
         int   fd  = -1;
         pid_t pid = -1;
@@ -356,16 +491,19 @@ gf_fuse_mount (const char *mountpoint, char *fsname,
                                 exit (pid == -1 ? 1 : 0);
                 }
 
-                ret = fuse_mount_sys (mountpoint, fsname, mountflags, mnt_param,
-                                      fd);
+                ret = fuse_mount_sys (mountpoint, fsname, mnt_param, fd);
                 if (ret == -1) {
                         gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                                "direct mount failed (%s) errno %d, "
-                                "retry to mount via fusermount",
+                                "direct mount failed (%s) errno %d",
                                 strerror (errno), errno);
 
-                        ret = fuse_mount_fusermount (mountpoint, fsname,
-                                                     mountflags, mnt_param, fd);
+                        if (errno == EPERM) {
+                                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                        "retry to mount via fusermount");
+
+                                ret = fuse_mount_fusermount (mountpoint, fsname,
+                                                             mnt_param, fd);
+                        }
                 }
 
                 if (ret == -1)
diff --git a/contrib/fuse-util/fusermount.c b/contrib/fuse-util/fusermount.c
index a64d8e102ff..ff743f75a21 100644
--- a/contrib/fuse-util/fusermount.c
+++ b/contrib/fuse-util/fusermount.c
@@ -520,20 +520,22 @@ static void parse_line(char *line, int linenum)
 
 static void read_conf(void)
 {
+	int  len;
 	FILE *fp = fopen(FUSE_CONF, "r");
 	if (fp != NULL) {
 		int linenum = 1;
 		char line[256];
 		int isnewline = 1;
 		while (fgets(line, sizeof(line), fp) != NULL) {
+			len = strlen (line);
 			if (isnewline) {
-				if (strlen(line) && line[strlen(line)-1] == '\n') {
+				if (len && line[len-1] == '\n') {
 					strip_line(line);
 					parse_line(line, linenum);
 				} else {
 					isnewline = 0;
 				}
-			} else if(strlen(line) && line[strlen(line)-1] == '\n') {
+			} else if (len && line[len-1] == '\n') {
 				fprintf(stderr, "%s: reading %s: line %i too long\n", progname, FUSE_CONF, linenum);
 
 				isnewline = 1;
diff --git a/contrib/ipaddr-py/COPYING b/contrib/ipaddr-py/COPYING
deleted file mode 100644
index d6456956733..00000000000
--- a/contrib/ipaddr-py/COPYING
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/contrib/ipaddr-py/MANIFEST.in b/contrib/ipaddr-py/MANIFEST.in
deleted file mode 100644
index f572804441b..00000000000
--- a/contrib/ipaddr-py/MANIFEST.in
+++ /dev/null
@@ -1,3 +0,0 @@
-include COPYING
-include ipaddr_test.py
-include RELEASENOTES
diff --git a/contrib/ipaddr-py/OWNERS b/contrib/ipaddr-py/OWNERS
deleted file mode 100644
index 501673e0395..00000000000
--- a/contrib/ipaddr-py/OWNERS
+++ /dev/null
@@ -1,4 +0,0 @@
-pmoody
-harro
-mshields
-smart
diff --git a/contrib/ipaddr-py/README b/contrib/ipaddr-py/README
deleted file mode 100644
index 1b54294bb10..00000000000
--- a/contrib/ipaddr-py/README
+++ /dev/null
@@ -1,8 +0,0 @@
-ipaddr.py is a library for working with IP addresses, both IPv4 and IPv6.
-It was developed by Google for internal use, and is now open source.
-
-Project home page: http://code.google.com/p/ipaddr-py/
-
-Please send contributions to ipaddr-py-dev@googlegroups.com.  Code should
-include unit tests and follow the Google Python style guide:
-http://code.google.com/p/soc/wiki/PythonStyleGuide
diff --git a/contrib/ipaddr-py/ipaddr.py b/contrib/ipaddr-py/ipaddr.py
deleted file mode 100644
index a89298a315d..00000000000
--- a/contrib/ipaddr-py/ipaddr.py
+++ /dev/null
@@ -1,1907 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2007 Google Inc.
-#  Licensed to PSF under a Contributor Agreement.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-"""A fast, lightweight IPv4/IPv6 manipulation library in Python.
-
-This library is used to create/poke/manipulate IPv4 and IPv6 addresses
-and networks.
-
-"""
-
-__version__ = 'trunk'
-
-import struct
-
-IPV4LENGTH = 32
-IPV6LENGTH = 128
-
-
-class AddressValueError(ValueError):
-    """A Value Error related to the address."""
-
-
-class NetmaskValueError(ValueError):
-    """A Value Error related to the netmask."""
-
-
-def IPAddress(address, version=None):
-    """Take an IP string/int and return an object of the correct type.
-
-    Args:
-        address: A string or integer, the IP address.  Either IPv4 or
-          IPv6 addresses may be supplied; integers less than 2**32 will
-          be considered to be IPv4 by default.
-        version: An Integer, 4 or 6. If set, don't try to automatically
-          determine what the IP address type is. important for things
-          like IPAddress(1), which could be IPv4, '0.0.0.1',  or IPv6,
-          '::1'.
-
-    Returns:
-        An IPv4Address or IPv6Address object.
-
-    Raises:
-        ValueError: if the string passed isn't either a v4 or a v6
-          address.
-
-    """
-    if version:
-        if version == 4:
-            return IPv4Address(address)
-        elif version == 6:
-            return IPv6Address(address)
-
-    try:
-        return IPv4Address(address)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    try:
-        return IPv6Address(address)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    raise ValueError('%r does not appear to be an IPv4 or IPv6 address' %
-                     address)
-
-
-def IPNetwork(address, version=None, strict=False):
-    """Take an IP string/int and return an object of the correct type.
-
-    Args:
-        address: A string or integer, the IP address.  Either IPv4 or
-          IPv6 addresses may be supplied; integers less than 2**32 will
-          be considered to be IPv4 by default.
-        version: An Integer, if set, don't try to automatically
-          determine what the IP address type is. important for things
-          like IPNetwork(1), which could be IPv4, '0.0.0.1/32', or IPv6,
-          '::1/128'.
-
-    Returns:
-        An IPv4Network or IPv6Network object.
-
-    Raises:
-        ValueError: if the string passed isn't either a v4 or a v6
-          address. Or if a strict network was requested and a strict
-          network wasn't given.
-
-    """
-    if version:
-        if version == 4:
-            return IPv4Network(address, strict)
-        elif version == 6:
-            return IPv6Network(address, strict)
-
-    try:
-        return IPv4Network(address, strict)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    try:
-        return IPv6Network(address, strict)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    raise ValueError('%r does not appear to be an IPv4 or IPv6 network' %
-                     address)
-
-
-def v4_int_to_packed(address):
-    """The binary representation of this address.
-
-    Args:
-        address: An integer representation of an IPv4 IP address.
-
-    Returns:
-        The binary representation of this address.
-
-    Raises:
-        ValueError: If the integer is too large to be an IPv4 IP
-          address.
-    """
-    if address > _BaseV4._ALL_ONES:
-        raise ValueError('Address too large for IPv4')
-    return struct.pack('!I', address)
-
-
-def v6_int_to_packed(address):
-    """The binary representation of this address.
-
-    Args:
-        address: An integer representation of an IPv4 IP address.
-
-    Returns:
-        The binary representation of this address.
-    """
-    return struct.pack('!QQ', address >> 64, address & (2**64 - 1))
-
-
-def _find_address_range(addresses):
-    """Find a sequence of addresses.
-
-    Args:
-        addresses: a list of IPv4 or IPv6 addresses.
-
-    Returns:
-        A tuple containing the first and last IP addresses in the sequence.
-
-    """
-    first = last = addresses[0]
-    for ip in addresses[1:]:
-        if ip._ip == last._ip + 1:
-            last = ip
-        else:
-            break
-    return (first, last)
-
-def _get_prefix_length(number1, number2, bits):
-    """Get the number of leading bits that are same for two numbers.
-
-    Args:
-        number1: an integer.
-        number2: another integer.
-        bits: the maximum number of bits to compare.
-
-    Returns:
-        The number of leading bits that are the same for two numbers.
-
-    """
-    for i in range(bits):
-        if number1 >> i == number2 >> i:
-            return bits - i
-    return 0
-
-def _count_righthand_zero_bits(number, bits):
-    """Count the number of zero bits on the right hand side.
-
-    Args:
-        number: an integer.
-        bits: maximum number of bits to count.
-
-    Returns:
-        The number of zero bits on the right hand side of the number.
-
-    """
-    if number == 0:
-        return bits
-    for i in range(bits):
-        if (number >> i) % 2:
-            return i
-
-def summarize_address_range(first, last):
-    """Summarize a network range given the first and last IP addresses.
-
-    Example:
-        >>> summarize_address_range(IPv4Address('1.1.1.0'),
-            IPv4Address('1.1.1.130'))
-        [IPv4Network('1.1.1.0/25'), IPv4Network('1.1.1.128/31'),
-        IPv4Network('1.1.1.130/32')]
-
-    Args:
-        first: the first IPv4Address or IPv6Address in the range.
-        last: the last IPv4Address or IPv6Address in the range.
-
-    Returns:
-        The address range collapsed to a list of IPv4Network's or
-        IPv6Network's.
-
-    Raise:
-        TypeError:
-            If the first and last objects are not IP addresses.
-            If the first and last objects are not the same version.
-        ValueError:
-            If the last object is not greater than the first.
-            If the version is not 4 or 6.
-
-    """
-    if not (isinstance(first, _BaseIP) and isinstance(last, _BaseIP)):
-        raise TypeError('first and last must be IP addresses, not networks')
-    if first.version != last.version:
-        raise TypeError("%s and %s are not of the same version" % (
-                str(first), str(last)))
-    if first > last:
-        raise ValueError('last IP address must be greater than first')
-
-    networks = []
-
-    if first.version == 4:
-        ip = IPv4Network
-    elif first.version == 6:
-        ip = IPv6Network
-    else:
-        raise ValueError('unknown IP version')
-
-    ip_bits = first._max_prefixlen
-    first_int = first._ip
-    last_int = last._ip
-    while first_int <= last_int:
-        nbits = _count_righthand_zero_bits(first_int, ip_bits)
-        current = None
-        while nbits >= 0:
-            addend = 2**nbits - 1
-            current = first_int + addend
-            nbits -= 1
-            if current <= last_int:
-                break
-        prefix = _get_prefix_length(first_int, current, ip_bits)
-        net = ip('%s/%d' % (str(first), prefix))
-        networks.append(net)
-        if current == ip._ALL_ONES:
-            break
-        first_int = current + 1
-        first = IPAddress(first_int, version=first._version)
-    return networks
-
-def _collapse_address_list_recursive(addresses):
-    """Loops through the addresses, collapsing concurrent netblocks.
-
-    Example:
-
-        ip1 = IPv4Network('1.1.0.0/24')
-        ip2 = IPv4Network('1.1.1.0/24')
-        ip3 = IPv4Network('1.1.2.0/24')
-        ip4 = IPv4Network('1.1.3.0/24')
-        ip5 = IPv4Network('1.1.4.0/24')
-        ip6 = IPv4Network('1.1.0.1/22')
-
-        _collapse_address_list_recursive([ip1, ip2, ip3, ip4, ip5, ip6]) ->
-          [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')]
-
-        This shouldn't be called directly; it is called via
-          collapse_address_list([]).
-
-    Args:
-        addresses: A list of IPv4Network's or IPv6Network's
-
-    Returns:
-        A list of IPv4Network's or IPv6Network's depending on what we were
-        passed.
-
-    """
-    ret_array = []
-    optimized = False
-
-    for cur_addr in addresses:
-        if not ret_array:
-            ret_array.append(cur_addr)
-            continue
-        if cur_addr in ret_array[-1]:
-            optimized = True
-        elif cur_addr == ret_array[-1].supernet().subnet()[1]:
-            ret_array.append(ret_array.pop().supernet())
-            optimized = True
-        else:
-            ret_array.append(cur_addr)
-
-    if optimized:
-        return _collapse_address_list_recursive(ret_array)
-
-    return ret_array
-
-
-def collapse_address_list(addresses):
-    """Collapse a list of IP objects.
-
-    Example:
-        collapse_address_list([IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) ->
-          [IPv4('1.1.0.0/23')]
-
-    Args:
-        addresses: A list of IPv4Network or IPv6Network objects.
-
-    Returns:
-        A list of IPv4Network or IPv6Network objects depending on what we
-        were passed.
-
-    Raises:
-        TypeError: If passed a list of mixed version objects.
-
-    """
-    i = 0
-    addrs = []
-    ips = []
-    nets = []
-
-    # split IP addresses and networks
-    for ip in addresses:
-        if isinstance(ip, _BaseIP):
-            if ips and ips[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            ips.append(ip)
-        elif ip._prefixlen == ip._max_prefixlen:
-            if ips and ips[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            ips.append(ip.ip)
-        else:
-            if nets and nets[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            nets.append(ip)
-
-    # sort and dedup
-    ips = sorted(set(ips))
-    nets = sorted(set(nets))
-
-    while i < len(ips):
-        (first, last) = _find_address_range(ips[i:])
-        i = ips.index(last) + 1
-        addrs.extend(summarize_address_range(first, last))
-
-    return _collapse_address_list_recursive(sorted(
-        addrs + nets, key=_BaseNet._get_networks_key))
-
-# backwards compatibility
-CollapseAddrList = collapse_address_list
-
-# Test whether this Python implementation supports byte objects that
-# are not identical to str ones.
-# We need to exclude platforms where bytes == str so that we can
-# distinguish between packed representations and strings, for example
-# b'12::' (the IPv4 address 49.50.58.58) and '12::' (an IPv6 address).
-try:
-    _compat_has_real_bytes = bytes is not str
-except NameError: # <Python2.6
-    _compat_has_real_bytes = False
-
-def get_mixed_type_key(obj):
-    """Return a key suitable for sorting between networks and addresses.
-
-    Address and Network objects are not sortable by default; they're
-    fundamentally different so the expression
-
-        IPv4Address('1.1.1.1') <= IPv4Network('1.1.1.1/24')
-
-    doesn't make any sense.  There are some times however, where you may wish
-    to have ipaddr sort these for you anyway. If you need to do this, you
-    can use this function as the key= argument to sorted().
-
-    Args:
-      obj: either a Network or Address object.
-    Returns:
-      appropriate key.
-
-    """
-    if isinstance(obj, _BaseNet):
-        return obj._get_networks_key()
-    elif isinstance(obj, _BaseIP):
-        return obj._get_address_key()
-    return NotImplemented
-
-class _IPAddrBase(object):
-
-    """The mother class."""
-
-    def __index__(self):
-        return self._ip
-
-    def __int__(self):
-        return self._ip
-
-    def __hex__(self):
-        return hex(self._ip)
-
-    @property
-    def exploded(self):
-        """Return the longhand version of the IP address as a string."""
-        return self._explode_shorthand_ip_string()
-
-    @property
-    def compressed(self):
-        """Return the shorthand version of the IP address as a string."""
-        return str(self)
-
-
-class _BaseIP(_IPAddrBase):
-
-    """A generic IP object.
-
-    This IP class contains the version independent methods which are
-    used by single IP addresses.
-
-    """
-
-    def __init__(self, address):
-        if (not (_compat_has_real_bytes and isinstance(address, bytes))
-            and '/' in str(address)):
-            raise AddressValueError(address)
-
-    def __eq__(self, other):
-        try:
-            return (self._ip == other._ip
-                    and self._version == other._version)
-        except AttributeError:
-            return NotImplemented
-
-    def __ne__(self, other):
-        eq = self.__eq__(other)
-        if eq is NotImplemented:
-            return NotImplemented
-        return not eq
-
-    def __le__(self, other):
-        gt = self.__gt__(other)
-        if gt is NotImplemented:
-            return NotImplemented
-        return not gt
-
-    def __ge__(self, other):
-        lt = self.__lt__(other)
-        if lt is NotImplemented:
-            return NotImplemented
-        return not lt
-
-    def __lt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseIP):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self._ip != other._ip:
-            return self._ip < other._ip
-        return False
-
-    def __gt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseIP):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self._ip != other._ip:
-            return self._ip > other._ip
-        return False
-
-    # Shorthand for Integer addition and subtraction. This is not
-    # meant to ever support addition/subtraction of addresses.
-    def __add__(self, other):
-        if not isinstance(other, int):
-            return NotImplemented
-        return IPAddress(int(self) + other, version=self._version)
-
-    def __sub__(self, other):
-        if not isinstance(other, int):
-            return NotImplemented
-        return IPAddress(int(self) - other, version=self._version)
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, str(self))
-
-    def __str__(self):
-        return  '%s' % self._string_from_ip_int(self._ip)
-
-    def __hash__(self):
-        return hash(hex(long(self._ip)))
-
-    def _get_address_key(self):
-        return (self._version, self)
-
-    @property
-    def version(self):
-        raise NotImplementedError('BaseIP has no version')
-
-
-class _BaseNet(_IPAddrBase):
-
-    """A generic IP object.
-
-    This IP class contains the version independent methods which are
-    used by networks.
-
-    """
-
-    def __init__(self, address):
-        self._cache = {}
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, str(self))
-
-    def iterhosts(self):
-        """Generate Iterator over usable hosts in a network.
-
-           This is like __iter__ except it doesn't return the network
-           or broadcast addresses.
-
-        """
-        cur = int(self.network) + 1
-        bcast = int(self.broadcast) - 1
-        while cur <= bcast:
-            cur += 1
-            yield IPAddress(cur - 1, version=self._version)
-
-    def __iter__(self):
-        cur = int(self.network)
-        bcast = int(self.broadcast)
-        while cur <= bcast:
-            cur += 1
-            yield IPAddress(cur - 1, version=self._version)
-
-    def __getitem__(self, n):
-        network = int(self.network)
-        broadcast = int(self.broadcast)
-        if n >= 0:
-            if network + n > broadcast:
-                raise IndexError
-            return IPAddress(network + n, version=self._version)
-        else:
-            n += 1
-            if broadcast + n < network:
-                raise IndexError
-            return IPAddress(broadcast + n, version=self._version)
-
-    def __lt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseNet):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self.network != other.network:
-            return self.network < other.network
-        if self.netmask != other.netmask:
-            return self.netmask < other.netmask
-        return False
-
-    def __gt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseNet):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self.network != other.network:
-            return self.network > other.network
-        if self.netmask != other.netmask:
-            return self.netmask > other.netmask
-        return False
-
-    def __le__(self, other):
-        gt = self.__gt__(other)
-        if gt is NotImplemented:
-            return NotImplemented
-        return not gt
-
-    def __ge__(self, other):
-        lt = self.__lt__(other)
-        if lt is NotImplemented:
-            return NotImplemented
-        return not lt
-
-    def __eq__(self, other):
-        try:
-            return (self._version == other._version
-                    and self.network == other.network
-                    and int(self.netmask) == int(other.netmask))
-        except AttributeError:
-            if isinstance(other, _BaseIP):
-                return (self._version == other._version
-                        and self._ip == other._ip)
-
-    def __ne__(self, other):
-        eq = self.__eq__(other)
-        if eq is NotImplemented:
-            return NotImplemented
-        return not eq
-
-    def __str__(self):
-        return  '%s/%s' % (str(self.ip),
-                           str(self._prefixlen))
-
-    def __hash__(self):
-        return hash(int(self.network) ^ int(self.netmask))
-
-    def __contains__(self, other):
-        # always false if one is v4 and the other is v6.
-        if self._version != other._version:
-          return False
-        # dealing with another network.
-        if isinstance(other, _BaseNet):
-            return (self.network <= other.network and
-                    self.broadcast >= other.broadcast)
-        # dealing with another address
-        else:
-            return (int(self.network) <= int(other._ip) <=
-                    int(self.broadcast))
-
-    def overlaps(self, other):
-        """Tell if self is partly contained in other."""
-        return self.network in other or self.broadcast in other or (
-            other.network in self or other.broadcast in self)
-
-    @property
-    def network(self):
-        x = self._cache.get('network')
-        if x is None:
-            x = IPAddress(self._ip & int(self.netmask), version=self._version)
-            self._cache['network'] = x
-        return x
-
-    @property
-    def broadcast(self):
-        x = self._cache.get('broadcast')
-        if x is None:
-            x = IPAddress(self._ip | int(self.hostmask), version=self._version)
-            self._cache['broadcast'] = x
-        return x
-
-    @property
-    def hostmask(self):
-        x = self._cache.get('hostmask')
-        if x is None:
-            x = IPAddress(int(self.netmask) ^ self._ALL_ONES,
-                          version=self._version)
-            self._cache['hostmask'] = x
-        return x
-
-    @property
-    def with_prefixlen(self):
-        return '%s/%d' % (str(self.ip), self._prefixlen)
-
-    @property
-    def with_netmask(self):
-        return '%s/%s' % (str(self.ip), str(self.netmask))
-
-    @property
-    def with_hostmask(self):
-        return '%s/%s' % (str(self.ip), str(self.hostmask))
-
-    @property
-    def numhosts(self):
-        """Number of hosts in the current subnet."""
-        return int(self.broadcast) - int(self.network) + 1
-
-    @property
-    def version(self):
-        raise NotImplementedError('BaseNet has no version')
-
-    @property
-    def prefixlen(self):
-        return self._prefixlen
-
-    def address_exclude(self, other):
-        """Remove an address from a larger block.
-
-        For example:
-
-            addr1 = IPNetwork('10.1.1.0/24')
-            addr2 = IPNetwork('10.1.1.0/26')
-            addr1.address_exclude(addr2) =
-                [IPNetwork('10.1.1.64/26'), IPNetwork('10.1.1.128/25')]
-
-        or IPv6:
-
-            addr1 = IPNetwork('::1/32')
-            addr2 = IPNetwork('::1/128')
-            addr1.address_exclude(addr2) = [IPNetwork('::0/128'),
-                IPNetwork('::2/127'),
-                IPNetwork('::4/126'),
-                IPNetwork('::8/125'),
-                ...
-                IPNetwork('0:0:8000::/33')]
-
-        Args:
-            other: An IPvXNetwork object of the same type.
-
-        Returns:
-            A sorted list of IPvXNetwork objects addresses which is self
-            minus other.
-
-        Raises:
-            TypeError: If self and other are of difffering address
-              versions, or if other is not a network object.
-            ValueError: If other is not completely contained by self.
-
-        """
-        if not self._version == other._version:
-            raise TypeError("%s and %s are not of the same version" % (
-                str(self), str(other)))
-
-        if not isinstance(other, _BaseNet):
-            raise TypeError("%s is not a network object" % str(other))
-
-        if other not in self:
-            raise ValueError('%s not contained in %s' % (str(other),
-                                                         str(self)))
-        if other == self:
-            return []
-
-        ret_addrs = []
-
-        # Make sure we're comparing the network of other.
-        other = IPNetwork('%s/%s' % (str(other.network), str(other.prefixlen)),
-                   version=other._version)
-
-        s1, s2 = self.subnet()
-        while s1 != other and s2 != other:
-            if other in s1:
-                ret_addrs.append(s2)
-                s1, s2 = s1.subnet()
-            elif other in s2:
-                ret_addrs.append(s1)
-                s1, s2 = s2.subnet()
-            else:
-                # If we got here, there's a bug somewhere.
-                assert True == False, ('Error performing exclusion: '
-                                       's1: %s s2: %s other: %s' %
-                                       (str(s1), str(s2), str(other)))
-        if s1 == other:
-            ret_addrs.append(s2)
-        elif s2 == other:
-            ret_addrs.append(s1)
-        else:
-            # If we got here, there's a bug somewhere.
-            assert True == False, ('Error performing exclusion: '
-                                   's1: %s s2: %s other: %s' %
-                                   (str(s1), str(s2), str(other)))
-
-        return sorted(ret_addrs, key=_BaseNet._get_networks_key)
-
-    def compare_networks(self, other):
-        """Compare two IP objects.
-
-        This is only concerned about the comparison of the integer
-        representation of the network addresses.  This means that the
-        host bits aren't considered at all in this method.  If you want
-        to compare host bits, you can easily enough do a
-        'HostA._ip < HostB._ip'
-
-        Args:
-            other: An IP object.
-
-        Returns:
-            If the IP versions of self and other are the same, returns:
-
-            -1 if self < other:
-              eg: IPv4('1.1.1.0/24') < IPv4('1.1.2.0/24')
-              IPv6('1080::200C:417A') < IPv6('1080::200B:417B')
-            0 if self == other
-              eg: IPv4('1.1.1.1/24') == IPv4('1.1.1.2/24')
-              IPv6('1080::200C:417A/96') == IPv6('1080::200C:417B/96')
-            1 if self > other
-              eg: IPv4('1.1.1.0/24') > IPv4('1.1.0.0/24')
-              IPv6('1080::1:200C:417A/112') >
-              IPv6('1080::0:200C:417A/112')
-
-            If the IP versions of self and other are different, returns:
-
-            -1 if self._version < other._version
-              eg: IPv4('10.0.0.1/24') < IPv6('::1/128')
-            1 if self._version > other._version
-              eg: IPv6('::1/128') > IPv4('255.255.255.0/24')
-
-        """
-        if self._version < other._version:
-            return -1
-        if self._version > other._version:
-            return 1
-        # self._version == other._version below here:
-        if self.network < other.network:
-            return -1
-        if self.network > other.network:
-            return 1
-        # self.network == other.network below here:
-        if self.netmask < other.netmask:
-            return -1
-        if self.netmask > other.netmask:
-            return 1
-        # self.network == other.network and self.netmask == other.netmask
-        return 0
-
-    def _get_networks_key(self):
-        """Network-only key function.
-
-        Returns an object that identifies this address' network and
-        netmask. This function is a suitable "key" argument for sorted()
-        and list.sort().
-
-        """
-        return (self._version, self.network, self.netmask)
-
-    def _ip_int_from_prefix(self, prefixlen=None):
-        """Turn the prefix length netmask into a int for comparison.
-
-        Args:
-            prefixlen: An integer, the prefix length.
-
-        Returns:
-            An integer.
-
-        """
-        if not prefixlen and prefixlen != 0:
-            prefixlen = self._prefixlen
-        return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen)
-
-    def _prefix_from_ip_int(self, ip_int, mask=32):
-        """Return prefix length from the decimal netmask.
-
-        Args:
-            ip_int: An integer, the IP address.
-            mask: The netmask.  Defaults to 32.
-
-        Returns:
-            An integer, the prefix length.
-
-        """
-        while mask:
-            if ip_int & 1 == 1:
-                break
-            ip_int >>= 1
-            mask -= 1
-
-        return mask
-
-    def _ip_string_from_prefix(self, prefixlen=None):
-        """Turn a prefix length into a dotted decimal string.
-
-        Args:
-            prefixlen: An integer, the netmask prefix length.
-
-        Returns:
-            A string, the dotted decimal netmask string.
-
-        """
-        if not prefixlen:
-            prefixlen = self._prefixlen
-        return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen))
-
-    def iter_subnets(self, prefixlen_diff=1, new_prefix=None):
-        """The subnets which join to make the current subnet.
-
-        In the case that self contains only one IP
-        (self._prefixlen == 32 for IPv4 or self._prefixlen == 128
-        for IPv6), return a list with just ourself.
-
-        Args:
-            prefixlen_diff: An integer, the amount the prefix length
-              should be increased by. This should not be set if
-              new_prefix is also set.
-            new_prefix: The desired new prefix length. This must be a
-              larger number (smaller prefix) than the existing prefix.
-              This should not be set if prefixlen_diff is also set.
-
-        Returns:
-            An iterator of IPv(4|6) objects.
-
-        Raises:
-            ValueError: The prefixlen_diff is too small or too large.
-                OR
-            prefixlen_diff and new_prefix are both set or new_prefix
-              is a smaller number than the current prefix (smaller
-              number means a larger network)
-
-        """
-        if self._prefixlen == self._max_prefixlen:
-            yield self
-            return
-
-        if new_prefix is not None:
-            if new_prefix < self._prefixlen:
-                raise ValueError('new prefix must be longer')
-            if prefixlen_diff != 1:
-                raise ValueError('cannot set prefixlen_diff and new_prefix')
-            prefixlen_diff = new_prefix - self._prefixlen
-
-        if prefixlen_diff < 0:
-            raise ValueError('prefix length diff must be > 0')
-        new_prefixlen = self._prefixlen + prefixlen_diff
-
-        if not self._is_valid_netmask(str(new_prefixlen)):
-            raise ValueError(
-                'prefix length diff %d is invalid for netblock %s' % (
-                    new_prefixlen, str(self)))
-
-        first = IPNetwork('%s/%s' % (str(self.network),
-                                     str(self._prefixlen + prefixlen_diff)),
-                         version=self._version)
-
-        yield first
-        current = first
-        while True:
-            broadcast = current.broadcast
-            if broadcast == self.broadcast:
-                return
-            new_addr = IPAddress(int(broadcast) + 1, version=self._version)
-            current = IPNetwork('%s/%s' % (str(new_addr), str(new_prefixlen)),
-                                version=self._version)
-
-            yield current
-
-    def masked(self):
-        """Return the network object with the host bits masked out."""
-        return IPNetwork('%s/%d' % (self.network, self._prefixlen),
-                         version=self._version)
-
-    def subnet(self, prefixlen_diff=1, new_prefix=None):
-        """Return a list of subnets, rather than an iterator."""
-        return list(self.iter_subnets(prefixlen_diff, new_prefix))
-
-    def supernet(self, prefixlen_diff=1, new_prefix=None):
-        """The supernet containing the current network.
-
-        Args:
-            prefixlen_diff: An integer, the amount the prefix length of
-              the network should be decreased by.  For example, given a
-              /24 network and a prefixlen_diff of 3, a supernet with a
-              /21 netmask is returned.
-
-        Returns:
-            An IPv4 network object.
-
-        Raises:
-            ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have a
-              negative prefix length.
-                OR
-            If prefixlen_diff and new_prefix are both set or new_prefix is a
-              larger number than the current prefix (larger number means a
-              smaller network)
-
-        """
-        if self._prefixlen == 0:
-            return self
-
-        if new_prefix is not None:
-            if new_prefix > self._prefixlen:
-                raise ValueError('new prefix must be shorter')
-            if prefixlen_diff != 1:
-                raise ValueError('cannot set prefixlen_diff and new_prefix')
-            prefixlen_diff = self._prefixlen - new_prefix
-
-
-        if self.prefixlen - prefixlen_diff < 0:
-            raise ValueError(
-                'current prefixlen is %d, cannot have a prefixlen_diff of %d' %
-                (self.prefixlen, prefixlen_diff))
-        return IPNetwork('%s/%s' % (str(self.network),
-                                    str(self.prefixlen - prefixlen_diff)),
-                         version=self._version)
-
-    # backwards compatibility
-    Subnet = subnet
-    Supernet = supernet
-    AddressExclude = address_exclude
-    CompareNetworks = compare_networks
-    Contains = __contains__
-
-
-class _BaseV4(object):
-
-    """Base IPv4 object.
-
-    The following methods are used by IPv4 objects in both single IP
-    addresses and networks.
-
-    """
-
-    # Equivalent to 255.255.255.255 or 32 bits of 1's.
-    _ALL_ONES = (2**IPV4LENGTH) - 1
-    _DECIMAL_DIGITS = frozenset('0123456789')
-
-    def __init__(self, address):
-        self._version = 4
-        self._max_prefixlen = IPV4LENGTH
-
-    def _explode_shorthand_ip_string(self, ip_str=None):
-        if not ip_str:
-            ip_str = str(self)
-        return ip_str
-
-    def _ip_int_from_string(self, ip_str):
-        """Turn the given IP string into an integer for comparison.
-
-        Args:
-            ip_str: A string, the IP ip_str.
-
-        Returns:
-            The IP ip_str as an integer.
-
-        Raises:
-            AddressValueError: if ip_str isn't a valid IPv4 Address.
-
-        """
-        octets = ip_str.split('.')
-        if len(octets) != 4:
-            raise AddressValueError(ip_str)
-
-        packed_ip = 0
-        for oc in octets:
-            try:
-                packed_ip = (packed_ip << 8) | self._parse_octet(oc)
-            except ValueError:
-                raise AddressValueError(ip_str)
-        return packed_ip
-
-    def _parse_octet(self, octet_str):
-        """Convert a decimal octet into an integer.
-
-        Args:
-            octet_str: A string, the number to parse.
-
-        Returns:
-            The octet as an integer.
-
-        Raises:
-            ValueError: if the octet isn't strictly a decimal from [0..255].
-
-        """
-        # Whitelist the characters, since int() allows a lot of bizarre stuff.
-        if not self._DECIMAL_DIGITS.issuperset(octet_str):
-            raise ValueError
-        octet_int = int(octet_str, 10)
-        # Disallow leading zeroes, because no clear standard exists on
-        # whether these should be interpreted as decimal or octal.
-        if octet_int > 255 or (octet_str[0] == '0' and len(octet_str) > 1):
-            raise ValueError
-        return octet_int
-
-    def _string_from_ip_int(self, ip_int):
-        """Turns a 32-bit integer into dotted decimal notation.
-
-        Args:
-            ip_int: An integer, the IP address.
-
-        Returns:
-            The IP address as a string in dotted decimal notation.
-
-        """
-        octets = []
-        for _ in xrange(4):
-            octets.insert(0, str(ip_int & 0xFF))
-            ip_int >>= 8
-        return '.'.join(octets)
-
-    @property
-    def max_prefixlen(self):
-        return self._max_prefixlen
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v4_int_to_packed(self._ip)
-
-    @property
-    def version(self):
-        return self._version
-
-    @property
-    def is_reserved(self):
-       """Test if the address is otherwise IETF reserved.
-
-        Returns:
-            A boolean, True if the address is within the
-            reserved IPv4 Network range.
-
-       """
-       return self in IPv4Network('240.0.0.0/4')
-
-    @property
-    def is_private(self):
-        """Test if this address is allocated for private networks.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 1918.
-
-        """
-        return (self in IPv4Network('10.0.0.0/8') or
-                self in IPv4Network('172.16.0.0/12') or
-                self in IPv4Network('192.168.0.0/16'))
-
-    @property
-    def is_multicast(self):
-        """Test if the address is reserved for multicast use.
-
-        Returns:
-            A boolean, True if the address is multicast.
-            See RFC 3171 for details.
-
-        """
-        return self in IPv4Network('224.0.0.0/4')
-
-    @property
-    def is_unspecified(self):
-        """Test if the address is unspecified.
-
-        Returns:
-            A boolean, True if this is the unspecified address as defined in
-            RFC 5735 3.
-
-        """
-        return self in IPv4Network('0.0.0.0')
-
-    @property
-    def is_loopback(self):
-        """Test if the address is a loopback address.
-
-        Returns:
-            A boolean, True if the address is a loopback per RFC 3330.
-
-        """
-        return self in IPv4Network('127.0.0.0/8')
-
-    @property
-    def is_link_local(self):
-        """Test if the address is reserved for link-local.
-
-        Returns:
-            A boolean, True if the address is link-local per RFC 3927.
-
-        """
-        return self in IPv4Network('169.254.0.0/16')
-
-
-class IPv4Address(_BaseV4, _BaseIP):
-
-    """Represent and manipulate single IPv4 Addresses."""
-
-    def __init__(self, address):
-
-        """
-        Args:
-            address: A string or integer representing the IP
-              '192.168.1.1'
-
-              Additionally, an integer can be passed, so
-              IPv4Address('192.168.1.1') == IPv4Address(3232235777).
-              or, more generally
-              IPv4Address(int(IPv4Address('192.168.1.1'))) ==
-                IPv4Address('192.168.1.1')
-
-        Raises:
-            AddressValueError: If ipaddr isn't a valid IPv4 address.
-
-        """
-        _BaseIP.__init__(self, address)
-        _BaseV4.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 4:
-                self._ip = struct.unpack('!I', address)[0]
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        self._ip = self._ip_int_from_string(addr_str)
-
-
-class IPv4Network(_BaseV4, _BaseNet):
-
-    """This class represents and manipulates 32-bit IPv4 networks.
-
-    Attributes: [examples for IPv4Network('1.2.3.4/27')]
-        ._ip: 16909060
-        .ip: IPv4Address('1.2.3.4')
-        .network: IPv4Address('1.2.3.0')
-        .hostmask: IPv4Address('0.0.0.31')
-        .broadcast: IPv4Address('1.2.3.31')
-        .netmask: IPv4Address('255.255.255.224')
-        .prefixlen: 27
-
-    """
-
-    # the valid octets for host and netmasks. only useful for IPv4.
-    _valid_mask_octets = set((255, 254, 252, 248, 240, 224, 192, 128, 0))
-
-    def __init__(self, address, strict=False):
-        """Instantiate a new IPv4 network object.
-
-        Args:
-            address: A string or integer representing the IP [& network].
-              '192.168.1.1/24'
-              '192.168.1.1/255.255.255.0'
-              '192.168.1.1/0.0.0.255'
-              are all functionally the same in IPv4. Similarly,
-              '192.168.1.1'
-              '192.168.1.1/255.255.255.255'
-              '192.168.1.1/32'
-              are also functionaly equivalent. That is to say, failing to
-              provide a subnetmask will create an object with a mask of /32.
-
-              If the mask (portion after the / in the argument) is given in
-              dotted quad form, it is treated as a netmask if it starts with a
-              non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it
-              starts with a zero field (e.g. 0.255.255.255 == /8), with the
-              single exception of an all-zero mask which is treated as a
-              netmask == /0. If no mask is given, a default of /32 is used.
-
-              Additionally, an integer can be passed, so
-              IPv4Network('192.168.1.1') == IPv4Network(3232235777).
-              or, more generally
-              IPv4Network(int(IPv4Network('192.168.1.1'))) ==
-                IPv4Network('192.168.1.1')
-
-            strict: A boolean. If true, ensure that we have been passed
-              A true network address, eg, 192.168.1.0/24 and not an
-              IP address on a network, eg, 192.168.1.1/24.
-
-        Raises:
-            AddressValueError: If ipaddr isn't a valid IPv4 address.
-            NetmaskValueError: If the netmask isn't valid for
-              an IPv4 address.
-            ValueError: If strict was True and a network address was not
-              supplied.
-
-        """
-        _BaseNet.__init__(self, address)
-        _BaseV4.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            self.ip = IPv4Address(self._ip)
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv4Address(self._ALL_ONES)
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 4:
-                self._ip = struct.unpack('!I', address)[0]
-                self.ip = IPv4Address(self._ip)
-                self._prefixlen = self._max_prefixlen
-                self.netmask = IPv4Address(self._ALL_ONES)
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP prefix string.
-        addr = str(address).split('/')
-
-        if len(addr) > 2:
-            raise AddressValueError(address)
-
-        self._ip = self._ip_int_from_string(addr[0])
-        self.ip = IPv4Address(self._ip)
-
-        if len(addr) == 2:
-            mask = addr[1].split('.')
-            if len(mask) == 4:
-                # We have dotted decimal netmask.
-                if self._is_valid_netmask(addr[1]):
-                    self.netmask = IPv4Address(self._ip_int_from_string(
-                            addr[1]))
-                elif self._is_hostmask(addr[1]):
-                    self.netmask = IPv4Address(
-                        self._ip_int_from_string(addr[1]) ^ self._ALL_ONES)
-                else:
-                    raise NetmaskValueError('%s is not a valid netmask'
-                                                     % addr[1])
-
-                self._prefixlen = self._prefix_from_ip_int(int(self.netmask))
-            else:
-                # We have a netmask in prefix length form.
-                if not self._is_valid_netmask(addr[1]):
-                    raise NetmaskValueError(addr[1])
-                self._prefixlen = int(addr[1])
-                self.netmask = IPv4Address(self._ip_int_from_prefix(
-                    self._prefixlen))
-        else:
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv4Address(self._ip_int_from_prefix(
-                self._prefixlen))
-        if strict:
-            if self.ip != self.network:
-                raise ValueError('%s has host bits set' %
-                                 self.ip)
-
-    def _is_hostmask(self, ip_str):
-        """Test if the IP string is a hostmask (rather than a netmask).
-
-        Args:
-            ip_str: A string, the potential hostmask.
-
-        Returns:
-            A boolean, True if the IP string is a hostmask.
-
-        """
-        bits = ip_str.split('.')
-        try:
-            parts = [int(x) for x in bits if int(x) in self._valid_mask_octets]
-        except ValueError:
-            return False
-        if len(parts) != len(bits):
-            return False
-        if parts[0] < parts[-1]:
-            return True
-        return False
-
-    def _is_valid_netmask(self, netmask):
-        """Verify that the netmask is valid.
-
-        Args:
-            netmask: A string, either a prefix or dotted decimal
-              netmask.
-
-        Returns:
-            A boolean, True if the prefix represents a valid IPv4
-            netmask.
-
-        """
-        mask = netmask.split('.')
-        if len(mask) == 4:
-            if [x for x in mask if int(x) not in self._valid_mask_octets]:
-                return False
-            if [y for idx, y in enumerate(mask) if idx > 0 and
-                y > mask[idx - 1]]:
-                return False
-            return True
-        try:
-            netmask = int(netmask)
-        except ValueError:
-            return False
-        return 0 <= netmask <= self._max_prefixlen
-
-    # backwards compatibility
-    IsRFC1918 = lambda self: self.is_private
-    IsMulticast = lambda self: self.is_multicast
-    IsLoopback = lambda self: self.is_loopback
-    IsLinkLocal = lambda self: self.is_link_local
-
-
-class _BaseV6(object):
-
-    """Base IPv6 object.
-
-    The following methods are used by IPv6 objects in both single IP
-    addresses and networks.
-
-    """
-
-    _ALL_ONES = (2**IPV6LENGTH) - 1
-    _HEXTET_COUNT = 8
-    _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef')
-
-    def __init__(self, address):
-        self._version = 6
-        self._max_prefixlen = IPV6LENGTH
-
-    def _ip_int_from_string(self, ip_str):
-        """Turn an IPv6 ip_str into an integer.
-
-        Args:
-            ip_str: A string, the IPv6 ip_str.
-
-        Returns:
-            A long, the IPv6 ip_str.
-
-        Raises:
-            AddressValueError: if ip_str isn't a valid IPv6 Address.
-
-        """
-        parts = ip_str.split(':')
-
-        # An IPv6 address needs at least 2 colons (3 parts).
-        if len(parts) < 3:
-            raise AddressValueError(ip_str)
-
-        # If the address has an IPv4-style suffix, convert it to hexadecimal.
-        if '.' in parts[-1]:
-            ipv4_int = IPv4Address(parts.pop())._ip
-            parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF))
-            parts.append('%x' % (ipv4_int & 0xFFFF))
-
-        # An IPv6 address can't have more than 8 colons (9 parts).
-        if len(parts) > self._HEXTET_COUNT + 1:
-            raise AddressValueError(ip_str)
-
-        # Disregarding the endpoints, find '::' with nothing in between.
-        # This indicates that a run of zeroes has been skipped.
-        try:
-            skip_index, = (
-                [i for i in xrange(1, len(parts) - 1) if not parts[i]] or
-                [None])
-        except ValueError:
-            # Can't have more than one '::'
-            raise AddressValueError(ip_str)
-
-        # parts_hi is the number of parts to copy from above/before the '::'
-        # parts_lo is the number of parts to copy from below/after the '::'
-        if skip_index is not None:
-            # If we found a '::', then check if it also covers the endpoints.
-            parts_hi = skip_index
-            parts_lo = len(parts) - skip_index - 1
-            if not parts[0]:
-                parts_hi -= 1
-                if parts_hi:
-                    raise AddressValueError(ip_str)  # ^: requires ^::
-            if not parts[-1]:
-                parts_lo -= 1
-                if parts_lo:
-                    raise AddressValueError(ip_str)  # :$ requires ::$
-            parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo)
-            if parts_skipped < 1:
-                raise AddressValueError(ip_str)
-        else:
-            # Otherwise, allocate the entire address to parts_hi.  The endpoints
-            # could still be empty, but _parse_hextet() will check for that.
-            if len(parts) != self._HEXTET_COUNT:
-                raise AddressValueError(ip_str)
-            parts_hi = len(parts)
-            parts_lo = 0
-            parts_skipped = 0
-
-        try:
-            # Now, parse the hextets into a 128-bit integer.
-            ip_int = 0L
-            for i in xrange(parts_hi):
-                ip_int <<= 16
-                ip_int |= self._parse_hextet(parts[i])
-            ip_int <<= 16 * parts_skipped
-            for i in xrange(-parts_lo, 0):
-                ip_int <<= 16
-                ip_int |= self._parse_hextet(parts[i])
-            return ip_int
-        except ValueError:
-            raise AddressValueError(ip_str)
-
-    def _parse_hextet(self, hextet_str):
-        """Convert an IPv6 hextet string into an integer.
-
-        Args:
-            hextet_str: A string, the number to parse.
-
-        Returns:
-            The hextet as an integer.
-
-        Raises:
-            ValueError: if the input isn't strictly a hex number from [0..FFFF].
-
-        """
-        # Whitelist the characters, since int() allows a lot of bizarre stuff.
-        if not self._HEX_DIGITS.issuperset(hextet_str):
-            raise ValueError
-        hextet_int = int(hextet_str, 16)
-        if hextet_int > 0xFFFF:
-            raise ValueError
-        return hextet_int
-
-    def _compress_hextets(self, hextets):
-        """Compresses a list of hextets.
-
-        Compresses a list of strings, replacing the longest continuous
-        sequence of "0" in the list with "" and adding empty strings at
-        the beginning or at the end of the string such that subsequently
-        calling ":".join(hextets) will produce the compressed version of
-        the IPv6 address.
-
-        Args:
-            hextets: A list of strings, the hextets to compress.
-
-        Returns:
-            A list of strings.
-
-        """
-        best_doublecolon_start = -1
-        best_doublecolon_len = 0
-        doublecolon_start = -1
-        doublecolon_len = 0
-        for index in range(len(hextets)):
-            if hextets[index] == '0':
-                doublecolon_len += 1
-                if doublecolon_start == -1:
-                    # Start of a sequence of zeros.
-                    doublecolon_start = index
-                if doublecolon_len > best_doublecolon_len:
-                    # This is the longest sequence of zeros so far.
-                    best_doublecolon_len = doublecolon_len
-                    best_doublecolon_start = doublecolon_start
-            else:
-                doublecolon_len = 0
-                doublecolon_start = -1
-
-        if best_doublecolon_len > 1:
-            best_doublecolon_end = (best_doublecolon_start +
-                                    best_doublecolon_len)
-            # For zeros at the end of the address.
-            if best_doublecolon_end == len(hextets):
-                hextets += ['']
-            hextets[best_doublecolon_start:best_doublecolon_end] = ['']
-            # For zeros at the beginning of the address.
-            if best_doublecolon_start == 0:
-                hextets = [''] + hextets
-
-        return hextets
-
-    def _string_from_ip_int(self, ip_int=None):
-        """Turns a 128-bit integer into hexadecimal notation.
-
-        Args:
-            ip_int: An integer, the IP address.
-
-        Returns:
-            A string, the hexadecimal representation of the address.
-
-        Raises:
-            ValueError: The address is bigger than 128 bits of all ones.
-
-        """
-        if not ip_int and ip_int != 0:
-            ip_int = int(self._ip)
-
-        if ip_int > self._ALL_ONES:
-            raise ValueError('IPv6 address is too large')
-
-        hex_str = '%032x' % ip_int
-        hextets = []
-        for x in range(0, 32, 4):
-            hextets.append('%x' % int(hex_str[x:x+4], 16))
-
-        hextets = self._compress_hextets(hextets)
-        return ':'.join(hextets)
-
-    def _explode_shorthand_ip_string(self, ip_str=None):
-        """Expand a shortened IPv6 address.
-
-        Args:
-            ip_str: A string, the IPv6 address.
-
-        Returns:
-            A string, the expanded IPv6 address.
-
-        """
-        if not ip_str:
-            ip_str = str(self)
-            if isinstance(self, _BaseNet):
-                ip_str = str(self.ip)
-
-        ip_int = self._ip_int_from_string(ip_str)
-        parts = []
-        for i in xrange(self._HEXTET_COUNT):
-            parts.append('%04x' % (ip_int & 0xFFFF))
-            ip_int >>= 16
-        parts.reverse()
-        return ':'.join(parts)
-
-    @property
-    def max_prefixlen(self):
-        return self._max_prefixlen
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v6_int_to_packed(self._ip)
-
-    @property
-    def version(self):
-        return self._version
-
-    @property
-    def is_multicast(self):
-        """Test if the address is reserved for multicast use.
-
-        Returns:
-            A boolean, True if the address is a multicast address.
-            See RFC 2373 2.7 for details.
-
-        """
-        return self in IPv6Network('ff00::/8')
-
-    @property
-    def is_reserved(self):
-        """Test if the address is otherwise IETF reserved.
-
-        Returns:
-            A boolean, True if the address is within one of the
-            reserved IPv6 Network ranges.
-
-        """
-        return (self in IPv6Network('::/8') or
-                self in IPv6Network('100::/8') or
-                self in IPv6Network('200::/7') or
-                self in IPv6Network('400::/6') or
-                self in IPv6Network('800::/5') or
-                self in IPv6Network('1000::/4') or
-                self in IPv6Network('4000::/3') or
-                self in IPv6Network('6000::/3') or
-                self in IPv6Network('8000::/3') or
-                self in IPv6Network('A000::/3') or
-                self in IPv6Network('C000::/3') or
-                self in IPv6Network('E000::/4') or
-                self in IPv6Network('F000::/5') or
-                self in IPv6Network('F800::/6') or
-                self in IPv6Network('FE00::/9'))
-
-    @property
-    def is_unspecified(self):
-        """Test if the address is unspecified.
-
-        Returns:
-            A boolean, True if this is the unspecified address as defined in
-            RFC 2373 2.5.2.
-
-        """
-        return self._ip == 0 and getattr(self, '_prefixlen', 128) == 128
-
-    @property
-    def is_loopback(self):
-        """Test if the address is a loopback address.
-
-        Returns:
-            A boolean, True if the address is a loopback address as defined in
-            RFC 2373 2.5.3.
-
-        """
-        return self._ip == 1 and getattr(self, '_prefixlen', 128) == 128
-
-    @property
-    def is_link_local(self):
-        """Test if the address is reserved for link-local.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 4291.
-
-        """
-        return self in IPv6Network('fe80::/10')
-
-    @property
-    def is_site_local(self):
-        """Test if the address is reserved for site-local.
-
-        Note that the site-local address space has been deprecated by RFC 3879.
-        Use is_private to test if this address is in the space of unique local
-        addresses as defined by RFC 4193.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 3513 2.5.6.
-
-        """
-        return self in IPv6Network('fec0::/10')
-
-    @property
-    def is_private(self):
-        """Test if this address is allocated for private networks.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 4193.
-
-        """
-        return self in IPv6Network('fc00::/7')
-
-    @property
-    def ipv4_mapped(self):
-        """Return the IPv4 mapped address.
-
-        Returns:
-            If the IPv6 address is a v4 mapped address, return the
-            IPv4 mapped address. Return None otherwise.
-
-        """
-        if (self._ip >> 32) != 0xFFFF:
-            return None
-        return IPv4Address(self._ip & 0xFFFFFFFF)
-
-    @property
-    def teredo(self):
-        """Tuple of embedded teredo IPs.
-
-        Returns:
-            Tuple of the (server, client) IPs or None if the address
-            doesn't appear to be a teredo address (doesn't start with
-            2001::/32)
-
-        """
-        if (self._ip >> 96) != 0x20010000:
-            return None
-        return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF),
-                IPv4Address(~self._ip & 0xFFFFFFFF))
-
-    @property
-    def sixtofour(self):
-        """Return the IPv4 6to4 embedded address.
-
-        Returns:
-            The IPv4 6to4-embedded address if present or None if the
-            address doesn't appear to contain a 6to4 embedded address.
-
-        """
-        if (self._ip >> 112) != 0x2002:
-            return None
-        return IPv4Address((self._ip >> 80) & 0xFFFFFFFF)
-
-
-class IPv6Address(_BaseV6, _BaseIP):
-
-    """Represent and manipulate single IPv6 Addresses.
-    """
-
-    def __init__(self, address):
-        """Instantiate a new IPv6 address object.
-
-        Args:
-            address: A string or integer representing the IP
-
-              Additionally, an integer can be passed, so
-              IPv6Address('2001:4860::') ==
-                IPv6Address(42541956101370907050197289607612071936L).
-              or, more generally
-              IPv6Address(IPv6Address('2001:4860::')._ip) ==
-                IPv6Address('2001:4860::')
-
-        Raises:
-            AddressValueError: If address isn't a valid IPv6 address.
-
-        """
-        _BaseIP.__init__(self, address)
-        _BaseV6.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 16:
-                tmp = struct.unpack('!QQ', address)
-                self._ip = (tmp[0] << 64) | tmp[1]
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        if not addr_str:
-            raise AddressValueError('')
-
-        self._ip = self._ip_int_from_string(addr_str)
-
-
-class IPv6Network(_BaseV6, _BaseNet):
-
-    """This class represents and manipulates 128-bit IPv6 networks.
-
-    Attributes: [examples for IPv6('2001:658:22A:CAFE:200::1/64')]
-        .ip: IPv6Address('2001:658:22a:cafe:200::1')
-        .network: IPv6Address('2001:658:22a:cafe::')
-        .hostmask: IPv6Address('::ffff:ffff:ffff:ffff')
-        .broadcast: IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff')
-        .netmask: IPv6Address('ffff:ffff:ffff:ffff::')
-        .prefixlen: 64
-
-    """
-
-
-    def __init__(self, address, strict=False):
-        """Instantiate a new IPv6 Network object.
-
-        Args:
-            address: A string or integer representing the IPv6 network or the IP
-              and prefix/netmask.
-              '2001:4860::/128'
-              '2001:4860:0000:0000:0000:0000:0000:0000/128'
-              '2001:4860::'
-              are all functionally the same in IPv6.  That is to say,
-              failing to provide a subnetmask will create an object with
-              a mask of /128.
-
-              Additionally, an integer can be passed, so
-              IPv6Network('2001:4860::') ==
-                IPv6Network(42541956101370907050197289607612071936L).
-              or, more generally
-              IPv6Network(IPv6Network('2001:4860::')._ip) ==
-                IPv6Network('2001:4860::')
-
-            strict: A boolean. If true, ensure that we have been passed
-              A true network address, eg, 192.168.1.0/24 and not an
-              IP address on a network, eg, 192.168.1.1/24.
-
-        Raises:
-            AddressValueError: If address isn't a valid IPv6 address.
-            NetmaskValueError: If the netmask isn't valid for
-              an IPv6 address.
-            ValueError: If strict was True and a network address was not
-              supplied.
-
-        """
-        _BaseNet.__init__(self, address)
-        _BaseV6.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            self.ip = IPv6Address(self._ip)
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv6Address(self._ALL_ONES)
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 16:
-                tmp = struct.unpack('!QQ', address)
-                self._ip = (tmp[0] << 64) | tmp[1]
-                self.ip = IPv6Address(self._ip)
-                self._prefixlen = self._max_prefixlen
-                self.netmask = IPv6Address(self._ALL_ONES)
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP prefix string.
-        addr = str(address).split('/')
-
-        if len(addr) > 2:
-            raise AddressValueError(address)
-
-        self._ip = self._ip_int_from_string(addr[0])
-        self.ip = IPv6Address(self._ip)
-
-        if len(addr) == 2:
-            if self._is_valid_netmask(addr[1]):
-                self._prefixlen = int(addr[1])
-            else:
-                raise NetmaskValueError(addr[1])
-        else:
-            self._prefixlen = self._max_prefixlen
-
-        self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen))
-
-        if strict:
-            if self.ip != self.network:
-                raise ValueError('%s has host bits set' %
-                                 self.ip)
-
-    def _is_valid_netmask(self, prefixlen):
-        """Verify that the netmask/prefixlen is valid.
-
-        Args:
-            prefixlen: A string, the netmask in prefix length format.
-
-        Returns:
-            A boolean, True if the prefix represents a valid IPv6
-            netmask.
-
-        """
-        try:
-            prefixlen = int(prefixlen)
-        except ValueError:
-            return False
-        return 0 <= prefixlen <= self._max_prefixlen
-
-    @property
-    def with_netmask(self):
-        return self.with_prefixlen
diff --git a/contrib/ipaddr-py/ipaddr_test.py b/contrib/ipaddr-py/ipaddr_test.py
deleted file mode 100755
index 09bece0e751..00000000000
--- a/contrib/ipaddr-py/ipaddr_test.py
+++ /dev/null
@@ -1,1099 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2007 Google Inc.
-#  Licensed to PSF under a Contributor Agreement.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittest for ipaddr module."""
-
-
-import unittest
-import time
-import ipaddr
-
-# Compatibility function to cast str to bytes objects
-if ipaddr._compat_has_real_bytes:
-    _cb = lambda bytestr: bytes(bytestr, 'charmap')
-else:
-    _cb = str
-
-class IpaddrUnitTest(unittest.TestCase):
-
-    def setUp(self):
-        self.ipv4 = ipaddr.IPv4Network('1.2.3.4/24')
-        self.ipv4_hostmask = ipaddr.IPv4Network('10.0.0.1/0.255.255.255')
-        self.ipv6 = ipaddr.IPv6Network('2001:658:22a:cafe:200:0:0:1/64')
-
-    def tearDown(self):
-        del(self.ipv4)
-        del(self.ipv4_hostmask)
-        del(self.ipv6)
-        del(self)
-
-    def testRepr(self):
-        self.assertEqual("IPv4Network('1.2.3.4/32')",
-                         repr(ipaddr.IPv4Network('1.2.3.4')))
-        self.assertEqual("IPv6Network('::1/128')",
-                         repr(ipaddr.IPv6Network('::1')))
-
-    def testAutoMasking(self):
-        addr1 = ipaddr.IPv4Network('1.1.1.255/24')
-        addr1_masked = ipaddr.IPv4Network('1.1.1.0/24')
-        self.assertEqual(addr1_masked, addr1.masked())
-
-        addr2 = ipaddr.IPv6Network('2000:cafe::efac:100/96')
-        addr2_masked = ipaddr.IPv6Network('2000:cafe::/96')
-        self.assertEqual(addr2_masked, addr2.masked())
-
-    # issue57
-    def testAddressIntMath(self):
-        self.assertEqual(ipaddr.IPv4Address('1.1.1.1') + 255,
-                         ipaddr.IPv4Address('1.1.2.0'))
-        self.assertEqual(ipaddr.IPv4Address('1.1.1.1') - 256,
-                         ipaddr.IPv4Address('1.1.0.1'))
-        self.assertEqual(ipaddr.IPv6Address('::1') + (2**16 - 2),
-                         ipaddr.IPv6Address('::ffff'))
-        self.assertEqual(ipaddr.IPv6Address('::ffff') - (2**16 - 2),
-                         ipaddr.IPv6Address('::1'))
-
-    def testInvalidStrings(self):
-        def AssertInvalidIP(ip_str):
-            self.assertRaises(ValueError, ipaddr.IPAddress, ip_str)
-        AssertInvalidIP("")
-        AssertInvalidIP("016.016.016.016")
-        AssertInvalidIP("016.016.016")
-        AssertInvalidIP("016.016")
-        AssertInvalidIP("016")
-        AssertInvalidIP("000.000.000.000")
-        AssertInvalidIP("000")
-        AssertInvalidIP("0x0a.0x0a.0x0a.0x0a")
-        AssertInvalidIP("0x0a.0x0a.0x0a")
-        AssertInvalidIP("0x0a.0x0a")
-        AssertInvalidIP("0x0a")
-        AssertInvalidIP("42.42.42.42.42")
-        AssertInvalidIP("42.42.42")
-        AssertInvalidIP("42.42")
-        AssertInvalidIP("42")
-        AssertInvalidIP("42..42.42")
-        AssertInvalidIP("42..42.42.42")
-        AssertInvalidIP("42.42.42.42.")
-        AssertInvalidIP("42.42.42.42...")
-        AssertInvalidIP(".42.42.42.42")
-        AssertInvalidIP("...42.42.42.42")
-        AssertInvalidIP("42.42.42.-0")
-        AssertInvalidIP("42.42.42.+0")
-        AssertInvalidIP(".")
-        AssertInvalidIP("...")
-        AssertInvalidIP("bogus")
-        AssertInvalidIP("bogus.com")
-        AssertInvalidIP("192.168.0.1.com")
-        AssertInvalidIP("12345.67899.-54321.-98765")
-        AssertInvalidIP("257.0.0.0")
-        AssertInvalidIP("42.42.42.-42")
-        AssertInvalidIP("3ffe::1.net")
-        AssertInvalidIP("3ffe::1::1")
-        AssertInvalidIP("1::2::3::4:5")
-        AssertInvalidIP("::7:6:5:4:3:2:")
-        AssertInvalidIP(":6:5:4:3:2:1::")
-        AssertInvalidIP("2001::db:::1")
-        AssertInvalidIP("FEDC:9878")
-        AssertInvalidIP("+1.+2.+3.4")
-        AssertInvalidIP("1.2.3.4e0")
-        AssertInvalidIP("::7:6:5:4:3:2:1:0")
-        AssertInvalidIP("7:6:5:4:3:2:1:0::")
-        AssertInvalidIP("9:8:7:6:5:4:3::2:1")
-        AssertInvalidIP("0:1:2:3::4:5:6:7")
-        AssertInvalidIP("3ffe:0:0:0:0:0:0:0:1")
-        AssertInvalidIP("3ffe::10000")
-        AssertInvalidIP("3ffe::goog")
-        AssertInvalidIP("3ffe::-0")
-        AssertInvalidIP("3ffe::+0")
-        AssertInvalidIP("3ffe::-1")
-        AssertInvalidIP(":")
-        AssertInvalidIP(":::")
-        AssertInvalidIP("::1.2.3")
-        AssertInvalidIP("::1.2.3.4.5")
-        AssertInvalidIP("::1.2.3.4:")
-        AssertInvalidIP("1.2.3.4::")
-        AssertInvalidIP("2001:db8::1:")
-        AssertInvalidIP(":2001:db8::1")
-        AssertInvalidIP(":1:2:3:4:5:6:7")
-        AssertInvalidIP("1:2:3:4:5:6:7:")
-        AssertInvalidIP(":1:2:3:4:5:6:")
-
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network, '')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          'google.com')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          '::1.2.3.4')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network, '')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'google.com')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '1.2.3.4')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'cafe:cafe::/128/190')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '1234:axy::b')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '1234:axy::b')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '2001:db8:::1')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '2001:888888::1')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Address(1)._ip_int_from_string,
-                          '1.a.2.3')
-        self.assertEqual(False, ipaddr.IPv4Network(1)._is_hostmask('1.a.2.3'))
-
-    def testGetNetwork(self):
-        self.assertEqual(int(self.ipv4.network), 16909056)
-        self.assertEqual(str(self.ipv4.network), '1.2.3.0')
-        self.assertEqual(str(self.ipv4_hostmask.network), '10.0.0.0')
-
-        self.assertEqual(int(self.ipv6.network),
-                         42540616829182469433403647294022090752)
-        self.assertEqual(str(self.ipv6.network),
-                         '2001:658:22a:cafe::')
-        self.assertEqual(str(self.ipv6.hostmask),
-                         '::ffff:ffff:ffff:ffff')
-
-    def testBadVersionComparison(self):
-        # These should always raise TypeError
-        v4addr = ipaddr.IPAddress('1.1.1.1')
-        v4net = ipaddr.IPNetwork('1.1.1.1')
-        v6addr = ipaddr.IPAddress('::1')
-        v6net = ipaddr.IPAddress('::1')
-
-        self.assertRaises(TypeError, v4addr.__lt__, v6addr)
-        self.assertRaises(TypeError, v4addr.__gt__, v6addr)
-        self.assertRaises(TypeError, v4net.__lt__, v6net)
-        self.assertRaises(TypeError, v4net.__gt__, v6net)
-
-        self.assertRaises(TypeError, v6addr.__lt__, v4addr)
-        self.assertRaises(TypeError, v6addr.__gt__, v4addr)
-        self.assertRaises(TypeError, v6net.__lt__, v4net)
-        self.assertRaises(TypeError, v6net.__gt__, v4net)
-
-    def testMixedTypeComparison(self):
-        v4addr = ipaddr.IPAddress('1.1.1.1')
-        v4net = ipaddr.IPNetwork('1.1.1.1/32')
-        v6addr = ipaddr.IPAddress('::1')
-        v6net = ipaddr.IPNetwork('::1/128')
-
-        self.assertFalse(v4net.__contains__(v6net))
-        self.assertFalse(v6net.__contains__(v4net))
-
-        self.assertRaises(TypeError, lambda: v4addr < v4net)
-        self.assertRaises(TypeError, lambda: v4addr > v4net)
-        self.assertRaises(TypeError, lambda: v4net < v4addr)
-        self.assertRaises(TypeError, lambda: v4net > v4addr)
-
-        self.assertRaises(TypeError, lambda: v6addr < v6net)
-        self.assertRaises(TypeError, lambda: v6addr > v6net)
-        self.assertRaises(TypeError, lambda: v6net < v6addr)
-        self.assertRaises(TypeError, lambda: v6net > v6addr)
-
-        # with get_mixed_type_key, you can sort addresses and network.
-        self.assertEqual([v4addr, v4net], sorted([v4net, v4addr],
-                                                 key=ipaddr.get_mixed_type_key))
-        self.assertEqual([v6addr, v6net], sorted([v6net, v6addr],
-                                                 key=ipaddr.get_mixed_type_key))
-
-    def testIpFromInt(self):
-        self.assertEqual(self.ipv4.ip, ipaddr.IPv4Network(16909060).ip)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, 2**32)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, -1)
-
-        ipv4 = ipaddr.IPNetwork('1.2.3.4')
-        ipv6 = ipaddr.IPNetwork('2001:658:22a:cafe:200:0:0:1')
-        self.assertEqual(ipv4, ipaddr.IPNetwork(int(ipv4)))
-        self.assertEqual(ipv6, ipaddr.IPNetwork(int(ipv6)))
-
-        v6_int = 42540616829182469433547762482097946625
-        self.assertEqual(self.ipv6.ip, ipaddr.IPv6Network(v6_int).ip)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, 2**128)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, -1)
-
-        self.assertEqual(ipaddr.IPNetwork(self.ipv4.ip).version, 4)
-        self.assertEqual(ipaddr.IPNetwork(self.ipv6.ip).version, 6)
-
-    if ipaddr._compat_has_real_bytes: # on python3+
-        def testIpFromPacked(self):
-            ip = ipaddr.IPNetwork
-
-            self.assertEqual(self.ipv4.ip,
-                             ip(_cb('\x01\x02\x03\x04')).ip)
-            self.assertEqual(ip('255.254.253.252'),
-                             ip(_cb('\xff\xfe\xfd\xfc')))
-            self.assertRaises(ValueError, ipaddr.IPNetwork, _cb('\x00' * 3))
-            self.assertRaises(ValueError, ipaddr.IPNetwork, _cb('\x00' * 5))
-            self.assertEqual(self.ipv6.ip,
-                             ip(_cb('\x20\x01\x06\x58\x02\x2a\xca\xfe'
-                               '\x02\x00\x00\x00\x00\x00\x00\x01')).ip)
-            self.assertEqual(ip('ffff:2:3:4:ffff::'),
-                             ip(_cb('\xff\xff\x00\x02\x00\x03\x00\x04' +
-                                   '\xff\xff' + '\x00' * 6)))
-            self.assertEqual(ip('::'),
-                             ip(_cb('\x00' * 16)))
-            self.assertRaises(ValueError, ip, _cb('\x00' * 15))
-            self.assertRaises(ValueError, ip, _cb('\x00' * 17))
-
-    def testGetIp(self):
-        self.assertEqual(int(self.ipv4.ip), 16909060)
-        self.assertEqual(str(self.ipv4.ip), '1.2.3.4')
-        self.assertEqual(str(self.ipv4_hostmask.ip), '10.0.0.1')
-
-        self.assertEqual(int(self.ipv6.ip),
-                         42540616829182469433547762482097946625)
-        self.assertEqual(str(self.ipv6.ip),
-                         '2001:658:22a:cafe:200::1')
-
-    def testGetNetmask(self):
-        self.assertEqual(int(self.ipv4.netmask), 4294967040L)
-        self.assertEqual(str(self.ipv4.netmask), '255.255.255.0')
-        self.assertEqual(str(self.ipv4_hostmask.netmask), '255.0.0.0')
-        self.assertEqual(int(self.ipv6.netmask),
-                         340282366920938463444927863358058659840)
-        self.assertEqual(self.ipv6.prefixlen, 64)
-
-    def testZeroNetmask(self):
-        ipv4_zero_netmask = ipaddr.IPv4Network('1.2.3.4/0')
-        self.assertEqual(int(ipv4_zero_netmask.netmask), 0)
-        self.assertTrue(ipv4_zero_netmask._is_valid_netmask(str(0)))
-
-        ipv6_zero_netmask = ipaddr.IPv6Network('::1/0')
-        self.assertEqual(int(ipv6_zero_netmask.netmask), 0)
-        self.assertTrue(ipv6_zero_netmask._is_valid_netmask(str(0)))
-
-    def testGetBroadcast(self):
-        self.assertEqual(int(self.ipv4.broadcast), 16909311L)
-        self.assertEqual(str(self.ipv4.broadcast), '1.2.3.255')
-
-        self.assertEqual(int(self.ipv6.broadcast),
-                         42540616829182469451850391367731642367)
-        self.assertEqual(str(self.ipv6.broadcast),
-                         '2001:658:22a:cafe:ffff:ffff:ffff:ffff')
-
-    def testGetPrefixlen(self):
-        self.assertEqual(self.ipv4.prefixlen, 24)
-
-        self.assertEqual(self.ipv6.prefixlen, 64)
-
-    def testGetSupernet(self):
-        self.assertEqual(self.ipv4.supernet().prefixlen, 23)
-        self.assertEqual(str(self.ipv4.supernet().network), '1.2.2.0')
-        self.assertEqual(ipaddr.IPv4Network('0.0.0.0/0').supernet(),
-                         ipaddr.IPv4Network('0.0.0.0/0'))
-
-        self.assertEqual(self.ipv6.supernet().prefixlen, 63)
-        self.assertEqual(str(self.ipv6.supernet().network),
-                         '2001:658:22a:cafe::')
-        self.assertEqual(ipaddr.IPv6Network('::0/0').supernet(),
-                         ipaddr.IPv6Network('::0/0'))
-
-    def testGetSupernet3(self):
-        self.assertEqual(self.ipv4.supernet(3).prefixlen, 21)
-        self.assertEqual(str(self.ipv4.supernet(3).network), '1.2.0.0')
-
-        self.assertEqual(self.ipv6.supernet(3).prefixlen, 61)
-        self.assertEqual(str(self.ipv6.supernet(3).network),
-                         '2001:658:22a:caf8::')
-
-    def testGetSupernet4(self):
-        self.assertRaises(ValueError, self.ipv4.supernet, prefixlen_diff=2,
-                          new_prefix=1)
-        self.assertRaises(ValueError, self.ipv4.supernet, new_prefix=25)
-        self.assertEqual(self.ipv4.supernet(prefixlen_diff=2),
-                         self.ipv4.supernet(new_prefix=22))
-
-        self.assertRaises(ValueError, self.ipv6.supernet, prefixlen_diff=2,
-                          new_prefix=1)
-        self.assertRaises(ValueError, self.ipv6.supernet, new_prefix=65)
-        self.assertEqual(self.ipv6.supernet(prefixlen_diff=2),
-                         self.ipv6.supernet(new_prefix=62))
-
-    def testIterSubnets(self):
-        self.assertEqual(self.ipv4.subnet(), list(self.ipv4.iter_subnets()))
-        self.assertEqual(self.ipv6.subnet(), list(self.ipv6.iter_subnets()))
-
-    def testFancySubnetting(self):
-        self.assertEqual(sorted(self.ipv4.subnet(prefixlen_diff=3)),
-                         sorted(self.ipv4.subnet(new_prefix=27)))
-        self.assertRaises(ValueError, self.ipv4.subnet, new_prefix=23)
-        self.assertRaises(ValueError, self.ipv4.subnet,
-                          prefixlen_diff=3, new_prefix=27)
-        self.assertEqual(sorted(self.ipv6.subnet(prefixlen_diff=4)),
-                         sorted(self.ipv6.subnet(new_prefix=68)))
-        self.assertRaises(ValueError, self.ipv6.subnet, new_prefix=63)
-        self.assertRaises(ValueError, self.ipv6.subnet,
-                          prefixlen_diff=4, new_prefix=68)
-
-    def testGetSubnet(self):
-        self.assertEqual(self.ipv4.subnet()[0].prefixlen, 25)
-        self.assertEqual(str(self.ipv4.subnet()[0].network), '1.2.3.0')
-        self.assertEqual(str(self.ipv4.subnet()[1].network), '1.2.3.128')
-
-        self.assertEqual(self.ipv6.subnet()[0].prefixlen, 65)
-
-    def testGetSubnetForSingle32(self):
-        ip = ipaddr.IPv4Network('1.2.3.4/32')
-        subnets1 = [str(x) for x in ip.subnet()]
-        subnets2 = [str(x) for x in ip.subnet(2)]
-        self.assertEqual(subnets1, ['1.2.3.4/32'])
-        self.assertEqual(subnets1, subnets2)
-
-    def testGetSubnetForSingle128(self):
-        ip = ipaddr.IPv6Network('::1/128')
-        subnets1 = [str(x) for x in ip.subnet()]
-        subnets2 = [str(x) for x in ip.subnet(2)]
-        self.assertEqual(subnets1, ['::1/128'])
-        self.assertEqual(subnets1, subnets2)
-
-    def testSubnet2(self):
-        ips = [str(x) for x in self.ipv4.subnet(2)]
-        self.assertEqual(
-            ips,
-            ['1.2.3.0/26', '1.2.3.64/26', '1.2.3.128/26', '1.2.3.192/26'])
-
-        ipsv6 = [str(x) for x in self.ipv6.subnet(2)]
-        self.assertEqual(
-            ipsv6,
-            ['2001:658:22a:cafe::/66',
-             '2001:658:22a:cafe:4000::/66',
-             '2001:658:22a:cafe:8000::/66',
-             '2001:658:22a:cafe:c000::/66'])
-
-    def testSubnetFailsForLargeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.subnet, 9)
-        self.assertRaises(ValueError, self.ipv6.subnet, 65)
-
-    def testSupernetFailsForLargeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.supernet, 25)
-        self.assertRaises(ValueError, self.ipv6.supernet, 65)
-
-    def testSubnetFailsForNegativeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.subnet, -1)
-        self.assertRaises(ValueError, self.ipv6.subnet, -1)
-
-    def testGetNumHosts(self):
-        self.assertEqual(self.ipv4.numhosts, 256)
-        self.assertEqual(self.ipv4.subnet()[0].numhosts, 128)
-        self.assertEqual(self.ipv4.supernet().numhosts, 512)
-
-        self.assertEqual(self.ipv6.numhosts, 18446744073709551616)
-        self.assertEqual(self.ipv6.subnet()[0].numhosts, 9223372036854775808)
-        self.assertEqual(self.ipv6.supernet().numhosts, 36893488147419103232)
-
-    def testContains(self):
-        self.assertTrue(ipaddr.IPv4Network('1.2.3.128/25') in self.ipv4)
-        self.assertFalse(ipaddr.IPv4Network('1.2.4.1/24') in self.ipv4)
-        self.assertTrue(self.ipv4 in self.ipv4)
-        self.assertTrue(self.ipv6 in self.ipv6)
-        # We can test addresses and string as well.
-        addr1 = ipaddr.IPv4Address('1.2.3.37')
-        self.assertTrue(addr1 in self.ipv4)
-        # issue 61, bad network comparison on like-ip'd network objects
-        # with identical broadcast addresses.
-        self.assertFalse(ipaddr.IPv4Network('1.1.0.0/16').__contains__(
-                ipaddr.IPv4Network('1.0.0.0/15')))
-
-    def testBadAddress(self):
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          'poop')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '1.2.3.256')
-
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'poopv6')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/32/24')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '10/8')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, '10/8')
-
-
-    def testBadNetMask(self):
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/33')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/254.254.255.256')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.1.1.1/240.255.0.0')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv6Network, '::1/')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv6Network, '::1/129')
-
-    def testNth(self):
-        self.assertEqual(str(self.ipv4[5]), '1.2.3.5')
-        self.assertRaises(IndexError, self.ipv4.__getitem__, 256)
-
-        self.assertEqual(str(self.ipv6[5]),
-                         '2001:658:22a:cafe::5')
-
-    def testGetitem(self):
-        # http://code.google.com/p/ipaddr-py/issues/detail?id=15
-        addr = ipaddr.IPv4Network('172.31.255.128/255.255.255.240')
-        self.assertEqual(28, addr.prefixlen)
-        addr_list = list(addr)
-        self.assertEqual('172.31.255.128', str(addr_list[0]))
-        self.assertEqual('172.31.255.128', str(addr[0]))
-        self.assertEqual('172.31.255.143', str(addr_list[-1]))
-        self.assertEqual('172.31.255.143', str(addr[-1]))
-        self.assertEqual(addr_list[-1], addr[-1])
-
-    def testEqual(self):
-        self.assertTrue(self.ipv4 == ipaddr.IPv4Network('1.2.3.4/24'))
-        self.assertFalse(self.ipv4 == ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertFalse(self.ipv4 == ipaddr.IPv6Network('::1.2.3.4/24'))
-        self.assertFalse(self.ipv4 == '')
-        self.assertFalse(self.ipv4 == [])
-        self.assertFalse(self.ipv4 == 2)
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1/32') ==
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1/24') ==
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertFalse(ipaddr.IPNetwork('1.1.1.0/24') ==
-                         ipaddr.IPAddress('1.1.1.1'))
-
-        self.assertTrue(self.ipv6 ==
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/64'))
-        self.assertTrue(ipaddr.IPNetwork('::1/128') ==
-                        ipaddr.IPAddress('::1'))
-        self.assertTrue(ipaddr.IPNetwork('::1/127') ==
-                        ipaddr.IPAddress('::1'))
-        self.assertFalse(ipaddr.IPNetwork('::0/127') ==
-                         ipaddr.IPAddress('::1'))
-        self.assertFalse(self.ipv6 ==
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/63'))
-        self.assertFalse(self.ipv6 == ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertFalse(self.ipv6 == '')
-        self.assertFalse(self.ipv6 == [])
-        self.assertFalse(self.ipv6 == 2)
-
-    def testNotEqual(self):
-        self.assertFalse(self.ipv4 != ipaddr.IPv4Network('1.2.3.4/24'))
-        self.assertTrue(self.ipv4 != ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertTrue(self.ipv4 != ipaddr.IPv6Network('::1.2.3.4/24'))
-        self.assertTrue(self.ipv4 != '')
-        self.assertTrue(self.ipv4 != [])
-        self.assertTrue(self.ipv4 != 2)
-
-        addr2 = ipaddr.IPAddress('2001:658:22a:cafe:200::1')
-        self.assertFalse(self.ipv6 !=
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/64'))
-        self.assertTrue(self.ipv6 !=
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/63'))
-        self.assertTrue(self.ipv6 != ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertTrue(self.ipv6 != '')
-        self.assertTrue(self.ipv6 != [])
-        self.assertTrue(self.ipv6 != 2)
-
-    def testSlash32Constructor(self):
-        self.assertEqual(str(ipaddr.IPv4Network('1.2.3.4/255.255.255.255')),
-                          '1.2.3.4/32')
-
-    def testSlash128Constructor(self):
-        self.assertEqual(str(ipaddr.IPv6Network('::1/128')),
-                                  '::1/128')
-
-    def testSlash0Constructor(self):
-        self.assertEqual(str(ipaddr.IPv4Network('1.2.3.4/0.0.0.0')),
-                          '1.2.3.4/0')
-
-    def testCollapsing(self):
-        # test only IP addresses including some duplicates
-        ip1 = ipaddr.IPv4Address('1.1.1.0')
-        ip2 = ipaddr.IPv4Address('1.1.1.1')
-        ip3 = ipaddr.IPv4Address('1.1.1.2')
-        ip4 = ipaddr.IPv4Address('1.1.1.3')
-        ip5 = ipaddr.IPv4Address('1.1.1.4')
-        ip6 = ipaddr.IPv4Address('1.1.1.0')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3, ip4, ip5, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.1.0/30'),
-                                     ipaddr.IPv4Network('1.1.1.4/32')])
-
-        # test a mix of IP addresses and networks including some duplicates
-        ip1 = ipaddr.IPv4Address('1.1.1.0')
-        ip2 = ipaddr.IPv4Address('1.1.1.1')
-        ip3 = ipaddr.IPv4Address('1.1.1.2')
-        ip4 = ipaddr.IPv4Address('1.1.1.3')
-        ip5 = ipaddr.IPv4Network('1.1.1.4/30')
-        ip6 = ipaddr.IPv4Network('1.1.1.4/30')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip5, ip1, ip2, ip3, ip4, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.1.0/29')])
-
-        # test only IP networks
-        ip1 = ipaddr.IPv4Network('1.1.0.0/24')
-        ip2 = ipaddr.IPv4Network('1.1.1.0/24')
-        ip3 = ipaddr.IPv4Network('1.1.2.0/24')
-        ip4 = ipaddr.IPv4Network('1.1.3.0/24')
-        ip5 = ipaddr.IPv4Network('1.1.4.0/24')
-        # stored in no particular order b/c we want CollapseAddr to call [].sort
-        ip6 = ipaddr.IPv4Network('1.1.0.0/22')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3, ip4, ip5, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.0.0/22'),
-                                     ipaddr.IPv4Network('1.1.4.0/24')])
-
-        # test that two addresses are supernet'ed properly
-        collapsed = ipaddr.collapse_address_list([ip1, ip2])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.0.0/23')])
-
-        # test same IP networks
-        ip_same1 = ip_same2 = ipaddr.IPv4Network('1.1.1.1/32')
-        self.assertEqual(ipaddr.collapse_address_list([ip_same1, ip_same2]),
-                         [ip_same1])
-
-        # test same IP addresses
-        ip_same1 = ip_same2 = ipaddr.IPv4Address('1.1.1.1')
-        self.assertEqual(ipaddr.collapse_address_list([ip_same1, ip_same2]),
-                         [ipaddr.IPNetwork('1.1.1.1/32')])
-        ip1 = ipaddr.IPv6Network('::2001:1/100')
-        ip2 = ipaddr.IPv6Network('::2002:1/120')
-        ip3 = ipaddr.IPv6Network('::2001:1/96')
-        # test that ipv6 addresses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3])
-        self.assertEqual(collapsed, [ip3])
-
-        # the toejam test
-        ip1 = ipaddr.IPAddress('1.1.1.1')
-        ip2 = ipaddr.IPAddress('::1')
-        self.assertRaises(TypeError, ipaddr.collapse_address_list,
-                          [ip1, ip2])
-
-    def testSummarizing(self):
-        #ip = ipaddr.IPAddress
-        #ipnet = ipaddr.IPNetwork
-        summarize = ipaddr.summarize_address_range
-        ip1 = ipaddr.IPAddress('1.1.1.0')
-        ip2 = ipaddr.IPAddress('1.1.1.255')
-        # test a /24 is sumamrized properly
-        self.assertEqual(summarize(ip1, ip2)[0], ipaddr.IPNetwork('1.1.1.0/24'))
-        # test an  IPv4 range that isn't on a network byte boundary
-        ip2 = ipaddr.IPAddress('1.1.1.8')
-        self.assertEqual(summarize(ip1, ip2), [ipaddr.IPNetwork('1.1.1.0/29'),
-                                               ipaddr.IPNetwork('1.1.1.8')])
-
-        ip1 = ipaddr.IPAddress('1::')
-        ip2 = ipaddr.IPAddress('1:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
-        # test a IPv6 is sumamrized properly
-        self.assertEqual(summarize(ip1, ip2)[0], ipaddr.IPNetwork('1::/16'))
-        # test an IPv6 range that isn't on a network byte boundary
-        ip2 = ipaddr.IPAddress('2::')
-        self.assertEqual(summarize(ip1, ip2), [ipaddr.IPNetwork('1::/16'),
-                                               ipaddr.IPNetwork('2::/128')])
-
-        # test exception raised when first is greater than last
-        self.assertRaises(ValueError, summarize, ipaddr.IPAddress('1.1.1.0'),
-            ipaddr.IPAddress('1.1.0.0'))
-        # test exception raised when first and last aren't IP addresses
-        self.assertRaises(TypeError, summarize,
-                          ipaddr.IPNetwork('1.1.1.0'),
-                          ipaddr.IPNetwork('1.1.0.0'))
-        self.assertRaises(TypeError, summarize,
-            ipaddr.IPNetwork('1.1.1.0'), ipaddr.IPNetwork('1.1.0.0'))
-        # test exception raised when first and last are not same version
-        self.assertRaises(TypeError, summarize, ipaddr.IPAddress('::'),
-            ipaddr.IPNetwork('1.1.0.0'))
-
-    def testAddressComparison(self):
-        self.assertTrue(ipaddr.IPAddress('1.1.1.1') <=
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertTrue(ipaddr.IPAddress('1.1.1.1') <=
-                        ipaddr.IPAddress('1.1.1.2'))
-        self.assertTrue(ipaddr.IPAddress('::1') <= ipaddr.IPAddress('::1'))
-        self.assertTrue(ipaddr.IPAddress('::1') <= ipaddr.IPAddress('::2'))
-
-    def testNetworkComparison(self):
-        # ip1 and ip2 have the same network address
-        ip1 = ipaddr.IPv4Network('1.1.1.0/24')
-        ip2 = ipaddr.IPv4Network('1.1.1.1/24')
-        ip3 = ipaddr.IPv4Network('1.1.2.0/24')
-
-        self.assertTrue(ip1 < ip3)
-        self.assertTrue(ip3 > ip2)
-
-        self.assertEqual(ip1.compare_networks(ip2), 0)
-        self.assertTrue(ip1._get_networks_key() == ip2._get_networks_key())
-        self.assertEqual(ip1.compare_networks(ip3), -1)
-        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
-
-        ip1 = ipaddr.IPv6Network('2001::2000/96')
-        ip2 = ipaddr.IPv6Network('2001::2001/96')
-        ip3 = ipaddr.IPv6Network('2001:ffff::2000/96')
-
-        self.assertTrue(ip1 < ip3)
-        self.assertTrue(ip3 > ip2)
-        self.assertEqual(ip1.compare_networks(ip2), 0)
-        self.assertTrue(ip1._get_networks_key() == ip2._get_networks_key())
-        self.assertEqual(ip1.compare_networks(ip3), -1)
-        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
-
-        # Test comparing different protocols.
-        # Should always raise a TypeError.
-        ipv6 = ipaddr.IPv6Network('::/0')
-        ipv4 = ipaddr.IPv4Network('0.0.0.0/0')
-        self.assertRaises(TypeError, ipv4.__lt__, ipv6)
-        self.assertRaises(TypeError, ipv4.__gt__, ipv6)
-        self.assertRaises(TypeError, ipv6.__lt__, ipv4)
-        self.assertRaises(TypeError, ipv6.__gt__, ipv4)
-
-        # Regression test for issue 19.
-        ip1 = ipaddr.IPNetwork('10.1.2.128/25')
-        self.assertFalse(ip1 < ip1)
-        self.assertFalse(ip1 > ip1)
-        ip2 = ipaddr.IPNetwork('10.1.3.0/24')
-        self.assertTrue(ip1 < ip2)
-        self.assertFalse(ip2 < ip1)
-        self.assertFalse(ip1 > ip2)
-        self.assertTrue(ip2 > ip1)
-        ip3 = ipaddr.IPNetwork('10.1.3.0/25')
-        self.assertTrue(ip2 < ip3)
-        self.assertFalse(ip3 < ip2)
-        self.assertFalse(ip2 > ip3)
-        self.assertTrue(ip3 > ip2)
-
-        # Regression test for issue 28.
-        ip1 = ipaddr.IPNetwork('10.10.10.0/31')
-        ip2 = ipaddr.IPNetwork('10.10.10.0')
-        ip3 = ipaddr.IPNetwork('10.10.10.2/31')
-        ip4 = ipaddr.IPNetwork('10.10.10.2')
-        sorted = [ip1, ip2, ip3, ip4]
-        unsorted = [ip2, ip4, ip1, ip3]
-        unsorted.sort()
-        self.assertEqual(sorted, unsorted)
-        unsorted = [ip4, ip1, ip3, ip2]
-        unsorted.sort()
-        self.assertEqual(sorted, unsorted)
-        self.assertRaises(TypeError, ip1.__lt__, ipaddr.IPAddress('10.10.10.0'))
-        self.assertRaises(TypeError, ip2.__lt__, ipaddr.IPAddress('10.10.10.0'))
-
-        # <=, >=
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1') <=
-                        ipaddr.IPNetwork('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1') <=
-                        ipaddr.IPNetwork('1.1.1.2'))
-        self.assertFalse(ipaddr.IPNetwork('1.1.1.2') <=
-                        ipaddr.IPNetwork('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('::1') <= ipaddr.IPNetwork('::1'))
-        self.assertTrue(ipaddr.IPNetwork('::1') <= ipaddr.IPNetwork('::2'))
-        self.assertFalse(ipaddr.IPNetwork('::2') <= ipaddr.IPNetwork('::1'))
-
-    def testStrictNetworks(self):
-        self.assertRaises(ValueError, ipaddr.IPNetwork, '192.168.1.1/24',
-                          strict=True)
-        self.assertRaises(ValueError, ipaddr.IPNetwork, '::1/120', strict=True)
-
-    def testOverlaps(self):
-        other = ipaddr.IPv4Network('1.2.3.0/30')
-        other2 = ipaddr.IPv4Network('1.2.2.0/24')
-        other3 = ipaddr.IPv4Network('1.2.2.64/26')
-        self.assertTrue(self.ipv4.overlaps(other))
-        self.assertFalse(self.ipv4.overlaps(other2))
-        self.assertTrue(other2.overlaps(other3))
-
-    def testEmbeddedIpv4(self):
-        ipv4_string = '192.168.0.1'
-        ipv4 = ipaddr.IPv4Network(ipv4_string)
-        v4compat_ipv6 = ipaddr.IPv6Network('::%s' % ipv4_string)
-        self.assertEqual(int(v4compat_ipv6.ip), int(ipv4.ip))
-        v4mapped_ipv6 = ipaddr.IPv6Network('::ffff:%s' % ipv4_string)
-        self.assertNotEqual(v4mapped_ipv6.ip, ipv4.ip)
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '2001:1.1.1.1:1.1.1.1')
-
-    # Issue 67: IPv6 with embedded IPv4 address not recognized.
-    def testIPv6AddressTooLarge(self):
-        # RFC4291 2.5.5.2
-        self.assertEqual(ipaddr.IPAddress('::FFFF:192.0.2.1'),
-                          ipaddr.IPAddress('::FFFF:c000:201'))
-        # RFC4291 2.2 (part 3) x::d.d.d.d 
-        self.assertEqual(ipaddr.IPAddress('FFFF::192.0.2.1'),
-                          ipaddr.IPAddress('FFFF::c000:201'))
-
-    def testIPVersion(self):
-        self.assertEqual(self.ipv4.version, 4)
-        self.assertEqual(self.ipv6.version, 6)
-
-    def testMaxPrefixLength(self):
-        self.assertEqual(self.ipv4.max_prefixlen, 32)
-        self.assertEqual(self.ipv6.max_prefixlen, 128)
-
-    def testPacked(self):
-        self.assertEqual(self.ipv4.packed,
-                         _cb('\x01\x02\x03\x04'))
-        self.assertEqual(ipaddr.IPv4Network('255.254.253.252').packed,
-                         _cb('\xff\xfe\xfd\xfc'))
-        self.assertEqual(self.ipv6.packed,
-                         _cb('\x20\x01\x06\x58\x02\x2a\xca\xfe'
-                             '\x02\x00\x00\x00\x00\x00\x00\x01'))
-        self.assertEqual(ipaddr.IPv6Network('ffff:2:3:4:ffff::').packed,
-                         _cb('\xff\xff\x00\x02\x00\x03\x00\x04\xff\xff'
-                            + '\x00' * 6))
-        self.assertEqual(ipaddr.IPv6Network('::1:0:0:0:0').packed,
-                         _cb('\x00' * 6 + '\x00\x01' + '\x00' * 8))
-
-    def testIpStrFromPrefixlen(self):
-        ipv4 = ipaddr.IPv4Network('1.2.3.4/24')
-        self.assertEqual(ipv4._ip_string_from_prefix(), '255.255.255.0')
-        self.assertEqual(ipv4._ip_string_from_prefix(28), '255.255.255.240')
-
-    def testIpType(self):
-        ipv4net = ipaddr.IPNetwork('1.2.3.4')
-        ipv4addr = ipaddr.IPAddress('1.2.3.4')
-        ipv6net = ipaddr.IPNetwork('::1.2.3.4')
-        ipv6addr = ipaddr.IPAddress('::1.2.3.4')
-        self.assertEqual(ipaddr.IPv4Network, type(ipv4net))
-        self.assertEqual(ipaddr.IPv4Address, type(ipv4addr))
-        self.assertEqual(ipaddr.IPv6Network, type(ipv6net))
-        self.assertEqual(ipaddr.IPv6Address, type(ipv6addr))
-
-    def testReservedIpv4(self):
-        # test networks
-        self.assertEqual(True, ipaddr.IPNetwork('224.1.1.1/31').is_multicast)
-        self.assertEqual(False, ipaddr.IPNetwork('240.0.0.0').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPNetwork('192.168.1.1/17').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('192.169.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork('10.255.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('11.0.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork('172.31.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('172.32.0.0').is_private)
-
-        self.assertEqual(True,
-                          ipaddr.IPNetwork('169.254.100.200/24').is_link_local)
-        self.assertEqual(False,
-                          ipaddr.IPNetwork('169.255.100.200/24').is_link_local)
-
-        self.assertEqual(True,
-                          ipaddr.IPNetwork('127.100.200.254/32').is_loopback)
-        self.assertEqual(True, ipaddr.IPNetwork('127.42.0.0/16').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('128.0.0.0').is_loopback)
-
-        # test addresses
-        self.assertEqual(True, ipaddr.IPAddress('224.1.1.1').is_multicast)
-        self.assertEqual(False, ipaddr.IPAddress('240.0.0.0').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPAddress('192.168.1.1').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('192.169.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPAddress('10.255.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('11.0.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPAddress('172.31.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('172.32.0.0').is_private)
-
-        self.assertEqual(True,
-                          ipaddr.IPAddress('169.254.100.200').is_link_local)
-        self.assertEqual(False,
-                          ipaddr.IPAddress('169.255.100.200').is_link_local)
-
-        self.assertEqual(True,
-                          ipaddr.IPAddress('127.100.200.254').is_loopback)
-        self.assertEqual(True, ipaddr.IPAddress('127.42.0.0').is_loopback)
-        self.assertEqual(False, ipaddr.IPAddress('128.0.0.0').is_loopback)
-        self.assertEqual(True, ipaddr.IPNetwork('0.0.0.0').is_unspecified)
-
-    def testReservedIpv6(self):
-
-        self.assertEqual(True, ipaddr.IPNetwork('ffff::').is_multicast)
-        self.assertEqual(True, ipaddr.IPNetwork(2**128-1).is_multicast)
-        self.assertEqual(True, ipaddr.IPNetwork('ff00::').is_multicast)
-        self.assertEqual(False, ipaddr.IPNetwork('fdff::').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fecf::').is_site_local)
-        self.assertEqual(True, ipaddr.IPNetwork(
-                'feff:ffff:ffff:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fbf:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPNetwork('ff00::').is_site_local)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fc00::').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork(
-                'fc00:ffff:ffff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('fbff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('fe00::').is_private)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fea0::').is_link_local)
-        self.assertEqual(True, ipaddr.IPNetwork('febf:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fe7f:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fec0::').is_link_local)
-
-        self.assertEqual(True, ipaddr.IPNetwork('0:0::0:01').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::1/127').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::2').is_loopback)
-
-        self.assertEqual(True, ipaddr.IPNetwork('0::0').is_unspecified)
-        self.assertEqual(False, ipaddr.IPNetwork('::1').is_unspecified)
-        self.assertEqual(False, ipaddr.IPNetwork('::/127').is_unspecified)
-
-        # test addresses
-        self.assertEqual(True, ipaddr.IPAddress('ffff::').is_multicast)
-        self.assertEqual(True, ipaddr.IPAddress(2**128-1).is_multicast)
-        self.assertEqual(True, ipaddr.IPAddress('ff00::').is_multicast)
-        self.assertEqual(False, ipaddr.IPAddress('fdff::').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPAddress('fecf::').is_site_local)
-        self.assertEqual(True, ipaddr.IPAddress(
-                'feff:ffff:ffff:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPAddress('fbf:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPAddress('ff00::').is_site_local)
-
-        self.assertEqual(True, ipaddr.IPAddress('fc00::').is_private)
-        self.assertEqual(True, ipaddr.IPAddress(
-                'fc00:ffff:ffff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('fbff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('fe00::').is_private)
-
-        self.assertEqual(True, ipaddr.IPAddress('fea0::').is_link_local)
-        self.assertEqual(True, ipaddr.IPAddress('febf:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPAddress('fe7f:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPAddress('fec0::').is_link_local)
-
-        self.assertEqual(True, ipaddr.IPAddress('0:0::0:01').is_loopback)
-        self.assertEqual(True, ipaddr.IPAddress('::1').is_loopback)
-        self.assertEqual(False, ipaddr.IPAddress('::2').is_loopback)
-
-        self.assertEqual(True, ipaddr.IPAddress('0::0').is_unspecified)
-        self.assertEqual(False, ipaddr.IPAddress('::1').is_unspecified)
-
-        # some generic IETF reserved addresses
-        self.assertEqual(True, ipaddr.IPAddress('100::').is_reserved)
-        self.assertEqual(True, ipaddr.IPNetwork('4000::1/128').is_reserved)
-
-    def testIpv4Mapped(self):
-        self.assertEqual(ipaddr.IPAddress('::ffff:192.168.1.1').ipv4_mapped,
-                         ipaddr.IPAddress('192.168.1.1'))
-        self.assertEqual(ipaddr.IPAddress('::c0a8:101').ipv4_mapped, None)
-        self.assertEqual(ipaddr.IPAddress('::ffff:c0a8:101').ipv4_mapped,
-                         ipaddr.IPAddress('192.168.1.1'))
-
-    def testAddrExclude(self):
-        addr1 = ipaddr.IPNetwork('10.1.1.0/24')
-        addr2 = ipaddr.IPNetwork('10.1.1.0/26')
-        addr3 = ipaddr.IPNetwork('10.2.1.0/24')
-        addr4 = ipaddr.IPAddress('10.1.1.0')
-        self.assertEqual(addr1.address_exclude(addr2),
-                         [ipaddr.IPNetwork('10.1.1.64/26'),
-                          ipaddr.IPNetwork('10.1.1.128/25')])
-        self.assertRaises(ValueError, addr1.address_exclude, addr3)
-        self.assertRaises(TypeError, addr1.address_exclude, addr4)
-        self.assertEqual(addr1.address_exclude(addr1), [])
-
-    def testHash(self):
-        self.assertEqual(hash(ipaddr.IPNetwork('10.1.1.0/24')),
-                          hash(ipaddr.IPNetwork('10.1.1.0/24')))
-        self.assertEqual(hash(ipaddr.IPAddress('10.1.1.0')),
-                          hash(ipaddr.IPAddress('10.1.1.0')))
-        # i70
-        self.assertEqual(hash(ipaddr.IPAddress('1.2.3.4')),
-                          hash(ipaddr.IPAddress(
-                    long(ipaddr.IPAddress('1.2.3.4')._ip))))
-        ip1 = ipaddr.IPAddress('10.1.1.0')
-        ip2 = ipaddr.IPAddress('1::')
-        dummy = {}
-        dummy[self.ipv4] = None
-        dummy[self.ipv6] = None
-        dummy[ip1] = None
-        dummy[ip2] = None
-        self.assertTrue(self.ipv4 in dummy)
-        self.assertTrue(ip2 in dummy)
-
-    def testCopyConstructor(self):
-        addr1 = ipaddr.IPNetwork('10.1.1.0/24')
-        addr2 = ipaddr.IPNetwork(addr1)
-        addr3 = ipaddr.IPNetwork('2001:658:22a:cafe:200::1/64')
-        addr4 = ipaddr.IPNetwork(addr3)
-        addr5 = ipaddr.IPv4Address('1.1.1.1')
-        addr6 = ipaddr.IPv6Address('2001:658:22a:cafe:200::1')
-
-        self.assertEqual(addr1, addr2)
-        self.assertEqual(addr3, addr4)
-        self.assertEqual(addr5, ipaddr.IPv4Address(addr5))
-        self.assertEqual(addr6, ipaddr.IPv6Address(addr6))
-
-    def testCompressIPv6Address(self):
-        test_addresses = {
-            '1:2:3:4:5:6:7:8': '1:2:3:4:5:6:7:8/128',
-            '2001:0:0:4:0:0:0:8': '2001:0:0:4::8/128',
-            '2001:0:0:4:5:6:7:8': '2001::4:5:6:7:8/128',
-            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
-            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
-            '0:0:3:0:0:0:0:ffff': '0:0:3::ffff/128',
-            '0:0:0:4:0:0:0:ffff': '::4:0:0:0:ffff/128',
-            '0:0:0:0:5:0:0:ffff': '::5:0:0:ffff/128',
-            '1:0:0:4:0:0:7:8': '1::4:0:0:7:8/128',
-            '0:0:0:0:0:0:0:0': '::/128',
-            '0:0:0:0:0:0:0:0/0': '::/0',
-            '0:0:0:0:0:0:0:1': '::1/128',
-            '2001:0658:022a:cafe:0000:0000:0000:0000/66':
-            '2001:658:22a:cafe::/66',
-            '::1.2.3.4': '::102:304/128',
-            '1:2:3:4:5:ffff:1.2.3.4': '1:2:3:4:5:ffff:102:304/128',
-            '::7:6:5:4:3:2:1': '0:7:6:5:4:3:2:1/128',
-            '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128',
-            '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128',
-            '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128',
-            }
-        for uncompressed, compressed in test_addresses.items():
-            self.assertEqual(compressed, str(ipaddr.IPv6Network(uncompressed)))
-
-    def testExplodeShortHandIpStr(self):
-        addr1 = ipaddr.IPv6Network('2001::1')
-        addr2 = ipaddr.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
-        self.assertEqual('2001:0000:0000:0000:0000:0000:0000:0001',
-                         addr1._explode_shorthand_ip_string(str(addr1.ip)))
-        self.assertEqual('0000:0000:0000:0000:0000:0000:0000:0001',
-                         ipaddr.IPv6Network('::1/128').exploded)
-        # issue 77
-        self.assertEqual('2001:0000:5ef5:79fd:0000:059d:a0e5:0ba1',
-                         addr2.exploded)
-
-    def testIntRepresentation(self):
-        self.assertEqual(16909060, int(self.ipv4))
-        self.assertEqual(42540616829182469433547762482097946625, int(self.ipv6))
-
-    def testHexRepresentation(self):
-        self.assertEqual(hex(0x1020304),
-                         hex(self.ipv4))
-
-        self.assertEqual(hex(0x20010658022ACAFE0200000000000001),
-                         hex(self.ipv6))
-
-    # backwards compatibility
-    def testBackwardsCompability(self):
-        self.assertEqual(ipaddr.CollapseAddrList(
-            [ipaddr.IPNetwork('1.1.0.0/24'), ipaddr.IPNetwork('1.1.1.0/24')]),
-                         [ipaddr.IPNetwork('1.1.0.0/23')])
-
-        self.assertEqual(ipaddr.IPNetwork('::42:0/112').AddressExclude(
-            ipaddr.IPNetwork('::42:8000/113')),
-                         [ipaddr.IPNetwork('::42:0/113')])
-
-        self.assertTrue(ipaddr.IPNetwork('1::/8').CompareNetworks(
-            ipaddr.IPNetwork('2::/9')) < 0)
-
-        self.assertEqual(ipaddr.IPNetwork('1::/16').Contains(
-            ipaddr.IPNetwork('2::/16')), False)
-
-        self.assertEqual(ipaddr.IPNetwork('0.0.0.0/0').Subnet(),
-                         [ipaddr.IPNetwork('0.0.0.0/1'),
-                          ipaddr.IPNetwork('128.0.0.0/1')])
-        self.assertEqual(ipaddr.IPNetwork('::/127').Subnet(),
-                         [ipaddr.IPNetwork('::/128'),
-                          ipaddr.IPNetwork('::1/128')])
-
-        self.assertEqual(ipaddr.IPNetwork('1.0.0.0/32').Supernet(),
-                         ipaddr.IPNetwork('1.0.0.0/31'))
-        self.assertEqual(ipaddr.IPNetwork('::/121').Supernet(),
-                         ipaddr.IPNetwork('::/120'))
-
-        self.assertEqual(ipaddr.IPNetwork('10.0.0.2').IsRFC1918(), True)
-        self.assertEqual(ipaddr.IPNetwork('10.0.0.0').IsMulticast(), False)
-        self.assertEqual(ipaddr.IPNetwork('127.255.255.255').IsLoopback(), True)
-        self.assertEqual(ipaddr.IPNetwork('169.255.255.255').IsLinkLocal(),
-                         False)
-
-    def testForceVersion(self):
-        self.assertEqual(ipaddr.IPNetwork(1).version, 4)
-        self.assertEqual(ipaddr.IPNetwork(1, version=6).version, 6)
-
-    def testWithStar(self):
-        self.assertEqual(str(self.ipv4.with_prefixlen), "1.2.3.4/24")
-        self.assertEqual(str(self.ipv4.with_netmask), "1.2.3.4/255.255.255.0")
-        self.assertEqual(str(self.ipv4.with_hostmask), "1.2.3.4/0.0.0.255")
-
-        self.assertEqual(str(self.ipv6.with_prefixlen),
-                         '2001:658:22a:cafe:200::1/64')
-        # rfc3513 sec 2.3 says that ipv6 only uses cidr notation for
-        # subnets
-        self.assertEqual(str(self.ipv6.with_netmask),
-                         '2001:658:22a:cafe:200::1/64')
-        # this probably don't make much sense, but it's included for
-        # compatibility with ipv4
-        self.assertEqual(str(self.ipv6.with_hostmask),
-                         '2001:658:22a:cafe:200::1/::ffff:ffff:ffff:ffff')
-
-    def testNetworkElementCaching(self):
-        # V4 - make sure we're empty
-        self.assertFalse(self.ipv4._cache.has_key('network'))
-        self.assertFalse(self.ipv4._cache.has_key('broadcast'))
-        self.assertFalse(self.ipv4._cache.has_key('hostmask'))
-
-        # V4 - populate and test
-        self.assertEqual(self.ipv4.network, ipaddr.IPv4Address('1.2.3.0'))
-        self.assertEqual(self.ipv4.broadcast, ipaddr.IPv4Address('1.2.3.255'))
-        self.assertEqual(self.ipv4.hostmask, ipaddr.IPv4Address('0.0.0.255'))
-
-        # V4 - check we're cached
-        self.assertTrue(self.ipv4._cache.has_key('network'))
-        self.assertTrue(self.ipv4._cache.has_key('broadcast'))
-        self.assertTrue(self.ipv4._cache.has_key('hostmask'))
-
-        # V6 - make sure we're empty
-        self.assertFalse(self.ipv6._cache.has_key('network'))
-        self.assertFalse(self.ipv6._cache.has_key('broadcast'))
-        self.assertFalse(self.ipv6._cache.has_key('hostmask'))
-
-        # V6 - populate and test
-        self.assertEqual(self.ipv6.network,
-                         ipaddr.IPv6Address('2001:658:22a:cafe::'))
-        self.assertEqual(self.ipv6.broadcast, ipaddr.IPv6Address(
-            '2001:658:22a:cafe:ffff:ffff:ffff:ffff'))
-        self.assertEqual(self.ipv6.hostmask,
-                         ipaddr.IPv6Address('::ffff:ffff:ffff:ffff'))
-
-        # V6 - check we're cached
-        self.assertTrue(self.ipv6._cache.has_key('network'))
-        self.assertTrue(self.ipv6._cache.has_key('broadcast'))
-        self.assertTrue(self.ipv6._cache.has_key('hostmask'))
-
-    def testTeredo(self):
-        # stolen from wikipedia
-        server = ipaddr.IPv4Address('65.54.227.120')
-        client = ipaddr.IPv4Address('192.0.2.45')
-        teredo_addr = '2001:0000:4136:e378:8000:63bf:3fff:fdd2'
-        self.assertEqual((server, client),
-                         ipaddr.IPAddress(teredo_addr).teredo)
-        bad_addr = '2000::4136:e378:8000:63bf:3fff:fdd2'
-        self.assertFalse(ipaddr.IPAddress(bad_addr).teredo)
-        bad_addr = '2001:0001:4136:e378:8000:63bf:3fff:fdd2'
-        self.assertFalse(ipaddr.IPAddress(bad_addr).teredo)
-
-        # i77
-        teredo_addr = ipaddr.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
-        self.assertEqual((ipaddr.IPv4Address('94.245.121.253'),
-                          ipaddr.IPv4Address('95.26.244.94')),
-                         teredo_addr.teredo)
-
-
-    def testsixtofour(self):
-        sixtofouraddr = ipaddr.IPAddress('2002:ac1d:2d64::1')
-        bad_addr = ipaddr.IPAddress('2000:ac1d:2d64::1')
-        self.assertEqual(ipaddr.IPv4Address('172.29.45.100'),
-                         sixtofouraddr.sixtofour)
-        self.assertFalse(bad_addr.sixtofour)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/contrib/ipaddr-py/setup.py b/contrib/ipaddr-py/setup.py
deleted file mode 100755
index 33564320e45..00000000000
--- a/contrib/ipaddr-py/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2008 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from distutils.core import setup
-
-import ipaddr
-
-
-setup(name='ipaddr',
-      maintainer='Google',
-      maintainer_email='ipaddr-py-dev@googlegroups.com',
-      version=ipaddr.__version__,
-      url='http://code.google.com/p/ipaddr-py/',
-      license='Apache License, Version 2.0',
-      classifiers=[
-          'Development Status :: 5 - Production/Stable',
-          'Intended Audience :: Developers',
-          'License :: OSI Approved :: Apache Software License',
-          'Operating System :: OS Independent',
-          'Topic :: Internet',
-          'Topic :: Software Development :: Libraries',
-          'Topic :: System :: Networking'],
-      py_modules=['ipaddr'])
diff --git a/contrib/ipaddr-py/test-2to3.sh b/contrib/ipaddr-py/test-2to3.sh
deleted file mode 100755
index 408d665bcc2..00000000000
--- a/contrib/ipaddr-py/test-2to3.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-# Converts the python2 ipaddr files to python3 and runs the unit tests
-# with both python versions.
-
-mkdir -p 2to3output && \
-cp -f *.py 2to3output && \
-( cd 2to3output && 2to3 . | patch -p0 ) && \
-py3version=$(python3 --version 2>&1) && \
-echo -e "\nTesting with ${py3version}" && \
-python3 2to3output/ipaddr_test.py && \
-rm -r 2to3output && \
-pyversion=$(python --version 2>&1) && \
-echo -e "\nTesting with ${pyversion}" && \
-./ipaddr_test.py
diff --git a/contrib/macfuse/mount_darwin.c b/contrib/macfuse/mount_darwin.c
index 10eff204bc6..d1d1c34e761 100644
--- a/contrib/macfuse/mount_darwin.c
+++ b/contrib/macfuse/mount_darwin.c
@@ -34,16 +34,15 @@
 #include "fuse_param.h"
 #include "fuse_ioctl.h"
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
 
 int
-gf_fuse_mount (const char *mountpoint, char *fsname,
-               unsigned long mountflags, char *mnt_param,
+gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
                pid_t *mnt_pid, int status_fd) /* Not used on OS X */
 {
         int fd       = 0;
@@ -95,7 +94,7 @@ gf_fuse_mount (const char *mountpoint, char *fsname,
         }
 
         /* sysctlbyname() includes the trailing '\0' in version_len */
-        version_len_desired = strlen("2.x.y") + 1;
+        version_len_desired = sizeof ("2.x.y");
 
         if (version_len != version_len_desired) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
diff --git a/contrib/mount/mntent.c b/contrib/mount/mntent.c
index e9b448845a7..9a7e5f39bdb 100644
--- a/contrib/mount/mntent.c
+++ b/contrib/mount/mntent.c
@@ -36,6 +36,7 @@
  */
 
 #if !defined(GF_LINUX_HOST_OS)
+
 #include <stdlib.h>
 #include <string.h>
 #include <sys/param.h>
@@ -49,166 +50,209 @@ typedef struct statvfs gf_statfs_t;
 typedef struct statfs gf_statfs_t;
 #endif
 
-static int pos = -1;
-static int mntsize = -1;
-static struct mntent _mntent;
+typedef struct _mntent_state {
+        struct mntent  mntent;
+        gf_statfs_t   *statfs;
+        int            count;
+        int            pos;
+        /* A buffer big enough to store all defined flags as a string.
+         * Increase it if necessary when more flags are defined. */
+        char           buf[256];
+} mntent_state_t;
+
+typedef struct _mntflag {
+        unsigned long  value;
+        const char    *on;
+        const char    *off;
+} mntflag_t;
+
+static mntflag_t mntflags[] = {
+        { MNT_RDONLY,      "ro",          "rw" },
+        { MNT_SYNCHRONOUS, "sync",        NULL },
+        { MNT_NOEXEC,      "noexec",      NULL },
+        { MNT_NOSUID,      "nosuid",      NULL },
+#if !defined(__FreeBSD__)
+        { MNT_NODEV,       "nodev",       NULL },
+#endif /* __FreeBSD__ */
+        { MNT_UNION,       "union",       NULL },
+        { MNT_ASYNC,       "async",       NULL },
+#if !defined(GF_DARWIN_HOST_OS)
+        { MNT_NOATIME,     "noatime",     NULL },
+#if !defined(__NetBSD__)
+        { MNT_NOCLUSTERR,  "noclusterr",  NULL },
+        { MNT_NOCLUSTERW,  "noclusterw",  NULL },
+        { MNT_NOSYMFOLLOW, "nosymfollow", NULL },
+        { MNT_SUIDDIR,     "suiddir",     NULL },
+#endif /* !__NetBSD__ */
+#endif /* !GF_DARWIN_HOST_OS */
+        { 0,               NULL,          NULL }
+};
 
 char *
 hasmntopt (const struct mntent *mnt, const char *option)
 {
-        int found;
         char *opt, *optbuf;
+        int len;
 
         optbuf = strdup(mnt->mnt_opts);
-        found = 0;
-        for (opt = optbuf; (opt = strtok(opt, " ")) != NULL; opt = NULL) {
-                if (!strcasecmp(opt, option)) {
-                        opt = opt - optbuf + mnt->mnt_opts;
-                        free (optbuf);
-                        return (opt);
+        if (optbuf == NULL) {
+                return NULL;
+        }
+
+        opt = optbuf;
+        len = 0;
+        while (*opt) {
+                while (opt[len] != 0) {
+                        if (opt[len] == ' ') {
+                                opt[len++] = 0;
+                                break;
+                        }
+                        len++;
+                }
+                if ((*opt != 0) && (strcasecmp(opt, option) == 0)) {
+                        break;
                 }
+                opt += len;
+                len = 0;
         }
-        free (optbuf);
-        return (NULL);
+        free(optbuf);
+        if (len == 0) {
+                return NULL;
+        }
+
+        return opt - optbuf + mnt->mnt_opts;
 }
 
-static char *
-concatopt (char *s0, const char *s1)
+static int
+writeopt(const char *text, char *buf, int buflen, int pos)
 {
-        size_t i;
-        char *cp;
-
-        if (s1 == NULL || *s1 == '\0')
-                return s0;
-        if (s0 && *s0) {
-                i = strlen(s0) + strlen(s1) + 1 + 1;
-                if ((cp = (char *)malloc(i)) == NULL)
-                        return (NULL);
-                (void)snprintf(cp, i, "%s %s", s0, s1);
-        } else
-                cp = strdup(s1);
-
-        if (s0)
-                free(s0);
-        return (cp);
-}
+        int len;
+
+        /* buflen must be > 0 */
+
+        if (text == NULL) {
+                return pos;
+        }
+
+        buf += pos;
+        if (pos > 0) {
+                /* We are sure we have at least one byte to store the space.
+                 * We don't need to check buflen here. */
+                *buf++ = ' ';
+                pos++;
+        }
+        len = strlen(text) + 1;
+        pos += len;
+        if (pos >= buflen) {
+                /* There won't be enough space for the text and the
+                 * terminating null character. We copy as much as we can
+                 * of the text and mark the end of the string with '...' */
+                memcpy(buf, text, buflen - pos + len);
+                if (buflen > 3) {
+                        strcpy(buf + buflen - 4, "...");
+                } else {
+                        strncpy(buf, "...", buflen - 1);
+                        buf[buflen - 1] = 0;
+                }
+                pos = buflen;
+        } else {
+                memcpy(buf, text, len);
+        }
 
+        return pos;
+}
 
 static char *
-flags2opts (int flags)
+flags2opts (int flags, char *buf, int buflen)
 {
-        char *res;
-        res = NULL;
-        res = concatopt(res, (flags & MNT_RDONLY) ? "ro" : "rw");
-        if (flags & MNT_SYNCHRONOUS)    res = concatopt(res, "sync");
-        if (flags & MNT_NOEXEC)         res = concatopt(res, "noexec");
-        if (flags & MNT_NOSUID)         res = concatopt(res, "nosuid");
-#if !defined(__FreeBSD__)
-        if (flags & MNT_NODEV)          res = concatopt(res, "nodev");
-#endif /* __FreeBSD__ */
-        if (flags & MNT_UNION)          res = concatopt(res, "union");
-        if (flags & MNT_ASYNC)          res = concatopt(res, "async");
-#if !defined(GF_DARWIN_HOST_OS)
-        if (flags & MNT_NOATIME)        res = concatopt(res, "noatime");
-#if !defined(__NetBSD__)
-        if (flags & MNT_NOCLUSTERR)     res = concatopt(res, "noclusterr");
-        if (flags & MNT_NOCLUSTERW)     res = concatopt(res, "noclusterw");
-        if (flags & MNT_NOSYMFOLLOW)    res = concatopt(res, "nosymfollow");
-        if (flags & MNT_SUIDDIR)        res = concatopt(res, "suiddir");
-#endif /* !__NetBSD__ */
-#endif /* !GF_DARWIN_HOS_OS */
-        return res;
+        char other[16];
+        mntflag_t *flg;
+        int pos;
+
+        if (buflen == 0) {
+                return NULL;
+        }
+
+        pos = 0;
+        for (flg = mntflags; flg->value != 0; flg++) {
+                pos = writeopt((flags & flg->value) == 0 ? flg->off : flg->on,
+                               buf, buflen, pos);
+                flags &= ~flg->value;
+        }
+
+        if (flags != 0) {
+                sprintf(other, "[0x%x]", flags);
+                writeopt(other, buf, buflen, pos);
+        }
+
+        return buf;
 }
 
-static struct mntent *
-statfs_to_mntent (gf_statfs_t *mntbuf)
+static void
+statfs_to_mntent (struct mntent *mntent, gf_statfs_t *mntbuf, char *buf,
+                  int buflen)
 {
-        static char opts_buf[40], *tmp;
         int f_flags;
 
-        _mntent.mnt_fsname = mntbuf->f_mntfromname;
-        _mntent.mnt_dir = mntbuf->f_mntonname;
-        _mntent.mnt_type = mntbuf->f_fstypename;
+        mntent->mnt_fsname = mntbuf->f_mntfromname;
+        mntent->mnt_dir = mntbuf->f_mntonname;
+        mntent->mnt_type = mntbuf->f_fstypename;
 
 #ifdef __NetBSD__
         f_flags = mntbuf->f_flag;
 #else
         f_flags = mntbuf->f_flags;
 #endif
-        tmp = flags2opts (f_flags);
-        if (tmp) {
-                opts_buf[sizeof(opts_buf)-1] = '\0';
-                strncpy (opts_buf, tmp, sizeof(opts_buf)-1);
-                free (tmp);
-        } else {
-                *opts_buf = '\0';
-        }
-        _mntent.mnt_opts = opts_buf;
-        _mntent.mnt_freq = _mntent.mnt_passno = 0;
-        return (&_mntent);
+        mntent->mnt_opts = flags2opts (f_flags, buf, buflen);
+
+        mntent->mnt_freq = mntent->mnt_passno = 0;
 }
 
 struct mntent *
-getmntent (FILE *fp)
+getmntent_r (FILE *fp, struct mntent *mntent, char *buf, int buflen)
 {
-        gf_statfs_t *mntbuf;
+        mntent_state_t *state = (mntent_state_t *)fp;
 
-        if (!fp)
+        if (state->pos >= state->count) {
                 return NULL;
-
-        if (pos == -1 || mntsize == -1)
-                mntsize = getmntinfo (&mntbuf, MNT_NOWAIT);
-
-        ++pos;
-        if (pos == mntsize) {
-                pos = mntsize = -1;
-                return (NULL);
         }
 
-        return (statfs_to_mntent (&mntbuf[pos]));
+        statfs_to_mntent(mntent, &state->statfs[state->pos++], buf, buflen);
+
+        return mntent;
 }
 
-/*
-  Careful using this function ``buffer`` and ``bufsize`` are
-  ignored since there is no stream with strings to populate
-  them on OSX or NetBSD, if one wishes to populate them then
-  perhaps a new function should be written in this source file
-  which uses 'getmntinfo()' to stringify the mntent's
-*/
-
-struct mntent *getmntent_r (FILE *fp, struct mntent *result,
-                            char *buffer, int bufsize)
+struct mntent *
+getmntent (FILE *fp)
 {
-        struct mntent *ment = NULL;
-
-        if (!fp)
-                return NULL;
+        mntent_state_t *state = (mntent_state_t *)fp;
 
-        flockfile (fp);
-        ment = getmntent (fp);
-        memcpy (result, ment, sizeof(*ment));
-        funlockfile (fp);
-
-        return result;
+        return getmntent_r(fp, &state->mntent, state->buf,
+                           sizeof(state->buf));
 }
 
 FILE *
 setmntent (const char *filename, const char *type)
 {
-        FILE *fp = NULL;
-#ifdef GF_DARWIN_HOST_OS
-        fp = fopen (filename, "w");
-#else
-        fp = fopen (filename, type);
-#endif
-        return fp;
+        mntent_state_t *state;
+
+        /* We don't really need to access any file so we'll use the FILE* as
+         * a fake file to store state information.
+         */
+
+        state = malloc(sizeof(mntent_state_t));
+        if (state != NULL) {
+                state->pos = 0;
+                state->count = getmntinfo(&state->statfs, MNT_NOWAIT);
+        }
+
+        return (FILE *)state;
 }
 
 int
 endmntent (FILE *fp)
 {
-        if (fp)
-                fclose (fp);
+        free(fp);
 
         return 1; /* endmntent() always returns 1 */
 }
diff --git a/contrib/stdlib/gf_mkostemp.c b/contrib/stdlib/gf_mkostemp.c
deleted file mode 100644
index 931249a4520..00000000000
--- a/contrib/stdlib/gf_mkostemp.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* Borrowed from glibc-2.16/sysdeps/posix/tempname.c */
-
-/* Copyright (C) 1991-2001, 2006, 2007, 2009 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/time.h>
-#include <fcntl.h>
-#include <sys/types.h>
-#include <time.h>
-#include <inttypes.h>
-
-static const char letters[] =
-"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-
-/* Generate a temporary file name based on TMPL.  TMPL must match the
-   rules for mk[s]temp (i.e. end in "XXXXXX", possibly with a suffix).
-*/
-
-#if !defined(TMP_MAX)
-#define TMP_MAX 238328
-#endif
-
-int
-gf_mkostemp (char *tmpl, int suffixlen, int flags)
-{
-        int len;
-        char *XXXXXX;
-        static uint64_t value;
-        uint64_t random_time_bits;
-        unsigned int count;
-        int fd = -1;
-
-  /* A lower bound on the number of temporary files to attempt to
-     generate.  The maximum total number of temporary file names that
-     can exist for a given template is 62**6.  It should never be
-     necessary to try all these combinations.  Instead if a reasonable
-     number of names is tried (we define reasonable as 62**3) fail to
-     give the system administrator the chance to remove the problems.  */
-
-        unsigned int attempts = TMP_MAX; /* TMP_MAX == 62³ */
-
-        len = strlen (tmpl);
-        if (len < 6 + suffixlen || memcmp (&tmpl[len - 6 - suffixlen],
-                                           "XXXXXX", 6))
-                return -1;
-
-  /* This is where the Xs start.  */
-        XXXXXX = &tmpl[len - 6 - suffixlen];
-
-  /* Get some more or less random data.  */
-# if HAVE_GETTIMEOFDAY
-        struct timeval tv;
-        gettimeofday (&tv, NULL);
-        random_time_bits = ((uint64_t) tv.tv_usec << 16) ^ tv.tv_sec;
-# else
-        random_time_bits = time (NULL);
-# endif
-
-        value += random_time_bits ^ getpid ();
-
-        for (count = 0; count < attempts; value += 7777, ++count) {
-                uint64_t v = value;
-
-                /* Fill in the random bits.  */
-                XXXXXX[0] = letters[v % 62];
-                v /= 62;
-                XXXXXX[1] = letters[v % 62];
-                v /= 62;
-                XXXXXX[2] = letters[v % 62];
-                v /= 62;
-                XXXXXX[3] = letters[v % 62];
-                v /= 62;
-                XXXXXX[4] = letters[v % 62];
-                v /= 62;
-                XXXXXX[5] = letters[v % 62];
-
-                fd = open (tmpl, (flags & ~O_ACCMODE)
-                           | O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
-
-                if (fd >= 0)
-                        return fd;
-                else if (errno != EEXIST)
-                        return -1;
-        }
-
-        /* We got out of the loop because we ran out of combinations to try.  */
-        return -1;
-}
diff --git a/contrib/sunrpc/xdr_sizeof.c b/contrib/sunrpc/xdr_sizeof.c
deleted file mode 100644
index ca1f7bf0a5e..00000000000
--- a/contrib/sunrpc/xdr_sizeof.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.1 (the "License").  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON- INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-/*
- * Sun RPC is a product of Sun Microsystems, Inc. and is provided for
- * unrestricted use provided that this legend is included on all tape
- * media and as a part of the software program in whole or part.  Users
- * may copy or modify Sun RPC without charge, but are not authorized
- * to license or distribute it to anyone else except as part of a product or
- * program developed by the user.
- *
- * SUN RPC IS PROVIDED AS IS WITH NO WARRANTIES OF ANY KIND INCLUDING THE
- * WARRANTIES OF DESIGN, MERCHANTIBILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE, OR ARISING FROM A COURSE OF DEALING, USAGE OR TRADE PRACTICE.
- *
- * Sun RPC is provided with no support and without any obligation on the
- * part of Sun Microsystems, Inc. to assist in its use, correction,
- * modification or enhancement.
- *
- * SUN MICROSYSTEMS, INC. SHALL HAVE NO LIABILITY WITH RESPECT TO THE
- * INFRINGEMENT OF COPYRIGHTS, TRADE SECRETS OR ANY PATENTS BY SUN RPC
- * OR ANY PART THEREOF.
- *
- * In no event will Sun Microsystems, Inc. be liable for any lost revenue
- * or profits or other special, indirect and consequential damages, even if
- * Sun has been advised of the possibility of such damages.
- *
- * Sun Microsystems, Inc.
- * 2550 Garcia Avenue
- * Mountain View, California  94043
- */
-
-/*
- * xdr_sizeof.c
- *
- * Copyright 1990 Sun Microsystems, Inc.
- *
- * General purpose routine to see how much space something will use
- * when serialized using XDR.
- */
-
-#ifdef GF_DARWIN_HOST_OS
-
-#include <rpc/types.h>
-#include <rpc/xdr.h>
-#include <sys/types.h>
-#include <sys/cdefs.h>
-
-#include <stdlib.h>
-
-/* ARGSUSED */
-#ifdef GF_DARWIN_HOST_OS
-static bool_t
-x_putlong (XDR *xdrs, const int *longp)
-{
-        xdrs->x_handy += BYTES_PER_XDR_UNIT;
-        return TRUE;
-}
-
-#else
-static bool_t
-x_putlong (XDR *xdrs, const long *longp)
-{
-        xdrs->x_handy += BYTES_PER_XDR_UNIT;
-        return TRUE;
-}
-#endif
-
-/* ARGSUSED */
-static bool_t
-x_putbytes (XDR *xdrs, const char *bp, u_int len)
-{
-        xdrs->x_handy += len;
-        return TRUE;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-static u_int
-x_getpostn (XDR *xdrs)
-{
-        return xdrs->x_handy;
-}
-#else
-static u_int
-x_getpostn (const XDR *xdrs)
-{
-        return xdrs->x_handy;
-}
-#endif
-
-/* ARGSUSED */
-static bool_t
-x_setpostn (XDR *xdrs, u_int len)
-{
-        /* This is not allowed */
-        return FALSE;
-}
-
-static int32_t *
-x_inline (XDR *xdrs, u_int len)
-{
-        if (len == 0)
-                return NULL;
-        if (xdrs->x_op != XDR_ENCODE)
-                return NULL;
-        if (len < (u_int) (long int) xdrs->x_base)
-        {
-                /* x_private was already allocated */
-                xdrs->x_handy += len;
-                return (int32_t *) xdrs->x_private;
-        }
-        else
-        {
-                /* Free the earlier space and allocate new area */
-                free (xdrs->x_private);
-                if ((xdrs->x_private = (caddr_t) malloc (len)) == NULL)
-                {
-                        xdrs->x_base = 0;
-                        return NULL;
-                }
-                xdrs->x_base = (void *) (long) len;
-                xdrs->x_handy += len;
-                return (int32_t *) xdrs->x_private;
-        }
-}
-
-static int
-harmless (void)
-{
-        /* Always return FALSE/NULL, as the case may be */
-        return 0;
-}
-
-static void
-x_destroy (XDR *xdrs)
-{
-        xdrs->x_handy = 0;
-        xdrs->x_base = 0;
-        if (xdrs->x_private)
-        {
-                free (xdrs->x_private);
-                xdrs->x_private = NULL;
-        }
-        return;
-}
-
-unsigned long
-xdr_sizeof (xdrproc_t func, void *data)
-{
-        XDR x;
-        struct xdr_ops ops;
-        bool_t stat;
-
-#ifdef GF_DARWIN_HOST_OS
-        typedef bool_t (*dummyfunc1) (XDR *, int *);
-#else
-        typedef bool_t (*dummyfunc1) (XDR *, long *);
-#endif
-        typedef bool_t (*dummyfunc2) (XDR *, caddr_t, u_int);
-
-        ops.x_putlong = x_putlong;
-        ops.x_putbytes = x_putbytes;
-        ops.x_inline = x_inline;
-        ops.x_getpostn = x_getpostn;
-        ops.x_setpostn = x_setpostn;
-        ops.x_destroy = x_destroy;
-
-        /* the other harmless ones */
-        ops.x_getlong = (dummyfunc1) harmless;
-        ops.x_getbytes = (dummyfunc2) harmless;
-
-        x.x_op = XDR_ENCODE;
-        x.x_ops = &ops;
-        x.x_handy = 0;
-        x.x_private = (caddr_t) NULL;
-        x.x_base = (caddr_t) 0;
-
-        stat = func (&x, data, 0);
-        if (x.x_private)
-                free (x.x_private);
-        return (stat == TRUE ? (unsigned) x.x_handy : 0);
-}
-#endif /* GF_DARWIN_HOST_OS */
diff --git a/contrib/timer-wheel/find_last_bit.c b/contrib/timer-wheel/find_last_bit.c
index 054e90a076f..192fee802a8 100644
--- a/contrib/timer-wheel/find_last_bit.c
+++ b/contrib/timer-wheel/find_last_bit.c
@@ -1,18 +1,20 @@
-/*
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-*/
+/* bit search implementation
+ *
+ * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * Copyright (C) 2008 IBM Corporation
+ * 'find_last_bit' is written by Rusty Russell <rusty@rustcorp.com.au>
+ * (Inspired by David Howell's find_next_bit implementation)
+ *
+ * Rewritten by Yury Norov <yury.norov@gmail.com> to decrease
+ * size and improve performance, 2015.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
 
 /**
  * @find_last_bit
@@ -20,63 +22,40 @@
  */
 
 #ifndef BITS_PER_LONG
+#ifdef __LP64__
 #define BITS_PER_LONG 64
+#else
+#define BITS_PER_LONG 32
+#endif
 #endif
 
-static inline int fls(int x)
-{
-	int r = 32;
-
-	if (!x)
-		return 0;
-	if (!(x & 0xffff0000u)) {
-		x <<= 16;
-		r -= 16;
-	}
-	if (!(x & 0xff000000u)) {
-		x <<= 8;
-		r -= 8;
-	}
-	if (!(x & 0xf0000000u)) {
-		x <<= 4;
-		r -= 4;
-	}
-	if (!(x & 0xc0000000u)) {
-		x <<= 2;
-		r -= 2;
-	}
-	if (!(x & 0x80000000u)) {
-		x <<= 1;
-		r -= 1;
-	}
-	return r;
-}
-
-
-unsigned long gf_tw_find_last_bit(const unsigned long *addr, unsigned long size)
+unsigned long gw_tw_fls (unsigned long word)
 {
-	unsigned long words;
-	unsigned long tmp;
-
-	/* Start at final word. */
-	words = size / BITS_PER_LONG;
+        int num = BITS_PER_LONG;
 
-	/* Partial final word? */
-	if (size & (BITS_PER_LONG-1)) {
-		tmp = (addr[words] & (~0UL >> (BITS_PER_LONG
-					 - (size & (BITS_PER_LONG-1)))));
-		if (tmp)
-			goto found;
-	}
-
-	while (words) {
-		tmp = addr[--words];
-		if (tmp) {
-found:
-			return words * BITS_PER_LONG + fls(tmp);
-		}
-	}
-
-	/* Not found */
-	return size;
+#if BITS_PER_LONG == 64
+        if (!(word & (~0ul << 32))) {
+                num -= 32;
+                word <<= 32;
+        }
+#endif
+        if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
+                num -= 16;
+                word <<= 16;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
+                num -= 8;
+                word <<= 8;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
+                num -= 4;
+                word <<= 4;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
+                num -= 2;
+                word <<= 2;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-1))))
+                num -= 1;
+        return num;
 }
diff --git a/contrib/timer-wheel/timer-wheel.c b/contrib/timer-wheel/timer-wheel.c
index 013c0f278a1..58e0607bf0c 100644
--- a/contrib/timer-wheel/timer-wheel.c
+++ b/contrib/timer-wheel/timer-wheel.c
@@ -1,4 +1,12 @@
 /*
+ *  linux/kernel/timer.c
+ *
+ *  Kernel internal timers
+ *
+ *  Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ */
+/*
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
@@ -57,9 +65,17 @@ __gf_tw_add_timer (struct tvec_base *base, struct gf_tw_timer_list *timer)
         list_add_tail (&timer->entry, vec);
 }
 
-/* optimized find_last_bit() */
 unsigned long gf_tw_find_last_bit(const unsigned long *, unsigned long);
 
+#if defined(__GNUC__) || defined(__clang__)
+static inline unsigned long gf_tw_fls (unsigned long word)
+{
+        return BITS_PER_LONG - __builtin_clzl(word);
+}
+#else
+extern unsigned long gf_tw_fls (unsigned long);
+#endif
+
 static inline unsigned long
 apply_slack(struct tvec_base *base, struct gf_tw_timer_list *timer)
 {
@@ -77,7 +93,7 @@ apply_slack(struct tvec_base *base, struct gf_tw_timer_list *timer)
         if (mask == 0)
                 return expires;
 
-        int bit = gf_tw_find_last_bit (&mask, BITS_PER_LONG);
+        int bit = gf_tw_fls (mask);
         mask = (1UL << bit) - 1;
 
         expires_limit = expires_limit & ~(mask);
@@ -143,7 +159,14 @@ run_timers (struct tvec_base *base)
                         data = timer->data;
 
                         __gf_tw_detach_timer (timer);
-                        fn (timer, data, call_time);
+                        pthread_spin_unlock(&base->lock);
+                        {
+                            /* It is required to run the actual function outside
+                               of the locked zone, so we don't bother about
+                               locked operations inside that function */
+                            fn(timer, data, call_time);
+                        }
+                        pthread_spin_lock(&base->lock);
                 }
         }
         pthread_spin_unlock (&base->lock);
diff --git a/contrib/timer-wheel/timer-wheel.h b/contrib/timer-wheel/timer-wheel.h
index baa029ebb30..5637735ec22 100644
--- a/contrib/timer-wheel/timer-wheel.h
+++ b/contrib/timer-wheel/timer-wheel.h
@@ -17,9 +17,9 @@
 #ifndef __TIMER_WHEEL_H
 #define __TIMER_WHEEL_H
 
-#include "locking.h"
+#include "glusterfs/locking.h"
 
-#include "list.h"
+#include "glusterfs/list.h"
 
 #define TVR_BITS  8
 #define TVN_BITS  6
diff --git a/contrib/umountd/umountd.c b/contrib/umountd/umountd.c
index 0d2c6f20b60..3f933ecb554 100644
--- a/contrib/umountd/umountd.c
+++ b/contrib/umountd/umountd.c
@@ -23,11 +23,11 @@
 #include <sys/stat.h>
 #include <sys/mount.h>
 
-#include "glusterfs.h"
-#include "globals.h"
-#include "logging.h"
-#include "syscall.h"
-#include "mem-types.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/mem-types.h"
 
 static void
 usage (void)
@@ -56,7 +56,8 @@ sanity_check (char *path, dev_t *devp)
                         break;
                 default:
                         gf_log ("umountd", GF_LOG_ERROR,
-                                "Cannot access %s\n", path, strerror (errno));
+                                "Cannot access %s: %s\n",
+                                path, strerror (errno));
                         goto out;
                 }
         }
@@ -65,12 +66,13 @@ sanity_check (char *path, dev_t *devp)
         if (*devp == -1 && ret == 0)
                 *devp = st.st_dev;
 
-        strncpy (pathtmp, path, PATH_MAX);
+        snprintf (pathtmp, PATH_MAX, "%s", path);
         parent = dirname (pathtmp);
 
         if (stat (parent, &parent_st) != 0) {
                 gf_log ("umountd", GF_LOG_ERROR,
-                        "Cannot access %s\n", parent, strerror (errno));
+                        "Cannot access %s: %s\n",
+                        parent, strerror (errno));
                 goto out;
         }
 
diff --git a/contrib/userspace-rcu/static-wfcqueue.h b/contrib/userspace-rcu/static-wfcqueue.h
new file mode 100644
index 00000000000..37d14ad674b
--- /dev/null
+++ b/contrib/userspace-rcu/static-wfcqueue.h
@@ -0,0 +1,685 @@
+#ifndef _URCU_WFCQUEUE_STATIC_H
+#define _URCU_WFCQUEUE_STATIC_H
+
+/*
+ * urcu/static/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfcqueue.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Copied from userspace-rcu 0.10 because version 0.7 doesn't contain it. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ *
+ * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ *     [1] [2] [3] [4] [5] [6]
+ * [1]  -   -   -   -   -   -
+ * [2]  -   -   -   -   -   -
+ * [3]  -   -   X   X   X   X
+ * [4]  -   -   X   -   X   X
+ * [5]  -   -   X   X   -   -
+ * [6]  -   -   X   X   -   -
+ *
+ * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
+ *
+ * For convenience, cds_wfcq_dequeue_blocking() and
+ * cds_wfcq_splice_blocking() hold the dequeue lock.
+ *
+ * Besides locking, mutual exclusion of dequeue, splice and iteration
+ * can be ensured by performing all of those operations from a single
+ * thread, without requiring any lock.
+ */
+
+#define WFCQ_ADAPT_ATTEMPTS		10	/* Retry if being set */
+#define WFCQ_WAIT			10	/* Wait 10 ms if being set */
+
+/*
+ * cds_wfcq_node_init: initialize wait-free queue node.
+ */
+static inline void _cds_wfcq_node_init(struct cds_wfcq_node *node)
+{
+	node->next = NULL;
+}
+
+/*
+ * cds_wfcq_init: initialize wait-free queue (with lock). Pair with
+ * cds_wfcq_destroy().
+ */
+static inline void _cds_wfcq_init(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	/* Set queue head and tail */
+	_cds_wfcq_node_init(&head->node);
+	tail->p = &head->node;
+	ret = pthread_mutex_init(&head->lock, NULL);
+	assert(!ret);
+}
+
+/*
+ * cds_wfcq_destroy: destroy wait-free queue (with lock). Pair with
+ * cds_wfcq_init().
+ */
+static inline void _cds_wfcq_destroy(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret = pthread_mutex_destroy(&head->lock);
+	assert(!ret);
+}
+
+/*
+ * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
+ * pair with any destroy function.
+ */
+static inline void ___cds_wfcq_init(struct __cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	/* Set queue head and tail */
+	_cds_wfcq_node_init(&head->node);
+	tail->p = &head->node;
+}
+
+/*
+ * cds_wfcq_empty: return whether wait-free queue is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ *
+ * We perform the test on head->node.next to check if the queue is
+ * possibly empty, but we confirm this by checking if the tail pointer
+ * points to the head node because the tail pointer is the linearisation
+ * point of the enqueuers. Just checking the head next pointer could
+ * make a queue appear empty if an enqueuer is preempted for a long time
+ * between xchg() and setting the previous node's next pointer.
+ */
+static inline bool _cds_wfcq_empty(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	/*
+	 * Queue is empty if no node is pointed by head->node.next nor
+	 * tail->p. Even though the tail->p check is sufficient to find
+	 * out of the queue is empty, we first check head->node.next as a
+	 * common case to ensure that dequeuers do not frequently access
+	 * enqueuer's tail->p cache line.
+	 */
+	return CMM_LOAD_SHARED(head->node.next) == NULL
+		&& CMM_LOAD_SHARED(tail->p) == &head->node;
+}
+
+static inline void _cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	ret = pthread_mutex_lock(&head->lock);
+	assert(!ret);
+}
+
+static inline void _cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	ret = pthread_mutex_unlock(&head->lock);
+	assert(!ret);
+}
+
+static inline bool ___cds_wfcq_append(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *new_head,
+		struct cds_wfcq_node *new_tail)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *old_tail;
+
+	/*
+	 * Implicit memory barrier before uatomic_xchg() orders earlier
+	 * stores to data structure containing node and setting
+	 * node->next to NULL before publication.
+	 */
+	old_tail = uatomic_xchg(&tail->p, new_tail);
+
+	/*
+	 * Implicit memory barrier after uatomic_xchg() orders store to
+	 * q->tail before store to old_tail->next.
+	 *
+	 * At this point, dequeuers see a NULL tail->p->next, which
+	 * indicates that the queue is being appended to. The following
+	 * store will append "node" to the queue from a dequeuer
+	 * perspective.
+	 */
+	CMM_STORE_SHARED(old_tail->next, new_head);
+	/*
+	 * Return false if queue was empty prior to adding the node,
+	 * else return true.
+	 */
+	return old_tail != &head->node;
+}
+
+/*
+ * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
+ *
+ * Issues a full memory barrier before enqueue. No mutual exclusion is
+ * required.
+ *
+ * Returns false if the queue was empty prior to adding the node.
+ * Returns true otherwise.
+ */
+static inline bool _cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *new_tail)
+{
+	return ___cds_wfcq_append(head, tail, new_tail, new_tail);
+}
+
+/*
+ * CDS_WFCQ_WAIT_SLEEP:
+ *
+ * By default, this sleeps for the given @msec milliseconds.
+ * This is a macro which LGPL users may #define themselves before
+ * including wfcqueue.h to override the default behavior (e.g.
+ * to log a warning or perform other background work).
+ */
+#ifndef CDS_WFCQ_WAIT_SLEEP
+#define CDS_WFCQ_WAIT_SLEEP(msec) ___cds_wfcq_wait_sleep(msec)
+#endif
+
+static inline void ___cds_wfcq_wait_sleep(int msec)
+{
+	(void) poll(NULL, 0, msec);
+}
+
+/*
+ * ___cds_wfcq_busy_wait: adaptative busy-wait.
+ *
+ * Returns 1 if nonblocking and needs to block, 0 otherwise.
+ */
+static inline bool
+___cds_wfcq_busy_wait(int *attempt, int blocking)
+{
+	if (!blocking)
+		return 1;
+	if (++(*attempt) >= WFCQ_ADAPT_ATTEMPTS) {
+		CDS_WFCQ_WAIT_SLEEP(WFCQ_WAIT);		/* Wait for 10ms */
+		*attempt = 0;
+	} else {
+		caa_cpu_relax();
+	}
+	return 0;
+}
+
+/*
+ * Waiting for enqueuer to complete enqueue and return the next node.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_node_sync_next(struct cds_wfcq_node *node, int blocking)
+{
+	struct cds_wfcq_node *next;
+	int attempt = 0;
+
+	/*
+	 * Adaptative busy-looping waiting for enqueuer to complete enqueue.
+	 */
+	while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		if (___cds_wfcq_busy_wait(&attempt, blocking))
+			return CDS_WFCQ_WOULDBLOCK;
+	}
+
+	return next;
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_first(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		int blocking)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *node;
+
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail))
+		return NULL;
+	node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+	/* Load head->node.next before loading node's content */
+	cmm_smp_read_barrier_depends();
+	return node;
+}
+
+/*
+ * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if queue is empty, first node otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_first(head, tail, 1);
+}
+
+
+/*
+ * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_first(head, tail, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_next(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node,
+		int blocking)
+{
+	struct cds_wfcq_node *next;
+
+	/*
+	 * Even though the following tail->p check is sufficient to find
+	 * out if we reached the end of the queue, we first check
+	 * node->next as a common case to ensure that iteration on nodes
+	 * do not frequently access enqueuer's tail->p cache line.
+	 */
+	if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		/* Load node->next before tail->p */
+		cmm_smp_rmb();
+		if (CMM_LOAD_SHARED(tail->p) == node)
+			return NULL;
+		next = ___cds_wfcq_node_sync_next(node, blocking);
+	}
+	/* Load node->next before loading next's content */
+	cmm_smp_read_barrier_depends();
+	return next;
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of queue, non-NULL next queue node
+ * otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node)
+{
+	return ___cds_wfcq_next(head, tail, node, 1);
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node)
+{
+	return ___cds_wfcq_next(head, tail, node, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		int *state,
+		int blocking)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *node, *next;
+
+	if (state)
+		*state = 0;
+
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail)) {
+		return NULL;
+	}
+
+	node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+	if (!blocking && node == CDS_WFCQ_WOULDBLOCK) {
+		return CDS_WFCQ_WOULDBLOCK;
+	}
+
+	if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		/*
+		 * @node is probably the only node in the queue.
+		 * Try to move the tail to &q->head.
+		 * q->head.next is set to NULL here, and stays
+		 * NULL if the cmpxchg succeeds. Should the
+		 * cmpxchg fail due to a concurrent enqueue, the
+		 * q->head.next will be set to the next node.
+		 * The implicit memory barrier before
+		 * uatomic_cmpxchg() orders load node->next
+		 * before loading q->tail.
+		 * The implicit memory barrier before uatomic_cmpxchg
+		 * orders load q->head.next before loading node's
+		 * content.
+		 */
+		_cds_wfcq_node_init(&head->node);
+		if (uatomic_cmpxchg(&tail->p, node, &head->node) == node) {
+			if (state)
+				*state |= CDS_WFCQ_STATE_LAST;
+			return node;
+		}
+		next = ___cds_wfcq_node_sync_next(node, blocking);
+		/*
+		 * In nonblocking mode, if we would need to block to
+		 * get node's next, set the head next node pointer
+		 * (currently NULL) back to its original value.
+		 */
+		if (!blocking && next == CDS_WFCQ_WOULDBLOCK) {
+			head->node.next = node;
+			return CDS_WFCQ_WOULDBLOCK;
+		}
+	}
+
+	/*
+	 * Move queue head forward.
+	 */
+	head->node.next = next;
+
+	/* Load q->head.next before loading node's content */
+	cmm_smp_read_barrier_depends();
+	return node;
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_blocking: dequeue node from queue, with state.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	return ___cds_wfcq_dequeue_with_state(head, tail, state, 1);
+}
+
+/*
+ * ___cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_blocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_nonblocking: dequeue node, with state.
+ *
+ * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
+ * if it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	return ___cds_wfcq_dequeue_with_state(head, tail, state, 0);
+}
+
+/*
+ * ___cds_wfcq_dequeue_nonblocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_nonblocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_splice: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice(
+		cds_wfcq_head_ptr_t u_dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t u_src_q_head,
+		struct cds_wfcq_tail *src_q_tail,
+		int blocking)
+{
+	struct __cds_wfcq_head *dest_q_head = u_dest_q_head._h;
+	struct __cds_wfcq_head *src_q_head = u_src_q_head._h;
+	struct cds_wfcq_node *head, *tail;
+	int attempt = 0;
+
+	/*
+	 * Initial emptiness check to speed up cases where queue is
+	 * empty: only require loads to check if queue is empty.
+	 */
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(src_q_head), src_q_tail))
+		return CDS_WFCQ_RET_SRC_EMPTY;
+
+	for (;;) {
+		/*
+		 * Open-coded _cds_wfcq_empty() by testing result of
+		 * uatomic_xchg, as well as tail pointer vs head node
+		 * address.
+		 */
+		head = uatomic_xchg(&src_q_head->node.next, NULL);
+		if (head)
+			break;	/* non-empty */
+		if (CMM_LOAD_SHARED(src_q_tail->p) == &src_q_head->node)
+			return CDS_WFCQ_RET_SRC_EMPTY;
+		if (___cds_wfcq_busy_wait(&attempt, blocking))
+			return CDS_WFCQ_RET_WOULDBLOCK;
+	}
+
+	/*
+	 * Memory barrier implied before uatomic_xchg() orders store to
+	 * src_q->head before store to src_q->tail. This is required by
+	 * concurrent enqueue on src_q, which exchanges the tail before
+	 * updating the previous tail's next pointer.
+	 */
+	tail = uatomic_xchg(&src_q_tail->p, &src_q_head->node);
+
+	/*
+	 * Append the spliced content of src_q into dest_q. Does not
+	 * require mutual exclusion on dest_q (wait-free).
+	 */
+	if (___cds_wfcq_append(__cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+			head, tail))
+		return CDS_WFCQ_RET_DEST_NON_EMPTY;
+	else
+		return CDS_WFCQ_RET_DEST_EMPTY;
+}
+
+/*
+ * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_blocking(
+		cds_wfcq_head_ptr_t dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+		src_q_head, src_q_tail, 1);
+}
+
+/*
+ * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Same as __cds_wfcq_splice_blocking, but returns
+ * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_nonblocking(
+		cds_wfcq_head_ptr_t dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+		src_q_head, src_q_tail, 0);
+}
+
+/*
+ * cds_wfcq_dequeue_with_state_blocking: dequeue a node from a wait-free queue.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_splice_blocking and dequeue lock is
+ * ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	struct cds_wfcq_node *retval;
+
+	_cds_wfcq_dequeue_lock(head, tail);
+	retval = ___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_cast(head),
+			tail, state);
+	_cds_wfcq_dequeue_unlock(head, tail);
+	return retval;
+}
+
+/*
+ * cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as cds_wfcq_dequeue_blocking, but without saving state.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	return _cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
+ * ensured.
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+_cds_wfcq_splice_blocking(
+		struct cds_wfcq_head *dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		struct cds_wfcq_head *src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	enum cds_wfcq_ret ret;
+
+	_cds_wfcq_dequeue_lock(src_q_head, src_q_tail);
+	ret = ___cds_wfcq_splice_blocking(cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+			cds_wfcq_head_cast(src_q_head), src_q_tail);
+	_cds_wfcq_dequeue_unlock(src_q_head, src_q_tail);
+	return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_STATIC_H */
diff --git a/contrib/userspace-rcu/static-wfstack.h b/contrib/userspace-rcu/static-wfstack.h
new file mode 100644
index 00000000000..29b81c3aac3
--- /dev/null
+++ b/contrib/userspace-rcu/static-wfstack.h
@@ -0,0 +1,455 @@
+#ifndef _URCU_STATIC_WFSTACK_H
+#define _URCU_STATIC_WFSTACK_H
+
+/*
+ * urcu/static/wfstack.h
+ *
+ * Userspace RCU library - Stack with with wait-free push, blocking traversal.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfstack.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't support a stack
+ * without mutex. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CDS_WFS_END			((void *) 0x1UL)
+#define CDS_WFS_ADAPT_ATTEMPTS		10	/* Retry if being set */
+#define CDS_WFS_WAIT			10	/* Wait 10 ms if being set */
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ *                       cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ *                      iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ *                      cds_wfs_push  __cds_wfs_pop  __cds_wfs_pop_all
+ * cds_wfs_push               -              -                  -
+ * __cds_wfs_pop              -              X                  X
+ * __cds_wfs_pop_all          -              X                  -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+/*
+ * cds_wfs_node_init: initialize wait-free stack node.
+ */
+static inline
+void _cds_wfs_node_init(struct cds_wfs_node *node)
+{
+	node->next = NULL;
+}
+
+/*
+ * __cds_wfs_init: initialize wait-free stack. Don't pair with
+ * any destroy function.
+ */
+static inline void ___cds_wfs_init(struct __cds_wfs_stack *s)
+{
+	s->head = CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_init: initialize wait-free stack. Pair with
+ * cds_wfs_destroy().
+ */
+static inline
+void _cds_wfs_init(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	s->head = CDS_WFS_END;
+	ret = pthread_mutex_init(&s->lock, NULL);
+	assert(!ret);
+}
+
+/*
+ * cds_wfs_destroy: destroy wait-free stack. Pair with
+ * cds_wfs_init().
+ */
+static inline
+void _cds_wfs_destroy(struct cds_wfs_stack *s)
+{
+	int ret = pthread_mutex_destroy(&s->lock);
+	assert(!ret);
+}
+
+static inline bool ___cds_wfs_end(void *node)
+{
+	return node == CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_empty: return whether wait-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+static inline bool _cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+
+	return ___cds_wfs_end(CMM_LOAD_SHARED(s->head));
+}
+
+/*
+ * cds_wfs_push: push a node into the stack.
+ *
+ * Issues a full memory barrier before push. No mutual exclusion is
+ * required.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+static inline
+int _cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+	struct cds_wfs_head *old_head, *new_head;
+
+	assert(node->next == NULL);
+	new_head = caa_container_of(node, struct cds_wfs_head, node);
+	/*
+	 * uatomic_xchg() implicit memory barrier orders earlier stores
+	 * to node (setting it to NULL) before publication.
+	 */
+	old_head = uatomic_xchg(&s->head, new_head);
+	/*
+	 * At this point, dequeuers see a NULL node->next, they should
+	 * busy-wait until node->next is set to old_head.
+	 */
+	CMM_STORE_SHARED(node->next, &old_head->node);
+	return !___cds_wfs_end(old_head);
+}
+
+/*
+ * Waiting for push to complete enqueue and return the next node.
+ */
+static inline struct cds_wfs_node *
+___cds_wfs_node_sync_next(struct cds_wfs_node *node, int blocking)
+{
+	struct cds_wfs_node *next;
+	int attempt = 0;
+
+	/*
+	 * Adaptative busy-looping waiting for push to complete.
+	 */
+	while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		if (!blocking)
+			return CDS_WFS_WOULDBLOCK;
+		if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
+			(void) poll(NULL, 0, CDS_WFS_WAIT);	/* Wait for 10ms */
+			attempt = 0;
+		} else {
+			caa_cpu_relax();
+		}
+	}
+
+	return next;
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop(cds_wfs_stack_ptr_t u_stack, int *state, int blocking)
+{
+	struct cds_wfs_head *head, *new_head;
+	struct cds_wfs_node *next;
+	struct __cds_wfs_stack *s = u_stack._s;
+
+	if (state)
+		*state = 0;
+	for (;;) {
+		head = CMM_LOAD_SHARED(s->head);
+		if (___cds_wfs_end(head)) {
+			return NULL;
+		}
+		next = ___cds_wfs_node_sync_next(&head->node, blocking);
+		if (!blocking && next == CDS_WFS_WOULDBLOCK) {
+			return CDS_WFS_WOULDBLOCK;
+		}
+		new_head = caa_container_of(next, struct cds_wfs_head, node);
+		if (uatomic_cmpxchg(&s->head, head, new_head) == head) {
+			if (state && ___cds_wfs_end(new_head))
+				*state |= CDS_WFS_STATE_LAST;
+			return &head->node;
+		}
+		if (!blocking) {
+			return CDS_WFS_WOULDBLOCK;
+		}
+		/* busy-loop if head changed under us */
+	}
+}
+
+/*
+ * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_wfs_pop_blocking needs to be synchronized using one of the
+ * following techniques:
+ *
+ * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
+ *    section. The caller must wait for a grace period to pass before
+ *    freeing the returned node or modifying the cds_wfs_node structure.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ *     __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ *    and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+	return ___cds_wfs_pop(u_stack, state, 1);
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
+{
+	return ___cds_wfs_pop_with_state_blocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_with_state_blocking, but returns
+ * CDS_WFS_WOULDBLOCK if it needs to block.
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+	return ___cds_wfs_pop(u_stack, state, 0);
+}
+
+/*
+ * __cds_wfs_pop_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
+{
+	return ___cds_wfs_pop_with_state_nonblocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_wfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_wfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_wfs_pop_blocking:
+ *
+ * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
+ *    section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
+ *    must wait for a grace period to pass before freeing the returned
+ *    node or modifying the cds_wfs_node structure. However, no RCU
+ *    read-side critical section is needed around __cds_wfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ *     __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ *    and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_wfs_head *
+___cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+	struct cds_wfs_head *head;
+
+	/*
+	 * Implicit memory barrier after uatomic_xchg() matches implicit
+	 * memory barrier before uatomic_xchg() in cds_wfs_push. It
+	 * ensures that all nodes of the returned list are consistent.
+	 * There is no need to issue memory barriers when iterating on
+	 * the returned list, because the full memory barrier issued
+	 * prior to each uatomic_cmpxchg, which each write to head, are
+	 * taking care to order writes to each node prior to the full
+	 * memory barrier after this uatomic_xchg().
+	 */
+	head = uatomic_xchg(&s->head, CDS_WFS_END);
+	if (___cds_wfs_end(head))
+		return NULL;
+	return head;
+}
+
+/*
+ * cds_wfs_pop_lock: lock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_lock(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	ret = pthread_mutex_lock(&s->lock);
+	assert(!ret);
+}
+
+/*
+ * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_unlock(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	ret = pthread_mutex_unlock(&s->lock);
+	assert(!ret);
+}
+
+/*
+ * Call __cds_wfs_pop_with_state_blocking with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
+{
+	struct cds_wfs_node *retnode;
+
+	_cds_wfs_pop_lock(s);
+	retnode = ___cds_wfs_pop_with_state_blocking(s, state);
+	_cds_wfs_pop_unlock(s);
+	return retnode;
+}
+
+/*
+ * Call _cds_wfs_pop_with_state_blocking without saving any state.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+	return _cds_wfs_pop_with_state_blocking(s, NULL);
+}
+
+/*
+ * Call __cds_wfs_pop_all with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_head *
+_cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
+{
+	struct cds_wfs_head *rethead;
+
+	_cds_wfs_pop_lock(s);
+	rethead = ___cds_wfs_pop_all(s);
+	_cds_wfs_pop_unlock(s);
+	return rethead;
+}
+
+/*
+ * cds_wfs_first: get first node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if popped stack is empty, top stack node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_first(struct cds_wfs_head *head)
+{
+	if (___cds_wfs_end(head))
+		return NULL;
+	return &head->node;
+}
+
+static inline struct cds_wfs_node *
+___cds_wfs_next(struct cds_wfs_node *node, int blocking)
+{
+	struct cds_wfs_node *next;
+
+	next = ___cds_wfs_node_sync_next(node, blocking);
+	/*
+	 * CDS_WFS_WOULDBLOCK != CSD_WFS_END, so we can check for end
+	 * even if ___cds_wfs_node_sync_next returns CDS_WFS_WOULDBLOCK,
+	 * and still return CDS_WFS_WOULDBLOCK.
+	 */
+	if (___cds_wfs_end(next))
+		return NULL;
+	return next;
+}
+
+/*
+ * cds_wfs_next_blocking: get next node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of popped stack, non-NULL next stack
+ * node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_blocking(struct cds_wfs_node *node)
+{
+	return ___cds_wfs_next(node, 1);
+}
+
+
+/*
+ * cds_wfs_next_nonblocking: get next node of a popped stack.
+ *
+ * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
+ * needs to block.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_nonblocking(struct cds_wfs_node *node)
+{
+	return ___cds_wfs_next(node, 0);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_WFSTACK_H */
diff --git a/contrib/userspace-rcu/wfcqueue.h b/contrib/userspace-rcu/wfcqueue.h
new file mode 100644
index 00000000000..0292585ac79
--- /dev/null
+++ b/contrib/userspace-rcu/wfcqueue.h
@@ -0,0 +1,216 @@
+#ifndef _URCU_WFCQUEUE_H
+#define _URCU_WFCQUEUE_H
+
+/*
+ * urcu/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't contain it.
+ * The non-LGPL section has been removed. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ */
+
+#define CDS_WFCQ_WOULDBLOCK	((struct cds_wfcq_node *) -1UL)
+
+enum cds_wfcq_ret {
+	CDS_WFCQ_RET_WOULDBLOCK =	-1,
+	CDS_WFCQ_RET_DEST_EMPTY =	0,
+	CDS_WFCQ_RET_DEST_NON_EMPTY =	1,
+	CDS_WFCQ_RET_SRC_EMPTY = 	2,
+};
+
+enum cds_wfcq_state {
+	CDS_WFCQ_STATE_LAST =		(1U << 0),
+};
+
+struct cds_wfcq_node {
+	struct cds_wfcq_node *next;
+};
+
+/*
+ * Do not put head and tail on the same cache-line if concurrent
+ * enqueue/dequeue are expected from many CPUs. This eliminates
+ * false-sharing between enqueue and dequeue.
+ */
+struct __cds_wfcq_head {
+	struct cds_wfcq_node node;
+};
+
+struct cds_wfcq_head {
+	struct cds_wfcq_node node;
+	pthread_mutex_t lock;
+};
+
+#ifndef __cplusplus
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfcq_head and struct __cds_wfcq_head on any of those two
+ * types.
+ */
+typedef union {
+	struct __cds_wfcq_head *_h;
+	struct cds_wfcq_head *h;
+} __attribute__((__transparent_union__)) cds_wfcq_head_ptr_t;
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct __cds_wfcq_head *__cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+	return head;
+}
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct cds_wfcq_head *cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+	return head;
+}
+#else /* #ifndef __cplusplus */
+
+/* C++ ignores transparent union. */
+typedef union {
+	struct __cds_wfcq_head *_h;
+	struct cds_wfcq_head *h;
+} cds_wfcq_head_ptr_t;
+
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t __cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+	cds_wfcq_head_ptr_t ret = { ._h = head };
+	return ret;
+}
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+	cds_wfcq_head_ptr_t ret = { .h = head };
+	return ret;
+}
+#endif /* #else #ifndef __cplusplus */
+
+struct cds_wfcq_tail {
+	struct cds_wfcq_node *p;
+};
+
+#include "static-wfcqueue.h"
+
+#define cds_wfcq_node_init		_cds_wfcq_node_init
+#define cds_wfcq_init			_cds_wfcq_init
+#define __cds_wfcq_init			___cds_wfcq_init
+#define cds_wfcq_destroy		_cds_wfcq_destroy
+#define cds_wfcq_empty			_cds_wfcq_empty
+#define cds_wfcq_enqueue		_cds_wfcq_enqueue
+
+/* Dequeue locking */
+#define cds_wfcq_dequeue_lock		_cds_wfcq_dequeue_lock
+#define cds_wfcq_dequeue_unlock		_cds_wfcq_dequeue_unlock
+
+/* Locking performed within cds_wfcq calls. */
+#define cds_wfcq_dequeue_blocking	_cds_wfcq_dequeue_blocking
+#define cds_wfcq_dequeue_with_state_blocking	\
+					_cds_wfcq_dequeue_with_state_blocking
+#define cds_wfcq_splice_blocking	_cds_wfcq_splice_blocking
+#define cds_wfcq_first_blocking		_cds_wfcq_first_blocking
+#define cds_wfcq_next_blocking		_cds_wfcq_next_blocking
+
+/* Locking ensured by caller by holding cds_wfcq_dequeue_lock() */
+#define __cds_wfcq_dequeue_blocking	___cds_wfcq_dequeue_blocking
+#define __cds_wfcq_dequeue_with_state_blocking	\
+					___cds_wfcq_dequeue_with_state_blocking
+#define __cds_wfcq_splice_blocking	___cds_wfcq_splice_blocking
+#define __cds_wfcq_first_blocking	___cds_wfcq_first_blocking
+#define __cds_wfcq_next_blocking	___cds_wfcq_next_blocking
+
+/*
+ * Locking ensured by caller by holding cds_wfcq_dequeue_lock().
+ * Non-blocking: deque, first, next return CDS_WFCQ_WOULDBLOCK if they
+ * need to block. splice returns nonzero if it needs to block.
+ */
+#define __cds_wfcq_dequeue_nonblocking	___cds_wfcq_dequeue_nonblocking
+#define __cds_wfcq_dequeue_with_state_nonblocking	\
+				___cds_wfcq_dequeue_with_state_nonblocking
+#define __cds_wfcq_splice_nonblocking	___cds_wfcq_splice_nonblocking
+#define __cds_wfcq_first_nonblocking	___cds_wfcq_first_nonblocking
+#define __cds_wfcq_next_nonblocking	___cds_wfcq_next_nonblocking
+
+/*
+ * __cds_wfcq_for_each_blocking: Iterate over all nodes in a queue,
+ * without dequeuing them.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking(head, tail, node)		\
+	for (node = __cds_wfcq_first_blocking(head, tail);	\
+		node != NULL;					\
+		node = __cds_wfcq_next_blocking(head, tail, node))
+
+/*
+ * __cds_wfcq_for_each_blocking_safe: Iterate over all nodes in a queue,
+ * without dequeuing them. Safe against deletion.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ * @n: struct cds_wfcq_node pointer holding the next pointer (used
+ *     internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking_safe(head, tail, node, n)		       \
+	for (node = __cds_wfcq_first_blocking(head, tail),		       \
+			n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL); \
+		node != NULL;						       \
+		node = n, n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_H */
diff --git a/contrib/userspace-rcu/wfstack.h b/contrib/userspace-rcu/wfstack.h
new file mode 100644
index 00000000000..738fd1cfd33
--- /dev/null
+++ b/contrib/userspace-rcu/wfstack.h
@@ -0,0 +1,178 @@
+#ifndef _URCU_WFSTACK_H
+#define _URCU_WFSTACK_H
+
+/*
+ * urcu/wfstack.h
+ *
+ * Userspace RCU library - Stack with wait-free push, blocking traversal.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't support a stack
+ * without mutex. The non-LGPL section has been removed. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ *                       cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ *                      iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ *                      cds_wfs_push  __cds_wfs_pop  __cds_wfs_pop_all
+ * cds_wfs_push               -              -                  -
+ * __cds_wfs_pop              -              X                  X
+ * __cds_wfs_pop_all          -              X                  -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+#define CDS_WFS_WOULDBLOCK	((void *) -1UL)
+
+enum cds_wfs_state {
+	CDS_WFS_STATE_LAST =		(1U << 0),
+};
+
+/*
+ * struct cds_wfs_node is returned by __cds_wfs_pop, and also used as
+ * iterator on stack. It is not safe to dereference the node next
+ * pointer when returned by __cds_wfs_pop_blocking.
+ */
+struct cds_wfs_node {
+	struct cds_wfs_node *next;
+};
+
+/*
+ * struct cds_wfs_head is returned by __cds_wfs_pop_all, and can be used
+ * to begin iteration on the stack. "node" needs to be the first field of
+ * cds_wfs_head, so the end-of-stack pointer value can be used for both
+ * types.
+ */
+struct cds_wfs_head {
+	struct cds_wfs_node node;
+};
+
+struct __cds_wfs_stack {
+	struct cds_wfs_head *head;
+};
+
+struct cds_wfs_stack {
+	struct cds_wfs_head *head;
+	pthread_mutex_t lock;
+};
+
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfs_stack and struct __cds_wfs_stack on any of those two
+ * types.
+ */
+typedef union {
+	struct __cds_wfs_stack *_s;
+	struct cds_wfs_stack *s;
+} __attribute__((__transparent_union__)) cds_wfs_stack_ptr_t;
+
+#include "static-wfstack.h"
+
+#define cds_wfs_node_init		_cds_wfs_node_init
+#define cds_wfs_init			_cds_wfs_init
+#define cds_wfs_destroy			_cds_wfs_destroy
+#define __cds_wfs_init			___cds_wfs_init
+#define cds_wfs_empty			_cds_wfs_empty
+#define cds_wfs_push			_cds_wfs_push
+
+/* Locking performed internally */
+#define cds_wfs_pop_blocking		_cds_wfs_pop_blocking
+#define cds_wfs_pop_with_state_blocking	_cds_wfs_pop_with_state_blocking
+#define cds_wfs_pop_all_blocking	_cds_wfs_pop_all_blocking
+
+/*
+ * For iteration on cds_wfs_head returned by __cds_wfs_pop_all or
+ * cds_wfs_pop_all_blocking.
+ */
+#define cds_wfs_first			_cds_wfs_first
+#define cds_wfs_next_blocking		_cds_wfs_next_blocking
+#define cds_wfs_next_nonblocking	_cds_wfs_next_nonblocking
+
+/* Pop locking with internal mutex */
+#define cds_wfs_pop_lock		_cds_wfs_pop_lock
+#define cds_wfs_pop_unlock		_cds_wfs_pop_unlock
+
+/* Synchronization ensured by the caller. See synchronization table. */
+#define __cds_wfs_pop_blocking		___cds_wfs_pop_blocking
+#define __cds_wfs_pop_with_state_blocking	\
+					___cds_wfs_pop_with_state_blocking
+#define __cds_wfs_pop_nonblocking	___cds_wfs_pop_nonblocking
+#define __cds_wfs_pop_with_state_nonblocking	\
+					___cds_wfs_pop_with_state_nonblocking
+#define __cds_wfs_pop_all		___cds_wfs_pop_all
+
+#ifdef __cplusplus
+}
+#endif
+
+/*
+ * cds_wfs_for_each_blocking: Iterate over all nodes returned by
+ * __cds_wfs_pop_all().
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking(head, node)			\
+	for (node = cds_wfs_first(head);			\
+		node != NULL;					\
+		node = cds_wfs_next_blocking(node))
+
+/*
+ * cds_wfs_for_each_blocking_safe: Iterate over all nodes returned by
+ * __cds_wfs_pop_all(). Safe against deletion.
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ * @n: struct cds_wfs_node pointer holding the next pointer (used
+ *     internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking_safe(head, node, n)			   \
+	for (node = cds_wfs_first(head),				   \
+			n = (node ? cds_wfs_next_blocking(node) : NULL);   \
+		node != NULL;						   \
+		node = n, n = (node ? cds_wfs_next_blocking(node) : NULL))
+
+#endif /* _URCU_WFSTACK_H */
diff --git a/contrib/uuid/clear.c b/contrib/uuid/clear.c
deleted file mode 100644
index 0362d073e3d..00000000000
--- a/contrib/uuid/clear.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * clear.c -- Clear a UUID
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "string.h"
-
-#include "uuidP.h"
-
-void gf_uuid_clear(uuid_t uu)
-{
-	memset(uu, 0, 16);
-}
-
diff --git a/contrib/uuid/compare.c b/contrib/uuid/compare.c
deleted file mode 100644
index dba4c5bf8cf..00000000000
--- a/contrib/uuid/compare.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * compare.c --- compare whether or not two UUID's are the same
- *
- * Returns 0 if the two UUID's are different, and 1 if they are the same.
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-#include <string.h>
-
-#define UUCMP(u1,u2) if (u1 != u2) return((u1 < u2) ? -1 : 1);
-
-int gf_uuid_compare(const uuid_t uu1, const uuid_t uu2)
-{
-	struct uuid	uuid1, uuid2;
-
-	uuid_unpack(uu1, &uuid1);
-	uuid_unpack(uu2, &uuid2);
-
-	UUCMP(uuid1.time_low, uuid2.time_low);
-	UUCMP(uuid1.time_mid, uuid2.time_mid);
-	UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version);
-	UUCMP(uuid1.clock_seq, uuid2.clock_seq);
-	return memcmp(uuid1.node, uuid2.node, 6);
-}
-
diff --git a/contrib/uuid/copy.c b/contrib/uuid/copy.c
deleted file mode 100644
index 45983bfd48b..00000000000
--- a/contrib/uuid/copy.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * copy.c --- copy UUIDs
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-
-void gf_uuid_copy(uuid_t dst, const uuid_t src)
-{
-	unsigned char		*cp1;
-	const unsigned char	*cp2;
-	int			i;
-
-	for (i=0, cp1 = dst, cp2 = src; i < 16; i++)
-		*cp1++ = *cp2++;
-}
diff --git a/contrib/uuid/gen_uuid.c b/contrib/uuid/gen_uuid.c
deleted file mode 100644
index 1ec156f76ff..00000000000
--- a/contrib/uuid/gen_uuid.c
+++ /dev/null
@@ -1,686 +0,0 @@
-/*
- * gen_uuid.c --- generate a DCE-compatible uuid
- *
- * Copyright (C) 1996, 1997, 1998, 1999 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-/*
- * Force inclusion of SVID stuff since we need it if we're compiling in
- * gcc-wall wall mode
- */
-#define _SVID_SOURCE
-
-#include "config.h"
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-#include <string.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <sys/types.h>
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#include <sys/wait.h>
-#include <sys/stat.h>
-#ifdef HAVE_SYS_FILE_H
-#include <sys/file.h>
-#endif
-#ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
-#endif
-#ifdef HAVE_NET_IF_H
-#include <net/if.h>
-#endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_NET_IF_DL_H
-#include <net/if_dl.h>
-#endif
-#if defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)
-#include <sys/syscall.h>
-#endif
-#ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
-#endif
-#include <limits.h>
-
-#include "uuidP.h"
-#include "uuidd.h"
-
-#ifdef HAVE_SRANDOM
-#define srand(x) 	srandom(x)
-#define rand() 		random()
-#endif
-
-#ifdef TLS
-#define THREAD_LOCAL static TLS
-#else
-#define THREAD_LOCAL static
-#endif
-
-#if defined(__linux__) && defined(__NR_gettid) && defined(HAVE_JRAND48)
-#define DO_JRAND_MIX
-THREAD_LOCAL unsigned short jrand_seed[3];
-#endif
-
-#ifndef OPEN_MAX
-#define OPEN_MAX 1024
-#endif
-
-#ifdef _WIN32
-static void gettimeofday (struct timeval *tv, void *dummy)
-{
-	FILETIME	ftime;
-	uint64_t	n;
-
-	GetSystemTimeAsFileTime (&ftime);
-	n = (((uint64_t) ftime.dwHighDateTime << 32)
-	     + (uint64_t) ftime.dwLowDateTime);
-	if (n) {
-		n /= 10;
-		n -= ((369 * 365 + 89) * (uint64_t) 86400) * 1000000;
-	}
-
-	tv->tv_sec = n / 1000000;
-	tv->tv_usec = n % 1000000;
-}
-
-static int getuid (void)
-{
-	return 1;
-}
-#endif
-
-static int get_random_fd(void)
-{
-	struct timeval	tv;
-	static int	fd = -2;
-	int		i;
-
-	if (fd == -2) {
-		gettimeofday(&tv, 0);
-#ifndef _WIN32
-		fd = open("/dev/urandom", O_RDONLY);
-		if (fd == -1)
-			fd = open("/dev/random", O_RDONLY | O_NONBLOCK);
-		if (fd >= 0) {
-			i = fcntl(fd, F_GETFD);
-			if (i >= 0)
-				fcntl(fd, F_SETFD, i | FD_CLOEXEC);
-		}
-#endif
-		srand((getpid() << 16) ^ getuid() ^ tv.tv_sec ^ tv.tv_usec);
-#ifdef DO_JRAND_MIX
-		jrand_seed[0] = getpid() ^ (tv.tv_sec & 0xFFFF);
-		jrand_seed[1] = getppid() ^ (tv.tv_usec & 0xFFFF);
-		jrand_seed[2] = (tv.tv_sec ^ tv.tv_usec) >> 16;
-#endif
-	}
-	/* Crank the random number generator a few times */
-	gettimeofday(&tv, 0);
-	for (i = (tv.tv_sec ^ tv.tv_usec) & 0x1F; i > 0; i--)
-		rand();
-	return fd;
-}
-
-
-/*
- * Generate a series of random bytes.  Use /dev/urandom if possible,
- * and if not, use srandom/random.
- */
-static void get_random_bytes(void *buf, int nbytes)
-{
-	int i, n = nbytes, fd = get_random_fd();
-	int lose_counter = 0;
-	unsigned char *cp = (unsigned char *) buf;
-#ifdef DO_JRAND_MIX
-	unsigned short tmp_seed[3];
-#endif
-	if (fd >= 0) {
-		while (n > 0) {
-			i = read(fd, cp, n);
-			if (i <= 0) {
-				if (lose_counter++ > 16)
-					break;
-				continue;
-			}
-			n -= i;
-			cp += i;
-			lose_counter = 0;
-		}
-	}
-
-	/*
-	 * We do this all the time, but this is the only source of
-	 * randomness if /dev/random/urandom is out to lunch.
-	 */
-	for (cp = buf, i = 0; i < nbytes; i++)
-		*cp++ ^= (rand() >> 7) & 0xFF;
-#ifdef DO_JRAND_MIX
-	memcpy(tmp_seed, jrand_seed, sizeof(tmp_seed));
-	jrand_seed[2] = jrand_seed[2] ^ syscall(__NR_gettid);
-	for (cp = buf, i = 0; i < nbytes; i++)
-		*cp++ ^= (jrand48(tmp_seed) >> 7) & 0xFF;
-	memcpy(jrand_seed, tmp_seed,
-	       sizeof(jrand_seed)-sizeof(unsigned short));
-#endif
-
-	return;
-}
-
-/*
- * Get the ethernet hardware address, if we can find it...
- *
- * XXX for a windows version, probably should use GetAdaptersInfo:
- * http://www.codeguru.com/cpp/i-n/network/networkinformation/article.php/c5451
- * commenting out get_node_id just to get gen_uuid to compile under windows
- * is not the right way to go!
- */
-static int get_node_id(unsigned char *node_id)
-{
-#ifdef HAVE_NET_IF_H
-	int 		sd;
-	struct ifreq 	ifr, *ifrp;
-	struct ifconf 	ifc;
-	char buf[1024];
-	int		n, i;
-	unsigned char 	*a;
-#ifdef HAVE_NET_IF_DL_H
-	struct sockaddr_dl *sdlp;
-#endif
-
-/*
- * BSD 4.4 defines the size of an ifreq to be
- * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len
- * However, under earlier systems, sa_len isn't present, so the size is
- * just sizeof(struct ifreq)
- */
-#ifdef HAVE_SA_LEN
-#ifndef max
-#define max(a,b) ((a) > (b) ? (a) : (b))
-#endif
-#define ifreq_size(i) max(sizeof(struct ifreq),\
-     sizeof((i).ifr_name)+(i).ifr_addr.sa_len)
-#else
-#define ifreq_size(i) sizeof(struct ifreq)
-#endif /* HAVE_SA_LEN*/
-
-	sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
-	if (sd < 0) {
-		return -1;
-	}
-	memset(buf, 0, sizeof(buf));
-	ifc.ifc_len = sizeof(buf);
-	ifc.ifc_buf = buf;
-	if (ioctl (sd, SIOCGIFCONF, (char *)&ifc) < 0) {
-		close(sd);
-		return -1;
-	}
-	n = ifc.ifc_len;
-	for (i = 0; i < n; i+= ifreq_size(*ifrp) ) {
-		ifrp = (struct ifreq *)((char *) ifc.ifc_buf+i);
-		strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ);
-#ifdef SIOCGIFHWADDR
-		if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0)
-			continue;
-		a = (unsigned char *) &ifr.ifr_hwaddr.sa_data;
-#else
-#ifdef SIOCGENADDR
-		if (ioctl(sd, SIOCGENADDR, &ifr) < 0)
-			continue;
-		a = (unsigned char *) ifr.ifr_enaddr;
-#else
-#ifdef HAVE_NET_IF_DL_H
-		sdlp = (struct sockaddr_dl *) &ifrp->ifr_addr;
-		if ((sdlp->sdl_family != AF_LINK) || (sdlp->sdl_alen != 6))
-			continue;
-		a = (unsigned char *) &sdlp->sdl_data[sdlp->sdl_nlen];
-#else
-		/*
-		 * XXX we don't have a way of getting the hardware
-		 * address
-		 */
-		close(sd);
-		return 0;
-#endif /* HAVE_NET_IF_DL_H */
-#endif /* SIOCGENADDR */
-#endif /* SIOCGIFHWADDR */
-		if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5])
-			continue;
-		if (node_id) {
-			memcpy(node_id, a, 6);
-			close(sd);
-			return 1;
-		}
-	}
-	close(sd);
-#endif
-	return 0;
-}
-
-/* Assume that the gettimeofday() has microsecond granularity */
-#define MAX_ADJUSTMENT 10
-
-static int get_clock(uint32_t *clock_high, uint32_t *clock_low,
-		     uint16_t *ret_clock_seq, int *num)
-{
-	THREAD_LOCAL int		adjustment = 0;
-	THREAD_LOCAL struct timeval	last = {0, 0};
-	THREAD_LOCAL int		state_fd = -2;
-	THREAD_LOCAL FILE		*state_f;
-	THREAD_LOCAL uint16_t		clock_seq;
-	struct timeval 			tv;
-	struct flock			fl;
-	uint64_t			clock_reg;
-	mode_t				save_umask;
-	int				len;
-
-	if (state_fd == -2) {
-		save_umask = umask(0);
-		state_fd = open("/var/lib/libuuid/clock.txt",
-				O_RDWR|O_CREAT, 0660);
-		(void) umask(save_umask);
-		state_f = fdopen(state_fd, "r+");
-		if (!state_f) {
-			close(state_fd);
-			state_fd = -1;
-		}
-	}
-	fl.l_type = F_WRLCK;
-	fl.l_whence = SEEK_SET;
-	fl.l_start = 0;
-	fl.l_len = 0;
-	fl.l_pid = 0;
-	if (state_fd >= 0) {
-		rewind(state_f);
-		while (fcntl(state_fd, F_SETLKW, &fl) < 0) {
-			if ((errno == EAGAIN) || (errno == EINTR))
-				continue;
-			fclose(state_f);
-			close(state_fd);
-			state_fd = -1;
-			break;
-		}
-	}
-	if (state_fd >= 0) {
-		unsigned int cl;
-		unsigned long tv1, tv2;
-		int a;
-
-		if (fscanf(state_f, "clock: %04x tv: %lu %lu adj: %d\n",
-			   &cl, &tv1, &tv2, &a) == 4) {
-			clock_seq = cl & 0x3FFF;
-			last.tv_sec = tv1;
-			last.tv_usec = tv2;
-			adjustment = a;
-		}
-	}
-
-	if ((last.tv_sec == 0) && (last.tv_usec == 0)) {
-		get_random_bytes(&clock_seq, sizeof(clock_seq));
-		clock_seq &= 0x3FFF;
-		gettimeofday(&last, 0);
-		last.tv_sec--;
-	}
-
-try_again:
-	gettimeofday(&tv, 0);
-	if ((tv.tv_sec < last.tv_sec) ||
-	    ((tv.tv_sec == last.tv_sec) &&
-	     (tv.tv_usec < last.tv_usec))) {
-		clock_seq = (clock_seq+1) & 0x3FFF;
-		adjustment = 0;
-		last = tv;
-	} else if ((tv.tv_sec == last.tv_sec) &&
-	    (tv.tv_usec == last.tv_usec)) {
-		if (adjustment >= MAX_ADJUSTMENT)
-			goto try_again;
-		adjustment++;
-	} else {
-		adjustment = 0;
-		last = tv;
-	}
-
-	clock_reg = tv.tv_usec*10 + adjustment;
-	clock_reg += ((uint64_t) tv.tv_sec)*10000000;
-	clock_reg += (((uint64_t) 0x01B21DD2) << 32) + 0x13814000;
-
-	if (num && (*num > 1)) {
-		adjustment += *num - 1;
-		last.tv_usec += adjustment / 10;
-		adjustment = adjustment % 10;
-		last.tv_sec += last.tv_usec / 1000000;
-		last.tv_usec = last.tv_usec % 1000000;
-	}
-
-	if (state_fd > 0) {
-		rewind(state_f);
-		len = fprintf(state_f, 
-			      "clock: %04x tv: %016lu %08lu adj: %08d\n",
-			      clock_seq, last.tv_sec, last.tv_usec, adjustment);
-		fflush(state_f);
-		if (ftruncate(state_fd, len) < 0) {
-			fprintf(state_f, "                   \n");
-			fflush(state_f);
-		}
-		rewind(state_f);
-		fl.l_type = F_UNLCK;
-		fcntl(state_fd, F_SETLK, &fl);
-	}
-
-	*clock_high = clock_reg >> 32;
-	*clock_low = clock_reg;
-	*ret_clock_seq = clock_seq;
-	return 0;
-}
-
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-static ssize_t read_all(int fd, char *buf, size_t count)
-{
-	ssize_t ret;
-	ssize_t c = 0;
-	int tries = 0;
-
-	memset(buf, 0, count);
-	while (count > 0) {
-		ret = read(fd, buf, count);
-		if (ret <= 0) {
-			if ((errno == EAGAIN || errno == EINTR || ret == 0) &&
-			    (tries++ < 5))
-				continue;
-			return c ? c : -1;
-		}
-		if (ret > 0)
-			tries = 0;
-		count -= ret;
-		buf += ret;
-		c += ret;
-	}
-	return c;
-}
-#endif
-
-/*
- * Close all file descriptors
- */
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-static void close_all_fds(void)
-{
-#ifdef F_CLOSEM
-	(void)fcntl(0, F_CLOSEM);
-	(void)open("/dev/null", O_RDWR); /* stdin */
-	(void)open("/dev/null", O_RDWR); /* stdout */
-	(void)open("/dev/null", O_RDWR); /* stderr */
-#else /* F_CLOSEM */
-	int i, max;
-
-#if defined(HAVE_SYSCONF) && defined(_SC_OPEN_MAX)
-	max = sysconf(_SC_OPEN_MAX);
-#elif defined(HAVE_GETDTABLESIZE)
-	max = getdtablesize();
-#elif defined(HAVE_GETRLIMIT) && defined(RLIMIT_NOFILE)
-	struct rlimit rl;
-
-	getrlimit(RLIMIT_NOFILE, &rl);
-	max = rl.rlim_cur;
-#else
-	max = OPEN_MAX;
-#endif
-
-	for (i=0; i < max; i++) {
-		close(i);
-		if (i <= 2)
-			open("/dev/null", O_RDWR);
-	}
-#endif /* F_CLOSEM */
-}
-#endif
-
-
-/*
- * Try using the uuidd daemon to generate the UUID
- *
- * Returns 0 on success, non-zero on failure.
- */
-static int get_uuid_via_daemon(int op, uuid_t out, int *num)
-{
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-	char op_buf[64];
-	int op_len;
-	int s;
-	ssize_t ret;
-	int32_t reply_len = 0, expected = 16;
-	struct sockaddr_un srv_addr;
-	struct stat st;
-	pid_t pid;
-	static const char *uuidd_path = UUIDD_PATH;
-	static int access_ret = -2;
-	static int start_attempts = 0;
-
-	if ((s = socket(AF_UNIX, SOCK_STREAM, 0)) < 0)
-		return -1;
-
-	srv_addr.sun_family = AF_UNIX;
-	strcpy(srv_addr.sun_path, UUIDD_SOCKET_PATH);
-
-	if (connect(s, (const struct sockaddr *) &srv_addr,
-		    sizeof(struct sockaddr_un)) < 0) {
-		if (access_ret == -2)
-			access_ret = access(uuidd_path, X_OK);
-		if (access_ret == 0)
-			access_ret = stat(uuidd_path, &st);
-		if (access_ret == 0 && (st.st_mode & (S_ISUID | S_ISGID)) == 0)
-			access_ret = access(UUIDD_DIR, W_OK);
-		if (access_ret == 0 && start_attempts++ < 5) {
-			if ((pid = fork()) == 0) {
-				close_all_fds();
-				execl(uuidd_path, "uuidd", "-qT", "300",
-				      (char *) NULL);
-				exit(1);
-			}
-			(void) waitpid(pid, 0, 0);
-			if (connect(s, (const struct sockaddr *) &srv_addr,
-				    sizeof(struct sockaddr_un)) < 0)
-				goto fail;
-		} else
-			goto fail;
-	}
-	op_buf[0] = op;
-	op_len = 1;
-	if (op == UUIDD_OP_BULK_TIME_UUID) {
-		memcpy(op_buf+1, num, sizeof(*num));
-		op_len += sizeof(*num);
-		expected += sizeof(*num);
-	}
-
-	ret = write(s, op_buf, op_len);
-	if (ret < 1)
-		goto fail;
-
-	ret = read_all(s, (char *) &reply_len, sizeof(reply_len));
-	if (ret < 0)
-		goto fail;
-
-	if (reply_len != expected)
-		goto fail;
-
-	ret = read_all(s, op_buf, reply_len);
-
-	if (op == UUIDD_OP_BULK_TIME_UUID)
-		memcpy(op_buf+16, num, sizeof(int));
-
-	memcpy(out, op_buf, 16);
-
-	close(s);
-	return ((ret == expected) ? 0 : -1);
-
-fail:
-	close(s);
-#endif
-	return -1;
-}
-
-void uuid__generate_time(uuid_t out, int *num)
-{
-	static unsigned char node_id[6];
-	static int has_init = 0;
-	struct uuid uu;
-	uint32_t	clock_mid;
-
-	if (!has_init) {
-		if (get_node_id(node_id) <= 0) {
-			get_random_bytes(node_id, 6);
-			/*
-			 * Set multicast bit, to prevent conflicts
-			 * with IEEE 802 addresses obtained from
-			 * network cards
-			 */
-			node_id[0] |= 0x01;
-		}
-		has_init = 1;
-	}
-	get_clock(&clock_mid, &uu.time_low, &uu.clock_seq, num);
-	uu.clock_seq |= 0x8000;
-	uu.time_mid = (uint16_t) clock_mid;
-	uu.time_hi_and_version = ((clock_mid >> 16) & 0x0FFF) | 0x1000;
-	memcpy(uu.node, node_id, 6);
-	uuid_pack(&uu, out);
-}
-
-void gf_uuid_generate_time(uuid_t out)
-{
-#ifdef TLS
-	THREAD_LOCAL int		num = 0;
-	THREAD_LOCAL struct uuid	uu;
-	THREAD_LOCAL time_t		last_time = 0;
-	time_t				now;
-
-	if (num > 0) {
-		now = time(0);
-		if (now > last_time+1)
-			num = 0;
-	}
-	if (num <= 0) {
-		num = 1000;
-		if (get_uuid_via_daemon(UUIDD_OP_BULK_TIME_UUID,
-					out, &num) == 0) {
-			last_time = time(0);
-			uuid_unpack(out, &uu);
-			num--;
-			return;
-		}
-		num = 0;
-	}
-	if (num > 0) {
-		uu.time_low++;
-		if (uu.time_low == 0) {
-			uu.time_mid++;
-			if (uu.time_mid == 0)
-				uu.time_hi_and_version++;
-		}
-		num--;
-		uuid_pack(&uu, out);
-		return;
-	}
-#else
-	if (get_uuid_via_daemon(UUIDD_OP_TIME_UUID, out, 0) == 0)
-		return;
-#endif
-
-	uuid__generate_time(out, 0);
-}
-
-
-void uuid__generate_random(uuid_t out, int *num)
-{
-	uuid_t	buf;
-	struct uuid uu;
-	int i, n;
-
-	if (!num || !*num)
-		n = 1;
-	else
-		n = *num;
-
-	for (i = 0; i < n; i++) {
-		get_random_bytes(buf, sizeof(buf));
-		uuid_unpack(buf, &uu);
-
-		uu.clock_seq = (uu.clock_seq & 0x3FFF) | 0x8000;
-		uu.time_hi_and_version = (uu.time_hi_and_version & 0x0FFF)
-			| 0x4000;
-		uuid_pack(&uu, out);
-		out += sizeof(uuid_t);
-	}
-}
-
-void gf_uuid_generate_random(uuid_t out)
-{
-	int	num = 1;
-	/* No real reason to use the daemon for random uuid's -- yet */
-
-	uuid__generate_random(out, &num);
-}
-
-
-/*
- * This is the generic front-end to gf_uuid_generate_random and
- * gf_uuid_generate_time.  It uses gf_uuid_generate_random only if
- * /dev/urandom is available, since otherwise we won't have
- * high-quality randomness.
- */
-void gf_uuid_generate(uuid_t out)
-{
-	if (get_random_fd() >= 0)
-		gf_uuid_generate_random(out);
-	else
-		gf_uuid_generate_time(out);
-}
diff --git a/contrib/uuid/gen_uuid_nt.c b/contrib/uuid/gen_uuid_nt.c
deleted file mode 100644
index 91828b7a13b..00000000000
--- a/contrib/uuid/gen_uuid_nt.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * gen_uuid_nt.c -- Use NT api to generate uuid
- *
- * Written by Andrey Shedel (andreys@ns.cr.cyco.com)
- */
-
-
-#include "uuidP.h"
-
-#pragma warning(push,4)
-
-#pragma comment(lib, "ntdll.lib")
-
-//
-// Here is a nice example why it's not a good idea
-// to use native API in ordinary applications.
-// Number of parameters in function below was changed from 3 to 4
-// for NT5.
-//
-//
-// NTSYSAPI
-// NTSTATUS
-// NTAPI
-// NtAllocateUuids(
-//     OUT PULONG p1,
-//     OUT PULONG p2,
-//     OUT PULONG p3,
-//     OUT PUCHAR Seed // 6 bytes
-//   );
-//
-//
-
-unsigned long
-__stdcall
-NtAllocateUuids(
-   void* p1,  // 8 bytes
-   void* p2,  // 4 bytes
-   void* p3   // 4 bytes
-   );
-
-typedef
-unsigned long
-(__stdcall*
-NtAllocateUuids_2000)(
-   void* p1,  // 8 bytes
-   void* p2,  // 4 bytes
-   void* p3,  // 4 bytes
-   void* seed // 6 bytes
-   );
-
-
-
-//
-// Nice, but instead of including ntddk.h ot winnt.h
-// I should define it here because they MISSED __stdcall in those headers.
-//
-
-__declspec(dllimport)
-struct _TEB*
-__stdcall
-NtCurrentTeb(void);
-
-
-//
-// The only way to get version information from the system is to examine
-// one stored in PEB. But it's pretty dangerouse because this value could
-// be altered in image header.
-//
-
-static
-int
-Nt5(void)
-{
-	//return NtCuttentTeb()->Peb->OSMajorVersion >= 5;
-	return (int)*(int*)((char*)(int)(*(int*)((char*)NtCurrentTeb() + 0x30)) + 0xA4) >= 5;
-}
-
-
-
-
-void gf_uuid_generate(uuid_t out)
-{
-	if(Nt5())
-	{
-		unsigned char seed[6];
-		((NtAllocateUuids_2000)NtAllocateUuids)(out, ((char*)out)+8, ((char*)out)+12, &seed[0] );
-	}
-	else
-	{
-		NtAllocateUuids(out, ((char*)out)+8, ((char*)out)+12);
-	}
-}
diff --git a/contrib/uuid/isnull.c b/contrib/uuid/isnull.c
deleted file mode 100644
index 20d8fcef6da..00000000000
--- a/contrib/uuid/isnull.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * isnull.c --- Check whether or not the UUID is null
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-
-/* Returns 1 if the uuid is the NULL uuid */
-int gf_uuid_is_null(const uuid_t uu)
-{
-	const unsigned char 	*cp;
-	int			i;
-
-	for (i=0, cp = uu; i < 16; i++)
-		if (*cp++)
-			return 0;
-	return 1;
-}
-
diff --git a/contrib/uuid/pack.c b/contrib/uuid/pack.c
deleted file mode 100644
index 097516d2e2f..00000000000
--- a/contrib/uuid/pack.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Internal routine for packing UUID's
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <string.h>
-#include "uuidP.h"
-
-void uuid_pack(const struct uuid *uu, uuid_t ptr)
-{
-	uint32_t	tmp;
-	unsigned char	*out = ptr;
-
-	tmp = uu->time_low;
-	out[3] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[2] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[1] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[0] = (unsigned char) tmp;
-
-	tmp = uu->time_mid;
-	out[5] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[4] = (unsigned char) tmp;
-
-	tmp = uu->time_hi_and_version;
-	out[7] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[6] = (unsigned char) tmp;
-
-	tmp = uu->clock_seq;
-	out[9] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[8] = (unsigned char) tmp;
-
-	memcpy(out+10, uu->node, 6);
-}
-
diff --git a/contrib/uuid/parse.c b/contrib/uuid/parse.c
deleted file mode 100644
index 059ae437805..00000000000
--- a/contrib/uuid/parse.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * parse.c --- UUID parsing
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-
-#include "uuidP.h"
-
-int gf_uuid_parse(const char *in, uuid_t uu)
-{
-	struct uuid	uuid;
-	int 		i;
-	const char	*cp;
-	char		buf[3];
-
-	if (strlen(in) != 36)
-		return -1;
-	for (i=0, cp = in; i <= 36; i++,cp++) {
-		if ((i == 8) || (i == 13) || (i == 18) ||
-		    (i == 23)) {
-			if (*cp == '-')
-				continue;
-			else
-				return -1;
-		}
-		if (i== 36)
-			if (*cp == 0)
-				continue;
-		if (!isxdigit(*cp))
-			return -1;
-	}
-	uuid.time_low = strtoul(in, NULL, 16);
-	uuid.time_mid = strtoul(in+9, NULL, 16);
-	uuid.time_hi_and_version = strtoul(in+14, NULL, 16);
-	uuid.clock_seq = strtoul(in+19, NULL, 16);
-	cp = in+24;
-	buf[2] = 0;
-	for (i=0; i < 6; i++) {
-		buf[0] = *cp++;
-		buf[1] = *cp++;
-		uuid.node[i] = strtoul(buf, NULL, 16);
-	}
-
-	uuid_pack(&uuid, uu);
-	return 0;
-}
diff --git a/contrib/uuid/tst_uuid.c b/contrib/uuid/tst_uuid.c
deleted file mode 100644
index 865564b0c34..00000000000
--- a/contrib/uuid/tst_uuid.c
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * tst_uuid.c --- test program from the UUID library
- *
- * Copyright (C) 1996, 1997, 1998 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "uuid.h"
-
-static int test_uuid(const char * uuid, int isValid)
-{
-	static const char * validStr[2] = {"invalid", "valid"};
-	uuid_t uuidBits;
-	int parsedOk;
-
-	parsedOk = gf_uuid_parse(uuid, uuidBits) == 0;
-
-	printf("%s is %s", uuid, validStr[isValid]);
-	if (parsedOk != isValid) {
-		printf(" but gf_uuid_parse says %s\n", validStr[parsedOk]);
-		return 1;
-	}
-	printf(", OK\n");
-	return 0;
-}
-
-#ifdef __GNUC__
-#define ATTR(x) __attribute__(x)
-#else
-#define ATTR(x)
-#endif
-
-int
-main(int argc ATTR((unused)) , char **argv ATTR((unused)))
-{
-	uuid_t		buf, tst;
-	char		str[100];
-	struct timeval	tv;
-	time_t		time_reg;
-	unsigned char	*cp;
-	int i;
-	int failed = 0;
-	int type, variant;
-
-	gf_uuid_generate(buf);
-	gf_uuid_unparse(buf, str);
-	printf("UUID generate = %s\n", str);
-	printf("UUID: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = gf_uuid_type(buf); 	variant = gf_uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	printf("\n");
-
-	gf_uuid_generate_random(buf);
-	gf_uuid_unparse(buf, str);
-	printf("UUID random string = %s\n", str);
-	printf("UUID: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = gf_uuid_type(buf); 	variant = gf_uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	if (type != 4) {
-		printf("Incorrect UUID type; was expecting "
-		       "4 (random type)!\n");
-		failed++;
-	}
-	printf("\n");
-
-	gf_uuid_generate_time(buf);
-	gf_uuid_unparse(buf, str);
-	printf("UUID string = %s\n", str);
-	printf("UUID time: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = gf_uuid_type(buf); 	variant = gf_uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	if (type != 1) {
-		printf("Incorrect UUID type; was expecting "
-		       "1 (time-based type)!\\n");
-		failed++;
-	}
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	time_reg = gf_uuid_time(buf, &tv);
-	printf("UUID time is: (%ld, %ld): %s\n", tv.tv_sec, tv.tv_usec,
-	       ctime(&time_reg));
-	gf_uuid_parse(str, tst);
-	if (!gf_uuid_compare(buf, tst))
-		printf("UUID parse and compare succeeded.\n");
-	else {
-		printf("UUID parse and compare failed!\n");
-		failed++;
-	}
-	gf_uuid_clear(tst);
-	if (gf_uuid_is_null(tst))
-		printf("UUID clear and is null succeeded.\n");
-	else {
-		printf("UUID clear and is null failed!\n");
-		failed++;
-	}
-	gf_uuid_copy(buf, tst);
-	if (!gf_uuid_compare(buf, tst))
-		printf("UUID copy and compare succeeded.\n");
-	else {
-		printf("UUID copy and compare failed!\n");
-		failed++;
-	}
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981b", 1);
-	failed += test_uuid("84949CC5-4701-4A84-895B-354C584A981B", 1);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981bc", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981", 0);
-	failed += test_uuid("84949cc5x4701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc504701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-470104a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a840895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b0354c584a981b", 0);
-	failed += test_uuid("g4949cc5-4701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981g", 0);
-
-	if (failed) {
-		printf("%d failures.\n", failed);
-		exit(1);
-	}
-	return 0;
-}
diff --git a/contrib/uuid/unpack.c b/contrib/uuid/unpack.c
deleted file mode 100644
index beaaff3ca8a..00000000000
--- a/contrib/uuid/unpack.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Internal routine for unpacking UUID
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <string.h>
-#include "uuidP.h"
-
-void uuid_unpack(const uuid_t in, struct uuid *uu)
-{
-	const uint8_t	*ptr = in;
-	uint32_t		tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_low = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_mid = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_hi_and_version = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->clock_seq = tmp;
-
-	memcpy(uu->node, ptr, 6);
-}
-
diff --git a/contrib/uuid/unparse.c b/contrib/uuid/unparse.c
deleted file mode 100644
index f6e29534140..00000000000
--- a/contrib/uuid/unparse.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * unparse.c -- convert a UUID to string
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <stdio.h>
-
-#include "uuidP.h"
-
-static const char *fmt_lower =
-	"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x";
-
-static const char *fmt_upper =
-	"%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X";
-
-#ifdef UUID_UNPARSE_DEFAULT_UPPER
-#define FMT_DEFAULT fmt_upper
-#else
-#define FMT_DEFAULT fmt_lower
-#endif
-
-static void gf_uuid_unparse_x(const uuid_t uu, char *out, const char *fmt)
-{
-	struct uuid uuid;
-
-	uuid_unpack(uu, &uuid);
-	sprintf(out, fmt,
-		uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,
-		uuid.clock_seq >> 8, uuid.clock_seq & 0xFF,
-		uuid.node[0], uuid.node[1], uuid.node[2],
-		uuid.node[3], uuid.node[4], uuid.node[5]);
-}
-
-void gf_uuid_unparse_lower(const uuid_t uu, char *out)
-{
-	gf_uuid_unparse_x(uu, out,	fmt_lower);
-}
-
-void gf_uuid_unparse_upper(const uuid_t uu, char *out)
-{
-	gf_uuid_unparse_x(uu, out,	fmt_upper);
-}
-
-void gf_uuid_unparse(const uuid_t uu, char *out)
-{
-	gf_uuid_unparse_x(uu, out, FMT_DEFAULT);
-}
diff --git a/contrib/uuid/uuid.h b/contrib/uuid/uuid.h
deleted file mode 100644
index 97de360ad52..00000000000
--- a/contrib/uuid/uuid.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Public include file for the UUID library
- *
- * Copyright (C) 1996, 1997, 1998 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifndef _UUID_UUID_H
-#define _UUID_UUID_H
-
-#include "config.h"
-#include <sys/types.h>
-#ifndef _WIN32
-#include <sys/time.h>
-#endif
-#include <time.h>
-
-typedef unsigned char uuid_t[16];
-
-/* UUID Variant definitions */
-#define UUID_VARIANT_NCS 	0
-#define UUID_VARIANT_DCE 	1
-#define UUID_VARIANT_MICROSOFT	2
-#define UUID_VARIANT_OTHER	3
-
-/* UUID Type definitions */
-#define UUID_TYPE_DCE_TIME   1
-#define UUID_TYPE_DCE_RANDOM 4
-
-/* Allow UUID constants to be defined */
-#ifdef __GNUC__
-#define UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \
-	static const uuid_t name __attribute__ ((unused)) = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15}
-#else
-#define UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \
-	static const uuid_t name = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15}
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* clear.c */
-void gf_uuid_clear(uuid_t uu);
-
-/* compare.c */
-int gf_uuid_compare(const uuid_t uu1, const uuid_t uu2);
-
-/* copy.c */
-void gf_uuid_copy(uuid_t dst, const uuid_t src);
-
-/* gen_uuid.c */
-void gf_uuid_generate(uuid_t out);
-void gf_uuid_generate_random(uuid_t out);
-void gf_uuid_generate_time(uuid_t out);
-
-/* isnull.c */
-int gf_uuid_is_null(const uuid_t uu);
-
-/* parse.c */
-int gf_uuid_parse(const char *in, uuid_t uu);
-
-/* unparse.c */
-void gf_uuid_unparse(const uuid_t uu, char *out);
-void gf_uuid_unparse_lower(const uuid_t uu, char *out);
-void gf_uuid_unparse_upper(const uuid_t uu, char *out);
-
-/* uuid_time.c */
-time_t gf_uuid_time(const uuid_t uu, struct timeval *ret_tv);
-int gf_uuid_type(const uuid_t uu);
-int gf_uuid_variant(const uuid_t uu);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _UUID_UUID_H */
diff --git a/contrib/uuid/uuidP.h b/contrib/uuid/uuidP.h
deleted file mode 100644
index 9a2de6132fe..00000000000
--- a/contrib/uuid/uuidP.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * uuid.h -- private header file for uuids
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuid.h"
-#ifdef HAVE_INTTYPES_H
-#include <inttypes.h>
-#else
-#include "uuid_types.h"
-#endif
-#include <sys/types.h>
-
-
-/*
- * Offset between 15-Oct-1582 and 1-Jan-70
- */
-#define TIME_OFFSET_HIGH 0x01B21DD2
-#define TIME_OFFSET_LOW  0x13814000
-
-struct uuid {
-	uint32_t	time_low;
-	uint16_t	time_mid;
-	uint16_t	time_hi_and_version;
-	uint16_t	clock_seq;
-	uint8_t	node[6];
-};
-
-
-/*
- * prototypes
- */
-void uuid_pack(const struct uuid *uu, uuid_t ptr);
-void uuid_unpack(const uuid_t in, struct uuid *uu);
diff --git a/contrib/uuid/uuid_time.c b/contrib/uuid/uuid_time.c
deleted file mode 100644
index 35f727018b1..00000000000
--- a/contrib/uuid/uuid_time.c
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * uuid_time.c --- Interpret the time field from a uuid.  This program
- * 	violates the UUID abstraction barrier by reaching into the guts
- *	of a UUID and interpreting it.
- *
- * Copyright (C) 1998, 1999 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <stdlib.h>
-#include <sys/types.h>
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#include <time.h>
-
-#include "uuidP.h"
-#include "logging.h"
-
-time_t gf_uuid_time(const uuid_t uu, struct timeval *ret_tv)
-{
-	struct timeval		tv;
-	struct uuid		uuid;
-	uint32_t		high;
-	uint64_t		clock_reg;
-
-	uuid_unpack(uu, &uuid);
-
-	high = uuid.time_mid | ((uuid.time_hi_and_version & 0xFFF) << 16);
-	clock_reg = uuid.time_low | ((uint64_t) high << 32);
-
-	clock_reg -= (((uint64_t) 0x01B21DD2) << 32) + 0x13814000;
-	tv.tv_sec = clock_reg / 10000000;
-	tv.tv_usec = (clock_reg % 10000000) / 10;
-
-	if (ret_tv)
-		*ret_tv = tv;
-
-	return tv.tv_sec;
-}
-
-int gf_uuid_type(const uuid_t uu)
-{
-	struct uuid		uuid;
-
-	uuid_unpack(uu, &uuid);
-	return ((uuid.time_hi_and_version >> 12) & 0xF);
-}
-
-int gf_uuid_variant(const uuid_t uu)
-{
-	struct uuid		uuid;
-	int			var;
-
-	uuid_unpack(uu, &uuid);
-	var = uuid.clock_seq;
-
-	if ((var & 0x8000) == 0)
-		return UUID_VARIANT_NCS;
-	if ((var & 0x4000) == 0)
-		return UUID_VARIANT_DCE;
-	if ((var & 0x2000) == 0)
-		return UUID_VARIANT_MICROSOFT;
-	return UUID_VARIANT_OTHER;
-}
-
-#ifdef DEBUG
-static const char *variant_string(int variant)
-{
-	switch (variant) {
-	case UUID_VARIANT_NCS:
-		return "NCS";
-	case UUID_VARIANT_DCE:
-		return "DCE";
-	case UUID_VARIANT_MICROSOFT:
-		return "Microsoft";
-	default:
-		return "Other";
-	}
-}
-
-
-int
-main(int argc, char **argv)
-{
-	uuid_t		buf;
-	time_t		time_reg;
-	struct timeval	tv;
-	int		type, variant;
-
-	if (argc != 2) {
-		fprintf(stderr, "Usage: %s uuid\n", argv[0]);
-		exit(1);
-	}
-	if (gf_uuid_parse(argv[1], buf)) {
-		fprintf(stderr, "Invalid UUID: %s\n", argv[1]);
-		exit(1);
-	}
-	variant = gf_uuid_variant(buf);
-	type = gf_uuid_type(buf);
-	time_reg = gf_uuid_time(buf, &tv);
-
-	printf("UUID variant is %d (%s)\n", variant, variant_string(variant));
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Warning: This program only knows how to interpret "
-		       "DCE UUIDs.\n\tThe rest of the output is likely "
-		       "to be incorrect!!\n");
-	}
-	printf("UUID type is %d", type);
-	switch (type) {
-	case 1:
-		printf(" (time based)\n");
-		break;
-	case 2:
-		printf(" (DCE)\n");
-		break;
-	case 3:
-		printf(" (name-based)\n");
-		break;
-	case 4:
-		printf(" (random)\n");
-		break;
-	default:
-		printf("\n");
-	}
-	if (type != 1) {
-		printf("Warning: not a time-based UUID, so UUID time "
-		       "decoding will likely not work!\n");
-	}
-	printf("UUID time is: (%" GF_PRI_SECOND ", %" GF_PRI_USEC "): %s\n", tv.tv_sec, tv.tv_usec, ctime(&time_reg));
-
-	return 0;
-}
-#endif
diff --git a/contrib/uuid/uuid_types.h.in b/contrib/uuid/uuid_types.h.in
deleted file mode 100644
index f21ff4ee183..00000000000
--- a/contrib/uuid/uuid_types.h.in
+++ /dev/null
@@ -1,50 +0,0 @@
-/* 
- * If linux/types.h is already been included, assume it has defined
- * everything we need.  (cross fingers)  Other header files may have 
- * also defined the types that we need.
- */
-#if (!defined(_STDINT_H) && !defined(_UUID_STDINT_H))
-#define _UUID_STDINT_H
-
-typedef unsigned char uint8_t;
-typedef signed char int8_t;
-
-#if (@SIZEOF_INT@ == 8)
-typedef int		int64_t;
-typedef unsigned int	uint64_t;
-#elif (@SIZEOF_LONG@ == 8)
-typedef long		int64_t;
-typedef unsigned long	uint64_t;
-#elif (@SIZEOF_LONG_LONG@ == 8)
-#if defined(__GNUC__)
-typedef __signed__ long long 	int64_t;
-#else
-typedef signed long long 	int64_t;
-#endif
-typedef unsigned long long	uint64_t;
-#endif
-
-#if (@SIZEOF_INT@ == 2)
-typedef	int		int16_t;
-typedef	unsigned int	uint16_t;
-#elif (@SIZEOF_SHORT@ == 2)
-typedef	short		int16_t;
-typedef	unsigned short	uint16_t;
-#else
-  ?==error: undefined 16 bit type
-#endif
-
-#if (@SIZEOF_INT@ == 4)
-typedef	int		int32_t;
-typedef	unsigned int	uint32_t;
-#elif (@SIZEOF_LONG@ == 4)
-typedef	long		int32_t;
-typedef	unsigned long	uint32_t;
-#elif (@SIZEOF_SHORT@ == 4)
-typedef	short		int32_t;
-typedef	unsigned short	uint32_t;
-#else
- ?== error: undefined 32 bit type
-#endif
-
-#endif
diff --git a/contrib/uuid/uuidd.h b/contrib/uuid/uuidd.h
deleted file mode 100644
index c71f4b78835..00000000000
--- a/contrib/uuid/uuidd.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Definitions used by the uuidd daemon
- *
- * Copyright (C) 2007 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifndef _UUID_UUIDD_H
-#define _UUID_UUIDD_H
-
-#define UUIDD_DIR		"/var/lib/libuuid"
-#define UUIDD_SOCKET_PATH	UUIDD_DIR "/request"
-#define UUIDD_PIDFILE_PATH	UUIDD_DIR "/uuidd.pid"
-#define UUIDD_PATH		"/usr/sbin/uuidd"
-
-#define UUIDD_OP_GETPID			0
-#define UUIDD_OP_GET_MAXOP		1
-#define UUIDD_OP_TIME_UUID		2
-#define UUIDD_OP_RANDOM_UUID		3
-#define UUIDD_OP_BULK_TIME_UUID		4
-#define UUIDD_OP_BULK_RANDOM_UUID	5
-#define UUIDD_MAX_OP			UUIDD_OP_BULK_RANDOM_UUID
-
-extern void uuid__generate_time(uuid_t out, int *num);
-extern void uuid__generate_random(uuid_t out, int *num);
-
-#endif /* _UUID_UUID_H */
diff --git a/contrib/xxhash/xxhash.c b/contrib/xxhash/xxhash.c
new file mode 100644
index 00000000000..56f80f8811d
--- /dev/null
+++ b/contrib/xxhash/xxhash.c
@@ -0,0 +1,1029 @@
+/*
+*  xxHash - Fast Hash algorithm
+*  Copyright (C) 2012-2016, Yann Collet
+*
+*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+*  Redistribution and use in source and binary forms, with or without
+*  modification, are permitted provided that the following conditions are
+*  met:
+*
+*  * Redistributions of source code must retain the above copyright
+*  notice, this list of conditions and the following disclaimer.
+*  * Redistributions in binary form must reproduce the above
+*  copyright notice, this list of conditions and the following disclaimer
+*  in the documentation and/or other materials provided with the
+*  distribution.
+*
+*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*  You can contact the author at :
+*  - xxHash homepage: http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ *            It can generate buggy code on targets which do not support unaligned memory accesses.
+ *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
+                        || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \
+                        || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define XXH_FORCE_MEMORY_ACCESS 2
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
+                    || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
+                    || defined(__ARM_ARCH_7S__) ))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault.
+ * When this macro is enabled, xxHash actively checks input for null pointer.
+ * It it is, result for null input pointers is the same as a null-length input.
+ */
+#ifndef XXH_ACCEPT_NULL_INPUT_POINTER   /* can be defined externally */
+#  define XXH_ACCEPT_NULL_INPUT_POINTER 0
+#endif
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
+#  define XXH_FORCE_NATIVE_FORMAT 0
+#endif
+
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash;
+ * set it to 0 when the input is guaranteed to be aligned,
+ * or when alignment doesn't matter for performance.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+/*! Modify the local functions below should you wish to use some other memory routines
+*   for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#include <assert.h>   /* assert */
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/* *************************************
+*  Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+# else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+# endif
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* *************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+static int XXH_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+static U32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
+
+/* *************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)  { enum { XXH_sa = 1/(int)(!!(c)) }; }  /* use after variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bit hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 =  668265263U;
+static const U32 PRIME32_5 =  374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+    seed += input * PRIME32_2;
+    seed  = XXH_rotl32(seed, 13);
+    seed *= PRIME32_1;
+    return seed;
+}
+
+/* mix all bits */
+static U32 XXH32_avalanche(U32 h32)
+{
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+    return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+static U32
+XXH32_finalize(U32 h32, const void* ptr, size_t len,
+                XXH_endianess endian, XXH_alignment align)
+
+{
+    const BYTE* p = (const BYTE*)ptr;
+#define PROCESS1             \
+    h32 += (*p) * PRIME32_5; \
+    p++;                     \
+    h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+
+#define PROCESS4                         \
+    h32 += XXH_get32bits(p) * PRIME32_3; \
+    p+=4;                                \
+    h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+
+    switch(len&15)  /* or switch(bEnd - p) */
+    {
+      case 12:      PROCESS4;
+                    /* fallthrough */
+      case 8:       PROCESS4;
+                    /* fallthrough */
+      case 4:       PROCESS4;
+                    return XXH32_avalanche(h32);
+
+      case 13:      PROCESS4;
+                    /* fallthrough */
+      case 9:       PROCESS4;
+                    /* fallthrough */
+      case 5:       PROCESS4;
+                    PROCESS1;
+                    return XXH32_avalanche(h32);
+
+      case 14:      PROCESS4;
+                    /* fallthrough */
+      case 10:      PROCESS4;
+                    /* fallthrough */
+      case 6:       PROCESS4;
+                    PROCESS1;
+                    PROCESS1;
+                    return XXH32_avalanche(h32);
+
+      case 15:      PROCESS4;
+                    /* fallthrough */
+      case 11:      PROCESS4;
+                    /* fallthrough */
+      case 7:       PROCESS4;
+                    /* fallthrough */
+      case 3:       PROCESS1;
+                    /* fallthrough */
+      case 2:       PROCESS1;
+                    /* fallthrough */
+      case 1:       PROCESS1;
+                    /* fallthrough */
+      case 0:       return XXH32_avalanche(h32);
+    }
+    assert(0);
+    return h32;   /* reaching this point is deemed impossible */
+}
+
+
+FORCE_INLINE U32
+XXH32_endian_align(const void* input, size_t len, U32 seed,
+                    XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16) {
+        const BYTE* const limit = bEnd - 15;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+        } while (p < limit);
+
+        h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
+            + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32)len;
+
+    return XXH32_finalize(h32, p, len&15, endian, align);
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+
+/*======   Hash streaming   ======*/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
+{
+    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + PRIME32_1 + PRIME32_2;
+    state.v2 = seed + PRIME32_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME32_1;
+    /* do not write into reserved, planned to be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+    return XXH_OK;
+}
+
+
+FORCE_INLINE
+XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    state->total_len_32 += (unsigned)len;
+    state->large_len |= (len>=16) | (state->total_len_32>=16);
+
+    if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (unsigned)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* some data left from previous update */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {   const U32* p32 = state->mem32;
+            state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+            state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+            state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+            state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian));
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do {
+            v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+            v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+            v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+            v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+FORCE_INLINE U32
+XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+{
+    U32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v1, 1)
+            + XXH_rotl32(state->v2, 7)
+            + XXH_rotl32(state->v3, 12)
+            + XXH_rotl32(state->v4, 18);
+    } else {
+        h32 = state->v3 /* == seed */ + PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned);
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*======   Canonical representation   ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+*   These functions allow transformation of hash result into and from its canonical format.
+*   This way, hash values can be written into a file or buffer, remaining comparable across different systems.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bit hash functions
+*********************************************************************/
+
+/*======   Memory access   ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint64_t U64;
+# else
+    /* if compiler doesn't support unsigned long long, replace by another 64-bit type */
+    typedef unsigned long long U64;
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*======   xxh64   ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 =  1609587929392839161ULL;
+static const U64 PRIME64_4 =  9650029242287828579ULL;
+static const U64 PRIME64_5 =  2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+    acc += input * PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= PRIME64_1;
+    return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * PRIME64_1 + PRIME64_4;
+    return acc;
+}
+
+static U64 XXH64_avalanche(U64 h64)
+{
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+    return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+static U64
+XXH64_finalize(U64 h64, const void* ptr, size_t len,
+               XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)ptr;
+
+#define PROCESS1_64          \
+    h64 ^= (*p) * PRIME64_5; \
+    p++;                     \
+    h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+
+#define PROCESS4_64          \
+    h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \
+    p+=4;                    \
+    h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+
+#define PROCESS8_64 {        \
+    U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \
+    p+=8;                    \
+    h64 ^= k1;               \
+    h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \
+}
+
+    switch(len&31) {
+      case 24: PROCESS8_64;
+                    /* fallthrough */
+      case 16: PROCESS8_64;
+                    /* fallthrough */
+      case  8: PROCESS8_64;
+               return XXH64_avalanche(h64);
+
+      case 28: PROCESS8_64;
+                    /* fallthrough */
+      case 20: PROCESS8_64;
+                    /* fallthrough */
+      case 12: PROCESS8_64;
+                    /* fallthrough */
+      case  4: PROCESS4_64;
+               return XXH64_avalanche(h64);
+
+      case 25: PROCESS8_64;
+                    /* fallthrough */
+      case 17: PROCESS8_64;
+                    /* fallthrough */
+      case  9: PROCESS8_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 29: PROCESS8_64;
+                    /* fallthrough */
+      case 21: PROCESS8_64;
+                    /* fallthrough */
+      case 13: PROCESS8_64;
+                    /* fallthrough */
+      case  5: PROCESS4_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 26: PROCESS8_64;
+                    /* fallthrough */
+      case 18: PROCESS8_64;
+                    /* fallthrough */
+      case 10: PROCESS8_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 30: PROCESS8_64;
+                    /* fallthrough */
+      case 22: PROCESS8_64;
+                    /* fallthrough */
+      case 14: PROCESS8_64;
+                    /* fallthrough */
+      case  6: PROCESS4_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 27: PROCESS8_64;
+                    /* fallthrough */
+      case 19: PROCESS8_64;
+                    /* fallthrough */
+      case 11: PROCESS8_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 31: PROCESS8_64;
+                    /* fallthrough */
+      case 23: PROCESS8_64;
+                    /* fallthrough */
+      case 15: PROCESS8_64;
+                    /* fallthrough */
+      case  7: PROCESS4_64;
+                    /* fallthrough */
+      case  3: PROCESS1_64;
+                    /* fallthrough */
+      case  2: PROCESS1_64;
+                    /* fallthrough */
+      case  1: PROCESS1_64;
+                    /* fallthrough */
+      case  0: return XXH64_avalanche(h64);
+    }
+
+    /* impossible to reach */
+    assert(0);
+    return 0;  /* unreachable, but some compilers complain without it */
+}
+
+FORCE_INLINE U64
+XXH64_endian_align(const void* input, size_t len, U64 seed,
+                XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+        } while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    return XXH64_finalize(h64, p, len, endian, align);
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*======   Hash Streaming   ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + PRIME64_1 + PRIME64_2;
+    state.v2 = seed + PRIME64_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME64_1;
+     /* do not write into reserved, planned to be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+    return XXH_OK;
+}
+
+FORCE_INLINE
+XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32) {  /* fill in tmp buffer */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* tmp buffer is full */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+        state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+        state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+        state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do {
+            v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+            v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+            v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+            v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+{
+    U64 h64;
+
+    if (state->total_len >= 32) {
+        U64 const v1 = state->v1;
+        U64 const v2 = state->v2;
+        U64 const v3 = state->v3;
+        U64 const v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+    } else {
+        h64  = state->v3 /*seed*/ + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned);
+}
+
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation   ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+#endif  /* XXH_NO_LONG_LONG */
diff --git a/contrib/xxhash/xxhash.h b/contrib/xxhash/xxhash.h
new file mode 100644
index 00000000000..d6bad943358
--- /dev/null
+++ b/contrib/xxhash/xxhash.h
@@ -0,0 +1,328 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2016, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bit version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bit applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************
+*  Definitions
+******************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/* ****************************
+ *  API modifier
+ ******************************/
+/** XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ *  This is useful to include xxhash functions in `static` mode
+ *  in order to inline them, and remove their symbol from the public list.
+ *  Inlining can offer dramatic performance improvement on small keys.
+ *  Methodology :
+ *     #define XXH_INLINE_ALL
+ *     #include "xxhash.h"
+ * `xxhash.c` is automatically included.
+ *  It's not useful to compile and link it as a separate module.
+ */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  ifndef XXH_STATIC_LINKING_ONLY
+#    define XXH_STATIC_LINKING_ONLY
+#  endif
+#  if defined(__GNUC__)
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
+#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#    define XXH_PUBLIC_API static inline
+#  elif defined(_MSC_VER)
+#    define XXH_PUBLIC_API static __inline
+#  else
+     /* this version may generate warnings for unused static functions */
+#    define XXH_PUBLIC_API static
+#  endif
+#else
+#  define XXH_PUBLIC_API   /* do nothing */
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+/*! XXH_NAMESPACE, aka Namespace Emulation :
+ *
+ * If you want to include _and expose_ xxHash functions from within your own library,
+ * but also want to avoid symbol collisions with other libraries which may also include xxHash,
+ *
+ * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+ * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+ *
+ * Note that no change is required within the calling program as long as it includes `xxhash.h` :
+ * regular symbol name will be automatically translated by this header.
+ */
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
+
+
+/* *************************************
+*  Version
+***************************************/
+#define XXH_VERSION_MAJOR    0
+#define XXH_VERSION_MINOR    6
+#define XXH_VERSION_RELEASE  5
+#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/*-**********************************************************************
+*  32-bit hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
+    Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
+
+/*
+ * Streaming functions generate the xxHash of an input provided in multiple segments.
+ * Note that, for small input, they are slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * XXH state must first be allocated, using XXH*_createState() .
+ *
+ * Start a new hash by initializing state with a seed, using XXH*_reset().
+ *
+ * Then, feed the hash state by calling XXH*_update() as many times as necessary.
+ * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using XXH*_digest().
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a digest,
+ * and generate some new hashes later on, by calling again XXH*_digest().
+ *
+ * When done, free XXH state space if it was allocated dynamically.
+ */
+
+/*======   Canonical representation   ======*/
+
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+ * The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+ * These functions allow transformation of hash result into and from its canonical format.
+ * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+ */
+
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+*  64-bit hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
+
+/*! XXH64() :
+    Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".
+    "seed" can be used to alter the result predictably.
+    This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).
+*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
+
+/*======   Canonical representation   ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif  /* XXH_NO_LONG_LONG */
+
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+   This section contains declarations which are not guaranteed to remain stable.
+   They may change in future versions, becoming incompatible with a different version of the library.
+   These declarations should only be used with static linking.
+   Never use them in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only present to allow
+ * static allocation of XXH state, on stack or in a struct for example.
+ * Never **ever** use members directly. */
+
+#if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+
+struct XXH32_state_s {
+   uint32_t total_len_32;
+   uint32_t large_len;
+   uint32_t v1;
+   uint32_t v2;
+   uint32_t v3;
+   uint32_t v4;
+   uint32_t mem32[4];
+   uint32_t memsize;
+   uint32_t reserved;   /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+struct XXH64_state_s {
+   uint64_t total_len;
+   uint64_t v1;
+   uint64_t v2;
+   uint64_t v3;
+   uint64_t v4;
+   uint64_t mem64[4];
+   uint32_t memsize;
+   uint32_t reserved[2];          /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+
+# else
+
+struct XXH32_state_s {
+   unsigned total_len_32;
+   unsigned large_len;
+   unsigned v1;
+   unsigned v2;
+   unsigned v3;
+   unsigned v4;
+   unsigned mem32[4];
+   unsigned memsize;
+   unsigned reserved;   /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+#   ifndef XXH_NO_LONG_LONG  /* remove 64-bit support */
+struct XXH64_state_s {
+   unsigned long long total_len;
+   unsigned long long v1;
+   unsigned long long v2;
+   unsigned long long v3;
+   unsigned long long v4;
+   unsigned long long mem64[4];
+   unsigned memsize;
+   unsigned reserved[2];     /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+#    endif
+
+# endif
+
+
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
+#endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* XXHASH_H_5627135585666179 */
diff --git a/contrib/xxhash/xxhsum.c b/contrib/xxhash/xxhsum.c
new file mode 100644
index 00000000000..69931f727f0
--- /dev/null
+++ b/contrib/xxhash/xxhsum.c
@@ -0,0 +1,1301 @@
+/*
+*  xxhsum - Command line interface for xxhash algorithms
+*  Copyright (C) Yann Collet 2012-2016
+*
+*  GPL v2 License
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License as published by
+*  the Free Software Foundation; either version 2 of the License, or
+*  (at your option) any later version.
+*
+*  This program is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU General Public License along
+*  with this program; if not, write to the Free Software Foundation, Inc.,
+*  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*
+*  You can contact the author at :
+*  - xxHash homepage : http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* xxhsum :
+ * Provides hash value of a file content, or a list of files, or stdin
+ * Display convention is Big Endian, for both 32 and 64 bits algorithms
+ */
+
+#ifndef XXHASH_C_2097394837
+#define XXHASH_C_2097394837
+
+/* ************************************
+ *  Compiler Options
+ **************************************/
+/* MS Visual */
+#if defined(_MSC_VER) || defined(_WIN32)
+#  define _CRT_SECURE_NO_WARNINGS   /* removes visual warnings */
+#endif
+
+/* Under Linux at least, pull in the *64 commands */
+#ifndef _LARGEFILE64_SOURCE
+#  define _LARGEFILE64_SOURCE
+#endif
+
+
+/* ************************************
+ *  Includes
+ **************************************/
+#include <stdlib.h>     /* malloc, calloc, free, exit */
+#include <stdio.h>      /* fprintf, fopen, ftello64, fread, stdin, stdout, _fileno (when present) */
+#include <string.h>     /* strcmp */
+#include <sys/types.h>  /* stat, stat64, _stat64 */
+#include <sys/stat.h>   /* stat, stat64, _stat64 */
+#include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>     /* assert */
+
+#define XXH_STATIC_LINKING_ONLY   /* *_state_t */
+#include "xxhash.h"
+
+
+/* ************************************
+ *  OS-Specific Includes
+ **************************************/
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>    /* _O_BINARY */
+#  include <io.h>       /* _setmode, _isatty */
+#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#  include <unistd.h>   /* isatty, STDIN_FILENO */
+#  define SET_BINARY_MODE(file)
+#  define IS_CONSOLE(stdStream) isatty(STDIN_FILENO)
+#endif
+
+#if !defined(S_ISREG)
+#  define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
+#endif
+
+
+/* ************************************
+*  Basic Types
+**************************************/
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+    typedef  int32_t S32;
+    typedef uint64_t U64;
+#  else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+    typedef   signed int       S32;
+    typedef unsigned long long U64;
+#  endif
+#endif
+
+static unsigned BMK_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+
+
+/* *************************************
+ *  Constants
+ ***************************************/
+#define LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE
+#define QUOTE(str) #str
+#define EXPAND_AND_QUOTE(str) QUOTE(str)
+#define PROGRAM_VERSION EXPAND_AND_QUOTE(LIB_VERSION)
+static const int g_nbBits = (int)(sizeof(void*)*8);
+static const char g_lename[] = "little endian";
+static const char g_bename[] = "big endian";
+#define ENDIAN_NAME (BMK_isLittleEndian() ? g_lename : g_bename)
+static const char author[] = "Yann Collet";
+#define WELCOME_MESSAGE(exename) "%s %s (%i-bits %s), by %s \n", \
+                    exename, PROGRAM_VERSION, g_nbBits, ENDIAN_NAME, author
+
+#define KB *( 1<<10)
+#define MB *( 1<<20)
+#define GB *(1U<<30)
+
+static size_t XXH_DEFAULT_SAMPLE_SIZE = 100 KB;
+#define NBLOOPS    3                              /* Default number of benchmark iterations */
+#define TIMELOOP_S 1
+#define TIMELOOP  (TIMELOOP_S * CLOCKS_PER_SEC)   /* Minimum timing per iteration */
+#define XXHSUM32_DEFAULT_SEED 0                   /* Default seed for algo_xxh32 */
+#define XXHSUM64_DEFAULT_SEED 0                   /* Default seed for algo_xxh64 */
+
+#define MAX_MEM    (2 GB - 64 MB)
+
+static const char stdinName[] = "-";
+typedef enum { algo_xxh32, algo_xxh64 } algoType;
+static const algoType g_defaultAlgo = algo_xxh64;    /* required within main() & usage() */
+
+/* <16 hex char> <SPC> <SPC> <filename> <'\0'>
+ * '4096' is typical Linux PATH_MAX configuration. */
+#define DEFAULT_LINE_LENGTH (sizeof(XXH64_hash_t) * 2 + 2 + 4096 + 1)
+
+/* Maximum acceptable line length. */
+#define MAX_LINE_LENGTH (32 KB)
+
+
+/* ************************************
+ *  Display macros
+ **************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYRESULT(...)   fprintf(stdout, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) do { if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); } while (0)
+static int g_displayLevel = 2;
+
+
+/* ************************************
+ *  Local variables
+ **************************************/
+static U32 g_nbIterations = NBLOOPS;
+
+
+/* ************************************
+ *  Benchmark Functions
+ **************************************/
+static clock_t BMK_clockSpan( clock_t start )
+{
+    return clock() - start;   /* works even if overflow; Typical max span ~ 30 mn */
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+    size_t const step = 64 MB;
+    void* testmem = NULL;
+
+    requiredMem = (((requiredMem >> 26) + 1) << 26);
+    requiredMem += 2*step;
+    if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+    while (!testmem) {
+        if (requiredMem > step) requiredMem -= step;
+        else requiredMem >>= 1;
+        testmem = malloc ((size_t)requiredMem);
+    }
+    free (testmem);
+
+    /* keep some space available */
+    if (requiredMem > step) requiredMem -= step;
+    else requiredMem >>= 1;
+
+    return (size_t)requiredMem;
+}
+
+
+static U64 BMK_GetFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+#endif
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    return (U64)statbuf.st_size;
+}
+
+typedef U32 (*hashFunction)(const void* buffer, size_t bufferSize, U32 seed);
+
+static U32 localXXH32(const void* buffer, size_t bufferSize, U32 seed) { return XXH32(buffer, bufferSize, seed); }
+
+static U32 localXXH64(const void* buffer, size_t bufferSize, U32 seed) { return (U32)XXH64(buffer, bufferSize, seed); }
+
+static void BMK_benchHash(hashFunction h, const char* hName, const void* buffer, size_t bufferSize)
+{
+    U32 nbh_perIteration = ((300 MB) / (bufferSize+1)) + 1;  /* first loop conservatively aims for 300 MB/s */
+    U32 iterationNb;
+    double fastestH = 100000000.;
+
+    DISPLAYLEVEL(2, "\r%70s\r", "");       /* Clean display line */
+    if (g_nbIterations<1) g_nbIterations=1;
+    for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) {
+        U32 r=0;
+        clock_t cStart;
+
+        DISPLAYLEVEL(2, "%1i-%-17.17s : %10u ->\r", iterationNb, hName, (U32)bufferSize);
+        cStart = clock();
+        while (clock() == cStart);   /* starts clock() at its exact beginning */
+        cStart = clock();
+
+        {   U32 i;
+            for (i=0; i<nbh_perIteration; i++)
+                r += h(buffer, bufferSize, i);
+        }
+        if (r==0) DISPLAYLEVEL(3,".\r");  /* do something with r to avoid compiler "optimizing" away hash function */
+        {   double const timeS = ((double)BMK_clockSpan(cStart) / CLOCKS_PER_SEC) / nbh_perIteration;
+            if (timeS < fastestH) fastestH = timeS;
+            DISPLAYLEVEL(2, "%1i-%-17.17s : %10u -> %8.0f it/s (%7.1f MB/s) \r",
+                    iterationNb, hName, (U32)bufferSize,
+                    (double)1 / fastestH,
+                    ((double)bufferSize / (1<<20)) / fastestH );
+        }
+        assert(fastestH > 1./2000000000);  /* avoid U32 overflow */
+        nbh_perIteration = (U32)(1 / fastestH) + 1;  /* adjust nbh_perIteration to last roughtly one second */
+    }
+    DISPLAYLEVEL(1, "%-19.19s : %10u -> %8.0f it/s (%7.1f MB/s) \n", hName, (U32)bufferSize,
+        (double)1 / fastestH,
+        ((double)bufferSize / (1<<20)) / fastestH);
+    if (g_displayLevel<1)
+        DISPLAYLEVEL(0, "%u, ", (U32)((double)1 / fastestH));
+}
+
+
+/* BMK_benchMem():
+ * specificTest : 0 == run all tests, 1+ run only specific test
+ * buffer : is supposed 8-bytes aligned (if malloc'ed, it should be)
+ * the real allocated size of buffer is supposed to be >= (bufferSize+3).
+ * @return : 0 on success, 1 if error (invalid mode selected) */
+static int BMK_benchMem(const void* buffer, size_t bufferSize, U32 specificTest)
+{
+    assert((((size_t)buffer) & 8) == 0);  /* ensure alignment */
+
+    /* XXH32 bench */
+    if ((specificTest==0) | (specificTest==1))
+        BMK_benchHash(localXXH32, "XXH32", buffer, bufferSize);
+
+    /* Bench XXH32 on Unaligned input */
+    if ((specificTest==0) | (specificTest==2))
+        BMK_benchHash(localXXH32, "XXH32 unaligned", ((const char*)buffer)+1, bufferSize);
+
+    /* Bench XXH64 */
+    if ((specificTest==0) | (specificTest==3))
+        BMK_benchHash(localXXH64, "XXH64", buffer, bufferSize);
+
+    /* Bench XXH64 on Unaligned input */
+    if ((specificTest==0) | (specificTest==4))
+        BMK_benchHash(localXXH64, "XXH64 unaligned", ((const char*)buffer)+3, bufferSize);
+
+    if (specificTest > 4) {
+        DISPLAY("benchmark mode invalid \n");
+        return 1;
+    }
+    return 0;
+}
+
+
+static size_t BMK_selectBenchedSize(const char* fileName)
+{   U64 const inFileSize = BMK_GetFileSize(fileName);
+    size_t benchedSize = (size_t) BMK_findMaxMem(inFileSize);
+    if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+    if (benchedSize < inFileSize) {
+        DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20));
+    }
+    return benchedSize;
+}
+
+
+static int BMK_benchFiles(const char** fileNamesTable, int nbFiles, U32 specificTest)
+{
+    int result = 0;
+    int fileIdx;
+
+    for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
+        const char* const inFileName = fileNamesTable[fileIdx];
+        FILE* const inFile = fopen( inFileName, "rb" );
+        size_t const benchedSize = BMK_selectBenchedSize(inFileName);
+        char* const buffer = (char*)calloc(benchedSize+16+3, 1);
+        void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF);  /* align on next 16 bytes */
+
+        /* Checks */
+        if ((inFile==NULL) || (inFileName==NULL)) {
+            DISPLAY("Pb opening %s\n", inFileName);
+            free(buffer);
+            return 11;
+        }
+        if(!buffer) {
+            DISPLAY("\nError: not enough memory!\n");
+            fclose(inFile);
+            return 12;
+        }
+
+        /* Fill input buffer */
+        DISPLAYLEVEL(1, "\rLoading %s...        \n", inFileName);
+        {   size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile);
+            fclose(inFile);
+            if(readSize != benchedSize) {
+                DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
+                free(buffer);
+                return 13;
+        }   }
+
+        /* bench */
+        result |= BMK_benchMem(alignedBuffer, benchedSize, specificTest);
+
+        free(buffer);
+    }
+
+    return result;
+}
+
+
+
+static int BMK_benchInternal(size_t keySize, int specificTest)
+{
+    void* const buffer = calloc(keySize+16+3, 1);
+    void* const alignedBuffer = ((char*)buffer+15) - (((size_t)((char*)buffer+15)) & 0xF);  /* align on next 16 bytes */
+    if(!buffer) {
+        DISPLAY("\nError: not enough memory!\n");
+        return 12;
+    }
+
+    /* bench */
+    DISPLAYLEVEL(1, "Sample of ");
+    if (keySize > 10 KB) {
+        DISPLAYLEVEL(1, "%u KB", (U32)(keySize >> 10));
+    } else {
+        DISPLAYLEVEL(1, "%u bytes", (U32)keySize);
+    }
+    DISPLAYLEVEL(1, "...        \n");
+
+    {   int const result = BMK_benchMem(alignedBuffer, keySize, specificTest);
+        free(buffer);
+        return result;
+    }
+}
+
+
+static void BMK_checkResult(U32 r1, U32 r2)
+{
+    static int nbTests = 1;
+    if (r1==r2) {
+        DISPLAYLEVEL(3, "\rTest%3i : %08X == %08X   ok   ", nbTests, r1, r2);
+    } else {
+        DISPLAY("\rERROR : Test%3i : %08X <> %08X   !!!!!   \n", nbTests, r1, r2);
+        exit(1);
+    }
+    nbTests++;
+}
+
+
+static void BMK_checkResult64(U64 r1, U64 r2)
+{
+    static int nbTests = 1;
+    if (r1!=r2) {
+        DISPLAY("\rERROR : Test%3i : 64-bit values non equals   !!!!!   \n", nbTests);
+        DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2);
+        exit(1);
+    }
+    nbTests++;
+}
+
+
+static void BMK_testSequence64(void* sentence, size_t len, U64 seed, U64 Nresult)
+{
+    XXH64_state_t state;
+    U64 Dresult;
+    size_t pos;
+
+    Dresult = XXH64(sentence, len, seed);
+    BMK_checkResult64(Dresult, Nresult);
+
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, sentence, len);
+    Dresult = XXH64_digest(&state);
+    BMK_checkResult64(Dresult, Nresult);
+
+    XXH64_reset(&state, seed);
+    for (pos=0; pos<len; pos++)
+        XXH64_update(&state, ((char*)sentence)+pos, 1);
+    Dresult = XXH64_digest(&state);
+    BMK_checkResult64(Dresult, Nresult);
+}
+
+
+static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult)
+{
+    XXH32_state_t state;
+    U32 Dresult;
+    size_t pos;
+
+    Dresult = XXH32(sequence, len, seed);
+    BMK_checkResult(Dresult, Nresult);
+
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, sequence, len);
+    Dresult = XXH32_digest(&state);
+    BMK_checkResult(Dresult, Nresult);
+
+    XXH32_reset(&state, seed);
+    for (pos=0; pos<len; pos++)
+        XXH32_update(&state, ((const char*)sequence)+pos, 1);
+    Dresult = XXH32_digest(&state);
+    BMK_checkResult(Dresult, Nresult);
+}
+
+
+#define SANITY_BUFFER_SIZE 101
+static void BMK_sanityCheck(void)
+{
+    static const U32 prime = 2654435761U;
+    BYTE sanityBuffer[SANITY_BUFFER_SIZE];
+    U32 byteGen = prime;
+
+    int i;
+    for (i=0; i<SANITY_BUFFER_SIZE; i++) {
+        sanityBuffer[i] = (BYTE)(byteGen>>24);
+        byteGen *= byteGen;
+    }
+
+    BMK_testSequence(NULL,          0, 0,     0x02CC5D05);
+    BMK_testSequence(NULL,          0, prime, 0x36B78AE7);
+    BMK_testSequence(sanityBuffer,  1, 0,     0xB85CBEE5);
+    BMK_testSequence(sanityBuffer,  1, prime, 0xD5845D64);
+    BMK_testSequence(sanityBuffer, 14, 0,     0xE5AA0AB4);
+    BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D);
+    BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0,     0x1F1AA412);
+    BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2);
+
+    BMK_testSequence64(NULL        ,  0, 0,     0xEF46DB3751D8E999ULL);
+    BMK_testSequence64(NULL        ,  0, prime, 0xAC75FDA2929B17EFULL);
+    BMK_testSequence64(sanityBuffer,  1, 0,     0x4FCE394CC88952D8ULL);
+    BMK_testSequence64(sanityBuffer,  1, prime, 0x739840CB819FA723ULL);
+    BMK_testSequence64(sanityBuffer, 14, 0,     0xCFFA8DB881BC3A3DULL);
+    BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL);
+    BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0,     0x0EAB543384F878ADULL);
+    BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL);
+
+    DISPLAYLEVEL(3, "\r%70s\r", "");       /* Clean display line */
+    DISPLAYLEVEL(3, "Sanity check -- all tests ok\n");
+}
+
+
+/* ********************************************************
+*  File Hashing
+**********************************************************/
+
+static void BMK_display_LittleEndian(const void* ptr, size_t length)
+{
+    const BYTE* p = (const BYTE*)ptr;
+    size_t idx;
+    for (idx=length-1; idx<length; idx--)    /* intentional underflow to negative to detect end */
+        DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_display_BigEndian(const void* ptr, size_t length)
+{
+    const BYTE* p = (const BYTE*)ptr;
+    size_t idx;
+    for (idx=0; idx<length; idx++)
+        DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* inFile, void* buffer, size_t blockSize)
+{
+    XXH64_state_t state64;
+    XXH32_state_t state32;
+    size_t readSize;
+
+    /* Init */
+    XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED);
+    XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED);
+
+    /* Load file & update hash */
+    readSize = 1;
+    while (readSize) {
+        readSize = fread(buffer, 1, blockSize, inFile);
+        switch(hashType)
+        {
+        case algo_xxh32:
+            XXH32_update(&state32, buffer, readSize);
+            break;
+        case algo_xxh64:
+            XXH64_update(&state64, buffer, readSize);
+            break;
+        default:
+            break;
+        }
+    }
+
+    switch(hashType)
+    {
+    case algo_xxh32:
+        {   U32 const h32 = XXH32_digest(&state32);
+            memcpy(xxhHashValue, &h32, sizeof(h32));
+            break;
+        }
+    case algo_xxh64:
+        {   U64 const h64 = XXH64_digest(&state64);
+            memcpy(xxhHashValue, &h64, sizeof(h64));
+            break;
+        }
+    default:
+            break;
+    }
+}
+
+
+typedef enum { big_endian, little_endian} endianess;
+
+static int BMK_hash(const char* fileName,
+                    const algoType hashType,
+                    const endianess displayEndianess)
+{
+    FILE*  inFile;
+    size_t const blockSize = 64 KB;
+    void*  buffer;
+    U32    h32 = 0;
+    U64    h64 = 0;
+
+    /* Check file existence */
+    if (fileName == stdinName) {
+        inFile = stdin;
+        SET_BINARY_MODE(stdin);
+    }
+    else
+        inFile = fopen( fileName, "rb" );
+    if (inFile==NULL) {
+        DISPLAY( "Pb opening %s\n", fileName);
+        return 1;
+    }
+
+    /* Memory allocation & restrictions */
+    buffer = malloc(blockSize);
+    if(!buffer) {
+        DISPLAY("\nError: not enough memory!\n");
+        fclose(inFile);
+        return 1;
+    }
+
+    /* loading notification */
+    {   const size_t fileNameSize = strlen(fileName);
+        const char* const fileNameEnd = fileName + fileNameSize;
+        const int maxInfoFilenameSize = (int)(fileNameSize > 30 ? 30 : fileNameSize);
+        int infoFilenameSize = 1;
+        while ((infoFilenameSize < maxInfoFilenameSize)
+            && (fileNameEnd[-1-infoFilenameSize] != '/')
+            && (fileNameEnd[-1-infoFilenameSize] != '\\') )
+              infoFilenameSize++;
+        DISPLAY("\rLoading %s...  \r", fileNameEnd - infoFilenameSize);
+
+        /* Load file & update hash */
+        switch(hashType)
+        {
+        case algo_xxh32:
+            BMK_hashStream(&h32, algo_xxh32, inFile, buffer, blockSize);
+            break;
+        case algo_xxh64:
+            BMK_hashStream(&h64, algo_xxh64, inFile, buffer, blockSize);
+            break;
+        default:
+            break;
+        }
+
+        fclose(inFile);
+        free(buffer);
+        DISPLAY("%s             \r", fileNameEnd - infoFilenameSize);  /* erase line */
+    }
+
+    /* display Hash */
+    switch(hashType)
+    {
+    case algo_xxh32:
+        {   XXH32_canonical_t hcbe32;
+            XXH32_canonicalFromHash(&hcbe32, h32);
+            displayEndianess==big_endian ?
+                BMK_display_BigEndian(&hcbe32, sizeof(hcbe32)) : BMK_display_LittleEndian(&hcbe32, sizeof(hcbe32));
+            DISPLAYRESULT("  %s\n", fileName);
+            break;
+        }
+    case algo_xxh64:
+        {   XXH64_canonical_t hcbe64;
+            XXH64_canonicalFromHash(&hcbe64, h64);
+            displayEndianess==big_endian ?
+                BMK_display_BigEndian(&hcbe64, sizeof(hcbe64)) : BMK_display_LittleEndian(&hcbe64, sizeof(hcbe64));
+            DISPLAYRESULT("  %s\n", fileName);
+            break;
+        }
+    default:
+            break;
+    }
+
+    return 0;
+}
+
+
+static int BMK_hashFiles(const char** fnList, int fnTotal,
+                         algoType hashType, endianess displayEndianess)
+{
+    int fnNb;
+    int result = 0;
+
+    if (fnTotal==0)
+        return BMK_hash(stdinName, hashType, displayEndianess);
+
+    for (fnNb=0; fnNb<fnTotal; fnNb++)
+        result += BMK_hash(fnList[fnNb], hashType, displayEndianess);
+    DISPLAY("\r%70s\r", "");
+    return result;
+}
+
+
+typedef enum {
+    GetLine_ok,
+    GetLine_eof,
+    GetLine_exceedMaxLineLength,
+    GetLine_outOfMemory,
+} GetLineResult;
+
+typedef enum {
+    CanonicalFromString_ok,
+    CanonicalFromString_invalidFormat,
+} CanonicalFromStringResult;
+
+typedef enum {
+    ParseLine_ok,
+    ParseLine_invalidFormat,
+} ParseLineResult;
+
+typedef enum {
+    LineStatus_hashOk,
+    LineStatus_hashFailed,
+    LineStatus_failedToOpen,
+} LineStatus;
+
+typedef union {
+    XXH32_canonical_t xxh32;
+    XXH64_canonical_t xxh64;
+} Canonical;
+
+typedef struct {
+    Canonical   canonical;
+    const char* filename;
+    int         xxhBits;    /* canonical type : 32:xxh32, 64:xxh64 */
+} ParsedLine;
+
+typedef struct {
+    unsigned long   nProperlyFormattedLines;
+    unsigned long   nImproperlyFormattedLines;
+    unsigned long   nMismatchedChecksums;
+    unsigned long   nOpenOrReadFailures;
+    unsigned long   nMixedFormatLines;
+    int             xxhBits;
+    int             quit;
+} ParseFileReport;
+
+typedef struct {
+    const char*     inFileName;
+    FILE*           inFile;
+    int             lineMax;
+    char*           lineBuf;
+    size_t          blockSize;
+    char*           blockBuf;
+    int             strictMode;
+    int             statusOnly;
+    int             warn;
+    int             quiet;
+    ParseFileReport report;
+} ParseFileArg;
+
+
+/*  Read line from stream.
+    Returns GetLine_ok, if it reads line successfully.
+    Returns GetLine_eof, if stream reaches EOF.
+    Returns GetLine_exceedMaxLineLength, if line length is longer than MAX_LINE_LENGTH.
+    Returns GetLine_outOfMemory, if line buffer memory allocation failed.
+ */
+static GetLineResult getLine(char** lineBuf, int* lineMax, FILE* inFile)
+{
+    GetLineResult result = GetLine_ok;
+    int len = 0;
+
+    if ((*lineBuf == NULL) || (*lineMax<1)) {
+        free(*lineBuf);  /* in case it's != NULL */
+        *lineMax = 0;
+        *lineBuf = (char*)malloc(DEFAULT_LINE_LENGTH);
+        if(*lineBuf == NULL) return GetLine_outOfMemory;
+        *lineMax = DEFAULT_LINE_LENGTH;
+    }
+
+    for (;;) {
+        const int c = fgetc(inFile);
+        if (c == EOF) {
+            /* If we meet EOF before first character, returns GetLine_eof,
+             * otherwise GetLine_ok.
+             */
+            if (len == 0) result = GetLine_eof;
+            break;
+        }
+
+        /* Make enough space for len+1 (for final NUL) bytes. */
+        if (len+1 >= *lineMax) {
+            char* newLineBuf = NULL;
+            int newBufSize = *lineMax;
+
+            newBufSize += (newBufSize/2) + 1; /* x 1.5 */
+            if (newBufSize > MAX_LINE_LENGTH) newBufSize = MAX_LINE_LENGTH;
+            if (len+1 >= newBufSize) return GetLine_exceedMaxLineLength;
+
+            newLineBuf = (char*) realloc(*lineBuf, newBufSize);
+            if (newLineBuf == NULL) return GetLine_outOfMemory;
+
+            *lineBuf = newLineBuf;
+            *lineMax = newBufSize;
+        }
+
+        if (c == '\n') break;
+        (*lineBuf)[len++] = (char) c;
+    }
+
+    (*lineBuf)[len] = '\0';
+    return result;
+}
+
+
+/*  Converts one hexadecimal character to integer.
+ *  Returns -1, if given character is not hexadecimal.
+ */
+static int charToHex(char c)
+{
+    int result = -1;
+    if (c >= '0' && c <= '9') {
+        result = (int) (c - '0');
+    } else if (c >= 'A' && c <= 'F') {
+        result = (int) (c - 'A') + 0x0a;
+    } else if (c >= 'a' && c <= 'f') {
+        result = (int) (c - 'a') + 0x0a;
+    }
+    return result;
+}
+
+
+/*  Converts XXH32 canonical hexadecimal string hashStr to big endian unsigned char array dst.
+ *  Returns CANONICAL_FROM_STRING_INVALID_FORMAT, if hashStr is not well formatted.
+ *  Returns CANONICAL_FROM_STRING_OK, if hashStr is parsed successfully.
+ */
+static CanonicalFromStringResult canonicalFromString(unsigned char* dst,
+                                                     size_t dstSize,
+                                                     const char* hashStr)
+{
+    size_t i;
+    for (i = 0; i < dstSize; ++i) {
+        int h0, h1;
+
+        h0 = charToHex(hashStr[i*2 + 0]);
+        if (h0 < 0) return CanonicalFromString_invalidFormat;
+
+        h1 = charToHex(hashStr[i*2 + 1]);
+        if (h1 < 0) return CanonicalFromString_invalidFormat;
+
+        dst[i] = (unsigned char) ((h0 << 4) | h1);
+    }
+    return CanonicalFromString_ok;
+}
+
+
+/*  Parse single line of xxHash checksum file.
+ *  Returns PARSE_LINE_ERROR_INVALID_FORMAT, if line is not well formatted.
+ *  Returns PARSE_LINE_OK if line is parsed successfully.
+ *  And members of parseLine will be filled by parsed values.
+ *
+ *  - line must be ended with '\0'.
+ *  - Since parsedLine.filename will point within given argument `line`,
+ *    users must keep `line`s content during they are using parsedLine.
+ *
+ *  Given xxHash checksum line should have the following format:
+ *
+ *      <8 or 16 hexadecimal char> <space> <space> <filename...> <'\0'>
+ */
+static ParseLineResult parseLine(ParsedLine* parsedLine, const char* line)
+{
+    const char* const firstSpace = strchr(line, ' ');
+    const char* const secondSpace = firstSpace + 1;
+
+    parsedLine->filename = NULL;
+    parsedLine->xxhBits = 0;
+
+    if (firstSpace == NULL || *secondSpace != ' ') return ParseLine_invalidFormat;
+
+    switch (firstSpace - line)
+    {
+    case 8:
+        {   XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32;
+            if (canonicalFromString(xxh32c->digest, sizeof(xxh32c->digest), line)
+                != CanonicalFromString_ok) {
+                return ParseLine_invalidFormat;
+            }
+            parsedLine->xxhBits = 32;
+            break;
+        }
+
+    case 16:
+        {   XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64;
+            if (canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), line)
+                != CanonicalFromString_ok) {
+                return ParseLine_invalidFormat;
+            }
+            parsedLine->xxhBits = 64;
+            break;
+        }
+
+    default:
+            return ParseLine_invalidFormat;
+            break;
+    }
+
+    parsedLine->filename = secondSpace + 1;
+    return ParseLine_ok;
+}
+
+
+/*!  Parse xxHash checksum file.
+ */
+static void parseFile1(ParseFileArg* parseFileArg)
+{
+    const char* const inFileName = parseFileArg->inFileName;
+    ParseFileReport* const report = &parseFileArg->report;
+
+    unsigned long lineNumber = 0;
+    memset(report, 0, sizeof(*report));
+
+    while (!report->quit) {
+        FILE* fp = NULL;
+        LineStatus lineStatus = LineStatus_hashFailed;
+        GetLineResult getLineResult;
+        ParsedLine parsedLine;
+        memset(&parsedLine, 0, sizeof(parsedLine));
+
+        lineNumber++;
+        if (lineNumber == 0) {
+            /* This is unlikely happen, but md5sum.c has this
+             * error check. */
+            DISPLAY("%s : too many checksum lines\n", inFileName);
+            report->quit = 1;
+            break;
+        }
+
+        getLineResult = getLine(&parseFileArg->lineBuf, &parseFileArg->lineMax,
+                                parseFileArg->inFile);
+        if (getLineResult != GetLine_ok) {
+            if (getLineResult == GetLine_eof) break;
+
+            switch (getLineResult)
+            {
+            case GetLine_ok:
+            case GetLine_eof:
+                /* These cases never happen.  See above getLineResult related "if"s.
+                   They exist just for make gcc's -Wswitch-enum happy. */
+                break;
+
+            default:
+                DISPLAY("%s : %lu: unknown error\n", inFileName, lineNumber);
+                break;
+
+            case GetLine_exceedMaxLineLength:
+                DISPLAY("%s : %lu: too long line\n", inFileName, lineNumber);
+                break;
+
+            case GetLine_outOfMemory:
+                DISPLAY("%s : %lu: out of memory\n", inFileName, lineNumber);
+                break;
+            }
+            report->quit = 1;
+            break;
+        }
+
+        if (parseLine(&parsedLine, parseFileArg->lineBuf) != ParseLine_ok) {
+            report->nImproperlyFormattedLines++;
+            if (parseFileArg->warn) {
+                DISPLAY("%s : %lu: improperly formatted XXHASH checksum line\n"
+                    , inFileName, lineNumber);
+            }
+            continue;
+        }
+
+        if (report->xxhBits != 0 && report->xxhBits != parsedLine.xxhBits) {
+            /* Don't accept xxh32/xxh64 mixed file */
+            report->nImproperlyFormattedLines++;
+            report->nMixedFormatLines++;
+            if (parseFileArg->warn) {
+                DISPLAY("%s : %lu: improperly formatted XXHASH checksum line (XXH32/64)\n"
+                    , inFileName, lineNumber);
+            }
+            continue;
+        }
+
+        report->nProperlyFormattedLines++;
+        if (report->xxhBits == 0) {
+            report->xxhBits = parsedLine.xxhBits;
+        }
+
+        fp = fopen(parsedLine.filename, "rb");
+        if (fp == NULL) {
+            lineStatus = LineStatus_failedToOpen;
+        } else {
+            lineStatus = LineStatus_hashFailed;
+            switch (parsedLine.xxhBits)
+            {
+            case 32:
+                {   XXH32_hash_t xxh;
+                    BMK_hashStream(&xxh, algo_xxh32, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+                    if (xxh == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) {
+                        lineStatus = LineStatus_hashOk;
+                }   }
+                break;
+
+            case 64:
+                {   XXH64_hash_t xxh;
+                    BMK_hashStream(&xxh, algo_xxh64, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+                    if (xxh == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) {
+                        lineStatus = LineStatus_hashOk;
+                }   }
+                break;
+
+            default:
+                break;
+            }
+            fclose(fp);
+        }
+
+        switch (lineStatus)
+        {
+        default:
+            DISPLAY("%s : unknown error\n", inFileName);
+            report->quit = 1;
+            break;
+
+        case LineStatus_failedToOpen:
+            report->nOpenOrReadFailures++;
+            if (!parseFileArg->statusOnly) {
+                DISPLAYRESULT("%s : %lu: FAILED open or read %s\n"
+                    , inFileName, lineNumber, parsedLine.filename);
+            }
+            break;
+
+        case LineStatus_hashOk:
+        case LineStatus_hashFailed:
+            {   int b = 1;
+                if (lineStatus == LineStatus_hashOk) {
+                    /* If --quiet is specified, don't display "OK" */
+                    if (parseFileArg->quiet) b = 0;
+                } else {
+                    report->nMismatchedChecksums++;
+                }
+
+                if (b && !parseFileArg->statusOnly) {
+                    DISPLAYRESULT("%s: %s\n", parsedLine.filename
+                        , lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
+            }   }
+            break;
+        }
+    }   /* while (!report->quit) */
+}
+
+
+/*  Parse xxHash checksum file.
+ *  Returns 1, if all procedures were succeeded.
+ *  Returns 0, if any procedures was failed.
+ *
+ *  If strictMode != 0, return error code if any line is invalid.
+ *  If statusOnly != 0, don't generate any output.
+ *  If warn != 0, print a warning message to stderr.
+ *  If quiet != 0, suppress "OK" line.
+ *
+ *  "All procedures are succeeded" means:
+ *    - Checksum file contains at least one line and less than SIZE_T_MAX lines.
+ *    - All files are properly opened and read.
+ *    - All hash values match with its content.
+ *    - (strict mode) All lines in checksum file are consistent and well formatted.
+ *
+ */
+static int checkFile(const char* inFileName,
+                     const endianess displayEndianess,
+                     U32 strictMode,
+                     U32 statusOnly,
+                     U32 warn,
+                     U32 quiet)
+{
+    int result = 0;
+    FILE* inFile = NULL;
+    ParseFileArg parseFileArgBody;
+    ParseFileArg* const parseFileArg = &parseFileArgBody;
+    ParseFileReport* const report = &parseFileArg->report;
+
+    if (displayEndianess != big_endian) {
+        /* Don't accept little endian */
+        DISPLAY( "Check file mode doesn't support little endian\n" );
+        return 0;
+    }
+
+    /* note : stdinName is special constant pointer.  It is not a string. */
+    if (inFileName == stdinName) {
+        /* note : Since we expect text input for xxhash -c mode,
+         * Don't set binary mode for stdin */
+        inFile = stdin;
+    } else {
+        inFile = fopen( inFileName, "rt" );
+    }
+
+    if (inFile == NULL) {
+        DISPLAY( "Pb opening %s\n", inFileName);
+        return 0;
+    }
+
+    parseFileArg->inFileName    = inFileName;
+    parseFileArg->inFile        = inFile;
+    parseFileArg->lineMax       = DEFAULT_LINE_LENGTH;
+    parseFileArg->lineBuf       = (char*) malloc((size_t) parseFileArg->lineMax);
+    parseFileArg->blockSize     = 64 * 1024;
+    parseFileArg->blockBuf      = (char*) malloc(parseFileArg->blockSize);
+    parseFileArg->strictMode    = strictMode;
+    parseFileArg->statusOnly    = statusOnly;
+    parseFileArg->warn          = warn;
+    parseFileArg->quiet         = quiet;
+
+    parseFile1(parseFileArg);
+
+    free(parseFileArg->blockBuf);
+    free(parseFileArg->lineBuf);
+
+    if (inFile != stdin) fclose(inFile);
+
+    /* Show error/warning messages.  All messages are copied from md5sum.c
+     */
+    if (report->nProperlyFormattedLines == 0) {
+        DISPLAY("%s: no properly formatted XXHASH checksum lines found\n", inFileName);
+    } else if (!statusOnly) {
+        if (report->nImproperlyFormattedLines) {
+            DISPLAYRESULT("%lu lines are improperly formatted\n"
+                , report->nImproperlyFormattedLines);
+        }
+        if (report->nOpenOrReadFailures) {
+            DISPLAYRESULT("%lu listed files could not be read\n"
+                , report->nOpenOrReadFailures);
+        }
+        if (report->nMismatchedChecksums) {
+            DISPLAYRESULT("%lu computed checksums did NOT match\n"
+                , report->nMismatchedChecksums);
+    }   }
+
+    /* Result (exit) code logic is copied from
+     * gnu coreutils/src/md5sum.c digest_check() */
+    result =   report->nProperlyFormattedLines != 0
+            && report->nMismatchedChecksums == 0
+            && report->nOpenOrReadFailures == 0
+            && (!strictMode || report->nImproperlyFormattedLines == 0)
+            && report->quit == 0;
+    return result;
+}
+
+
+static int checkFiles(const char** fnList, int fnTotal,
+                      const endianess displayEndianess,
+                      U32 strictMode,
+                      U32 statusOnly,
+                      U32 warn,
+                      U32 quiet)
+{
+    int ok = 1;
+
+    /* Special case for stdinName "-",
+     * note: stdinName is not a string.  It's special pointer. */
+    if (fnTotal==0) {
+        ok &= checkFile(stdinName, displayEndianess, strictMode, statusOnly, warn, quiet);
+    } else {
+        int fnNb;
+        for (fnNb=0; fnNb<fnTotal; fnNb++)
+            ok &= checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, warn, quiet);
+    }
+    return ok ? 0 : 1;
+}
+
+
+/* ********************************************************
+*  Main
+**********************************************************/
+
+static int usage(const char* exename)
+{
+    DISPLAY( WELCOME_MESSAGE(exename) );
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [arg] [filenames]\n", exename);
+    DISPLAY( "When no filename provided, or - provided : use stdin as input\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default: %i)\n", (int)g_defaultAlgo);
+    DISPLAY( " -c  : read xxHash sums from the [filenames] and check them\n");
+    DISPLAY( " -h  : help \n");
+    return 0;
+}
+
+
+static int usage_advanced(const char* exename)
+{
+    usage(exename);
+    DISPLAY( "Advanced :\n");
+    DISPLAY( " --little-endian : hash printed using little endian convention (default: big endian)\n");
+    DISPLAY( " -V, --version   : display version\n");
+    DISPLAY( " -h, --help      : display long help and exit\n");
+    DISPLAY( " -b  : benchmark mode \n");
+    DISPLAY( " -i# : number of iterations (benchmark mode; default %i)\n", g_nbIterations);
+    DISPLAY( "\n");
+    DISPLAY( "The following four options are useful only when verifying checksums (-c):\n");
+    DISPLAY( "--strict : don't print OK for each successfully verified file\n");
+    DISPLAY( "--status : don't output anything, status code shows success\n");
+    DISPLAY( "--quiet  : exit non-zero for improperly formatted checksum lines\n");
+    DISPLAY( "--warn   : warn about improperly formatted checksum lines\n");
+    return 0;
+}
+
+static int badusage(const char* exename)
+{
+    DISPLAY("Wrong parameters\n");
+    usage(exename);
+    return 1;
+}
+
+/*! readU32FromChar() :
+   @return : unsigned integer value read from input in `char` format,
+             0 is no figure at *stringPtr position.
+    Interprets K, KB, KiB, M, MB and MiB suffix.
+    Modifies `*stringPtr`, advancing it to position where reading stopped.
+    Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+    unsigned result = 0;
+    while ((**stringPtr >='0') && (**stringPtr <='9'))
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        result <<= 10;
+        if (**stringPtr=='M') result <<= 10;
+        (*stringPtr)++ ;
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
+    return result;
+}
+
+int main(int argc, const char** argv)
+{
+    int i, filenamesStart = 0;
+    const char* const exename = argv[0];
+    U32 benchmarkMode = 0;
+    U32 fileCheckMode = 0;
+    U32 strictMode    = 0;
+    U32 statusOnly    = 0;
+    U32 warn          = 0;
+    U32 quiet         = 0;
+    U32 specificTest  = 0;
+    size_t keySize    = XXH_DEFAULT_SAMPLE_SIZE;
+    algoType algo     = g_defaultAlgo;
+    endianess displayEndianess = big_endian;
+
+    /* special case : xxh32sum default to 32 bits checksum */
+    if (strstr(exename, "xxh32sum") != NULL) algo = algo_xxh32;
+
+    for(i=1; i<argc; i++) {
+        const char* argument = argv[i];
+
+        if(!argument) continue;   /* Protection, if argument empty */
+
+        if (!strcmp(argument, "--little-endian")) { displayEndianess = little_endian; continue; }
+        if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; }
+        if (!strcmp(argument, "--strict")) { strictMode = 1; continue; }
+        if (!strcmp(argument, "--status")) { statusOnly = 1; continue; }
+        if (!strcmp(argument, "--quiet")) { quiet = 1; continue; }
+        if (!strcmp(argument, "--warn")) { warn = 1; continue; }
+        if (!strcmp(argument, "--help")) { return usage_advanced(exename); }
+        if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE(exename)); return 0; }
+
+        if (*argument!='-') {
+            if (filenamesStart==0) filenamesStart=i;   /* only supports a continuous list of filenames */
+            continue;
+        }
+
+        /* command selection */
+        argument++;   /* note : *argument=='-' */
+
+        while (*argument!=0) {
+            switch(*argument)
+            {
+            /* Display version */
+            case 'V':
+                DISPLAY(WELCOME_MESSAGE(exename)); return 0;
+
+            /* Display help on usage */
+            case 'h':
+                return usage_advanced(exename);
+
+            /* select hash algorithm */
+            case 'H':
+                algo = (algoType)(argument[1] - '0');
+                argument+=2;
+                break;
+
+            /* File check mode */
+            case 'c':
+                fileCheckMode=1;
+                argument++;
+                break;
+
+            /* Warning mode (file check mode only, alias of "--warning") */
+            case 'w':
+                warn=1;
+                argument++;
+                break;
+
+            /* Trigger benchmark mode */
+            case 'b':
+                argument++;
+                benchmarkMode = 1;
+                specificTest = readU32FromChar(&argument);   /* select one specific test (hidden option) */
+                break;
+
+            /* Modify Nb Iterations (benchmark only) */
+            case 'i':
+                argument++;
+                g_nbIterations = readU32FromChar(&argument);
+                break;
+
+            /* Modify Block size (benchmark only) */
+            case 'B':
+                argument++;
+                keySize = readU32FromChar(&argument);
+                break;
+
+            /* Modify verbosity of benchmark output (hidden option) */
+            case 'q':
+                argument++;
+                g_displayLevel--;
+                break;
+
+            default:
+                return badusage(exename);
+            }
+        }
+    }   /* for(i=1; i<argc; i++) */
+
+    /* Check benchmark mode */
+    if (benchmarkMode) {
+        DISPLAYLEVEL(2, WELCOME_MESSAGE(exename) );
+        BMK_sanityCheck();
+        if (filenamesStart==0) return BMK_benchInternal(keySize, specificTest);
+        return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart, specificTest);
+    }
+
+    /* Check if input is defined as console; trigger an error in this case */
+    if ( (filenamesStart==0) && IS_CONSOLE(stdin) ) return badusage(exename);
+
+    if (filenamesStart==0) filenamesStart = argc;
+    if (fileCheckMode) {
+        return checkFiles(argv+filenamesStart, argc-filenamesStart,
+                          displayEndianess, strictMode, statusOnly, warn, quiet);
+    } else {
+        return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess);
+    }
+}
+
+#endif /* XXHASH_C_2097394837 */
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 1103b607dba..de68c20b4d7 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,6 +1,9 @@
 EXTRA_DIST = glusterfs.8 mount.glusterfs.8 gluster.8 \
 			glusterd.8 glusterfsd.8
 
-man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 glusterfsd.8
+man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8
+if WITH_SERVER
+man8_MANS += glusterd.8 glusterfsd.8
+endif
 
 CLEANFILES =
diff --git a/doc/README.md b/doc/README.md
index e057437fcba..6aa28642ef4 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -1,6 +1,14 @@
+## Developer Guide
+
+Gluster's contributors can check about the internals by visiting [Developer Guide Section](developer-guide). While it is not 'comprehensive', it can help you to get started.
+
+Also while coding, keep [Coding Standard](developer-guide/coding-standard.md) in mind.
+
+When you are ready to commit the changes, make sure you meet our [Commit message standard](developer-guide/commit-guidelines.md).
+
 ## Admin Guide ##
 
-The gluster administration guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable admin guide can be found [here](http://gluster.readthedocs.org/en/latest/Administrator%20Guide/README/).
+The gluster administration guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable admin guide can be found [here](http://docs.gluster.org/en/latest/Administrator%20Guide/).
 
 The doc patch has to be sent against the above mentioned repository.
 
@@ -10,7 +18,7 @@ The Gluster features which are 'in progress' or implemented can be found at [git
 
 ## Upgrade Guide ##
 
-The gluster upgrade guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable upgrade guide can be found [here](http://gluster.readthedocs.org/en/latest/Upgrade-Guide/README/)
+The gluster upgrade guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable upgrade guide can be found [here](http://docs.gluster.org/en/latest/Upgrade-Guide)
 
 The doc patch has to be sent against the above mentioned repository.
 
diff --git a/doc/developer-guide/coredump-analysis.md b/doc/debugging/analyzing-regression-cores.md
index 16fa9165fd0..5e10f41c6eb 100644
--- a/doc/developer-guide/coredump-analysis.md
+++ b/doc/debugging/analyzing-regression-cores.md
@@ -1,36 +1,35 @@
-This document explains how to analyze core-dumps obtained from regression
-machines, with examples.  
-1) Download the core-tarball and extract it.  
-2) 'cd' into directory where the tarball is extracted.
-~~~
-[root@atalur Downloads]# pwd
-/home/atalur/Downloads
-[root@atalur Downloads]# ls
+# Analyzing Regression Cores
+This document explains how to analyze core-dumps obtained from regression machines, with examples.
+1. Download the core-tarball and extract it.
+2. `cd` into directory where the tarball is extracted.
+```
+[sh]# pwd
+/home/user/Downloads
+[sh]# ls
 build  build-install-20150625_05_42_39.tar.bz2  lib64  usr
-~~~
-3) Determine the core file you need to examine. There can be more than one core file.  
-You can list them from './build/install/cores' directory.
-~~~
-[root@atalur Downloads]# ls build/install/cores/
+```
+3. Determine the core file you need to examine. There can be more than one core file. You can list them from './build/install/cores' directory.
+```
+[sh]# ls build/install/cores/
 core.9341  liblist.txt  liblist.txt.tmp
-~~~
+```
 In case you are unsure which binary generated the core-file, executing 'file' command on it will help.
-~~~
-[root@atalur Downloads]# file ./build/install/cores/core.9341 
+```
+[sh]# file ./build/install/cores/core.9341 
 ./build/install/cores/core.9341: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from '/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'
-~~~
-As seen, the core file was generated by glusterfsd binary, and path to it is provided (/build/install/sbin/glusterfsd).  
-4) Now, run the following command on the core:
-~~~
+```
+As seen, the core file was generated by glusterfsd binary, and path to it is provided (/build/install/sbin/glusterfsd).
+
+4. Now, run the following command on the core:
+```
 gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.xxx' <target, say ./build/install/sbin/glusterd>
 In this case,
 gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.9341' ./build/install/sbin/glusterfsd
-~~~
-5) You can cross check if all shared libraries are available and loaded by using 'info sharedlibrary' command from
-inside gdb.  
-6) Once verified, usual gdb commands based on requirement can be used to debug the core.
-'bt' or 'backtrace' from gdb of core used in examples:
-~~~
+```
+5. You can cross check if all shared libraries are available and loaded by using 'info sharedlibrary' command from inside gdb.
+6. Once verified, usual gdb commands based on requirement can be used to debug the core.
+   `bt` or `backtrace` from gdb of core used in examples:
+```
 Core was generated by `/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'.
 Program terminated with signal SIGABRT, Aborted.
 #0  0x00007f512a54e625 in raise () from ./lib64/libc.so.6
@@ -52,4 +51,4 @@ Program terminated with signal SIGABRT, Aborted.
 #12 0x00007f512a55f8f0 in ?? () from ./lib64/libc.so.6
 #13 0x0000000000000000 in ?? ()
 (gdb) 
-~~~
+```
diff --git a/doc/debugging/coredump-analysis.md b/doc/debugging/coredump-analysis.md
deleted file mode 100644
index f9ecf73216e..00000000000
--- a/doc/debugging/coredump-analysis.md
+++ /dev/null
@@ -1,31 +0,0 @@
-This document explains how to analyze core-dumps obtained from regression
-machines, with examples.  
-1) Download the core-tarball and extract it.  
-2) 'cd' into the root of extracted tarball.
-~~~
-[root@atalur Downloads]# pwd
-/home/atalur/Downloads
-[root@atalur Downloads]# ls
-build  build-install-20150625_05_42_39.tar.bz2  lib64  usr
-~~~
-3) Determine the core file you need to examine. There can be more than one core file.  
-You can list them from './build/install/cores' directory.
-~~~
-[root@atalur Downloads]# ls build/install/cores/
-core.9341  liblist.txt  liblist.txt.tmp
-~~~
-In case you are unsure which binary generated the core-file, executing 'file' command on it will help.
-~~~
-[root@atalur Downloads]# file ./build/install/cores/core.9341 
-./build/install/cores/core.9341: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from '/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'
-~~~
-As seen, the core file was generated by glusterfsd binary, and path to it is provide (/build/install/sbin/glusterfsd).  
-4) Now, run the following command on the core:
-~~~
-gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.xxx' <target, say ./build/install/sbin/glusterd>
-In this case,
-gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.9341' ./build/install/sbin/glusterfsd
-~~~
-5) You can cross check if all shared libraries are available and loaded by using 'info sharedlibrary' command from
-inside gdb.  
-6) Once verified, usual gdb commands based on requirement can be used to debug the core.
diff --git a/doc/debugging/gfid-to-path.md b/doc/debugging/gfid-to-path.md
index 09c459e52c8..1917bf2cca1 100644
--- a/doc/debugging/gfid-to-path.md
+++ b/doc/debugging/gfid-to-path.md
@@ -1,37 +1,37 @@
-#Convert GFID to Path
+# Convert GFID to Path
 
 GlusterFS internal file identifier (GFID) is a uuid that is unique to each
 file across the entire cluster. This is analogous to inode number in a
 normal filesystem. The GFID of a file is stored in its xattr named
 `trusted.gfid`.
 
-####Special mount using [gfid-access translator][1]:
-~~~
+#### Special mount using [gfid-access translator][1]:
+```
 mount -t glusterfs -o aux-gfid-mount vm1:test /mnt/testvol
-~~~
+```
 
 Assuming, you have `GFID` of a file from changelog (or somewhere else).
 For trying this out, you can get `GFID` of a file from mountpoint:
-~~~
+```
 getfattr -n glusterfs.gfid.string /mnt/testvol/dir/file
-~~~
+```
 
 
 ---
-###Get file path from GFID (Method 1):
+### Get file path from GFID (Method 1):
 **(Lists hardlinks delimited by `:`, returns path as seen from mountpoint)**
 
-####Turn on build-pgfid option
-~~~
+#### Turn on build-pgfid option
+```
 gluster volume set test build-pgfid on
-~~~
+```
 Read virtual xattr `glusterfs.ancestry.path` which contains the file path
-~~~
+```
 getfattr -n glusterfs.ancestry.path -e text /mnt/testvol/.gfid/<GFID>
-~~~
+```
 
 **Example:**
-~~~
+```
 [root@vm1 glusterfs]# ls -il /mnt/testvol/dir/
 total 1
 10610563327990022372 -rw-r--r--. 2 root root 3 Jul 17 18:05 file
@@ -46,28 +46,23 @@ glusterfs.gfid.string="11118443-1894-4273-9340-4b212fa1c0e4"
 getfattr: Removing leading '/' from absolute path names
 # file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
 glusterfs.ancestry.path="/dir/file:/dir/file3"
-~~~
+```
 
 ---
-###Get file path from GFID (Method 2):
+### Get file path from GFID (Method 2):
 **(Does not list all hardlinks, returns backend brick path)**
-~~~
+```
 getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/<GFID>
-~~~
+```
 
 **Example:**
-~~~
+```
 [root@vm1 glusterfs]# getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
 getfattr: Removing leading '/' from absolute path names
 # file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
 trusted.glusterfs.pathinfo="(<DISTRIBUTE:test-dht> <POSIX(/mnt/brick-test/b):vm1:/mnt/brick-test/b/dir//file3>)"
-~~~
+```
 
 ---
-###Get file path from GFID (Method 3):
-https://gist.github.com/semiosis/4392640
-
----
-####References and links:
+#### References and links:
 [posix: placeholders for GFID to path conversion](http://review.gluster.org/5951)
-[1]: https://github.com/gluster/glusterfs/blob/master/doc/features/gfid-access.md
diff --git a/doc/debugging/mem-alloc-list.md b/doc/debugging/mem-alloc-list.md
new file mode 100644
index 00000000000..1c68e65d323
--- /dev/null
+++ b/doc/debugging/mem-alloc-list.md
@@ -0,0 +1,19 @@
+## Viewing Memory Allocations
+
+While statedumps provide stats of the number of allocations, size etc for a
+particular mem type, there is no easy way to examine all the allocated objects of that type
+in memory.Being able to view this information could help with determining how an object is used,
+and if there are any memory leaks.
+
+The mem_acct_rec structures have been updated to include lists to which the allocated object is
+added. These can be examined in gdb using simple scripts.
+
+`gdb> plist xl->mem_acct.rec[$type]->obj_list`
+
+will print out the pointers of all allocations of $type.
+
+These changes are primarily targeted at developers and need to enabled
+at compile-time using `configure --enable-debug`.
+
+
+
diff --git a/doc/debugging/split-brain.md b/doc/debugging/split-brain.md
index b0d938e26bc..6b122c40551 100644
--- a/doc/debugging/split-brain.md
+++ b/doc/debugging/split-brain.md
@@ -1,33 +1,36 @@
-Steps to recover from File split-brain.
-======================================
-
-Quick Start:
-============
-1. Get the path of the file that is in split-brain:  
->  It can be obtained either by  
->       a) The command `gluster volume heal info split-brain`.  
->       b) Identify the files for which file operations performed
-           from the client keep failing with Input/Output error.
-
-2. Close the applications that opened this file from the mount point.
+# Steps to recover from File split-brain
+This document contains steps to recover from a file split-brain.
+## Quick Start:
+### Step 1. Get the path of the file that is in split-brain:
+It can be obtained either by
+1. The command `gluster volume heal info split-brain`.
+2. Identify the files for which file operations performed from the client keep failing with Input/Output error.
+
+### Step 2. Close the applications that opened this file from the mount point.
 In case of VMs, they need to be powered-off.
 
-3. Decide on the correct copy:  
-> This is done by observing the afr changelog extended attributes of the file on
+### Step 3. Decide on the correct copy:  
+This is done by observing the afr changelog extended attributes of the file on
 the bricks using the getfattr command; then identifying the type of split-brain 
 (data split-brain, metadata split-brain, entry split-brain or split-brain due to
 gfid-mismatch); and finally determining which of the bricks contains the 'good copy'
 of the file.  
->   `getfattr -d -m . -e hex <file-path-on-brick>`.  
+```
+getfattr -d -m . -e hex <file-path-on-brick>
+```
+  
 It is also possible that one brick might contain the correct data while the
 other might contain the correct metadata.
 
-4. Reset the relevant extended attribute on the brick(s) that contains the
-'bad copy' of the file data/metadata using the setfattr command.  
->   `setfattr -n <attribute-name> -v <attribute-value> <file-path-on-brick>`
+### Step 4. Reset the relevant extended attribute on the brick(s) that contains the 'bad copy' of the file data/metadata using the setfattr command.  
+```
+setfattr -n <attribute-name> -v <attribute-value> <file-path-on-brick>
+```
 
-5. Trigger self-heal on the file by performing lookup from the client:  
->   `ls -l <file-path-on-gluster-mount>`
+### Step 5. Trigger self-heal on the file by performing lookup from the client:  
+```
+ls -l <file-path-on-gluster-mount>
+```
 
 Detailed Instructions for steps 3 through 5:  
 ===========================================
@@ -36,13 +39,15 @@ afr changelog extended attributes.
 
 Execute `getfattr -d -m . -e hex <file-path-on-brick>`
 
-* Example:  
+Example:  
+```
 [root@store3 ~]# getfattr -d -e hex -m. brick-a/file.txt  
 \#file: brick-a/file.txt  
 security.selinux=0x726f6f743a6f626a6563745f723a66696c655f743a733000  
 trusted.afr.vol-client-2=0x000000000000000000000000  
 trusted.afr.vol-client-3=0x000000000200000000000000  
 trusted.gfid=0x307a5c9efddd4e7c96e94fd4bcdcbd1b  
+```
 
 The extended attributes with `trusted.afr.<volname>-client-<subvolume-index>`
 are used by afr to maintain changelog of the file.The values of the
@@ -51,10 +56,11 @@ client (fuse or nfs-server) processes. When the glusterfs client modifies a file
 or directory, the client contacts each brick and updates the changelog extended 
 attribute according to the response of the brick.
 
-'subvolume-index' is nothing but (brick number - 1) in
+`subvolume-index` is nothing but (brick number - 1) in
 `gluster volume info <volname>` output.
 
-* Example:  
+Example:
+```
 [root@pranithk-laptop ~]# gluster volume info vol  
  Volume Name: vol  
  Type: Distributed-Replicate  
@@ -71,6 +77,7 @@ attribute according to the response of the brick.
  brick-f: pranithk-laptop:/gfs/brick-f  
  brick-g: pranithk-laptop:/gfs/brick-g  
  brick-h: pranithk-laptop:/gfs/brick-h  
+```
 
 In the example above:  
 ```
@@ -91,12 +98,15 @@ present in all the other bricks in it's replica set as seen by that brick.
 
 In the example volume given above, all files in brick-a will have 2 entries, 
 one for itself and the other for the file present in it's replica pair, i.e.brick-b:  
+```
 trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for itself (brick-a)  
 trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for brick-b as seen by brick-a  
-
+```
 Likewise, all files in brick-b will have:  
+```
 trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for brick-a as seen by brick-b  
 trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for itself (brick-b)  
+```
 
 The same can be extended for other replica pairs.  
 
@@ -122,7 +132,8 @@ When a file split-brain happens it could be either data split-brain or
 meta-data split-brain or both. When a split-brain happens the changelog of the
 file would be something like this:  
 
-* Example:(Lets consider both data, metadata split-brain on same file).  
+Example:(Lets consider both data, metadata split-brain on same file).  
+```
 [root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
 getfattr: Removing leading '/' from absolute path names  
 \#file: gfs/brick-a/a  
@@ -133,10 +144,11 @@ trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57
 trusted.afr.vol-client-0=0x000003b00000000100000000  
 trusted.afr.vol-client-1=0x000000000000000000000000  
 trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+```
 
-###Observations:
+### Observations:
 
-####According to changelog extended attributes on file /gfs/brick-a/a:  
+#### According to changelog extended attributes on file /gfs/brick-a/a:  
 The first 8 digits of trusted.afr.vol-client-0 are all
 zeros (0x00000000................), and the first 8 digits of
 trusted.afr.vol-client-1 are not all zeros (0x000003d7................).
@@ -149,7 +161,7 @@ trusted.afr.vol-client-1 are not all zeros (0x........00000001........).
 So the changelog on /gfs/brick-a/a implies that some metadata operations succeeded 
 on itself but failed on /gfs/brick-b/a.
 
-####According to Changelog extended attributes on file /gfs/brick-b/a:  
+#### According to Changelog extended attributes on file /gfs/brick-b/a:  
 The first 8 digits of trusted.afr.vol-client-0 are not all
 zeros (0x000003b0................), and the first 8 digits of
 trusted.afr.vol-client-1 are all zeros (0x00000000................).
@@ -205,6 +217,7 @@ Hence execute
 `setfattr -n trusted.afr.vol-client-1 -v 0x000003d70000000000000000 /gfs/brick-a/a`
 
 Thus after the above operations are done, the changelogs look like this:  
+```
 [root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
 getfattr: Removing leading '/' from absolute path names  
 \#file: gfs/brick-a/a  
@@ -216,7 +229,7 @@ trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57
 trusted.afr.vol-client-0=0x000000000000000100000000  
 trusted.afr.vol-client-1=0x000000000000000000000000  
 trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
-
+```
 
 Triggering Self-heal:
 ---------------------
@@ -243,9 +256,9 @@ needs to be removed.The gfid-link files are present in the .glusterfs folder
 in the top-level directory of the brick. If the gfid of the file is
 0x307a5c9efddd4e7c96e94fd4bcdcbd1b (the trusted.gfid extended attribute got
 from the getfattr command earlier),the gfid-link file can be found at
-> /gfs/brick-a/.glusterfs/30/7a/307a5c9efddd4e7c96e94fd4bcdcbd1b
+`/gfs/brick-a/.glusterfs/30/7a/307a5c9efddd4e7c96e94fd4bcdcbd1b`
 
-####Word of caution:
+#### Word of caution:
 Before deleting the gfid-link, we have to ensure that there are no hard links
 to the file present on that brick. If hard-links exist,they must be deleted as
 well.
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
index f34a5c3436a..9dfdce15fad 100644
--- a/doc/debugging/statedump.md
+++ b/doc/debugging/statedump.md
@@ -1,31 +1,53 @@
-#Statedump
+# Statedump
 Statedump is a file generated by glusterfs process with different data structure state which may contain the active inodes, fds, mempools, iobufs, memory allocation stats of different types of datastructures per xlator etc.
 
-##How to generate statedump
-We can find the directory where statedump files are created using 'gluster --print-statedumpdir' command.
+## How to generate statedump
+We can find the directory where statedump files are created using `gluster --print-statedumpdir` command.
 Create that directory if not already present based on the type of installation.
 Lets call this directory `statedump-directory`.
 
-We can generate statedump using 'kill -USR1 <pid-of-gluster-process>'.
+We can generate statedump using `kill -USR1 <pid-of-gluster-process>`.
 gluster-process is nothing but glusterd/glusterfs/glusterfsd process.
 
 There are also commands to generate statedumps for brick processes/nfs server/quotad
 
-For bricks:     `gluster volume statedump <volname>`
+For bricks: 
+```
+gluster volume statedump <volname>
+```
 
-For nfs server: `gluster volume statedump <volname> nfs`
+For nfs server:
+```
+gluster volume statedump <volname> nfs
+```
 
-For quotad:     `gluster volume statedump <volname> quotad`
+For quotad:
+```
+gluster volume statedump <volname> quotad
+```
 
 For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
 
-##How to read statedump
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+```
+gluster volume statedump <volname> client <hostname>:<process id>
+```
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
+## How to read statedump
 We shall see snippets of each type of statedump.
 
 First and last lines of the file have starting and ending time of writing the statedump file. Times will be in UTC timezone.
 
 mallinfo return status is printed in the following format. Please read man mallinfo for more information about what each field means.
-###Mallinfo
+### Mallinfo
 ```
 [mallinfo]
 mallinfo_arena=100020224 /* Non-mmapped space allocated (bytes) */
@@ -40,7 +62,7 @@ mallinfo_fordblks=3310112 /* Total free space (bytes) */
 mallinfo_keepcost=133712 /* Top-most, releasable space (bytes) */
 ```
 
-###Data structure allocation stats
+### Data structure allocation stats
 For every xlator data structure memory per translator loaded in the call-graph is displayed in the following format:
 
 For xlator with name: glusterfs
@@ -61,7 +83,7 @@ max_num_allocs=3 #Maximum number of active allocations at any point in the life
 total_allocs=7 #Number of times this data is allocated in the life of the process.
 ```
 
-###Mempools
+### Mempools
 
 Mempools are optimization to reduce the number of allocations of a data type. If we create a mem-pool of lets say 1024 elements for a data-type, new elements will be allocated from heap using syscalls like calloc, only if all the 1024 elements in the pool are in active use.
 
@@ -81,7 +103,7 @@ cur-stdalloc=0 #Denotes the number of allocations made from heap once cold-count
 max-stdalloc=0 #Maximum number of allocations from heap that are in active use at any point in the life of the process.
 ```
 
-###Iobufs
+### Iobufs
 ```
 [iobuf.global]
 iobuf_pool=0x1f0d970 #The memory pool for iobufs
@@ -92,7 +114,7 @@ iobuf_pool.arena_cnt=8 #Total number of arenas in the pool
 iobuf_pool.request_misses=0 #The number of iobufs that were stdalloc'd (as they exceeded the default max page size provided by iobuf_pool).
 ```
 
-There are 3 lists of arenas
+There are 3 lists of arenas:
 
 1. Arena list: arenas allocated during iobuf pool creation and the arenas that are in use(active_cnt != 0) will be part of this list.
 2. Purge list: arenas that can be purged(no active iobufs, active_cnt == 0).
@@ -129,7 +151,7 @@ arena.6.active_iobuf.2.ptr=0x7fdb92189000
 
 At any given point in time if there are lots of filled arenas then that could be a sign of iobuf leaks.
 
-###Call stack
+### Call stack
 All the fops received by gluster are handled using call-stacks. Call stack contains the information about uid/gid/pid etc of the process that is executing the fop. Each call-stack contains different call-frames per xlator which handles that fop.
 
 ```
@@ -144,7 +166,7 @@ op=LOOKUP #Fop
 type=1 #Type of the op i.e. FOP/MGMT-OP
 cnt=9 #Number of frames in this stack.
 ```
-###Call-frame
+### Call-frame
 Each frame will have information about which xlator the frame belongs to, what is the function it wound to/from and will be unwind to. It also mentions if the unwind happened or not. If we observe hangs in the system and want to find out which xlator is causing it. Take a statedump and see what is the final xlator which is yet to be unwound.
 
 ```
@@ -159,7 +181,7 @@ wind_to=priv->children[i]->fops->lookup
 unwind_to=afr_lookup_cbk #Parent xlator function to which unwind happened
 ```
 
-###History of operations in Fuse
+### History of operations in Fuse
 
 Fuse maintains history of operations that happened in fuse.
 
@@ -175,7 +197,7 @@ TIME=2014-07-09 16:44:57.523394
 message=[0] fuse_getattr_resume: 4591, STAT, path: (/iozone.tmp), gfid: (3afb4968-5100-478d-91e9-76264e634c9f)
 ```
 
-###Xlator configuration
+### Xlator configuration
 ```
 [cluster/replicate.r2-replicate-0] #Xlator type, name information
 child_count=2 #Number of children to the xlator
@@ -195,7 +217,7 @@ favorite_child=-1
 wait_count=1
 ```
 
-###Graph/inode table
+### Graph/inode table
 ```
 [active graph - 1]
 
@@ -207,7 +229,7 @@ conn.1.bound_xl./data/brick01a/homegfs.lru_size=183 #Number of inodes present in
 conn.1.bound_xl./data/brick01a/homegfs.purge_size=0 #Number of inodes present in purge list
 ```
 
-###Inode
+### Inode
 ```
 [conn.1.bound_xl./data/brick01a/homegfs.active.324] #324th inode in active inode list
 gfid=e6d337cf-97eb-44b3-9492-379ba3f6ad42 #Gfid of the inode
@@ -215,6 +237,7 @@ nlookup=13 #Number of times lookups happened from the client or from fuse kernel
 fd-count=4 #Number of fds opened on the inode
 ref=11 #Number of refs taken on the inode
 ia_type=1 #Type of the inode. This should be changed to some string :-(
+Ref by xl:.patchy-md-cache=11 #Further this there will be a list of xlators, and the ref count taken by each of them on this inode at the time of statedump
 
 [conn.1.bound_xl./data/brick01a/homegfs.lru.1] #1st inode in lru list. Note that ref count is zero for these inodes.
 gfid=5114574e-69bc-412b-9e52-f13ff087c6fc
@@ -222,8 +245,10 @@ nlookup=5
 fd-count=0
 ref=0
 ia_type=2
+Ref by xl:.fuse=1
+Ref by xl:.patchy-client-0=-1
 ```
-###Inode context
+### Inode context
 For each inode per xlator some context could be stored. This context can also be printed in the statedump. Here is the inode ctx of locks xlator
 ```
 [xlator.features.locks.homegfs-locks.inode]
@@ -240,12 +265,12 @@ lock-dump.domain.domain=homegfs-replicate-0 #Domain name where entry/data operat
 inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=11141120, len=131072, pid = 18446744073709551615, owner=080b1ada117f0000, client=0xb7fc30, connection-id=compute-30-029.com-3505-2014/06/29-14:46:12:477358-homegfs-client-0-0-1, granted at Sun Jun 29 11:10:36 2014 #Active lock information
 ```
 
-##FAQ
-###How to debug Memory leaks using statedump?
+## FAQ
+### How to debug Memory leaks using statedump?
 
-####Using memory accounting feature:
+#### Using memory accounting feature:
 
-`https://bugzilla.redhat.com/show_bug.cgi?id=1120151` is one of the bugs which was debugged using statedump to see which data-structure is leaking. Here is the process used to find what the leak is using statedump. According to the bug the observation is that the process memory usage is increasing whenever one of the bricks is wiped in a replicate volume and a `full` self-heal is invoked to heal the contents. Statedump of the process is taken using kill -USR1 `<pid-of-gluster-self-heal-daemon>`.
+[Bug 1120151](https://bugzilla.redhat.com/show_bug.cgi?id=1120151) is one of the bugs which was debugged using statedump to see which data-structure is leaking. Here is the process used to find what the leak is using statedump. According to the bug the observation is that the process memory usage is increasing whenever one of the bricks is wiped in a replicate volume and a `full` self-heal is invoked to heal the contents. Statedump of the process is taken using `kill -USR1 <pid-of-gluster-self-heal-daemon>`.
 ```
 grep -w num_allocs glusterdump.5225.dump.1405493251
 num_allocs=77078
@@ -268,10 +293,10 @@ grep of the statedump revealed too many allocations for the following data-types
 3. gf_common_mt_mem_pool.
 
 After checking afr-code for allocations with tag `gf_common_mt_char` found `data-self-heal` code path does not free one such allocated memory. `gf_common_mt_mem_pool` suggests that there is a leak in pool memory. `replicate-0:dict_t`, `glusterfs:data_t` and `glusterfs:data_pair_t` pools are using lot of memory, i.e. cold_count is `0` and too many allocations. Checking source code of dict.c revealed that `key` in `dict` is allocated with `gf_common_mt_char` i.e. `2.` tag and value is created using gf_asprintf which in-turn uses `gf_common_mt_asprintf` i.e. `1.`. Browsing the code for leak in self-heal code paths lead to a line which over-writes a variable with new dictionary even when it was already holding a reference to another dictionary. After fixing these leaks, ran the same test to verify that none of the `num_allocs` are increasing even after healing 10,000 files directory hierarchy in statedump of self-heal daemon.
-Please check http://review.gluster.org/8316 for more info about patch/code.
+Please check this [patch](http://review.gluster.org/8316) for more info about the fix.
 
-####Debugging leaks in memory pools:
-Statedump output of memory pools was used to test and verify the fixes to https://bugzilla.redhat.com/show_bug.cgi?id=1134221. On code analysis, dict_t objects were found to be leaking (in terms of not being unref'd enough number of times, during name self-heal. The test involved creating 100 files on plain replicate volume, removing them from one of the bricks's backend, and then triggering lookup on them from the mount point. Statedump of the mount process was taken before executing the test case and after it, after compiling glusterfs with -DDEBUG flags (to have cold count set to 0 by default).
+#### Debugging leaks in memory pools:
+Statedump output of memory pools was used to test and verify the fixes to [Bug 1134221](https://bugzilla.redhat.com/show_bug.cgi?id=1134221). On code analysis, dict_t objects were found to be leaking (in terms of not being unref'd enough number of times, during name self-heal. The test involved creating 100 files on plain replicate volume, removing them from one of the brick's backend, and then triggering lookup on them from the mount point. Statedump of the mount process was taken before executing the test case and after it, after compiling glusterfs with -DDEBUG flags (to have cold count set to 0 by default).
 
 Statedump output of the fuse mount process before the test case was executed:
 
@@ -303,7 +328,7 @@ cur-stdalloc=214
 max-stdalloc=220
 
 ```
-Here, with cold count being 0 by default, cur-stdalloc indicated the number of dict_t objects that were allocated in heap using mem_get(), and yet to be freed using mem_put() (refer to https://github.com/gluster/glusterfs/blob/master/doc/data-structures/mem-pool.md for more details on how mempool works). After the test case (name selfheal of 100 files), there was a rise in the cur-stdalloc value (from 14 to 214) for dict_t.
+Here, with cold count being 0 by default, `cur-stdalloc` indicated the number of `dict_t` objects that were allocated in heap using `mem_get()`, and yet to be freed using `mem_put()` (refer to this [page](../developer-guide/datastructure-mem-pool.md) for more details on how mempool works). After the test case (name selfheal of 100 files), there was a rise in the cur-stdalloc value (from 14 to 214) for `dict_t`.
 
 After these leaks were fixed, glusterfs was again compiled with -DDEBUG flags, and the same steps were performed again and statedump was taken before and after executing the test case, of the mount. This was done to ascertain the validity of the fix. And the following are the results:
 
@@ -337,8 +362,8 @@ max-stdalloc=119
 ```
 The value of cur-stdalloc remained 14 before and after the test, indicating that the fix indeed does what it's supposed to do.
 
-###How to debug hangs because of frame-loss?
-`https://bugzilla.redhat.com/show_bug.cgi?id=994959` is one of the bugs where statedump was helpful in finding where the frame was lost. Here is the process used to find where the hang is using statedump.
+### How to debug hangs because of frame-loss?
+[Bug 994959](https://bugzilla.redhat.com/show_bug.cgi?id=994959) is one of the bugs where statedump was helpful in finding where the frame was lost. Here is the process used to find where the hang is using statedump.
 When the hang was observed, statedumps are taken for all the processes. On mount's statedump the following stack is shown:
 ```
 [global.callpool.stack.1.frame.1]
@@ -386,4 +411,4 @@ unwind_to=qr_readdirp_cbk
 ```
 `unwind_to` shows that call was unwound to `afr_readdirp_cbk` from client xlator.
 Inspecting that function revealed that afr is not unwinding the stack when fop failed.
-Check http://review.gluster.org/5531 for more info about patch/code changes.
+Check this [patch](http://review.gluster.org/5531) for more info about the fix.
diff --git a/doc/developer-guide/Language-Bindings.md b/doc/developer-guide/Language-Bindings.md
index 89ef6df3d78..951f5fae2f6 100644
--- a/doc/developer-guide/Language-Bindings.md
+++ b/doc/developer-guide/Language-Bindings.md
@@ -1,10 +1,11 @@
+# Language Bindings
 GlusterFS 3.4 introduced the libgfapi client API for C programs. This
 page lists bindings to the libgfapi C library from other languages.
 
 Go
 --
 
--   [gogfapi](https://forge.gluster.org/gogfapi) - Go language bindings
+-   [gogfapi](https://github.com/gluster/gogfapi) - Go language bindings
     for libgfapi, aiming to provide an api consistent with the default
     Go file apis.
 
@@ -37,3 +38,8 @@ Rust
 -   [gfapi-sys](https://github.com/cholcombe973/Gfapi-sys) - Libgfapi
     bindings for Rust using FFI
 
+Perl
+----
+
+-   [libgfapi-perl](https://github.com/gluster/libgfapi-perl) - Libgfapi
+    bindings for Perl using FFI
diff --git a/doc/developer-guide/Developers-Index.md b/doc/developer-guide/README.md
index 9bcbcdc4cbe..aaf9c7476b0 100644
--- a/doc/developer-guide/Developers-Index.md
+++ b/doc/developer-guide/README.md
@@ -18,11 +18,9 @@ code check-in.
     the GPL v2 and the LGPL v3 or later
 -   [GlusterFS Coding Standards](./coding-standard.md)
 
-Developing
-----------
+-   If you are not sure of where to start, and what to do, we have a small
+    write-up on what you can pick. [Check it out](./options-to-contribute.md)
 
--   [Language Bindings](./Language Bindings.md) - Connect to
-    GlusterFS using various language bindings
 
 Adding File operations
 ----------------------
@@ -53,20 +51,30 @@ Daemon Management Framework
 Translators
 -----------
 
--   [Block Device Tanslator](./bd-xlator.md)
 -   [Performance/write-Behind Translator](./write-behind.md)
 -   [Translator Development](./translator-development.md)
 -   [Storage/posix Translator](./posix.md)
--   [Compression translator](./network_compression.md)
+
+
+Brick multiplex
+---------------
+
+-   [Brick mux resource reduction](./brickmux-thread-reduction.md)
+
+Fuse
+----
+
+-   [Interrupt handling](./fuse-interrupt.md)
 
 Testing/Debugging
 -----------------
 
 -   [Unit Tests in GlusterFS](./unittest.md)
 -   [Using the Gluster Test
-    Framework](./Using Gluster Test Framework.md) - Step by
+    Framework](./Using-Gluster-Test-Framework.md) - Step by
     step instructions for running the Gluster Test Framework
--   [Coredump Analysis](./coredump-analysis.md) - Steps to analize coredumps generated by regression machines.
+-   [Coredump Analysis](../debugging/analyzing-regression-cores.md) - Steps to analize coredumps generated by regression machines.
+-   [Identifying Resource Leaks](./identifying-resource-leaks.md)
 
 Release Process
 ---------------
diff --git a/doc/developer-guide/Using-Gluster-Test-Framework.md b/doc/developer-guide/Using-Gluster-Test-Framework.md
index 96fa9247e84..d2bb1c391da 100644
--- a/doc/developer-guide/Using-Gluster-Test-Framework.md
+++ b/doc/developer-guide/Using-Gluster-Test-Framework.md
@@ -1,3 +1,4 @@
+# USing Gluster Test Framwork
 Description
 -----------
 
diff --git a/doc/developer-guide/afr-locks-evolution.md b/doc/developer-guide/afr-locks-evolution.md
index 7d2a136d871..2dabbcfeb13 100644
--- a/doc/developer-guide/afr-locks-evolution.md
+++ b/doc/developer-guide/afr-locks-evolution.md
@@ -32,10 +32,10 @@ AFR makes use of locks xlator extensively:
 * For Entry self-heal, it is `entrylk(NULL name, parent inode)`. Specifying NULL for the name takes full lock on the directory referred to by the inode.
 * For data self-heal, there is a bit of history as to how locks evolved:
 
-###Initial version (say version 1) :
+### Initial version (say version 1) :
 There was no concept of selfheal daemon (shd). Only client lookups triggered heals. so AFR always took `inodelk(0,0,DATA_DOMAIN)` for healing. The issue with this approach was that when heal was in progress, I/O from clients was blocked .
 
-###version 2:
+### version 2:
 shd was introduced. We needed to allow I/O to go through when heal was going,provided the ranges did not overlap. To that extent, the following approach was adopted:
 
 + 1.shd takes (full inodelk in DATA_DOMAIN). Thus client FOPS are blocked and cannot modify changelog-xattrs
@@ -79,7 +79,7 @@ It modifies data but the FOP succeeds only on brick 2. writev returns success, a
 and thus goes ahead and copies stale 128Kb from brick 1 to brick2. Thus as far as application is concerned, `writev` returned success but bricks have stale data.
 What needs to be done is `writev` must return success only if it succeeded on atleast one source brick (brick b1 in this case). Otherwise  The heal still happens in reverse direction but as far as the application is concerned, it received an error.  
 
-###Note on lock **domains**
+### Note on lock **domains**
 We have used conceptual names in this document like DATA_DOMAIN/ METADATA_DOMAIN/ SELF_HEAL_DOMAIN. In the code, these are mapped to strings that are based on the AFR xlator name like so:
 
 DATA_DOMAIN     --->"vol_name-replicate-n"
diff --git a/doc/developer-guide/afr-self-heal-daemon.md b/doc/developer-guide/afr-self-heal-daemon.md
index b85ddd1c856..65940d420b7 100644
--- a/doc/developer-guide/afr-self-heal-daemon.md
+++ b/doc/developer-guide/afr-self-heal-daemon.md
@@ -39,7 +39,7 @@ When a client (mount) performs an operation on the file, the index xlator presen
 and removes it in post-op phase if the operation is successful. Thus if an entry is present inside the .glusterfs/indices/xattrop/ directory when there is no I/O 
 happening on the file, it means the file needs healing (or atleast an examination if the brick crashed after the post-op completed but just before the removal of the hardlink).
 
-####Index heal steps:
+#### Index heal steps:
 <pre><code>
 In shd process of *each node* {
         opendir +readdir (.glusterfs/indices/xattrop/)
diff --git a/doc/developer-guide/bd-xlator.md b/doc/developer-guide/bd-xlator.md
deleted file mode 100644
index 1771fb6e24b..00000000000
--- a/doc/developer-guide/bd-xlator.md
+++ /dev/null
@@ -1,469 +0,0 @@
-#Block device translator
-
-Block device translator (BD xlator) is a translator added to GlusterFS  which provides block backend for GlusterFS. This replaces the existing bd_map translator in GlusterFS that provided similar but very limited functionality. GlusterFS expects the underlying brick to be formatted with a POSIX compatible file system. BD xlator changes that and allows for having bricks that are raw block devices like LVM which needn’t have any file systems on them. Hence with BD xlator, it becomes possible to build a GlusterFS volume comprising of bricks that are logical volumes (LV).
-
-##bd
-
-BD xlator maps underlying LVs to files and hence the LVs appear as files to GlusterFS clients. Though BD volume externally appears very similar to the usual Posix volume, not all operations are supported or possible for the files on a BD volume. Only those operations that make sense for a block device are supported and the exact semantics are described in subsequent sections.
-
-While Posix volume takes a file system directory as brick, BD volume needs a volume group (VG) as brick. In the usual use case of BD volume, a file created on BD volume will result in an LV being created in the brick VG. In addition to a VG, BD volume also needs a file system directory that should be specified at the volume creation time. This directory is necessary for supporting the notion of directories and directory hierarchy for the BD volume. Metadata about LVs (size, mapping info) is stored in this directory.
-
-BD xlator was mainly developed to use block devices directly as VM images when GlusterFS is used as storage for KVM virtualization. Some of the salient points of BD xlator are
-
-* Since BD supports file level snapshots and clones by leveraging the snapshot and clone capabilities of LVM, it can be used to fully off-load snapshot and cloning operations from QEMU to the storage (GlusterFS) itself.
-
-* BD understands dm-thin LVs and hence can support files that are backed by thinly provisioned LVs. This capability of BD xlator translates to having thinly provisioned raw VM images.
-
-* BD enables thin LVs from a thin pool to be used from multiple nodes that have visibility to GlusterFS BD volume. Thus thin pool can be used as a VM image repository allowing access/visibility to it from multiple nodes.
-
-* BD supports true zerofill by using BLKZEROOUT ioctl on underlying block devices. Thus BD allows SCSI WRITESAME to be used on underlying block device if the device supports it.
-
-Though BD xlator is primarily intended to be used with block devices, it does provide full Posix xlator compatibility for files that are created on BD volume but are not backed by or mapped to a block device. Such files which don’t have a block device mapping exist on the Posix directory that is specified during BD volume creation. BD xlator is available from GlusterFS-3.5 release.
-
-###Compiling BD translator
-
-BD xlator needs lvm2 development library. –enable-bd-xlator option can be used with `./configure` script to explicitly enable BD translator. The following snippet from the output of configure script shows that BD xlator is enabled for compilation.
-
-
-#####GlusterFS configure summary
-
- …
- Block Device xlator  : yes
-
-
-###Creating a BD volume
-
-BD supports hosting of both linear LV and thin LV within the same volume. However seperate examples are provided below. As noted above, the prerequisite for a BD volume is VG which is created from a loop device here, but it can be any other device too.
-
-
-* Creating BD volume with linear LV backend
-
-* Create a loop device
-
-
-    [root@node ~]# dd if=/dev/zero of=bd-loop count=1024 bs=1M
-
-    [root@node ~]# losetup /dev/loop0 bd-loop
-
-
-* Prepare a brick by creating a VG
-
-    [root@node ~]# pvcreate /dev/loop0
-
-    [root@node ~]# vgcreate bd-vg /dev/loop0
-
-
-* Create the BD volume
-
-* Create a POSIX directory first
-
-
-    [root@node ~]# mkdir /bd-meta
-
-It is recommended that this directory is created on an LV in the brick VG itself so that both data and metadata live together on the same device.
-
-
-* Create and mount the volume
-
-    [root@node ~]# gluster volume create bd node:/bd-meta?bd-vg force
-
-
-The general syntax for specifying the brick is `host:/posix-dir?volume-group-name` where “?” is the separator.
-
-
-
-    [root@node ~]# gluster volume start bd
-    [root@node ~]# gluster volume info bd
-    Volume Name: bd
-    Type: Distribute
-    Volume ID: cb042d2a-f435-4669-b886-55f5927a4d7f
-    Status: Started
-    Xlator 1: BD
-    Capability 1: offload_copy
-    Capability 2: offload_snapshot
-    Number of Bricks: 1
-    Transport-type: tcp
-    Bricks:
-    Brick1: node:/bd-meta
-    Brick1 VG: bd-vg
-
-
-
-    [root@node ~]# mount -t glusterfs node:/bd /mnt
-
-* Create a file that is backed by an LV
-
-    [root@node ~]# ls /mnt
-
-    [root@node ~]#
-
-Since the volume is empty now, so is the underlying VG.
-
-    [root@node ~]# lvdisplay bd-vg
-    [root@node ~]#
-
-Creating a file that is mapped to an LV is a 2 step operation. First the file should be created on the mount point and a specific extended attribute should be set to map the file to LV.
-
-    [root@node ~]# touch /mnt/lv
-    [root@node ~]# setfattr -n “user.glusterfs.bd” -v “lv” /mnt/lv
-
-Now an LV got created in the VG brick and the file /mnt/lv maps to this LV. Any read/write to this file ends up as read/write to the underlying LV.
-
-    [root@node ~]# lvdisplay bd-vg
-    — Logical volume —
-    LV Path                 /dev/bd-vg/6ff0f25f-2776-4d19-adfb-df1a3cab8287
-    LV Name                 6ff0f25f-2776-4d19-adfb-df1a3cab8287
-    VG Name                 bd-vg
-    LV UUID                 PjMPcc-RkD5-RADz-6ixG-UYsk-oclz-vL0nv6
-    LV Write Access         read/write
-    LV Creation host, time node, 2013-11-26 16:15:45 +0530
-    LV Status               available
-    open                    0
-    LV Size                 4.00 MiB
-    Current LE              1
-    Segments                1
-    Allocation              inherit
-    Read ahead sectors      0
-    Block device            253:6
-
-The file gets created with default LV size which is 1 LE which is 4MB in this case.
-
-    [root@node ~]# ls -lh /mnt/lv
-    -rw-r–r–. 1 root root 4.0M Nov 26 16:15 /mnt/lv
-
-truncate can be used to set the required file size.
-
-    [root@node ~]# truncate /mnt/lv -s 256M
-    [root@node ~]# lvdisplay bd-vg
-    — Logical volume —
-    LV Path               /dev/bd-vg/6ff0f25f-2776-4d19-adfb-df1a3cab8287
-    LV Name               6ff0f25f-2776-4d19-adfb-df1a3cab8287
-    VG Name               bd-vg
-    LV UUID               PjMPcc-RkD5-RADz-6ixG-UYsk-oclz-vL0nv6
-    LV Write Access       read/write
-    LV Creation host, time node, 2013-11-26 16:15:45 +0530
-    LV Status             available
-    # open                   0
-    LV Size               256.00 MiB
-    Current LE             64
-    Segments                1
-    Allocation            inherit
-    Read ahead sectors     0
-    Block device          253:6
-
-
-    [root@node ~]# ls -lh /mnt/lv
-    -rw-r–r–. 1 root root 256M Nov 26 16:15 /mnt/lv
-
-    currently LV size has been set to        256
-
-The size of the file/LV can be specified during creation/mapping time itself like this:
-
-    setfattr -n “user.glusterfs.bd” -v “lv:256MB” /mnt/lv
-
-2. Creating BD volume with thin LV backend
-
-* Create a loop device
-
-
-    [root@node ~]# dd if=/dev/zero of=bd-loop-thin count=1024 bs=1M
-
-    [root@node ~]# losetup /dev/loop0 bd-loop-thin
-
-
-* Prepare a brick by creating a VG and thin pool
-
-
-    [root@node ~]# pvcreate /dev/loop0
-
-    [root@node ~]# vgcreate bd-vg-thin /dev/loop0
-
-
-* Create a thin pool
-
-
-    [root@node ~]# lvcreate –thin bd-vg-thin -L 1000M
-
-    Rounding up size to full physical extent 4.00 MiB
-    Logical volume “lvol0″ created
-
-lvdisplay shows the thin pool
-
-    [root@node ~]# lvdisplay bd-vg-thin
-    — Logical volume —
-    LV Name                      lvol0
-    VG Name                      bd-vg-thin
-    LV UUID                      HVa3EM-IVMS-QG2g-oqU6-1UxC-RgqS-g8zhVn
-    LV Write Access              read/write
-    LV Creation host, time node, 2013-11-26 16:39:06 +0530
-    LV Pool transaction ID       0
-    LV Pool metadata             lvol0_tmeta
-    LV Pool data                 lvol0_tdata
-    LV Pool chunk size           64.00 KiB
-    LV Zero new blocks           yes
-    LV Status                    available
-    # open                       0
-    LV Size                      1000.00 MiB
-    Allocated pool data          0.00%
-    Allocated metadata           0.88%
-    Current LE                   250
-    Segments                     1
-    Allocation                     inherit
-    Read ahead sectors     auto
-    Block device                253:9
-
-* Create the BD volume
-
-* Create a POSIX directory first
-
-
-    [root@node ~]# mkdir /bd-meta-thin
-
-* Create and mount the volume
-
-    [root@node ~]# gluster volume create bd-thin node:/bd-meta-thin?bd-vg-thin force
-
-    [root@node ~]# gluster volume start bd-thin
-
-
-    [root@node ~]# gluster volume info bd-thin
-    Volume Name: bd-thin
-    Type: Distribute
-    Volume ID: 27aa7eb0-4ffa-497e-b639-7cbda0128793
-    Status: Started
-    Xlator 1: BD
-    Capability 1: thin
-    Capability 2: offload_copy
-    Capability 3: offload_snapshot
-    Number of Bricks: 1
-    Transport-type: tcp
-    Bricks:
-    Brick1: node:/bd-meta-thin
-    Brick1 VG: bd-vg-thin
-
-
-    [root@node ~]# mount -t glusterfs node:/bd-thin /mnt
-
-* Create a file that is backed by a thin LV
-
-
-    [root@node ~]# ls /mnt
-
-    [root@node ~]#
-
-Creating a file that is mapped to a thin LV is a 2 step operation. First the file should be created on the mount point and a specific extended attribute should be set to map the file to a thin LV.
-
-    [root@node ~]# touch /mnt/thin-lv
- 
-    [root@node ~]# setfattr -n “user.glusterfs.bd” -v “thin:256MB” /mnt/thin-lv
-
-Now /mnt/thin-lv is a thin provisioned file that is backed by a thin LV and size has been set to 256.
-
-    [root@node ~]# lvdisplay bd-vg-thin
-    — Logical volume —
-    LV Name             lvol0
-    VG Name             bd-vg-thin
-    LV UUID             HVa3EM-IVMS-QG2g-oqU6-1UxC-RgqS-g8zhVn
-    LV Write Access     read/write
-    LV Creation host, time node, 2013-11-26 16:39:06 +0530
-    LV Pool transaction ID 1
-    LV Pool metadata    lvol0_tmeta
-    LV Pool data        lvol0_tdata
-    LV Pool chunk size  64.00 KiB
-    LV Zero new blocks  yes
-    LV Status           available
-    # open               0
-    LV Size             000.00 MiB
-    Allocated pool data 0.00%
-    Allocated metadata  0.98%
-    Current LE          250
-    Segments            1
-    Allocation          inherit
-    Read ahead sectors  auto
-    Block device         253:9
-
-
-
-
-    — Logical volume —
-    LV Path             dev/bd-vg-thin/081b01d1-1436-4306-9baf-41c7bf5a2c73
-    LV Name             081b01d1-1436-4306-9baf-41c7bf5a2c73
-    VG Name             bd-vg-thin
-    LV UUID             coxpTY-2UZl-9293-8H2X-eAZn-wSp6-csZIeB
-    LV Write Access     read/write
-    LV Creation host, time node, 2013-11-26 16:43:19 +0530
-    LV Pool name        lvol0
-    LV Status           available
-    # open               0
-    LV Size             256.00 MiB
-    Mapped size          0.00%
-    Current LE           64
-    Segments             1
-    Allocation          inherit
-    Read ahead sectors   auto
-    Block device                 253:10
-
-
-
-
-
-As can be seen from above, creation of a file resulted in creation of a thin LV in the brick.
-
-
-###Improvisation on BD translator:
-
-First version of BD xlator  ( block backend) had few limitations  such as
-
-* Creation of directories not supported
-* Supports only single brick
-* Does not use extended attributes (and client gfid) like posix xlator
-* Creation of special files (symbolic links, device nodes etc) not
-  supported
-
-Basic limitation of not allowing directory creation was blocking
-oVirt/VDSM to consume BD xlator as part of Gluster domain since VDSM
-creates multi-level directories when GlusterFS is used as storage
-backend for storing VM images.
-
-To overcome these limitations a new BD xlator with following
-improvements are implemented.
-
-* New hybrid BD xlator that handles both regular files and block device
-  files
-* The volume will have both POSIX and BD bricks. Regular files are
-  created on POSIX bricks, block devices are created on the BD brick (VG)
-* BD xlator leverages exiting POSIX xlator for most POSIX calls and
-  hence sits above the POSIX xlator
-* Block device file is differentiated from regular file by an extended
-  attribute
-* The xattr 'user.glusterfs.bd' (BD_XATTR) plays a role in mapping a
-  posix file to Logical Volume (LV).
-* When a client sends a request to set BD_XATTR on a posix file, a new
-  LV is created and mapped to posix file. So every block device will
-  have a representative file in POSIX brick with 'user.glusterfs.bd'
-  (BD_XATTR) set.
-* Here after all operations on this file results in LV related
-  operations.
-
-For example, opening a file that has BD_XATTR set results in opening
-the LV block device, reading results in reading the corresponding LV
-block device.
-
-When BD xlator gets request to set BD_XATTR via setxattr call, it
-creates a LV and information about this LV is placed in the xattr of the
-posix file. xattr "user.glusterfs.bd" used to identify that posix file
-is mapped to BD.
-
-Usage:
-Server side:
-
-    [root@host1 ~]# gluster volume create bdvol host1:/storage/vg1_info?vg1 host2:/storage/vg2_info?vg2
-
-It creates a distributed gluster volume 'bdvol' with Volume Group vg1
-using posix brick /storage/vg1_info in host1 and Volume Group vg2 using
-/storage/vg2_info in host2.
-
-
-    [root@host1 ~]# gluster volume start bdvol
-
-Client side:
-
-    [root@node ~]# mount -t glusterfs host1:/bdvol /media
-    [root@node ~]# touch /media/posix
-
-It creates regular posix file 'posix' in either host1:/vg1 or host2:/vg2 brick
-
-    [root@node ~]# mkdir /media/image
-
-    [root@node ~]# touch /media/image/lv1
-
-
-It also creates regular posix file 'lv1' in either host1:/vg1 or
-host2:/vg2 brick
-
-    [root@node ~]# setfattr -n "user.glusterfs.bd" -v "lv" /media/image/lv1
-
-    [root@node ~]#
-
-
-Above setxattr results in creating a new LV in corresponding brick's VG
-and it sets 'user.glusterfs.bd' with value 'lv:<default-extent-size''
-
-
-    [root@node ~]# truncate -s5G /media/image/lv1
-
-
-It results in resizig LV 'lv1'to 5G
-
-New BD xlator code is placed in `xlators/storage/bd` directory.
-
-Also add volume-uuid to the VG so that same VG cannot be used for other
-bricks/volumes. After deleting a gluster volume, one has to manually
-remove the associated tag using vgchange <vg-name> --deltag
-`<trusted.glusterfs.volume-id:<volume-id>>`
-
-
-#### Exposing volume capabilities
-
-With multiple storage translators (posix and bd) being supported in GlusterFS, it becomes
-necessary to know the volume type so that user can issue appropriate calls that are relevant
-only to the a given volume type. Hence there needs to be a way to expose the type of
-the storage translator of the volume to the user.
-
-BD xlator is capable of providing server offloaded file copy, server/storage offloaded
-zeroing of a file etc. This capabilities should be visible to the client/user, so that these
-features can be exploited.
-
-BD xlator exports capability information through gluster volume info (and --xml) output. For eg:
-
-`snip of gluster volume info output for a BD based volume`
-
-        Xlator 1: BD
-        Capability 1: thin
-
-`snip of gluster volume info --xml output for a BD based volume`
-
-        <xlators>
-          <xlator>
-                <name>BD</name>
-                <capabilities>
-                        <capability>thin</capability>
-                </capabilities>
-          </xlator>
-        </xlators>
-
-But this capability information should also exposed through some other means so that a host
-which is not part of Gluster peer could also avail this capabilities.
-
-* Type
-
-BD translator supports both regular files and block device, i,e., one can create files on
-GlusterFS volume backed by BD translator and this file could end up as regular posix file or
-a logical volume (block device) based on the user''s choice. User can do a setxattr on the
-created file to convert it to a logical volume.
-
-Users of BD backed volume like QEMU would like to know that it is working with BD type of volume
-so that it can issue an additional setxattr call after creating a VM image on GlusterFS backend.
-This is necessary to ensure that the created VM image is backed by LV instead of file.
-
-There are different ways to expose this information (BD type of volume) to user.
-One way is to export it via a `getxattr` call. That said, When a client issues getxattr("volume_type")
-on a root gfid, bd xlator will return 1 implying its BD xlator. But posix xlator will return ENODATA
-and client code can interpret this as posix xlator. Also capability list can be returned via
-getxattr("caps") for root gfid.
-
-* Capabilities
-
-BD xlator supports new features such as server offloaded file copy, thin provisioned VM images etc.
-
-There is no standard way of exploiting these features from client side (such as syscall
-to exploit server offloaded copy). So these features need to be exported to the client so that
-they can be used. BD xlator latest version exports these capabilities information through
-gluster volume info (and --xml) output. But if a client is not part of GlusterFS peer
-it can''t run volume info command to get the list of capabilities of a given GlusterFS volume.
-For example, GlusterFS block driver in qemu need to get the capability list so that these features are used.
-
-
-
-Parts of this documentation were originally published here
-#http://raobharata.wordpress.com/2013/11/27/glusterfs-block-device-translator/
diff --git a/doc/developer-guide/brickmux-thread-reduction.md b/doc/developer-guide/brickmux-thread-reduction.md
new file mode 100644
index 00000000000..7d76e8ff579
--- /dev/null
+++ b/doc/developer-guide/brickmux-thread-reduction.md
@@ -0,0 +1,64 @@
+# Resource usage reduction in brick multiplexing
+
+Each brick is regresented with a graph of translators in a brick process.
+Each translator in the graph has its own set of threads and mem pools
+and other system resources allocations. Most of the times all these
+resources are not put to full use. Reducing the resource consumption
+of each brick is a problem in itself that needs to be addressed. The other
+aspect to it is, sharing of resources across brick graph, this becomes
+critical in brick multiplexing scenario. In this document we will be discussing
+only about the threads.
+
+If a brick mux process hosts 50 bricks there are atleast 600+ threads created
+in that process. Some of these are global threads that are shared by all the
+brick graphs, and others are per translator threads. The global threads like
+synctask threads, timer threads, sigwaiter, poller etc. are configurable and
+do not needs to be reduced. The per translator threads keeps growing as the
+number of bricks in the process increases. Each brick spawns atleast 10+
+threads:
+- io-threads
+- posix threads:
+     1. Janitor
+     2. Fsyncer
+     3. Helper
+     4. aio-thread
+- changelog and bitrot threads(even when the features are not enabled)
+
+## io-threads
+
+io-threads should be made global to the process, having 16+ threads for
+each brick does not make sense. But io-thread translator is loaded in
+the graph, and the position of io-thread translator decides from when
+the fops will be parallelised across threads. We cannot entirely move
+the io-threads to libglusterfs and say the multiplexing happens from
+the master translator or so. Hence, the io-thread orchestrator code
+is moved to libglusterfs, which ensures there is only one set of
+io-threads that is shared among the io-threads translator in each brick.
+This poses performance issues due to lock-contention in the io-threds
+layer. This also shall be addressed by having multiple locks instead of
+one global lock for io-threads.
+
+## Posix threads
+Most of the posix threads execute tasks in a timely manner, hence it can be
+replaced with a timer whose handler register a task to synctask framework, once
+the task is complete, the timer is registered again. With this we can eliminate
+the need of one thread for each task. The problem with using synctasks is
+the performance impact it will have due to make/swapcontext. For task that
+does not involve network wait, we need not do makecontext, instead the task
+function with arg can be stored and executed when a synctask thread is free.
+We need to implement an api in synctask to execute atomic tasks(no network wait)
+without the overhead of make/swapcontext. This will solve the performance
+impact associated with using synctask framework.
+
+And the other challenge, is to cancel all the tasks pending from a translator.
+This is important to cleanly detach brick. For this, we need to implement an
+api in synctask that can cancel all the tasks from a given translator.
+
+For future, this will be replced to use global thread-pool(once implemented).
+
+## Changelog and bitrot threads
+
+In the initial implementation, the threads are not created if the feature is
+not enabled. We need to share threads across changelog instances if we plan
+to enable these features in brick mux scenario.
+
diff --git a/doc/developer-guide/coding-standard.md b/doc/developer-guide/coding-standard.md
index 368c5553464..031c6c0da99 100644
--- a/doc/developer-guide/coding-standard.md
+++ b/doc/developer-guide/coding-standard.md
@@ -1,11 +1,38 @@
 GlusterFS Coding Standards
 ==========================
 
+Before you get started
+----------------------
+Before starting with other part of coding standard, install `clang-format`
+
+On Fedora:
+```
+$ dnf install clang
+```
+On debian/Ubuntu:
+```
+$ apt-get install clang
+```
+Once you are done with all the local changes, you need to run below set of commands,
+before submitting the patch for review.
+```
+$ git add $file # if any
+$ git commit -a -s -m "commit message"
+$ git show --pretty="format:" --name-only | grep -v "contrib/" | egrep "*\.[ch]$" | xargs clang-format -i
+$ git diff # see if there are any changes
+$ git commit -a --amend # get the format changes done
+$ ./submit-for-review.sh
+```
+
+
 Structure definitions should have a comment per member
 ------------------------------------------------------
 
-Every member in a structure definition must have a comment about its
-purpose. The comment should be descriptive without being overly verbose.
+Every member in a structure definition must have a comment about its purpose.
+The comment should be descriptive without being overly verbose.  For pointer
+members, lifecycle concerns for the pointed-to object should be noted.  For lock
+members, the relationship between the lock member and the other members it
+protects should be explicit.
 
 *Bad:*
 
@@ -23,59 +50,182 @@ DBTYPE      access_mode;    /* access mode for accessing
                              */
 ```
 
-Declare all variables at the beginning of the function
-------------------------------------------------------
+Structure members should be aligned based on the padding requirements
+---------------------------------------------------------------------
 
-All local variables in a function must be declared immediately after the
-opening brace. This makes it easy to keep track of memory that needs to be freed
-during exit. It also helps debugging, since gdb cannot handle variables
-declared inside loops or other such blocks.
+The compiler will make sure that structure members have optimum alignment,
+but at the expense of suboptimal padding. More important is to optimize the
+padding. The compiler won't do that for you.
 
-Always initialize local variables
----------------------------------
+This also will help utilize the memory better
 
-Every local variable should be initialized to a sensible default value
-at the point of its declaration. All pointers should be initialized to NULL,
-and all integers should be zero or (if it makes sense) an error value.
+*Bad:*
+```
+struct bad {
+    bool  b; /* 0 */
+             /* 1..7 pad */
+    void *p; /* 8..15 */
+    char  c; /* 16 */
+    char  a[16]; /* 17..33 */
+             /* 34..39 pad */
+    int64_t ii; /* 40..47 */
+    int32_t i; /* 48..51 */
+             /* 52..55 pad */
+    int64_t iii; /* 56..63 */
+};
+```
 
+*Good:*
+```
+struct good {
+   int64_t ii; /* explicit 64-bit types */
+   void *p; /* may be 64- or 32-bit */
+   long  l;   /* may be 64- or 32-bit */
+   int   i;   /* 32-bit */
+   short s;   /* 16-bit */
+   char  c;   /* 8-bit */
+   bool  b;   /* 8-bit */
+   char  a[1024];
+);
+```
+Make sure the items with the most stringent alignment requirements will need
+to come earliest (ie, pointers and  perhaps uint64_t etc), and those with less
+stringent alignment requirements at the end (uint16/uint8 and char). Also note
+that the long array (if any) should be at the end of the structure, regardless
+of the type.
+
+Also note, if your structure's overall size is crossing 1k-4k limit, it is
+recommended to mention the reason why the particular structure needs so much
+memory as a comment at the top.
+
+Use \_typename for struct tags and typename\_t for typedefs
+---------------------------------------------------------
+
+Being consistent here makes it possible to automate navigation from use of a
+type to its true definition (not just the typedef).
+
+*Bad:*
+
+```
+struct thing {...};
+struct thing_t {...};
+typedef struct _thing thing;
+```
 
 *Good:*
 
 ```
-int ret       = 0;
-char *databuf = NULL;
-int _fd       = -1;
+typedef struct _thing {...} thing_t;
 ```
 
-Initialization should always be done with a constant value
-----------------------------------------------------------
+No double underscores
+---------------------
+
+Identifiers beginning with double underscores are supposed to reserved for the
+compiler.
 
-Never use a non-constant expression as the initialization value for a variable.
+http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf
 
+When you need to define inner/outer functions, use a different prefix/suffix.
 
 *Bad:*
 
 ```
+void __do_something (void);
+
+void
+do_something (void)
+{
+        LOCK ();
+        __do_something ();
+        UNLOCK ();
+}
+```
+
+*Good:*
+
+```
+void do_something_locked (void);
+```
+
+Only use safe pointers in initializers
+----------------------------------------------------------
+
+Some pointers, such as `this` in a fop function, can be assumed to be non-NULL.
+However, other parameters and further-derived values might be NULL.
+
+*Good:*
+
+```
 pid_t pid     = frame->root->pid;
-char *databuf = malloc (1024);
 ```
 
+
+*Bad:*
+
+```
+data_t *my_data = dict_get (xdata, "fubar");
+```
+
+No giant stack allocations
+--------------------------
+
+Synctasks have small finite stacks.  To avoid overflowing these stacks, avoid
+allocating any large data structures on the stack.  Use dynamic allocation
+instead.
+
+*Bad:*
+
+```
+gf_boolean_t port_inuse[65536]; /* 256KB, this actually happened */
+```
+
+NOTE: Ideal is to limit the stack array to less than 256 bytes.
+
+
+Character array initializing
+----------------------------
+
+It is recommended to keep the character array initializing to empty string.
+
+*Good:*
+```
+char msg[1024] = "";
+```
+
+Not so much recommended, even though it means the same.
+
+```
+char msg[1024] = {0,};
+```
+
+We recommend above to structure initialization.
+
+
+
 Validate all arguments to a function
 ------------------------------------
 
 All pointer arguments to a function must be checked for `NULL`.
-A macro named `VALIDATE` (in `common-utils.h`)
-takes one argument, and if it is `NULL`, writes a log message and
-jumps to a label called `err` after setting op_ret and op_errno
-appropriately. It is recommended to use this template.
+A macro named `GF_VALIDATE_OR_GOTO` (in `common-utils.h`)
+takes two arguments; if the first is `NULL`, it writes a log message and
+jumps to a label specified by the second aergument after setting errno
+appropriately. There are several variants of this function for more
+specific purposes, and their use is recommended.
+
+*Bad:*
 
+```
+/* top of function */
+ret = dict_get (xdata, ...)
+```
 
 *Good:*
 
 ```
-VALIDATE(frame);
-VALIDATE(this);
-VALIDATE(inode);
+/* top of function */
+GF_VALIDATE_OR_GOTO(xdata,out);
+ret = dict_get (xdata, ...)
 ```
 
 Never rely on precedence of operators
@@ -83,25 +233,34 @@ Never rely on precedence of operators
 
 Never write code that relies on the precedence of operators to execute
 correctly.  Such code can be hard to read and someone else might not
-know the precedence of operators as accurately as you do.
+know the precedence of operators as accurately as you do.  This includes
+precedence of increment/decrement vs. field/subscript.  The only exceptions are
+arithmetic operators (which have had defined precedence since before computers
+even existed) and boolean negation.
 
 *Bad:*
 
 ```
 if (op_ret == -1 && errno != ENOENT)
+++foo->bar      /* incrementing foo, or incrementing foo->bar? */
+a && b || !c
 ```
 
 *Good:*
 
 ```
 if ((op_ret == -1) && (errno != ENOENT))
+(++foo)->bar
+++(foo->bar)
+(a && b) || !c
+a && (b || !c)
 ```
 
 Use exactly matching types
 --------------------------
 
 Use a variable of the exact type declared in the manual to hold the
-return value of a function. Do not use an ``equivalent'' type.
+return value of a function. Do not use an 'equivalent' type.
 
 
 *Bad:*
@@ -116,42 +275,56 @@ int len = strlen (path);
 size_t len = strlen (path);
 ```
 
-Never write code such as `foo->bar->baz`; check every pointer
+Avoid code such as `foo->bar->baz`; check every pointer
 -------------------------------------------------------------
 
-Do not write code that blindly follows a chain of pointer
-references. Any pointer in the chain may be `NULL` and thus
-cause a crash. Verify that each pointer is non-null before following
-it.
+Do not write code that blindly follows a chain of pointer references. Any
+pointer in the chain may be `NULL` and thus cause a crash. Verify that each
+pointer is non-null before following it.  Even if `foo->bar` has been checked
+and is known safe, repeating it can make code more verbose and less clear.
 
-Check return value of all functions and system calls
+This rule includes `[]` as well as `->` because both dereference pointers.
+
+*Bad:*
+
+```
+foo->bar->field1 = value1;
+xyz = foo->bar->field2 + foo->bar->field3 * foo->bar->field4;
+foo->bar[5].baz
+```
+
+*Good:*
+
+```
+my_bar = foo->bar;
+if (!my_bar) ... return;
+my_bar->field1 = value1;
+xyz = my_bar->field2 + my_bar->field3 * my_bar->field4;
+```
+
+Document unchecked return values
 ----------------------------------------------------
 
-The return value of all system calls and API functions must be checked
-for success or failure.
+In general, return values should be checked.  If a function is being called
+for its side effects and the return value really doesn't matter, an explicit
+cast to void is required (to keep static analyzers happy) and a comment is
+recommended.
 
 *Bad:*
 
 ```
 close (fd);
+do_important_thing ();
 ```
 
-*Good:*
+*Good (or at least OK):*
 
 ```
-op_ret = close (_fd);
-if (op_ret == -1) {
-        gf_log (this->name, GF_LOG_ERROR,
-                "close on file %s failed (%s)", real_path,
-                strerror (errno));
-        op_errno = errno;
-        goto out;
-}
+(void) sleep (1);
 ```
 
-
-Gracefully handle failure of malloc
------------------------------------
+Gracefully handle failure of malloc (and other allocation functions)
+--------------------------------------------------------------------
 
 GlusterFS should never crash or exit due to lack of memory. If a
 memory allocation fails, the call should be unwound and an error
@@ -176,7 +349,7 @@ int32_t dict_get_int32 (dict_t *this, char *key);
 int dict_get_int32 (dict_t *this, char *key, int32_t *val);
 ```
 
-Always use the `n' versions of string functions
+Always use the 'n' versions of string functions
 -----------------------------------------------
 
 Unless impossible, use the length-limited versions of the string functions.
@@ -193,18 +366,43 @@ strcpy (entry_path, real_path);
 strncpy (entry_path, real_path, entry_path_len);
 ```
 
+Do not use memset prior to sprintf/snprintf/vsnprintf etc...
+------------------------------------------------------------
+snprintf(and other similar string functions) terminates the buffer with a
+'\0'(null character). Hence, there is no need to do a memset before using
+snprintf. (Of course you need to account one extra byte for the null character
+in your allocation).
+
+Note: Similarly if you are doing pre-memory allocation for the buffer, use
+GF_MALLOC instead of GF_CALLOC, since the later is bit costlier.
+
+*Bad:*
+
+```
+char buffer[x];
+memset (buffer, 0, x);
+bytes_read = snprintf (buffer, sizeof buffer, "bad standard");
+```
+
+*Good:*
+```
+char buffer[x];
+bytes_read = snprintf (buffer, sizeof (buffer), "good standard");
+```
+
+And it is always to good initialize the char array if the string is static.
+
+E.g.
+```
+char buffer[] = "good standard";
+```
+
 No dead or commented code
 -------------------------
 
 There must be no dead code (code to which control can never be passed) or
 commented out code in the codebase.
 
-Only one unwind and return per function
----------------------------------------
-
-There must be only one exit out of a function. `UNWIND` and return
-should happen at only point in the function.
-
 Function length or Keep functions small
 ---------------------------------------
 
@@ -226,20 +424,35 @@ same_owner (posix_lock_t *l1, posix_lock_t *l2)
 }
 ```
 
-Defining functions as static
-----------------------------
+Define functions as static
+--------------------------
+
+Declare functions as static unless they're exposed via a module-level API for
+use from other modules.
+
+No nested functions
+-------------------
+
+Nested functions have proven unreliable, e.g. as callbacks in code that uses
+ucontext (green) threads,
+
+Use inline functions instead of macros whenever possible
+--------------------------------------------------------
 
-Define internal functions as static only if you're
-very sure that there will not be a crash(..of any kind..) emanating in
-that function. If there is even a remote possibility, perhaps due to
-pointer derefering, etc, declare the function as non-static. This
-ensures that when a crash does happen, the function name shows up the
-in the back-trace generated by libc. However, doing so has potential
-for polluting the function namespace, so to avoid conflicts with other
-components in other parts, ensure that the function names are
-prepended with a prefix that identify the component to which it
-belongs. For eg. non-static functions in io-threads translator start
-with iot_.
+Inline functions enforce type safety; macros do not.  Use macros only for things
+that explicitly need to be type-agnostic (e.g. cases where one might use
+generics or templates in other languages), or that use other preprocessor
+features such as `#` for stringification or `##` for token pasting.  In general,
+"static inline" is the preferred form.
+
+Avoid copypasta
+---------------
+
+Code that is copied and then pasted into multiple functions often creates
+maintenance problems later, e.g. updating all but one instance for a subsequent
+change.  If you find yourself copying the same "boilerplate" many places,
+consider refactoring to use helper functions (including inline) or macros, or
+code generation.
 
 Ensure function calls wrap around after 80-columns
 --------------------------------------------------
@@ -335,13 +548,95 @@ pthread_mutex_lock (&mutex);
 pthread_mutex_unlock (&mutex);
 ```
 
-*A skeleton fop function:*
+### Always use braces
+
+Even around single statements.
+
+*Bad:*
+
+```
+if (condition) action ();
+
+if (condition)
+        action ();
+```
+
+*Good:*
+
+```
+if (condition) {
+        action ();
+}
+```
+
+### Avoid multi-line conditionals
+
+These can be hard to read and even harder to modify later.  Predicate functions
+and helper variables are always better for maintainability.
+
+*Bad:*
+
+```
+if ((thing1 && other_complex_condition (thing1, lots, of, args))
+    || (!thing2 || even_more_complex_condition (thing2))
+    || all_sorts_of_stuff_with_thing3) {
+    return;
+}
+
+```
+
+*Better:*
+
+```
+thing1_ok = predicate1 (thing1, lots, of, args
+thing2_ok = predicate2 (thing2);
+thing3_ok = predicate3 (thing3);
+
+if (!thing1_ok || !thing2_ok || !thing3_ok) {
+        return;
+}
+```
+
+*Best:*
+
+```
+if (thing1 && other_complex_condition (thing1, lots, of, args)) {
+        return;
+}
+if (!thing2 || even_more_complex_condition (thing2)) {
+        /* Note potential for a different message here. */
+        return;
+}
+if (all_sorts_of_stuff_with_thing3) {
+        /* And here too. */
+        return;
+}
+```
+
+### Use 'const' liberally
+
+If a value isn't supposed/expected to change, there's no cost to adding a
+'const' keyword and it will help prevent violation of expectations.
+
+### Avoid global variables (including 'static' auto variables)
+Almost all state in Gluster is contextual and should be contained in the
+appropriate structure reflecting its scope (e.g. `call\_frame\_t`, `call\_stack\_t`,
+`xlator\_t`, `glusterfs\_ctx\_t`).  With dynamic loading and graph switches in play,
+each global requires careful consideration of when it should be initialized or
+reinitialized, when it might _accidentally_ be reinitialized, when its value
+might become stale, and so on.  A few global variables are needed to serve as
+'anchor points' for these structures, and more exceptions to the rule might be
+approved in the future, but new globals should not be added to the codebase
+without explicit approval.
+
+## A skeleton fop function
 
-This is the recommended template for any fop. In the beginning come
-the initializations. After that, the `success' control flow should be
-linear.  Any error conditions should cause a `goto` to a single
-point, `out`.  At that point, the code should detect the error
-that has occurred and do appropriate cleanup.
+This is the recommended template for any fop. In the beginning come the
+initializations. After that, the 'success' control flow should be linear.  Any
+error conditions should cause a `goto` to a label at the end.  By convention
+this is 'out' if there is only one such label, but a cascade of such labels is
+allowable to support multi-stage cleanup.  At that point, the code should detect
+the error that has occurred and do appropriate cleanup.
 
 ```
 int32_t
diff --git a/doc/developer-guide/commit-guidelines.md b/doc/developer-guide/commit-guidelines.md
new file mode 100644
index 00000000000..38bbe525cbd
--- /dev/null
+++ b/doc/developer-guide/commit-guidelines.md
@@ -0,0 +1,136 @@
+## Git Commit Good Practice
+
+The following document is based on experience doing code development, bug troubleshooting and code review across a number of projects using Git. The document is mostly borrowed from [Open Stack](https://wiki.openstack.org/wiki/GitCommitMessages), but made more meaningful in the context of GlusterFS project.
+
+This topic can be split into two areas of concern
+
+* The structured set/split of the code changes
+* The information provided in the commit message
+
+### Executive Summary
+The points and examples that will be raised in this document ought to clearly demonstrate the value in splitting up changes into a sequence of individual commits, and the importance in writing good commit messages to go along with them. If these guidelines were widely applied it would result in a significant improvement in the quality of the GlusterFS Git history. Both a carrot & stick will be required to effect changes. This document intends to be the carrot by alerting people to the benefits, while anyone doing Gerrit code review can act as the stick ;-P
+
+In other words, when reviewing a change in Gerrit:
+* Do not simply look at the correctness of the code.
+* Review the commit message itself and request improvements to its content.
+* Look out for commits which are mixing multiple logical changes and require the submitter to split them into separate commits.
+* Ensure whitespace changes are not mixed in with functional changes.
+* Ensure no-op code refactoring is done separately from functional changes.
+
+And so on.
+
+It might be mentioned that Gerrit's handling of patch series is not entirely perfect. Let that not become a valid reason to avoid creating patch series. The tools being used should be subservient to developers needs, and since they are open source they can be fixed / improved. Software source code is "read mostly, write occassionally" and thus the most important criteria is to improve the long term maintainability by the large pool of developers in the community, and not to sacrifice too much for the sake of the single author who may never touch the code again.
+
+And now the long detailed guidelines & examples of good & bad practice
+
+### Structural split of changes
+The cardinal rule for creating good commits is to ensure there is only one "logical change" per commit. There are many reasons why this is an important rule:
+
+* The smaller the amount of code being changed, the quicker & easier it is to review & identify potential flaws.
+* If a change is found to be flawed later, it may be necessary to revert the broken commit. This is much easier to do if there are not other unrelated code changes entangled with the original commit.
+* When troubleshooting problems using Git's bisect capability, small well defined changes will aid in isolating exactly where the code problem was introduced.
+* When browsing history using Git annotate/blame, small well defined changes also aid in isolating exactly where & why a piece of code came from.
+
+#### Things to avoid when creating commits
+With the above points in mind, there are some commonly encountered examples of bad things to avoid
+
+* Mixing whitespace changes with functional code changes.
+
+The whitespace changes will obscure the important functional changes, making it harder for a reviewer to correctly determine whether the change is correct. Solution: Create 2 commits, one with the whitespace changes, one with the functional changes. Typically the whitespace change would be done first, but that need not be a hard rule.
+
+* Mixing two unrelated functional changes.
+
+Again the reviewer will find it harder to identify flaws if two unrelated changes are mixed together. If it becomes necessary to later revert a broken commit, the two unrelated changes will need to be untangled, with further risk of bug creation.
+
+* Sending large new features in a single giant commit.
+
+It may well be the case that the code for a new feature is only useful when all of it is present. This does not, however, imply that the entire feature should be provided in a single commit. New features often entail refactoring existing code. It is highly desirable that any refactoring is done in commits which are separate from those implementing the new feature. This helps reviewers and test suites validate that the refactoring has no unintentional functional changes.
+
+Even the newly written code can often be split up into multiple pieces that can be independently reviewed. For example, changes which add new internal fops or library functions, can be in self-contained commits. Again this leads to easier code review. It also allows other developers to cherry-pick small parts of the work, if the entire new feature is not immediately ready for merge. This will encourage the author & reviewers to think about the generic library functions' design, and not simply pick a design that is easier for their currently chosen internal implementation.
+
+The basic rule to follow is
+
+If a code change can be split into a sequence of patches/commits, then it should be split. Less is not more. More is more.
+
+##### Examples of bad practice
+
+TODO: Pick glusterfs specific example.
+
+
+##### Examples of good practice
+
+
+### Information in commit messages
+As important as the content of the change, is the content of the commit message describing it. When writing a commit message there are some important things to remember
+
+* Do not assume the reviewer understands what the original problem was.
+
+When reading bug reports, after a number of back & forth comments, it is often as clear as mud, what the root cause problem is. The commit message should have a clear statement as to what the original problem is. The bug is merely interesting historical background on /how/ the problem was identified. It should be possible to review a proposed patch for correctness without needing to read the bug ticket.
+
+* Do not assume the reviewer has access to external web services/site.
+
+In 6 months time when someone is on a train/plane/coach/beach/pub troubleshooting a problem & browsing Git history, there is no guarantee they will have access to the online bug tracker, or online blueprint documents. The great step forward with distributed SCM is that you no longer need to be "online" to have access to all information about the code repository. The commit message should be totally self-contained, to maintain that benefit.
+
+* Do not assume the code is self-evident/self-documenting.
+
+What is self-evident to one person, might be clear as mud to another person. Always document what the original problem was and how it is being fixed, for any change except the most obvious typos, or whitespace only commits.
+
+* Describe why a change is being made.
+
+A common mistake is to just document how the code has been written, without describing /why/ the developer chose to do it that way. By all means describe the overall code structure, particularly for large changes, but more importantly describe the intent/motivation behind the changes.
+
+* Read the commit message to see if it hints at improved code structure.
+
+Often when describing a large commit message, it becomes obvious that a commit should have in fact been split into 2 or more parts. Don't be afraid to go back and rebase the change to split it up into separate commits.
+
+* Ensure sufficient information to decide whether to review.
+
+When Gerrit sends out email alerts for new patch submissions there is minimal information included, principally the commit message and the list of files changes. Given the high volume of patches, it is not reasonable to expect all reviewers to examine the patches in detail. The commit message must thus contain sufficient information to alert the potential reviewers to the fact that this is a patch they need to look at.
+
+* The first commit line is the most important.
+
+In Git commits the first line of the commit message has special significance. It is used as email subject line, git annotate messages, gitk viewer annotations, merge commit messages and many more places where space is at a premium. As well as summarizing the change itself, it should take care to detail what part of the code is affected. eg if it is 'afr', 'dht' or any translator. Or in some cases, it can be touching all these components, but the commit message can be 'coverity:', 'txn-framework:', 'new-fop: ', etc.
+
+* Describe any limitations of the current code.
+
+If the code being changed still has future scope for improvements, or any known limitations then mention these in the commit message. This demonstrates to the reviewer that the broader picture has been considered and what tradeoffs have been done in terms of short term goals vs. long term wishes.
+
+* Do not include patch set-specific comments.
+
+In other words, if you rebase your change please don't add "Patch set 2: rebased" to your commit message. That isn't going to be relevant once your change has merged. Please do make a note of that in Gerrit as a comment on your change, however. It helps reviewers know what changed between patch sets. This also applies to comments such as "Added unit tests", "Fixed localization problems", or any other such patch set to patch set changes that don't affect the overall intent of your commit.
+
+**The main rule to follow is:**
+
+The commit message must contain all the information required to fully understand & review the patch for correctness. Less is not more. More is more.
+
+
+#### Including external references
+
+The commit message is primarily targeted towards human interpretation, but there is always some metadata provided for machine use. In the case of GlusterFS this includes at least the 'Change-id', "bug"/"feature" ID references and "Signed-off-by" tag (generated by 'git commit -s').
+
+The 'Change-id' line is a unique hash describing the change, which is generated by a Git commit hook. This should not be changed when rebasing a commit following review feedback, since it is used by Gerrit, to track versions of a patch.
+
+The 'bug' line can reference a bug in a few ways. Gerrit creates a link to the bug when viewing the patch on review.gluster.org so that reviewers can quickly access the bug/issue on Bugzilla or Github.
+
+**Fixes: bz#1601166** -- use 'Fixes: bz#NNNNN' if the commit is intended to fully fix and close the bug being referenced.
+**Fixes: #411** -- use 'Fixes: #NNN' if the patch fixes the github issue completely.
+
+**Updates: bz#1193929** -- use 'Updates: bz#NNNN' if the commit is only a partial fix and more work is needed.
+**Updates: #175** -- use 'Updates: #NNNN' if the commit is only a partial fix and more work is needed for the feature completion.
+
+We encourage the use of `Co-Authored-By: name <name@example.com>` in commit messages to indicate people who worked on a particular patch. It's a convention for recognizing multiple authors, and our projects would encourage the stats tools to observe it when collecting statistics.
+
+### Summary of Git commit message structure
+
+* Provide a brief description of the change in the first line.
+* The first line should be limited to 50 characters and should not end with a period.
+
+* Insert a single blank line after the first line.
+
+* Provide a detailed description of the change in the following lines, breaking paragraphs where needed.
+
+* Subsequent lines should be wrapped at 72 characters.
+
+Put the 'Change-id', 'Fixes bz#NNNNN' and 'Signed-off-by: <>' lines at the very end.
+
+TODO: Add good examples
diff --git a/doc/developer-guide/datastructure-inode.md b/doc/developer-guide/datastructure-inode.md
index a340ab9ca8e..45d7a941e5f 100644
--- a/doc/developer-guide/datastructure-inode.md
+++ b/doc/developer-guide/datastructure-inode.md
@@ -1,6 +1,6 @@
-#Inode and dentry management in GlusterFS:
+# Inode and dentry management in GlusterFS:
 
-##Background
+## Background
 Filesystems internally refer to files and directories via inodes. Inodes
 are unique identifiers of the entities stored in a filesystem. Whenever an
 application has to operate on a file/directory (read/modify), the filesystem
@@ -41,11 +41,10 @@ struct _inode_table {
 };
 ```
 
-#Life-cycle
+# Life-cycle
 ```
-
 inode_table_new (size_t lru_limit, xlator_t *xl)
-
+```
 This is a function which allocates a new inode table. Usually the top xlators in
 the graph such as protocol/server (for bricks), fuse and nfs (for fuse and nfs
 mounts) and libgfapi do inode managements. Hence they are the ones which will
@@ -59,11 +58,8 @@ new inode table.
 Thus an allocated inode table is destroyed only when the filesystem daemon is
 killed or unmounted.
 
-```
-
-#what it contains.
-```
 
+# what it contains.
 Inode table in glusterfs mainly contains a hash table for maintaining inodes.
 In general a file/directory is considered to be existing if there is a
 corresponding inode present in the inode table. If a inode for a file/directory
@@ -76,21 +72,21 @@ size of the hash table (as of now it is hard coded to 14057. The hash value of
 a inode is calculated using its gfid).
 
 Apart from the hash table, inode table also maintains 3 important list of inodes
-1) Active list:
+1. Active list:
 Active list contains all the active inodes (i.e inodes which are currently part
 of some fop).
-2) Lru list:
+2. Lru list:
 Least recently used inodes list. A limit can be set for the size of the lru
 list. For bricks it is 16384 and for clients it is infinity.
-3) Purge list:
+3. Purge list:
 List of all the inodes which have to be purged (i.e inodes which have to be
 deleted from the inode table due to unlink/rmdir/forget).
 
 And at last it also contains the mem-pool for allocating inodes, dentries so
 that frequent malloc/calloc and free of the data structures can be avoided.
-```
 
-#Data structure (inode)
+
+# Data structure (inode)
 ```
 struct _inode {
         inode_table_t       *table;         /* the table this inode belongs to */
@@ -108,7 +104,7 @@ struct _inode {
         struct _inode_ctx   *_ctx;          /* place holder for keeping the
         information about the inode by different xlators */
 };
-
+```
 As said above, inodes are internal way of identifying the files/directories. A
 inode uniquely represents a file/directory. A new inode is created whenever a
 create/mkdir/symlink/mknod operations are performed. Apart from that a new inode
@@ -128,9 +124,9 @@ inodes are those inodes whose refcount is greater than zero. Whenever some
 operation comes on a file/directory, and the resolver tries to find the inode
 for it, it increments the refcount of the inode before returning the inode. The
 refcount of an inode can be incremented by calling the below function
-
+```
 inode_ref (inode_t *inode)
-
+```
 Any xlator which wants to operate on a inode as part of some fop (or wants the
 inode in the callback), should hold a ref on the inode.
 Once the fop is completed before sending the reply of the fop to the above
@@ -139,18 +135,18 @@ zero, it is removed from the active inodes list and put into LRU list maintained
 by the inode table. Thus in short if some fop is happening on a file/directory,
 the corresponding inode will be in the active list or it will be in the LRU
 list.
-```
 
-#Life Cycle
+
+# Life Cycle
 
 A new inode is created whenever a new file/directory/symlink is created OR a
 successful lookup of an existing entry is done. The xlators which does inode
 management (as of now protocol/server, fuse, nfs, gfapi) will perform inode_link
 operation upon successful lookup or successful creation of a new entry.
-
+```
 inode_link (inode_t *inode, inode_t *parent, const char *name,
             struct iatt *buf);
-
+```
 inode_link actually adds the inode to the inode table (to be precise it adds
 the inode to the hash table maintained by the inode table. The hash value is
 calculated based on the gfid). Copies the gfid to the inode (the gfid is
@@ -160,7 +156,7 @@ A inode is removed from the inode table and eventually destroyed when unlink
 or rmdir operation is performed on a file/directory, or the the lru limit of
 the inode table has been exceeded.
 
-#Data structure (dentry)
+# Data structure (dentry)
 ```
 
 struct _dentry {
@@ -170,22 +166,22 @@ struct _dentry {
         char              *name;         /* name of the directory entry */
         inode_t           *parent;       /* directory of the entry */
 };
-
+```
 A dentry is the presence of an entry for a file/directory within its parent
 directory. A dentry usually points to the inode to which it belongs to. In
 glusterfs a dentry contains the following fields.
-1) a hook using which it can add itself to the list of
+1. a hook using which it can add itself to the list of
 the dentries maintained by the inode to which it points to.
-2) A hash table pointer.
-3) Pointer to the inode to which it belongs to.
-4) Name of the dentry
-5) Pointer to the inode of the parent directory in which the dentry is present
+2. A hash table pointer.
+3. Pointer to the inode to which it belongs to.
+4. Name of the dentry
+5. Pointer to the inode of the parent directory in which the dentry is present
 
 A new dentry is created when a new file/directory/symlink is created or a hard
 link to an existing file is created.
-
+```
 __dentry_create (inode_t *inode, inode_t *parent, const char *name);
-
+```
 A dentry holds a refcount on the parent
 directory so that the parent inode is never removed from the active inode's list
 and put to the lru list (If the lru limit of the lru list is exceeded, there is
@@ -212,15 +208,14 @@ deleted due to file removal or lru limit being exceeded  the inode is retired
 purge list maintained by the inode table), the nlookup count is set to 0 via
 inode_forget api. The inode table, then prunes all the inodes from the purge
 list by destroying the inode contexts maintained by each xlator.
-
+```
 unlinking of the dentry is done via inode_unlink;
 
 void
 inode_unlink (inode_t *inode, inode_t *parent, const char *name);
-
+```
 If the inode has multiple hard links, then the unlink operation performed by
 the application results just in the removal of the dentry with the name provided
 by the application. For the inode to be removed, all the dentries of the inode
 should be unlinked.
-```
 
diff --git a/doc/developer-guide/datastructure-iobuf.md b/doc/developer-guide/datastructure-iobuf.md
index 5f521f1485f..03604e3672c 100644
--- a/doc/developer-guide/datastructure-iobuf.md
+++ b/doc/developer-guide/datastructure-iobuf.md
@@ -1,6 +1,6 @@
-#Iobuf-pool
-##Datastructures
-###iobuf
+# Iobuf-pool
+## Datastructures
+### iobuf
 Short for IO Buffer. It is one allocatable unit for the consumers of the IOBUF
 API, each unit hosts @page_size(defined in arena structure) bytes of memory. As
 initial step of processing a fop, the IO buffer passed onto GlusterFS by the
@@ -28,7 +28,7 @@ struct iobuf {
 };
 ```
 
-###iobref
+### iobref
 There may be need of multiple iobufs for a single fop, like in vectored read/write.
 Hence multiple iobufs(default 16) are encapsulated under one iobref.
 ```
@@ -40,7 +40,7 @@ struct iobref {
         int                used;    /* number of iobufs added to this iobref */
 };
 ```
-###iobuf_arenas
+### iobuf_arenas
 One region of memory MMAPed from the operating system. Each region MMAPs
 @arena_size bytes of memory, and hosts @arena_size / @page_size IOBUFs.
 The same sized iobufs are grouped into one arena, for sanity of access.
@@ -77,7 +77,7 @@ struct iobuf_arena {
 };
 
 ```
-###iobuf_pool
+### iobuf_pool
 Pool of Iobufs. As there may be many Io buffers required by the filesystem,
 a pool of iobufs are preallocated and kept, if these preallocated ones are
 exhausted only then the standard malloc/free is called, thus improving the
@@ -139,8 +139,8 @@ arenas in the purge list are destroyed only if there is  atleast one arena in
 (e.g: If there is an arena (page_size=128KB, count=32) in purge list, this arena
 is destroyed(munmap) only if there is an arena in 'arenas' list with page_size=128KB).
 
-##APIs
-###iobuf_get
+## APIs
+### iobuf_get
 
 ```
 struct iobuf *iobuf_get (struct iobuf_pool *iobuf_pool);
@@ -149,7 +149,7 @@ Creates a new iobuf of the default page size(128KB hard coded as of yet).
 Also takes a reference(increments ref count), hence no need of doing it
 explicitly after getting iobuf.
 
-###iobuf_get2
+### iobuf_get2
 
 ```
 struct iobuf * iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size);
@@ -179,7 +179,7 @@ if (requested iobuf size > Max iobuf size in the pool(1MB as of yet))
 Also takes a reference(increments ref count), hence no need of doing it
 explicitly after getting iobuf.
 
-###iobuf_ref
+### iobuf_ref
 
 ```
 struct iobuf *iobuf_ref (struct iobuf *iobuf);
@@ -188,7 +188,7 @@ struct iobuf *iobuf_ref (struct iobuf *iobuf);
 xlator/function/, its a good practice to take a reference so that iobuf is not
 deleted by the allocator.
 
-###iobuf_unref
+### iobuf_unref
 ```
 void iobuf_unref (struct iobuf *iobuf);
 ```
@@ -203,33 +203,33 @@ Unreference the iobuf, if the ref count is zero iobuf is considered free.
 Every iobuf_ref should have a corresponding iobuf_unref, and also every
 iobuf_get/2 should have a correspondning iobuf_unref.
 
-###iobref_new
+### iobref_new
 ```
 struct iobref *iobref_new ();
 ```
 Creates a new iobref structure and returns its pointer.
 
-###iobref_ref
+### iobref_ref
 ```
 struct iobref *iobref_ref (struct iobref *iobref);
 ```
 Take a reference on the iobref.
 
-###iobref_unref
+### iobref_unref
 ```
 void iobref_unref (struct iobref *iobref);
 ```
 Decrements the reference count of the iobref. If the ref count is 0, then unref
 all the iobufs(iobuf_unref) in the iobref, and destroy the iobref.
 
-###iobref_add
+### iobref_add
 ```
 int iobref_add (struct iobref *iobref, struct iobuf *iobuf);
 ```
 Adds the given iobuf into the iobref, it takes a ref on the iobuf before adding
 it, hence explicit iobuf_ref is not required if adding to the iobref.
 
-###iobref_merge
+### iobref_merge
 ```
 int iobref_merge (struct iobref *to, struct iobref *from);
 ```
@@ -239,13 +239,13 @@ on all the iobufs added to the 'to' iobref. Hence iobref_unref should be
 performed both on 'from' and 'to' iobrefs (performing iobref_unref only on 'to'
 will not free the iobufs and may result in leak).
 
-###iobref_clear
+### iobref_clear
 ```
 void iobref_clear (struct iobref *iobref);
 ```
 Unreference all the iobufs in the iobref, and also unref the iobref.
 
-##Iobuf Leaks
+## Iobuf Leaks
 If all iobuf_refs/iobuf_new do not have correspondning iobuf_unref, then the
 iobufs are not freed and recurring execution of such code path may lead to huge
 memory leaks. The easiest way to identify if a memory leak is caused by iobufs
diff --git a/doc/developer-guide/datastructure-mem-pool.md b/doc/developer-guide/datastructure-mem-pool.md
index c71aa2a8ddd..225567cbf9f 100644
--- a/doc/developer-guide/datastructure-mem-pool.md
+++ b/doc/developer-guide/datastructure-mem-pool.md
@@ -1,5 +1,5 @@
-#Mem-pool
-##Background
+# Mem-pool
+## Background
 There was a time when every fop in glusterfs used to incur cost of allocations/de-allocations for every stack wind/unwind between xlators because stack/frame/*_localt_t in every wind/unwind was allocated and de-allocated. Because of all these system calls in the fop path there was lot of latency and the worst part is that most of the times the number of frames/stacks active at any time wouldn't cross a threshold. So it was decided that this threshold number of frames/stacks would be allocated in the beginning of the process only once. Get one of them from the pool of stacks/frames whenever `STACK_WIND` is performed and put it back into the pool in `STACK_UNWIND`/`STACK_DESTROY` without incurring any extra system calls. The data structures are allocated only when threshold number of such items are in active use i.e. pool is in complete use.% increase in the performance once this was added to all the common data structures (inode/fd/dict etc) in xlators throughout the stack was tremendous.
 
 ## Data structure
@@ -27,7 +27,7 @@ will be served from here until all the elements in the pool are in use i.e. cold
 };
 ```
 
-##Life-cycle
+## Life-cycle
 ```
 mem_pool_new (data_type, unsigned long count)
 
@@ -120,5 +120,5 @@ mem_pool_destroy (struct mem_pool *pool)
 Deletes this pool from the `global_list` maintained by `glusterfs-ctx` and frees all the memory allocated in `mem_pool_new`.
 
 
-###How to pick pool-size
+### How to pick pool-size
 This varies from work-load to work-load. Create the mem-pool with some random size and run the work-load. Take the statedump after the work-load is complete. In the statedump if `max_alloc` is always less than `cold_count` may be reduce the size of the pool closer to `max_alloc`. On the otherhand if there are lots of `pool-misses` then increase the `pool_size` by `max_stdalloc` to achieve better 'hit-rate' of the pool.
diff --git a/doc/developer-guide/dirops-transactions-in-dht.md b/doc/developer-guide/dirops-transactions-in-dht.md
new file mode 100644
index 00000000000..909a97001aa
--- /dev/null
+++ b/doc/developer-guide/dirops-transactions-in-dht.md
@@ -0,0 +1,273 @@
+# dirops transactions in dht
+Need for transactions during operations on directories arise from two
+basic design elements of DHT:
+
+ 1. A directory is created on all subvolumes of dht. Since glusterfs
+ associates each file-system object with an unique gfid, every
+ subvolume should have the same unique mapping of (path of directory,
+ gfid). To elaborate,
+    * Each subvolume should've same gfid associated with a path to
+ directory.
+    * A gfid should not be associated with more than one path in any
+ subvolume.
+
+ So, entire operations like mkdir, renamedir, rmdir and creation of
+ directories during self-heal need to be atomic in dht. In other words,
+ any of these operations shouldn't begin on an inode if one of them is
+ already in progress on the same inode, till it completes on all
+ subvolumes of dht. If not, more than one of these operations
+ happening in parallel can break any or all of the two requirements
+ listed above. This is referred in the rest of the document by the
+ name _Atomicity during namespace operations_.
+   
+ 2. Each directory has an independent layout persisted on
+ subvolumes. Each subvolume contains only part of the layout relevant
+ to it. For performance reasons _and_ since _only_ dht has aggregated
+ view, this layout is cached in memory of client. To make sure dht
+ reads or modifies a single complete layout while parallel modifications of    the layout are in progress, we need atomicity during layout modification and reading. This is referred in the rest of the document as  _Atomicity during layout modification and reading_.
+
+Rest of the document explains how atomicity is achieved for each of
+the case above.
+
+**Atomicity during layout modification and reading**
+File operations a.k.a fops can be classified into two categories based on how they consume layout.
+
+ - Layout writer. Setting of layout during selfheal of a directory is
+ layout writer of _that_ directory. 
+ - Layout reader. 
+     * Any entry fop like create, unlink, rename, link, symlink,
+ unlink, mknod, rename, mkdir, rmdir, renamedir which needs layout of the parent directory. Each of these fops are readers of layout on parent directory.
+     *  setting of layout during mkdir of a directory is considered as
+ a reader of the same directory's layout. The reason for this is that
+ only a parallel lookup on that directory can be a competing fop that modifies the layout (Other fops need gfid of the directory which can be got only after either lookup or mkdir completes). However, healing of layout is considered as a writer and a single writer blocks all readers.
+ 
+*Algorithm*
+Atomicity is achieved by locking on the inode of directory whose
+layout is being modified or read. The fop used is inodelk.
+ -  Writer acquires blocking inodelk (directory-inode, write-lock) on
+ all subvolumes serially. The order of subvols in which they are
+ locked by different clients remains constant for a directory. If locking fails on any subvolume, layout modification is abandoned.
+ - Reader acquires an inodelk (directory-inode, read-lock) on _any_
+ one subvolume. If locking fails on a subvolume (say with
+ ESTALE/ENOTCONN error), locking can be tried on other subvolumes till
+ we get one lock. If we cannot get lock on at least one subvolume,
+ consistency of layout is not guaranteed. Based on the consistency
+ requirements of fops, they can be failed or continued.
+
+Reasons why writer has to lock on _all_ subvols:
+
+ - DHT don't have a metadata server and locking is implemented by brick. So, there is no well-defined subvol/brick that can be used as an arbitrator by different clients while acquiring locks.
+ - readers should acquire as minimum number of locks as possible. In
+ other words, the algorithm aims to have less synchronization cost to
+ readers.
+ - The subvolume to which a directory hashes could be used as a
+ lock server. However, in the case of an entry fop like create
+ (/a/b/c) where we want to block modification of layout of b for the
+ duration of create, we would be required to acquire lock on the
+ subvol to which /a/b hashes. To find out the hashed-subvol of
+ /a/b, we would need layout of /a. Note that how there is a dependency
+ of locking the layouts of ancestors all the way to root. So this
+ locking is not preferred. Also, note that only the immediate parent
+ inode is available in arguments of a fop like create.
+
+**Atomicity during namespace operations**
+
+ -  We use locks on inode of parent directory in the namespace of
+ _"basename"_  during mkdir, rmdir, renamedir and directory
+ creation phase of self-heal. The exact fop we use is _entrylk
+ (parent-inode, "basename")_.
+ - refresh in-memory layout of parent-inode from values stored on backend
+ -  _entrylk (parent-inode, "basename")_ is done on subvolume to which
+ _"basename" hashes_. So, this operation is a _reader_ of the
+ layout on _parent-inode_. Which means an _inodelk (parent-inode,
+ read-lock)_ has to be completed before _entrylk (parent-inode,
+ "basename")_ is issued. Both the locks have to be held till the
+ operation is tried on all subvolumes. If acquiring of any/all of
+ these locks fail, the operation should be failed.
+
+With the above details, algorithms for mkdir, rmdir, renamedir,
+self-heal of directory are explicitly detailed below.
+
+**Self-heal of a directory**
+  
+ -  creation of directories on subvolumes is done only during
+ _named-lookup_ of a directory as we need < parent-inode,
+ "basename" >.
+ -  If a directory is missing on one or more subvolumes,
+    *  acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+    * refresh the in-memory layout of parent-inode from values stored on backend
+    *  acquire _entrylk (parent-inode, "basename")_ on the subvolume
+ to which _"basename"_ hashes.
+    *  If any/all of the locks fail, self-heal is aborted.
+    *  create directories on missing subvolumes.
+    *  release _entrylk (parent-inode, "basename")_.
+    *  release _inodelk (parent-inode, read-lock)_.
+    
+ -  If layout of a directory needs healing
+     * acquire _inodelk (directory-inode, write-lock)_ on _all_ the
+ subvolumes. If locking fails on any of the subvolumes,
+ self-heal is aborted. Blocking Locks are acquired serially across subvolumes in a _well-defined_ order which is _constant_ across all the healers of a directory. One order could be the order in which subvolumes are stored in the array _children_ of dht xlator.
+     *  heal the layout.
+     *  release _inodelk (directory-inode, write-lock)_ on _all_ the
+ subvolumes in parallel.
+     *  Note that healing of layout can be done in both _named_ and
+ _nameless_ lookups of a directory as _only directory-inode_ is needed
+ for healing and it is available during both.
+
+**mkdir (parent-inode, "basename")**
+
+* while creating directory across subvolumes,
+
+ - acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+ - refresh in-memory layout of parent-inode from values stored on backend
+ - acquire _entrylk (parent-inode, "basename")_ on the subvolume to
+ which _"basename"_ hashes.
+ -  If any/all of the above two locks fail, release the locks that
+ were acquired successfully and mkdir should be failed (as perceived by application).
+ - do _mkdir (parent-inode, "basename")_ on the subvolume to which
+ _"basename"_ hashes. If this mkdir fails, mkdir is failed.
+ - do _mkdir (parent-inode, "basename")_ on the remaining subvolumes.
+ - release _entrylk (parent-inode, "basename")_.
+ - release _inodelk (parent-inode, "read-lock")_.
+*  while setting the layout of a directory,
+ - acquire _inodelk (directory-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+ -  If locking fails, cleanup the locks that were acquired
+ successfully and abort layout setting. Note that we'll have a
+ directory without a layout till a lookup happens on the
+ directory. This means entry operations within this directory fail
+ in this time window. We can also consider failing mkdir. The
+ problem of dealing with a directory without layout is out of the
+ scope of this document. 
+ -  set the layout on _directory-inode_.
+ -  release _inodelk (directory-inode, read-lock)_.
+* Note that during layout setting we consider mkdir as a  _reader_ not
+ _writer_, though it is setting the layout. Reasons are:
+ - Before any of other readers like create, link etc that operate on
+ this directory to happen, _gfid_ of this directory has to be
+ resolved. But _gfid_ is only available only if either of following
+ conditions are true: 
+      * after mkdir is complete.
+      * a lookup on the same path happens parallel to in-progress
+ mkdir.
+
+     But, on completion of any of the above two operations, layout
+     will be healed. So, none of the _readers_ will happen on a
+     directory with partial layout.
+
+* Note that since we've an _entrylk (parent-inode, "basename")_ for
+   the entire duration of (attempting) creating directories, parallel
+   mkdirs will no longer contend on _mkdir_ on subvolume _to which  "basename"  hashes_. But instead, contend on _entrylk (parent-inode, "basename")_ on the subvolume _to which "basename" hashes_. So, we can attempt the _mkdir_ in _parallel_ on all subvolumes instead of two stage mkdir on hashed first and the rest of them in parallel later. However, we need to make sure that mkdir is successful on the subvolume _to which "basename" hashes_ for mkdir to be successful (as perceived by application). In the case of failed mkdir (as perceived by application), a cleanup should be performed on all the subvolumes before _entrylk (parent-inode, "basename")_ is released.
+
+**rmdir (parent-inode, "basename", directory-inode)**
+
+ -  acquire _inodelk (parent-inode, read-lock)_ on _any one_
+ subvolume.
+ -  refresh in-memory layout of parent-inode from values stored on backend
+ -  acquire _entrylk (parent-inode, "basename")_ on the subvolume to
+ which _"basename" hashes_.
+ -  If any/all of the above locks fail, rmdir is failed after cleanup
+ of the locks that were acquired successfully.
+ -  do _rmdir (parent-inode, "basename")_ on the subvolumes to which
+ _"basename" doesn't hash to_.
+    * If successful, continue. 
+    * Else,
+     *  recreate directories on those subvolumes where rmdir
+  succeeded.
+     *  heal the layout of _directory-inode_. Note that this will have
+  same synchronization requirements as discussed during layout
+  healing part of the section "Directoy self-heal" above.
+     *  release _entrylk (parent-inode, "basename")_.
+     *  release _inodelk (parent-inode, read-lock)_.
+     *  fail _rmdir (parent-inode, "basename")_ to application.
+ - do _rmdir (parent-inode, "basename")_ on the subvolume to which
+ _"basename" hashes_. 
+ - If successful, continue.
+ -  Else, Go to the failure part of _rmdir (parent-inode, "basename")_
+ on subvolumes to which "basename" _doesn't hash to_. 
+ - release _entrylk (parent-inode, "basename")_.
+ - release _inodelk (parent-inode, read-lock)_.
+ - return success to application.
+ 
+**renamedir (src-parent-inode, "src-basename", src-directory-inode, dst-parent-inode, "dst-basename", dst-directory-inode)**
+
+ -  requirement is to prevent any operation in both _src-namespace_
+ and _dst-namespace_. So, we need to acquire locks on both
+ namespaces.We also need to have constant ordering while acquiring
+ locks during parallel renames of the form _rename (src, dst)_ and
+ _rename (dst, src)_ to prevent deadlocks. We can sort gfids of
+ _src-parent-inode_ and _dst-parent-inode_ and use that order to
+ acquire locks. For the sake of explanation lets say we ended up
+ with order of _src_ followed by _dst_.
+ -  acquire _inodelk (src-parent-inode, read-lock)_.
+ -  refresh in-memory layout of src-parent-inode from values stored on backend
+ -  acquire _entrylk (src-parent-inode, "src-basename")_.
+ -  acquire _inodelk (dst-parent-inode, read-lock)_.
+ -  refresh in-memory layout of dst-parent-inode from values stored on backend
+ -  acquire _entrylk (dst-parent-inode, "dst-basename")_.
+ - If acquiring any/all of the locks above fail,
+      * release the locks that were successfully acquired.
+      * fail the renamedir operation to application
+      *  done
+ - do _renamedir ("src", "dst")_ on the subvolume _to which "dst-basename" hashes_.
+    *  If failure, Goto point _If acquiring any/all of the locks above fail_.
+    *  else, continue.
+ - do _renamedir ("src", "dst")_ on rest of the subvolumes.
+    *  If there is any failure, 
+      * revert the successful renames.
+      *  Goto to point  _If acquiring any/all of the locks above fail_.
+    *  else,
+        -  release all the locks acquired.
+        -  return renamedir as success to application.
+
+**Some examples of races**
+This section gives concrete examples of races that can result in inconsistencies explained in the beginning of the document. 
+
+Some assumptions are:
+
+*  We consider an example distribute of three subvols s1, s2 and s3.
+*  For examples of renamedir ("src", "dst"), _src_ hashes to s1 and _dst_ hashes to s2. _src_ and _dst_ are associated with _gfid-src_ and _gfid-dst_ respectively
+*  For non renamedir examples, _dir_ is the name of directory and it hashes to s1.
+
+And the examples are:  
+
+ -  mkdir vs rmdir - inconsistency in namespace.
+   *  mkdir ("dir", gfid1) is complete on s1
+   *  rmdir is issued on same directory. Note that, since rmdir needs a gfid, a lookup should be complete before rmdir. lookup creates the directory on rest of the subvols as part of self-heal.
+   *  rmdir (gfid1) deletes directory from all subvols.
+   *  A new mkdir ("dir", gfid2) is issued. It is successful on s1 associating "dir" with gfid2.
+   *  mkdir ("dir", gfid1) resumes and creates directory on s2 and s3 associating "dir" with gfid1.
+   *  mkdir ("dir", gfid2) fails with EEXIST on s2 and s3. Since, EEXIST errors are ignored, mkdir is considered successful to application.
+   *  In this example we have multiple inconsitencies
+      *  "dir" is associated with gfid1 on s2, s3 and with gfid2 on s1
+      *  Even if mkdir ("dir", gfid2) was not issued, we would've a case of a directory magically reappearing after a successful rmdir.
+ -  lookup heal vs rmdir
+   * rmdir ("dir", gfid1) is issued. It is successful on s2 and s3 (non-hashed subvols for name "dir")
+   *  lookup ("dir") is issued. Since directory is present on s1 yet, it is created on s2 and s3 associating with gfid1 as part of self-heal
+   *  rmdir ("dir", gfid1) is complete on s1 and it is successful
+   *  Another lookup ("dir") creates the directory on s1 too
+   *  "dir" magically reappears after a successful rmdir
+ - lookup heal (src) vs renamedir ("src", "dst")
+   *  renamedir ("src", "dst") complete on s2
+   *  lookup ("src") recreates _src_ with _gfid-src_ on s2
+   *  renamedir ("src", "dst") completes on s1, s3. After rename is complete path _dst_ will be associated with gfid _gfid-src_
+   *  Another lookup ("src") recreates _src_ on subvols s1 and s3, associating it with gfid _gfid-src_
+   *  Inconsistencies are
+      * after a successful renamedir ("src", "dst"), both src and dst exist
+      *  Two directories - src and dst - are associated with same gfid. One common symptom is that some entries (of the earlier _src_ and current _dst_ directory) being missed out in readdir listing as the gfid handle might be pointing to the empty healed directory than the actual directory containing entries
+ - lookup heal (dst) vs renamedir ("src", "dst")
+   *  dst exists and empty when renamdir started
+   *  dst doesn't exist when renamedir started
+      -  renamedir ("src", "dst") complete on s2 and s3
+      -  lookup ("dst") creates _dst_ associating it with _gfid-src_ on s1
+      -  An entry is created in _dst_ on either s1
+      -  renamedir ("src", "dst") on s1 will result in a directory _dst/dst_ as _dst_ is no longer empty and _man 2 rename_ states that if _dst_ is not empty, _src_ is renamed _as a subdirectory of dst_
+      -  A lookup ( _dst/dst_) creates _dst/dst_ on s2 and s3 associating with _gfid-src_ as part of self-heal
+      -  Inconsistencies are:
+         * Two directories - _dst_ and _dst/dst_ - exist even though both of them didn't exist at the beginning of renamedir
+         *  Both _dst_ and _dst/dst_ have same gfid - _gfid-src_. As observed earlier, symptom might be directory listing being incomplete
+ - mkdir (dst) vs renamedir ("src", "dst")
+ - rmdir (src) vs renamedir ("src", "dst")
+ - rmdir (dst) vs renamedir ("src", "dst")
diff --git a/doc/developer-guide/ec-implementation.md b/doc/developer-guide/ec-implementation.md
new file mode 100644
index 00000000000..77e62583caa
--- /dev/null
+++ b/doc/developer-guide/ec-implementation.md
@@ -0,0 +1,588 @@
+Erasure coding implementation
+=============================
+
+This document provides information about how [erasure code][1] has
+been implemented into ec translator. It describes the algorithm used
+and the optimizations made, but it doesn't contain a full description
+of the mathematical background needed to understand erasure coding in
+general. It either describes the other parts of ec not directly
+related to the encoding/decoding procedure, like synchronization or
+fop management.
+
+
+Introduction
+------------
+
+EC is based on [Reed-Solomon][2] erasure code. It's a very old code.
+It's not considered the best one nowadays, but is good enough and it's
+one of the few codes that is not covered by any patent and can be
+freely used.
+
+To define the Reed-Solomon code we use 3 parameters:
+
+  * __Key fragments (K)__  
+    It represents the minimum number of healthy fragments that will be
+    needed to be able to recover the original data. Any subset of K
+    out of the total number of fragments will serve.
+
+  * __Redundancy fragments (R)__  
+    It represents the number of extra fragments to compute for each
+    original data block. This value determines how many fragments can
+    be lost before being unable to recover the original data.
+
+  * __Fragment size (S)__  
+    This determines the size of each fragment. The original data
+    block size is computed as S * K. Currently this values is fixed
+    to 512 bytes.
+
+  * __Total number of fragments (N = K + R)__  
+    This isn't a real parameter but it will be useful to simplify
+    the following descriptions.
+
+From the point of view of the implementation, it only consists on
+matrix multiplications. There are two kinds of matrices to use for
+Reed-Solomon:
+
+  * __[Systematic][3]__  
+    This kind of matrix has the particularity that K of the encoded
+    fragments are simply a copy of the original data, divided into K
+    pieces. Thus no real encoding needs to be done for them and only
+    the R redundancy fragments need to be computed.
+
+    This kind of matrices contain one KxK submatrix that is the
+    [identity matrix][4].
+
+  * __Non-systematic__  
+    This kind of matrix doesn't contain an identity submatrix. This
+    means that all of the N fragments need to be encoded, requiring
+    more computation. On the other hand, these matrices have some nice
+    properties that allow faster implementations of some algorithms,
+    like the matrix inversion used to decode the data.
+
+    Another advantage of non-systematic matrices is that the decoding
+    time is constant, independently of how many fragments are lost,
+    while systematic approach can suffer from performance degradation
+    when one fragment is lost.
+
+All non-systematic matrices can be converted to systematic ones, but
+then we lose the good properties of the non-systematic. We have to
+choose betwee best peek performance (systematic) and performance
+stability (non-systematic).
+
+
+Encoding procedure
+------------------
+
+To encode a block of data we need a KxN matrix where each subset of K
+rows is [linearly independent][5]. In other words, the determinant of
+each KxK submatrix is not 0.
+
+There are some known ways to obtain this kind of matrices. EC uses a
+small variation of a matrix known as [Vandermonde Matrix][6] where
+each element of the matrix is defined as:
+
+    a(i, j) = i ^ (K - j)
+
+    where i is the row from 1 to N, and j is the column from 1 to K.
+
+This is exactly the Vandermonde Matrix but with the elements of each
+row in reverse order. This change is made to be able to implement a
+small optimization in the matrix multiplication.
+
+Once we have the matrix, we only need to compute the multiplication
+of this matrix by a vector composed of K elements of data coming from
+the original data block.
+
+     /                 \             /                               \
+    |    1   1  1  1  1 |    / \    |     a +    b +   c +  d + e = t |
+    |   16   8  4  2  1 |   | a |   |   16a +   8b +  4c + 2d + e = u |
+    |   81  27  9  3  1 |   | b | = |   81a +  27b +  9c + 3d + e = v |
+    |  256  64 16  4  1 | * | c |   |  256a +  64b + 16c + 4d + e = w |
+    |  625 125 25  5  1 |   | d |   |  625a + 125b + 25c + 5d + e = x |
+    | 1296 216 36  6  1 |   | e |   | 1296a + 216b + 36c + 6d + e = y |
+    | 2401 343 49  7  1 |    \ /    | 2401a + 343b + 49c + 7d + e = z |
+     \                 /             \                               /
+
+The optimization that can be done here is this:
+
+    16a + 8b + 4c + 2d + e = 2(2(2(2a + b) + c) + d) + e
+
+So all the multiplications are always by the number of the row (2 in
+this case) and we don't need temporal storage for intermediate
+results:
+
+    a *= 2
+    a += b
+    a *= 2
+    a += c
+    a *= 2
+    a += d
+    a *= 2
+    a += e
+
+Once we have the result vector, each element is a fragment that needs
+to be stored in a separate place.
+
+
+Decoding procedure
+------------------
+
+To recover the data we need exactly K of the fragments. We need to
+know which K fragments we have (i.e. the original row number from
+which each fragment was calculated). Once we have this data we build
+a square KxK matrix composed by the rows corresponding to the given
+fragments and invert it.
+
+With the inverted matrix, we can recover the original data by
+multiplying it with the vector composed by the K fragments.
+
+In our previous example, if we consider that we have recovered
+fragments t, u, v, x and z, corresponding to rows 1, 2, 3, 5 and 7,
+we can build the following matrix:
+
+     /                 \
+    |    1   1  1  1  1 |
+    |   16   8  4  2  1 |
+    |   81  27  9  3  1 |
+    |  625 125 25  5  1 |
+    | 2401 343 49  7  1 |
+     \                 /
+
+And invert it:
+
+     /                                     \
+    |    1/48  -1/15    1/16  -1/48   1/240 |
+    |  -17/48  16/15  -15/16  13/48 -11/240 |
+    |  101/48 -86/15   73/16 -53/48  41/240 |
+    | -247/48 176/15 -129/16  83/48 -61/240 |
+    |   35/8   -7      35/8   -7/8    1/8   |
+     \                                     /
+
+Multiplying it by the vector (t, u, v, x, z) we recover the original
+data (a, b, c, d, e):
+
+     /                                     \     / \     / \
+    |    1/48  -1/15    1/16  -1/48   1/240 |   | t |   | a |
+    |  -17/48  16/15  -15/16  13/48 -11/240 |   | u |   | b |
+    |  101/48 -86/15   73/16 -53/48  41/240 | * | v | = | c |
+    | -247/48 176/15 -129/16  83/48 -61/240 |   | x |   | d |
+    |   35/8   -7      35/8   -7/8    1/8   |   | z |   | e |
+     \                                     /     \ /     \ /
+
+
+Galois Field
+------------
+
+This encoding/decoding procedure is quite complex to compute using
+regular mathematical operations and it's not well suited for what
+we want to do (note that matrix elements can grow unboundly).
+
+To solve this problem, exactly the same procedure is done inside a
+[Galois Field][7] of characteristic 2, which is a finite field with
+some interesting properties that make it specially useful for fast
+operations using computers.
+
+There are two main differences when we use this specific Galois Field:
+
+  * __All regular additions are replaced by bitwise xor's__  
+    For todays computers it's not really faster to execute an xor
+    compared to an addition, however replacing additions by xor's
+    inside a multiplication has many advantages (we will make use of
+    this to optimize the multiplication).
+
+    Another consequence of this change is that additions and
+    substractions are really the same xor operation.
+
+  * __The elements of the matrix are bounded__  
+    The field uses a modulus that keep all possible elements inside
+    a delimited region, avoiding really big numbers and fixing the
+    number of bits needed to represent each value.
+
+    In the current implementation EC uses 8 bits per field element.
+
+It's very important to understand how multiplications are computed
+inside a Galois Field to be able to understand how has it been
+optimized.
+
+We'll start with a simple 'old school' multiplication but in base 2.
+For example, if we want to multiply 7 * 5 (111b * 101b in binary), we
+do the following:
+
+          1 1 1 (= 7)
+    *     1 0 1 (= 5)
+    -----------
+          1 1 1 (= 7)
+    +   0 0 0   (= 0)
+    + 1 1 1     (= 7)
+    -----------
+    1 0 0 0 1 1 (= 35)
+
+This is quite simple. Note that the addition of the third column
+generates a carry that is propagated to all the other left columns.
+
+The next step is to define the modulus of the field. Suppose we use
+11 as the modulus. Then we convert the result into an element of the
+field by dividing by the modulus and taking the residue. We also use
+the 'old school' method in binary:
+
+
+      1 0 0 0 1 1 (= 35) | 1 0 1 1 (= 11)
+    - 0 0 0 0            ----------------
+    ---------              0 1 1 (= 3)
+      1 0 0 0 1
+    -   1 0 1 1
+    -----------
+      0 0 1 1 0 1
+    -     1 0 1 1
+    -------------
+          0 0 1 0 (= 2)
+
+So, 7 * 5 in a field with modulus 11 is 2. Note that the main
+objective in each iteration of the division is to make higher bits
+equal to 0 when possible (if it's not possible in one iteration, it
+will be zeroed on the next).
+
+If we do the same but changing additions with xors we get this:
+
+          1 1 1 (= 7)
+    *     1 0 1 (= 5)
+    -----------
+          1 1 1 (= 7)
+    x   0 0 0   (= 0)
+    x 1 1 1     (= 7)
+    -----------
+      1 1 0 1 1 (= 27)
+
+In this case, the xor of the third column doesn't generate any carry.
+
+Now we need to divide by the modulus. We can also use 11 as the
+modulus since it still satisfies the needed conditions to work on a
+Galois Field of characteristic 2 with 3 bits:
+
+      1 1 0 1 1 (= 27) | 1 0 1 1 (= 11)
+    x 1 0 1 1          ----------------
+    ---------            1 1 1 (= 7)
+      0 1 1 0 1
+    x   1 0 1 1
+    -----------
+        0 1 1 0 1
+    x     1 0 1 1
+    -------------
+          0 1 1 0 (= 6)
+
+Note that, in this case, to make zero the higher bit we need to
+consider the result of the xor operation, not the addition operation.
+
+So, 7 * 5 in a Galois Field of 3 bits with modulus 11 is 6.
+
+
+Optimization
+------------
+
+To compute all these operations in a fast way some methods have been
+traditionally used. Maybe the most common is the [lookup table][8].
+
+The problem with this method is that it requires 3 lookups for each
+byte multiplication, greatly amplifying the needed memory bandwidth
+and making it difficult to take advantage of any SIMD support on the
+processor.
+
+What EC does to improve the performance is based on the following
+property (using the 3 bits Galois Field of the last example):
+
+    A * B mod N = (A * b{2} * 4 mod N) x
+                  (A * b{1} * 2 mod N) x
+                  (A * b{0} mod N)
+
+This is basically a rewrite of the steps made in the previous example
+to multiply two numbers but moving the modulus calculation inside each
+intermediate result. What we can see here is that each term of the
+xor can be zeroed if the corresponding bit of B is 0, so we can ignore
+that factor. If the bit is 1, we need to compute A multiplied by a
+power of two and take the residue of the division by the modulus. We
+can precompute these values:
+
+    A0 = A (we don't need to compute the modulus here)
+    A1 = A0 * 2 mod N
+    A2 = A1 * 2 mod N
+
+Having these values we only need to add those corresponding to bits
+set to 1 in B. Using our previous example:
+
+    A = 1 1 1 (= 7)
+    B = 1 0 1 (= 5)
+
+    A0 = 1 1 1 (= 7)
+    A1 = 1 1 1 * 1 0 mod 1 0 1 1 = 1 0 1 (= 5)
+    A2 = 1 0 1 * 1 0 mod 1 0 1 1 = 0 0 1 (= 1)
+
+    Since only bits 0 and 2 are 1 in B, we add A0 and A2:
+
+    A0 + A2 = 1 1 1 x 0 0 1 = 1 1 0 (= 6)
+
+If we carefully look at what we are doing when computing each Ax, we
+see that we do two basic things:
+
+  - Shift the original value one bit to the left
+  - If the highest bit is 1, xor with the modulus
+
+Let's write this in a detailed way (representing each bit):
+
+    Original value:      a{2} a{1} a{0}
+    Shift 1 bit:    a{2} a{1} a{0}  0
+
+    If a{2} is 0 we already have the result:
+        a{1} a{0} 0
+
+    If a{2} is 1 we need to xor with the modulus:
+        1 a{1} a{0} 0 x 1 0 1 1 = a{1} (a{0} x 1) 1
+
+An important thing to see here is that if a{2} is 0, we can get the
+same result by xoring with all 0 instead of the modulus. For this
+reason we can rewrite the modulus as this:
+
+    Modulus: a{2} 0 a{2} a{2}
+
+This means that the modulus will be 0 0 0 0 is a{2} is 0, so the value
+won't change, and it will be 1 0 1 1 if a{2} is 1, giving the correct
+result. So, the computation is simply:
+
+    Original value:      a{2} a{1} a{0}
+    Shift 1 bit:    a{2} a{1} a{0}  0
+    Apply modulus:  a{1} (a{0} x a{2}) a{2}
+
+We can compute all Ax using this method. We'll get this:
+
+    A0 = a{2} a{1} a{0}
+    A1 = a{1} (a{0} x a{2}) a{2}
+    A2 = (a{0} x a{2}) (a{1} x a{2}) a{1}
+
+Once we have all terms, we xor the ones corresponding to the bits set
+to 1 in B. In out example this will be A0 and A2:
+
+   Result: (a{2} x a{0} x a{2}) (a{1} x a{1} x a{2}) (a{0} x a{1})
+
+We can easily see that we can remove some redundant factors:
+
+   Result: a{0} a{2} (a{0} x a{1})
+
+This way we have come up with a simply set of equations to compute the
+multiplication of any number by 5. If A is 1 1 1 (= 7), the result
+must be 1 1 0 (= 6) using the equations, as we expected. If we try
+another numbe for A, like 0 1 0 (= 2), the result must be 0 0 1 (= 1).
+
+This seems a really fast way to compute the multiplication without
+using any table lookup. The problem is that this is only valid for
+B = 5. For other values of B another set of equations will be found.
+To solve this problem we can pregenerate the equations for all
+possible values of B. Since the Galois Field we use is small, this is
+feasible.
+
+One thing to be aware of is that, in general, two equations for
+different bits of the same B can share common subexpressions. This
+gives space for further optimizations to reduce the total number of
+xors used in the final equations for a given B. However this is not
+easy to find, since finding the smallest number of xors that give the
+correct result is an NP-Problem. For EC an exhaustive search has been
+made to find the best combinations for each possible value.
+
+
+Implementation
+--------------
+
+All this seems great from the hardware point of view, but implementing
+this using normal processor instructions is not so easy because we
+would need a lot of shifts, ands, xors and ors to move the bits of
+each number to the correct position to compute the equation and then
+another shift to put each bit back to its final place.
+
+For example, to implement the functions to multiply by 5, we would
+need something like this:
+
+    Bit 2:  T2 = (A & 1) << 2
+    Bit 1:  T1 = (A & 4) >> 1
+    Bit 0:  T0 = ((A >> 1) x A) & 1
+    Result: T2 + T1 + T0
+
+This doesn't look good. So here we make a change in the way we get
+and process the data: instead of reading full numbers into variables
+and operate with them afterwards, we use a single independent variable
+for each bit of the number.
+
+Assume that we can read and write independent bits from memory (later
+we'll see how to solve this problem when this is not possible). In
+this case, the code would look something like this:
+
+    Bit 2:        T2 = Mem[2]
+    Bit 1:        T1 = Mem[1]
+    Bit 0:        T0 = Mem[0]
+    Computation:  T1 ^= T0
+    Store result: Mem[2] = T0
+                  Mem[1] = T2
+                  Mem[0] = T1
+
+Note that in this case we handle the final reordering of bits simply
+by storing the right variable to the right place, without any shifts,
+ands nor ors. In fact we only have memory loads, memory stores and
+xors. Note also that we can do all the computations directly using the
+variables themselves, without additional storage. This true for most
+of the values, but in some cases an additional one or two temporal
+variables will be needed to store intermediate results.
+
+The drawback of this approach is that additions, that are simply a
+xor of two numbers will need as many xors as bits are in each number.
+
+
+SIMD optimization
+-----------------
+
+So we have a good way to compute the multiplications, but even using
+this we'll need several operations for each byte of the original data.
+We can improve this by doing multiple multiplications using the same
+set of instructions.
+
+With the approach taken in the implementation section, we can see that
+in fact it's really easy to add SIMD support to this method. We only
+need to store in each variable one bit from multiple numbers. For
+example, when we load T2 from memory, instead of reading the bit 2 of
+the first number, we can read the bit 2 of the first, second, third,
+fourth, ... numbers. The same can be done when loading T1 and T0.
+
+Obviously this needs to have a special encoding of the numbers into
+memory to be able to do that in a single operation, but since we can
+choose whatever encoding we want for EC, we have chosen to have
+exactly that. We interpret the original data as a stream of bits, and
+we split it into subsequences of length L, each containing one bit of
+a number. Every S subsequences form a set of numbers of S bits that
+are encoded and decoded as a single group. This repeats for any
+remaining data.
+
+For example, in a simple case with L = 8 and S = 3, the original data
+would contain something like this (interpreted as a sequence of bits,
+offsets are also bit-based):
+
+    Offset  0: a{0} b{0} c{0} d{0} e{0} f{0} g{0} h{0}
+    Offset  8: a{1} b{1} c{1} d{1} e{1} f{1} g{1} h{1}
+    Offset 16: a{2} b{2} c{2} d{2} e{2} f{2} g{2} h{2}
+    Offset 24: i{0} j{0} k{0} l{0} m{0} n{0} o{0} p{0}
+    Offset 32: i{1} j{1} k{1} l{1} m{1} n{1} o{1} p{1}
+    Offset 40: i{2} j{2} k{2} l{2} m{2} n{2} o{2} p{2}
+
+Note: If the input file is not a multiple of S * L, 0-padding is done.
+
+Here we have 16 numbers encoded, from A to P. This way we can easily
+see that reading the first byte of the file will read all bits 0 of
+number A, B, C, D, E, F, G and H. The same happens with bits 1 and 2
+when we read the second and third bytes respectively. Using this
+encoding and the implementation described above, we can see that the
+same set of instructions will be computing the multiplication of 8
+numbers at the same time.
+
+This can be further improved if we use L = 64 with 64 bits variables
+on 64-bits processor. It's even faster if we use L = 128 using SSE
+registers or L = 256 using AVX registers on Intel processors.
+
+Currently EC uses L = 512 and S = 8. This means that numbers are
+packed in blocks of 512 bytes and gives space for even bigger
+processor registers up to 512 bits.
+
+
+Conclusions
+-----------
+
+This method requires a single variable/processor register for each
+bit. This can be challenging if we want to avoid additional memory
+accesses, even if we use modern processors that have many registers.
+However, the implementation we chose for the Vandermonde Matrix
+doesn't require temporary storage, so we don't need a full set of 8
+new registers (one for each bit) to store partial computations.
+Additionally, the computation of the multiplications requires, at
+most, 2 extra registers, but this is afordable.
+
+Xors are a really fast operation in modern processors. Intel CPU's
+can dispatch up to 3 xors per CPU cycle if there are no dependencies
+with ongoing previous instructions. Worst case is 1 xor per cycle. So,
+in some configurations, this method could be very near to the memory
+speed.
+
+Another interesting thing of this method is that all data it needs to
+operate is packed in small sequential blocks of memory, meaning that
+it can take advantage of the faster internal CPU caches.
+
+
+Results
+-------
+
+For the particular case of 8 bits, EC can compute each multiplication
+using 12.8 xors on average (without counting 0 and 1 that do not
+require any xor). Some numbers require less, like 2 that only requires
+3 xors.
+
+Having all this, we can check some numbers to see the performance of
+this method.
+
+Maybe the most interesting thing is the average number of xors needed
+to encode a single byte of data. To compute this we'll need to define
+some variables:
+
+  * K: Number of data fragments  
+  * R: Number of redundancy fragments  
+  * N: K + R  
+  * B: Number of bits per number  
+  * A: Average number of xors per number  
+  * Z: Bits per CPU register (can be up to 256 for AVX registers)  
+  * X: Average number of xors per CPU cycle  
+  * L: Average cycles per load  
+  * S: Average cycles per store  
+  * G: Core speed in Hz  
+
+_Total number of bytes processed for a single matrix multiplication_:
+
+  * __Read__:    K * B * Z / 8  
+  * __Written__: N * B * Z / 8  
+
+_Total number of memory accesses_:
+
+  * __Loads__:  K * B * N  
+  * __Stores__: B * N  
+
+>  We need to read the same K * B * Z bits, in registers of Z bits, N
+>  times, one for each row of the matrix. However the last N - 1 reads
+>  could be made from the internal CPU caches if conditions are good.
+
+_Total number of operations_:
+
+  * __Additions__:       (K - 1) * N  
+  * __Multiplications__: K * N  
+
+__Total number of xors__: B * (K - 1) * N + A * K * N =
+                          N * ((A + B) * K - B)
+
+__Xors per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z)
+
+__CPU cycles per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z * X) +
+                         8 * L * N / Z +      (loads)
+                         8 * S * N / (K * Z)  (stores)
+
+__Bytes per second__: G / {CPU cycles per byte}
+
+Some xors per byte numbers for specific configurations (B=8):
+
+              Z=64  Z=128  Z=256
+    K=2/R=1   0.79  0.39   0.20
+    K=4/R=2   1.76  0.88   0.44
+    K=4/R=3   2.06  1.03   0.51
+    K=8/R=3   3.40  1.70   0.85
+    K=8/R=4   3.71  1.86   0.93
+    K=16/R=4  6.34  3.17   1.59
+
+
+
+[1]: https://en.wikipedia.org/wiki/Erasure_code
+[2]: https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction
+[3]: https://en.wikipedia.org/wiki/Systematic_code
+[4]: https://en.wikipedia.org/wiki/Identity_matrix
+[5]: https://en.wikipedia.org/wiki/Linear_independence
+[6]: https://en.wikipedia.org/wiki/Vandermonde_matrix
+[7]: https://en.wikipedia.org/wiki/Finite_field
+[8]: https://en.wikipedia.org/wiki/Finite_field_arithmetic#Implementation_tricks
diff --git a/doc/developer-guide/fuse-interrupt.md b/doc/developer-guide/fuse-interrupt.md
new file mode 100644
index 00000000000..ec991b81ec5
--- /dev/null
+++ b/doc/developer-guide/fuse-interrupt.md
@@ -0,0 +1,211 @@
+# Fuse interrupt handling
+
+## Conventions followed
+
+- *FUSE* refers to the "wire protocol" between kernel and userspace and
+  related specifications.
+- *fuse* refers to the kernel subsystem and also to the GlusterFs translator.
+
+## FUSE interrupt handling spec
+
+The [Linux kernel FUSE documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fuse.txt?h=v4.18#n148)
+desrcibes how interrupt handling happens in fuse.
+
+## Interrupt handling in the fuse translator
+
+### Declarations
+
+This document describes the internal API in the fuse translator with which
+interrupt can be handled.
+
+The API being internal (to be used only in fuse-bridge.c; the functions are
+not exported to a header file).
+
+```
+enum fuse_interrupt_state {
+    /* ... */
+    INTERRUPT_SQUELCHED,
+    INTERRUPT_HANDLED,
+    /* ... */
+};
+typedef enum fuse_interrupt_state fuse_interrupt_state_t;
+struct fuse_interrupt_record;
+typedef struct fuse_interrupt_record fuse_interrupt_record_t;
+typedef void (*fuse_interrupt_handler_t)(xlator_t *this,
+                                         fuse_interrupt_record_t *);
+struct fuse_interrupt_record {
+    fuse_in_header_t fuse_in_header;
+    void *data;
+    /*
+       ...
+     */
+};
+
+fuse_interrupt_record_t *
+fuse_interrupt_record_new(fuse_in_header_t *finh,
+                          fuse_interrupt_handler_t handler);
+
+void
+fuse_interrupt_record_insert(xlator_t *this, fuse_interrupt_record_t *fir);
+
+gf_boolean_t
+fuse_interrupt_finish_fop(call_frame_t *frame, xlator_t *this,
+                          gf_boolean_t sync, void **datap);
+
+void
+fuse_interrupt_finish_interrupt(xlator_t *this, fuse_interrupt_record_t *fir,
+                                fuse_interrupt_state_t intstat,
+                                gf_boolean_t sync, void **datap);
+```
+
+The code demonstrates the usage of the API through `fuse_flush()`. (It's a
+dummy implementation only for demonstration purposes.) Flush is chosen
+because a `FLUSH` interrupt is easy to trigger (see
+*tests/features/interrupt.t*). Interrupt handling for flush is switched on
+by `--fuse-flush-handle-interrupt` (a hidden glusterfs command line flag).
+The implementation of flush interrupt is contained in the
+`fuse_flush_interrupt_handler()` function and blocks guarded by the
+
+```
+if (priv->flush_handle_interrupt) { ...
+```
+
+conditional (where `priv` is a `*fuse_private_t`).
+
+### Overview
+
+"Regular" fuse fops and interrupt handlers interact via a list containing
+interrupt records.
+
+If a fop wishes to have its interrupts handled, it needs to set up an
+interrupt record and insert it into the list; also when it's to finish
+(ie. in its "cbk" stage) it needs to delete the record from the list.
+
+If no interrupt happens, basically that's all to it - a list insertion
+and deletion.
+
+However, if an interrupt comes for the fop, the interrupt FUSE request
+will carry the data identifying an ongoing fop (that is, its `unique`),
+and based on that, the interrupt record will be looked up in the list, and
+the specific interrupt handler (a member of the interrupt record) will be
+called.
+
+Usually the fop needs to share some data with the interrupt handler to
+enable it to perform its task (also shared via the interrupt record).
+The interrupt API offers two approaches to manage shared data:
+- _Async or reference-counting strategy_: from the point on when the interrupt
+  record is inserted to the list, it's owned jointly by the regular fop and
+  the prospective interrupt handler. Both of them need to check before they
+  return if the other is still holding a reference; if not, then they are
+  responsible for reclaiming the shared data.
+- _Sync or borrow strategy_: the interrupt handler is considered a borrower
+  of the shared data. The interrupt handler should not reclaim the shared
+  data. The fop will wait for the interrupt handler to finish (ie., the borrow
+  to be returned), then it has to reclaim the shared data.
+
+The user of the interrupt API need to call the following functions to
+instrument this control flow:
+- `fuse_interrupt_record_insert()` in the fop to insert the interrupt record to
+   the list;
+- `fuse_interrupt_finish_fop()`in the fop (cbk) and
+- `fuse_interrupt_finish_interrupt()`in the interrupt handler
+
+to perform needed synchronization at the end their tenure. The data management
+strategies are implemented by the `fuse_interrupt_finish_*()` functions (which
+have an argument to specify which strategy to use); these routines take care
+of freeing the interrupt record itself, while the reclamation of the shared data
+is left to the API user.
+
+### Usage
+
+A given FUSE fop can be enabled to handle interrupts via the following
+steps:
+
+- Define a handler function (of type `fuse_interrupt_handler_t`).
+  It should implement the interrupt handling logic and in the end
+  call (directly or as async callback) `fuse_interrupt_finish_interrupt()`.
+  The `intstat` argument to `fuse_interrupt_finish_interrupt` should be
+  either `INTERRUPT_SQUELCHED` or `INTERRUPT_HANDLED`.
+    - `INTERRUPT_SQUELCHED` means that the interrupt could not be delivered
+      and the fop is going on uninterrupted.
+    - `INTERRUPT_HANDLED` means that the interrupt was actually handled. In
+      this case the fop will be answered from interrupt context with errno
+      `EINTR` (that is, the fop should not send a response to the kernel).
+
+  (the enum `fuse_interrupt_state` includes further members, which are reserved
+  for internal use).
+
+  We return to the `sync` and `datap` arguments later.
+- In the `fuse_<FOP>` function create an interrupt record using
+  `fuse_interrupt_record_new()`, passing the incoming `fuse_in_header` and
+  the above handler function to it.
+    - Arbitrary further data can be referred to via the `data` member of the
+      interrupt record that is to be passed on from fop context to
+      interrupt context.
+- When it's set up, pass the interrupt record to
+  `fuse_interrupt_record_insert()`.
+- In `fuse_<FOP>_cbk` call `fuse_interrupt_finish_fop()`.
+    - `fuse_interrupt_finish_fop()` returns a Boolean according to whether the
+      interrupt was handled. If it was, then the FUSE request is already
+      answered and the stack gets destroyed in `fuse_interrupt_finish_fop` so
+      `fuse_<FOP>_cbk()` can just return (zero). Otherwise follow the standard
+      cbk logic (answer the FUSE request and destroy the stack -- these are
+      typically accomplished by `fuse_err_cbk()`).
+- The last two argument of `fuse_interrupt_finish_fop()` and
+  `fuse_interrupt_finish_interrupt()` are `gf_boolean_t sync` and
+  `void **datap`.
+    - `sync` represents the strategy for freeing the interrupt record. The
+      interrupt handler and the fop handler are in race to get at the interrupt
+      record first (interrupt handler for purposes of doing the interrupt
+      handling, fop handler for purposes of deactivating the interrupt record
+      upon completion of the fop handling).
+        - If `sync` is true, then the fop handler will wait for the interrupt
+          handler to finish and it takes care of freeing.
+        - If `sync` is false, the loser of the above race will perform freeing.
+
+      Freeing is done within the respective interrupt finish routines, except
+      for the `data` field of the interrupt record; with respect to that, see
+      the discussion of the `datap` parameter below. The strategy has to be
+      consensual, that is, `fuse_interrupt_finish_fop()` and
+      `fuse_interrupt_finish_interrupt()` must pass the same value for `sync`.
+      If dismantling the resources associated with the interrupt record is
+      simple, `sync = _gf_false` is the suggested choice; `sync = _gf_true` can
+      be useful in the opposite case, when dismantling those resources would
+      be inconvenient to implement in two places or to enact in non-fop context.
+    - If `datap` is `NULL`, the `data` member of the interrupt record will be
+      freed within the interrupt finish routine.  If it points to a valid
+      `void *` pointer, and if caller is doing the cleanup (see `sync` above),
+      then that pointer will be directed to the `data` member of the interrupt
+      record and it's up to the caller what it's doing with it.
+        - If `sync` is true, interrupt handler can use `datap = NULL`, and
+          fop handler will have `datap` point to a valid pointer.
+        - If `sync` is false, and handlers pass a pointer to a pointer for
+          `datap`, they should check if the pointed pointer is NULL before
+          attempting to deal with the data.
+
+### FUSE answer for the interrupted fop
+
+The kernel acknowledges a successful interruption for a given FUSE request
+if the filesystem daemon answers it with errno EINTR; upon that, the syscall
+which induced the request will be abruptly terminated with an interrupt, rather
+than returning a value.
+
+In glusterfs, this can be arranged in two ways.
+
+- If the interrupt handler wins the race for the interrupt record, ie.
+  `fuse_interrupt_finish_fop()` returns true to `fuse_<FOP>_cbk()`, then, as
+  said above, `fuse_<FOP>_cbk()` does not need to answer the FUSE request.
+  That's because then the interrupt handler will take care about answering
+  it (with errno EINTR).
+- If `fuse_interrupt_finish_fop()` returns false to `fuse_<FOP>_cbk()`, then
+  this return value does not inform the fop handler whether there was an interrupt
+  or not. This return value occurs both when fop handler won the race for the
+  interrupt record against the interrupt handler, and when there was no interrupt
+  at all.
+
+  However, the internal logic of the fop handler might detect from other
+  circumstances that an interrupt was delivered. For example, the fop handler
+  might be sleeping, waiting for some data to arrive, so that a premature
+  wakeup (with no data present) occurs if the interrupt handler intervenes. In
+  such cases it's the responsibility of the fop handler to reply the FUSE
+  request with errro EINTR.
diff --git a/doc/developer-guide/identifying-resource-leaks.md b/doc/developer-guide/identifying-resource-leaks.md
new file mode 100644
index 00000000000..950cae79b0a
--- /dev/null
+++ b/doc/developer-guide/identifying-resource-leaks.md
@@ -0,0 +1,200 @@
+# Identifying Resource Leaks
+
+Like most other pieces of software, GlusterFS is not perfect in how it manages
+its resources like memory, threads and the like. Gluster developers try hard to
+prevent leaking resources but releasing and unallocating the used structures.
+Unfortunately every now and then some resource leaks are unintentionally added.
+
+This document tries to explain a few helpful tricks to identify resource leaks
+so that they can be addressed.
+
+
+## Debug Builds
+
+There are certain techniques used in GlusterFS that make it difficult to use
+tools like Valgrind for memory leak detection. There are some build options
+that make it more practical to use Valgrind and other tools. When running
+Valgrind, it is important to have GlusterFS builds that contain the
+debuginfo/symbols. Some distributions (try to) strip the debuginfo to get
+smaller executables. Fedora and RHEL based distributions have sub-packages
+called ...-debuginfo that need to be installed for symbol resolving.
+
+
+### Memory Pools
+
+By using memory pools, there are no allocation/freeing of single structures
+needed. This improves performance, but also makes it impossible to track the
+allocation and freeing of srtuctures.
+
+It is possible to disable the use of memory pools, and use standard `malloc()`
+and `free()` functions provided by the C library. Valgrind is then able to
+track the allocated areas and verify if they have been free'd. In order to
+disable memory pools, the Gluster sources needs to be configured with the
+`--enable-debug` option:
+
+```shell
+./configure --enable-debug
+```
+
+When building RPMs, the `.spec` handles the `--with=debug` option too:
+
+```shell
+make dist
+rpmbuild -ta --with=debug glusterfs-....tar.gz
+```
+
+### Dynamically Loaded xlators
+
+Valgrind tracks the call chain of functions that do memory allocations. The
+addresses of the functions are stored and before Valgrind exits the addresses
+are resolved into human readable function names and offsets (line numbers in
+source files). Because Gluster loads xlators dynamically, and unloads then
+before exiting, Valgrind is not able to resolve the function addresses into
+symbols anymore. Whenever this happend, Valgrind shows `???` in the output,
+like
+
+```
+  ==25170== 344 bytes in 1 blocks are definitely lost in loss record 233 of 324
+  ==25170==    at 0x4C29975: calloc (vg_replace_malloc.c:711)
+  ==25170==    by 0x52C7C0B: __gf_calloc (mem-pool.c:117)
+  ==25170==    by 0x12B0638A: ???
+  ==25170==    by 0x528FCE6: __xlator_init (xlator.c:472)
+  ==25170==    by 0x528FE16: xlator_init (xlator.c:498)
+  ...
+```
+
+These `???` can be prevented by not calling `dlclose()` for unloading the
+xlator. This will cause a small leak of the handle that was returned with
+`dlopen()`, but for improved debugging this can be acceptible. For this and
+other Valgrind features, a `--enable-valgrind` option is available to
+`./configure`. When GlusterFS is built with this option, Valgrind will be able
+to resolve the symbol names of the functions that do memory allocations inside
+xlators.
+
+```shell
+./configure --enable-valgrind
+```
+
+When building RPMs, the `.spec` handles the `--with=valgrind` option too:
+
+```shell
+make dist
+rpmbuild -ta --with=valgrind glusterfs-....tar.gz
+```
+
+## Running Valgrind against a single xlator
+
+Debugging a single xlator is not trivial. But there are some tools to make it
+easier. The `sink` xlator does not do any memory allocations itself, but
+contains just enough functionality to mount a volume with only the `sink`
+xlator. There is a little gfapi application under `tests/basic/gfapi/` in the
+GlusterFS sources that can be used to run only gfapi and the core GlusterFS
+infrastructure with the `sink` xlator. By extending the `.vol` file to load
+more xlators, each xlator can be debugged pretty much separately (as long as
+the xlators have no dependencies on each other). A basic Valgrind run with the
+suitable configure options looks like this:
+
+```shell
+./autogen.sh
+./configure --enable-debug --enable-valgrind
+make && make install
+cd tests/basic/gfapi/
+make gfapi-load-volfile
+valgrind ./gfapi-load-volfile sink.vol
+```
+
+Combined with other very useful options to Valgrind, the following execution
+shows many more useful details:
+
+```shell
+valgrind \
+        --fullpath-after= --leak-check=full --show-leak-kinds=all \
+        ./gfapi-load-volfile sink.vol
+```
+
+Note that the `--fullpath-after=` option is left empty, this makes Valgrind
+print the full path and filename that contains the functions:
+
+```
+==2450== 80 bytes in 1 blocks are definitely lost in loss record 8 of 60
+==2450==    at 0x4C29975: calloc (/builddir/build/BUILD/valgrind-3.11.0/coregrind/m_replacemalloc/vg_replace_malloc.c:711)
+==2450==    by 0x52C6F73: __gf_calloc (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/mem-pool.c:117)
+==2450==    by 0x12F10CDA: init (/usr/src/debug/glusterfs-3.11dev/xlators/meta/src/meta.c:231)
+==2450==    by 0x528EFD5: __xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:472)
+==2450==    by 0x528F105: xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:498)
+==2450==    by 0x52D9D8B: glusterfs_graph_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/graph.c:321)
+...
+```
+
+In the above example, the `init` function in `xlators/meta/src/meta.c` does a
+memory allocation on line 231. This memory is never free'd again, and hence
+Valgrind logs this call stack. When looking in the code, it seems that the
+allocation of `priv` is assigned to the `this->private` member of the
+`xlator_t` structure. Because the allocation is done in `init()`, free'ing is
+expected to happen in `fini()`. Both functions are shown below, with the
+inclusion of the empty `fini()`:
+
+
+```
+226 int
+227 init (xlator_t *this)
+228 {
+229         meta_priv_t *priv = NULL;
+230 
+231         priv = GF_CALLOC (sizeof(*priv), 1, gf_meta_mt_priv_t);
+232         if (!priv)
+233                 return -1;
+234 
+235         GF_OPTION_INIT ("meta-dir-name", priv->meta_dir_name, str, out);
+236 
+237         this->private = priv;
+238 out:
+239         return 0;
+240 }
+241 
+242 
+243 int
+244 fini (xlator_t *this)
+245 {
+246         return 0;
+247 }
+```
+
+In this case, the resource leak can be addressed by adding a single line to the
+`fini()` function:
+
+```
+243 int
+244 fini (xlator_t *this)
+245 {
+246         GF_FREE (this->private);
+247         return 0;
+248 }
+```
+
+Running the same Valgrind command and comparing the output will show that the
+memory leak in `xlators/meta/src/meta.c:init` is not reported anymore.
+
+### Running DRD, the Valgrind thread error detector
+
+When configuring GlusterFS with:
+
+```shell
+./configure --enable-valgrind
+```
+
+the default Valgrind tool (Memcheck) is enabled. But it's also possble to select
+one of Memcheck or DRD by using:
+
+```shell
+./configure --enable-valgrind=memcheck
+```
+
+or:
+
+```shell
+./configure --enable-valgrind=drd
+```
+
+respectively. When using DRD, it's recommended to consult
+https://valgrind.org/docs/manual/drd-manual.html before running.
diff --git a/doc/developer-guide/logging-guidelines.md b/doc/developer-guide/logging-guidelines.md
index 58adf944b67..0e6b2588535 100644
--- a/doc/developer-guide/logging-guidelines.md
+++ b/doc/developer-guide/logging-guidelines.md
@@ -62,7 +62,7 @@ There are 2 interfaces provided to log messages,
       headers (like the time stamp, dom, errnum etc.). The primary users of
       the above interfaces are, when printing the final graph, or printing
       the configuration when a process is about dump core or abort, or
-      printing the backtrace when a process recieves a critical signal
+      printing the backtrace when a process receives a critical signal
       - These interfaces should not be used outside the scope of the users
       above, unless you know what you are doing
 
diff --git a/doc/developer-guide/network_compression.md b/doc/developer-guide/network_compression.md
index 7327591ef63..1222a765276 100644
--- a/doc/developer-guide/network_compression.md
+++ b/doc/developer-guide/network_compression.md
@@ -1,9 +1,9 @@
-#On-Wire Compression + Decompression
+# On-Wire Compression + Decompression
 
 The 'compression translator' compresses and decompresses data in-flight
 between client and bricks.
 
-###Working
+### Working
 When a writev call occurs, the client compresses the data before sending it to
 brick. On the brick, compressed data is decompressed. Similarly, when a readv
 call occurs, the brick compresses the data before sending it to client. On the
@@ -19,7 +19,7 @@ During normal operation, this is the format of data sent over wire:
 The trailer contains the CRC32 checksum and length of original uncompressed
 data. This is used for validation.
 
-###Usage
+### Usage
 
 Turning on compression xlator:
 
@@ -27,7 +27,7 @@ Turning on compression xlator:
 gluster volume set <vol_name> network.compression on
 ~~~
 
-###Configurable parameters (optional)
+### Configurable parameters (optional)
 
 **Compression level**
 ~~~
@@ -35,10 +35,10 @@ gluster volume set <vol_name> network.compression.compression-level 8
 ~~~
 
 ~~~
-0  : no compression
-1  : best speed
-9  : best compression
--1 : default compression
+ 0  : no compression
+ 1  : best speed
+ 9  : best compression
+-1  : default compression
 ~~~
 
 **Minimum file size**
@@ -55,7 +55,7 @@ Other less frequently used parameters include `network.compression.mem-level`
 and `network.compression.window-size`. More details can about these options
 can be found by running `gluster volume set help` command.
 
-###Known Issues and Limitations
+### Known Issues and Limitations
 
 * Compression translator cannot work with striped volumes.
 * Mount point hangs when writing a file with write-behind xlator turned on. To
@@ -65,7 +65,7 @@ set`performance.strict-write-ordering` to on.
 distribute volumes. This limitation is caused by AFR not being able to
 propagate xdata. This issue has been fixed in glusterfs versions > 3.5
 
-###TODO
+### TODO
 Although zlib offers high compression ratio, it is very slow. We can make the
 translator pluggable to add support for other compression methods such as
 [lz4 compression](https://code.google.com/p/lz4/)
diff --git a/doc/developer-guide/options-to-contribute.md b/doc/developer-guide/options-to-contribute.md
new file mode 100644
index 00000000000..3f0d84e7645
--- /dev/null
+++ b/doc/developer-guide/options-to-contribute.md
@@ -0,0 +1,212 @@
+# A guide for contributors
+
+While you have gone through 'how to contribute' guides, if you are
+not sure what to work on, but really want to help the project, you
+have now landed on the right document :-)
+
+### Basic
+
+Instead of planning to fix **all** the below issues in one patch,
+we recommend you to have a a constant, continuous flow of improvements
+for the project. We recommend you to pick 1 file (or just few files) at
+a time to address below issues.
+Pick any `.c` (or `.h`) file, and you can send a patch which fixes **any**
+of the below themes. Ideally, fix all such occurrences in the file, even
+though, the reviewers would review even a single line change patch
+from you.
+
+1. Check for variable definitions, and if there is an array definition,
+which is very large at the top of the function, see if you can re-scope
+the variable to relevant sections (if it helps).
+
+Most of the time, some of these arrays may be used for 'error' handling,
+and it is possible to use them only in that scope.
+
+Reference: https://review.gluster.org/20846/
+
+
+2. Check for complete string initialization at the beginning of a function.
+Ideally, there is no reason to initialize a string. Fix it across the file.
+
+Example:
+
+`char new_path_name[PATH_MAX] = {0};` to `char new_path_name[PATH_MAX];`
+
+
+3. Change `calloc()` to `malloc()` wherever it makes sense.
+
+In a case of allocating a structures, where you expect certain (or most of)
+variables to be 0 (or NULL), it makes sense to use calloc(). But otherwise,
+there is an extra cost to `memset()` the whole object after allocating it.
+While it is not a significant improvement in performance, code which gets
+hit 1000s of times in a second, it would add some value.
+
+Reference: https://review.gluster.org/20878/
+
+
+4. You can consider using `snprintf()`, instead of `strncpy()` while dealing
+with strings.
+
+strncpy() won't null terminate if the dest buffer isn't big enough; snprintf()
+does. While most of the string operations in the code is on array, and larger
+size than required, strncpy() does an extra copy of 0s at the end of
+string till the size of the array. It makes sense to use `snprintf()`,
+which doesn't suffer from that behavior.
+
+Also check the return value from snprintf() for buffer overflow and handle
+accordingly
+
+Reference: https://review.gluster.org/20925/
+
+
+5. Now, pick a `.h` file, and see if a structure is very large, and see
+if re-aligning them as per [coding-standard](./coding-standard.md) gives any size benefit,
+if yes, go ahead and change it. Make sure you check all the structures
+in the file for similar pattern.
+
+Reference: [Check this section](https://github.com/gluster/glusterfs/blob/master/doc/developer-guide/coding-standard.md#structure-members-should-be-aligned-based-on-the-padding-requirements
+
+
+### If you are up for more :-)
+
+Good progress! Glad you are interested to know more. We are surely interested
+in next level of contributions from you!
+
+#### Coverity
+
+Visit [Coverity Dashboard](https://scan.coverity.com/projects/gluster-glusterfs?tab=overview).
+
+Now, if the number of defect is not 0, you have an opportunity to contribute.
+
+You get all the detail on why the particular defect is mentioned there, and
+most probable hint on how to fix it. Do it!
+
+Reference: https://review.gluster.org/21394/
+
+Use the same reference Id (789278) as the patch, so we can capture it is in
+single bugzilla.
+
+#### Clang-Scan
+
+Clang-Scan is a tool which scans the .c files and reports the possible issues,
+similar to coverity, but a different tool. Over the years we have seen, they
+both report very different set of issues, and hence there is a value in fixing it.
+
+GlusterFS project gets tested with clang-scan job every night, and the report is
+posted in the [job details page](https://build.gluster.org/job/clang-scan/lastCompletedBuild/clangScanBuildBugs/).
+As long as the number is not 0 in the report here, you have an opportunity to
+contribute! Similar to coverity dashboard, click on 'Details' to find out the
+reason behind that report, and send a patch.
+
+Reference: https://review.gluster.org/21025
+
+Again, you can use reference Id (1622665) for these patches!
+
+
+### I am good with programming, I would like to do more than above!
+
+#### Locked regions / Critical sections
+
+In the file you open, see if the lock is taken only to increment or decrement
+a flag, counter etc. If yes, then recommend you to convert it to ATOMIC locks.
+It is simple activity, but, if you know programing, you would know the benefit
+here.
+
+NOTE: There may not always a possibility to do this! You may have to check
+with developers first before going ahead.
+
+Reference: https://review.gluster.org/21221/
+
+
+#### ASan (address sanitizer)
+
+[The job](https://build.gluster.org/job/asan/) runs regression with asan builds,
+and you can also run glusterfs with asan on your workload to identify the leaks.
+If there are any leaks reported, feel free to check it, and send us patch.
+
+You can also run `valgrind` and let us know what it reports.
+
+Reference: https://review.gluster.org/21397
+
+
+#### Porting to different architecture
+
+This is something which we are not focusing right now, happy to collaborate!
+
+Reference: https://review.gluster.org/21276
+
+
+#### Fix 'TODO/FIXME' in codebase
+
+There are few cases of pending features, or pending validations, which are
+pending from sometime. You can pick them in the given file, and choose to
+fix it.
+
+
+### I don't know C, but I am interested to contribute in some way!
+
+You are most welcome! Our community is open for your contribution! First thing
+which comes to our mind is **documentation**. Next is, **testing** or validation.
+
+If you have some hardware, and want to run some performance comparisons with
+different version, or options, and help us to tune better is also a great help.
+
+
+#### Documentation
+
+1. We have some documentation in [glusterfs repo](../), go through these, and
+see if you can help us to keep up-to-date.
+
+2. The https://docs.gluster.org is powered by https://github.com/gluster/glusterdocs
+repo. You can check out the repo, and help in keeping that up-to-date.
+
+3. [Our website](https://gluster.org) is maintained by https://github.com/gluster/glusterweb
+repo. Help us to keep this up-to-date, and add content there.
+
+4. Write blogs about Gluster, and your experience, and make world know little
+more about Gluster, and your use-case, and how it helped to solve the problem.
+
+
+#### Testing
+
+1. There is a regression test suite in glusterfs, which runs with every patch, and is
+triggered by just running `./run-tests.sh` from the root of the project repo.
+
+You can add more test case to match your use-case, and send it as a patch, so you
+can make sure all future patches in glusterfs would keep your usecase intact.
+
+2. [Glusto-Tests](https://github.com/gluster/glusto-tests): This is another testing
+framework written for gluster, and makes use of clustered setup to test different
+use-cases, and helps to validate many bugs.
+
+
+#### Ansible
+
+Gluster Organization has rich set of ansible roles, which are actively maintained.
+Feel free to check them out here - https://github.com/gluster/gluster-ansible
+
+
+#### Monitoring
+
+We have prometheus repo, and are actively working on adding more metrics. Add what
+you need @ https://github.com/gluster/gluster-prometheus
+
+
+#### Health-Report
+
+This is a project, where at any given point in time, you want to run some set of
+commands locally, and get an output to analyze the status, it can be added.
+Contribute @ https://github.com/gluster/gluster-health-report
+
+
+### All these C/bash/python is old-school, I want something in containers.
+
+We have something for you too :-)
+
+Please visit our https://github.com/gluster/gcs repo for checking how you can help,
+and how gluster can help you in container world.
+
+
+### Note
+
+For any queries, best way is to contact us through mailing-list, <mailto:gluster-devel@gluster.org>
diff --git a/doc/developer-guide/rpc-for-glusterfs.new-versions.md b/doc/developer-guide/rpc-for-glusterfs.new-versions.md
new file mode 100644
index 00000000000..e3da5efa4a2
--- /dev/null
+++ b/doc/developer-guide/rpc-for-glusterfs.new-versions.md
@@ -0,0 +1,32 @@
+# GlusterFS RPC program versions
+
+## Compatibility
+
+RPC layer of glusterfs is implemented with possible changes over the protocol layers in mind. If there are any changes in the FOPs from what is assumed to be client side, and whats in serverside, they are to be added as a separate program table.
+
+### Program tables and Versions
+
+A given RPC program has a specific Task, and Version along with actors belonging to the program. In any of the programs, if a new actor is added, it is very important to define one more program with different version, and then keep both, if both are supported. Or else, it is important to handle the 'handshake' properly.
+
+#### Server details
+
+More info on RPC program is at `rpc/rpc-lib/src/rpcsvc.h` and check for structure `rpcsvc_actor_t` and `struct rpcsvc_program`. For usage, check `xlators/protocol/server/src/server-rpc-fops.c`
+
+#### Client details
+
+For details on client structures check `rpc/rpc-lib/src/rpc-clnt.h` for `rpc_clnt_procedure_t` and `rpc_clnt_program_t`. For usage, check `xlators/protocol/client/src/client-rpc-fops.c`
+
+## Protocol
+
+A protocol is what is agreed between two parties. In glusterfs, a RPC protocol is defined as .x file, which then gets converted to .c/.h file using `rpcgen`. There are different protocols defined for communication between `xlators/mgmt/glusterd <==> glusterfsd`, `gluster CLI <==> glusterd`, and `client-protocol <==> server-protocol`
+
+Once a protocol is defined and a release is made with that protocol, make sure no one changes it. Any edits to a given structure there should be a new version of the structure, and also it should get used in new actor, and thus new program version.
+
+## Server and Client Handshake
+
+When a client succeeds to establish a connect (it can be any transport, socket, ib-verbs or unix), client sends a dump (GF_DUMP_DUMP) request to server, which will respond back with all the supported versions of the server RPC (the supported programs which are registered with `rpcsvc_program_register()`).
+
+A client which expects certain programs to be present in server, it should be taking care of looking for it in the handshake methods, and take appropriate action depending on what to do next. In general a compatibility issue should be handled at handshake level itself, thus we can clearly let user/admin know of any 'in-compatibilities'. 
+As a developer of GlusterFS protocol layer, one just has to make sure *never to make changes to existing program structures*, but they have to add new programs if required. New programs can have the same actors as present in existing, and also little more. Or it can even have same actor behave differently, take different parameter.
+
+If this is followed properly, there would be smooth upgrade / downgrade of versions. If not, technically, it is 100% guarantee of getting compatibility related issues.
diff --git a/doc/developer-guide/syncop.md b/doc/developer-guide/syncop.md
new file mode 100644
index 00000000000..bcc8bd08e01
--- /dev/null
+++ b/doc/developer-guide/syncop.md
@@ -0,0 +1,72 @@
+# syncop framework
+A coroutines-based, cooperative multi-tasking framework.
+
+## Topics
+
+- Glossary
+- Lifecycle of a synctask
+- Existing usage
+
+
+## Glossary
+
+### syncenv
+
+syncenv is an object that provides access to a pool of worker threads.
+synctasks execute in a syncenv.
+
+### synctask
+
+synctask can be informally defined as a pair of function pointers, namely _the
+call_ and _the callback_ (see syncop.h for more details).
+
+        synctask_fn_t - 'the call'
+        synctask_cbk_t - 'the callback'
+
+synctask has two modes of operation,
+
+1. The calling thread waits for the synctask to complete.
+2. The calling thread schedules the synctask and continues.
+
+synctask guarantees that the callback is called _after_ the call completes.
+
+### Lifecycle of a synctask
+
+A synctask could go into the following stages while in execution.
+
+- CREATED  - On calling synctask_create/synctask_new.
+
+- RUNNABLE - synctask is queued in env->runq.
+
+- RUNNING  - When one of syncenv's worker threads calls synctask_switch_to.
+
+- WAITING  - When a synctask calls synctask_yield.
+
+- DONE     - When a synctask has run to completion.
+
+
+                                +-------------------------------+
+                                |            CREATED            |
+                                +-------------------------------+
+                                  |
+                                  | synctask_new/synctask_create
+                                  v
+                                +-------------------------------+
+                                |    RUNNABLE (in env->runq)    | <+
+                                +-------------------------------+  |
+                                  |                                |
+                                  | synctask_switch_to             |
+                                  v                                |
+ +------+  on task completion   +-------------------------------+  |
+ | DONE | <-------------------- |            RUNNING            |  | synctask_wake/wake
+ +------+                       +-------------------------------+  |
+                                  |                                |
+                                  | synctask_yield/yield           |
+                                  v                                |
+                                +-------------------------------+  |
+                                |    WAITING (in env->waitq)    | -+
+                                +-------------------------------+
+
+Note: A synctask is not guaranteed to run on the same thread throughout its
+lifetime. Every time a synctask yields, it is possible for it to run on a
+different thread.
diff --git a/doc/developer-guide/thread-naming.md b/doc/developer-guide/thread-naming.md
new file mode 100644
index 00000000000..513140d4437
--- /dev/null
+++ b/doc/developer-guide/thread-naming.md
@@ -0,0 +1,104 @@
+Thread Naming
+================
+Gluster processes spawn many threads; some threads are created by libglusterfs
+library, while others are created by xlators. When gfapi library is used in an
+application, some threads belong to the application and some are spawned by
+gluster libraries. We also have features where n number of threads are spawned
+to act as worker threads for same operation.
+
+In all the above cases, it is useful to be able to determine the list of threads
+that exist in runtime. Naming threads when you create them is the easiest way to
+provide that information to kernel so that it can then be queried by any means.
+
+How to name threads
+-------------------
+We have two wrapper functions in libglusterfs for creating threads. They take
+name as an argument and set thread name after its creation.
+
+```C
+gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
+                  void *(*start_routine)(void *), void *arg, const char *name)
+```
+
+```C
+gf_thread_create_detached (pthread_t *thread,
+                           void *(*start_routine)(void *), void *arg,
+                           const char *name)
+```
+
+As max name length for a thread in POSIX is only 16 characters including the
+'\0' character, you have to be a little creative with naming. Also, it is
+important that all Gluster threads have common prefix. Considering these
+conditions, we have "glfs_" as prefix for all the threads created by these
+wrapper functions. It is responsibility of the owner of thread to provide the
+suffix part of the name. It does not have to be a descriptive name, as it has
+only 10 letters to work with. However, it should be unique enough such that it
+can be matched with a table which describes it.
+
+If n number of threads are spwaned to perform same function, it is must that the
+threads are numbered.
+
+Table of thread names
+---------------------
+Thread names don't have to be a descriptive; however, it should be unique enough
+such that it can be matched with a table below without ambiguity.
+
+- bdaio    - block device aio
+- brfsscan - bit rot fs scanner
+- brhevent - bit rot event handler
+- brmon    - bit rot monitor
+- brosign  - bit rot one shot signer
+- brpobj   - bit rot object processor
+- brsproc  - bit rot scrubber
+- brssign  - bit rot stub signer
+- brswrker - bit rot worker
+- clogc    - changelog consumer
+- clogcbki - changelog callback invoker
+- clogd    - changelog dispatcher
+- clogecon - changelog reverse connection
+- clogfsyn - changelog fsync
+- cloghcon - changelog history consumer
+- clogjan  - changelog janitor
+- clogpoll - changelog poller
+- clogproc - changelog process
+- clogro   - changelog rollover
+- ctrcomp  - change time recorder compaction
+- dhtdf    - dht defrag task
+- dhtdg    - dht defrag start
+- dhtfcnt  - dht rebalance file counter
+- ecshd    - ec heal daemon
+- epollN   -  epoll thread
+- fdlwrker - fdl worker
+- fusenoti - fuse notify
+- fuseproc - fuse main thread
+- gdhooks  - glusterd hooks
+- glfspoll -  gfapi poller thread
+- idxwrker - index worker
+- iosdump  - io stats dump
+- iotwr    - io thread worker
+- jbrflush - jbr flush
+- leasercl - lease recall
+- memsweep - sweeper thread for mem pools
+- nfsauth  - nfs auth
+- nfsnsm   - nfs nsm
+- nfsudp   - nfs udp mount
+- nlmmon   - nfs nlm/nsm mon
+- posixaio - posix aio
+- posixfsy - posix fsync
+- posixhc  - posix heal
+- posixjan - posix janitor
+- posixrsv - posix reserve
+- quiesce  - quiesce dequeue
+- rdmaAsyn - rdma async event handler
+- rdmaehan - rdma completion handler
+- rdmarcom - rdma receive completion handler
+- rdmascom - rdma send completion handler
+- rpcsvcrh - rpcsvc request handler
+- scleanup - socket cleanup
+- shdheal  - self heal daemon
+- sigwait  -  glusterfsd sigwaiter
+- spoller  - socket poller
+- sprocN   - syncop worker thread
+- tbfclock - token bucket filter token generator thread
+- timer    - timer thread
+- upreaper - upcall reaper
diff --git a/doc/developer-guide/translator-development.md b/doc/developer-guide/translator-development.md
index 3bf7e153354..f75935519f6 100644
--- a/doc/developer-guide/translator-development.md
+++ b/doc/developer-guide/translator-development.md
@@ -472,7 +472,7 @@ hello
 Now let's interrupt the process and see where we are.
 
 ```
-^C
+
 Program received signal SIGINT, Interrupt.
 0x0000003a0060b3dc in pthread_cond_wait@@GLIBC_2.3.2 ()
                    from /lib64/libpthread.so.0
@@ -680,4 +680,4 @@ Original author's site:
 
 Gluster community site:
 
- * [Translators](http://www.gluster.org/community/documentation/index.php/Translators)
+ * [Translators](https://docs.gluster.org/en/latest/Quick-Start-Guide/Architecture/#translators)
diff --git a/doc/developer-guide/writing-a-cloudsync-plugin.md b/doc/developer-guide/writing-a-cloudsync-plugin.md
new file mode 100644
index 00000000000..907860aaed8
--- /dev/null
+++ b/doc/developer-guide/writing-a-cloudsync-plugin.md
@@ -0,0 +1,164 @@
+## How to write your Cloudsync Plugin
+
+### Background
+
+Cloudsync translator is part of the archival feature in Gluster. This translator
+does the retrieval/download part. Each cold  file will be archived to a remote
+storage (public or private cloud). On future access to the file, it will be
+retrieved from the remote storage by Cloudsync translator. Each remote storage
+would need a unique plugin. Cloudsync translator will load this plugin and
+call the necessary plugin functions.
+
+Upload can be done by a script or program. There are some basic mandatory steps
+for uploading the data. There is a sample script for crawl and upload given at
+the end of this guide.
+
+### Necessary changes to create a plugin
+
+1. Define store_methods:
+
+* This structure is the container of basic functions that will be called by
+  cloudsync xlator.
+
+        typedef struct store_methodds {
+                int (*fop_download) (call_frame_t *frame, void *config);
+                /* return type should be the store config */
+                void *(*fop_init) (xlator_t *this);
+                int (*fop_reconfigure) (xlator_t *this, dict_t *options);
+                void (*fop_fini) (void *config);
+        } store_methods_t;
+
+
+        Member details:
+        fop_download:
+                This is the download function pointer.
+
+                frame: This will have the fd to write data downloaded from
+                       cloud to GlusterFS.(frame->local->fd)
+
+                config: This is the plugin configuration variable.
+
+                Note: Structure cs_local_t has member dlfd and dloffset which
+                      can be used to manage the writes to Glusterfs.
+                      Include cloudsync-common.h to access these structures.
+
+        fop_init:
+                This is similar to xlator init. But here the return value is
+                the plugin configuration pointer. This pointer will be stored
+                in the cloudsync private object (priv->stores->config). And
+                the cloudsync private object can be accessed by "this->private"
+                where "this" is of type xlator_t.
+
+        fop_reconfigure:
+                This is similar to xlator_reconfigure.
+
+        fop_fini:
+                Free plugin resources.
+
+        Note: Store_methods_t is part of cs_private_t which in turn part of
+              xlator_t. Create a store_methods_t object named "store_ops" in
+              your plugin. For example
+
+              store_methods_t store_ops = {
+                .fop_download = aws_download_s3,
+                .fop_init     = aws_init,
+                .fop_reconfigure = aws_reconfigure,
+                .fop_fini     = aws_fini,
+              };
+
+
+2 - Making Cloudsync xlator aware of the plugin:
+
+        Add an entry in to the cs_plugin structure. For example
+        struct cs_plugin plugins[] = {
+                {
+                  .name = "amazons3",
+                  .library = "libamazons3.so",
+                  .description = "amazon s3 store."
+                },
+
+                {.name = NULL},
+        };
+
+        Description about individual members:
+                name: name of the plugin
+                library: This is the shared object created. Cloudsync will load
+                         this library during init.
+                description: Describe about the plugin.
+
+3- Makefile Changes in Cloudsync:
+
+		Add <plugin.la> to cloudsync_la_LIBADD variable.
+
+4 - Configure.ac changes:
+
+                In cloudsync section add the necessary dependency checks for
+                the plugin.
+
+5 - Export symbols:
+
+        Cloudsync needs "store_ops" to resolve all plugin functions.
+        Create a file <plugin>.sym and add write "store_ops" to it.
+
+
+### Sample script for upload
+This script assumes amazon s3 is the target cloud and bucket name is
+gluster-bucket. User can do necessary aws configuration using command
+"aws configure". Currently for amazons3 there are four gluster settings
+available.
+1- features.s3plugin-seckey -> s3 secret key
+2- features.s3plugin-keyid -> s3 key id
+3- features.s3plugin-bucketid -> bucketid
+4- features.s3plugin-hostname -> hostname e.g. s3.amazonaws.com
+
+Additionally set cloudsync storetype to amazons3.
+
+gluster v set <VOLNAME> cloudsync-storetype amazons3
+
+Now create a mount dedicated for this upload task.
+
+That covers necessary configurations needed.
+
+Below is the sample script for upload. The script will crawl directly on the
+brick and will upload those files which are not modified for last one month.
+It needs two arguments.
+1st arguement - Gluster Brick path
+2nd arguement - coldness that is how many days since the file was modified.
+3rd argument - dedicated gluster mount point created for uploading.
+
+Once the cloud setup is done, run the following script on individual bricks.
+Note: For an AFR volume, pick only the fully synchronized brick among the
+replica bricks.
+
+```
+target_folder=$1
+coldness=$2
+mnt=$3
+
+cd $target_folder
+for i in `find . -type f  | grep -v "glusterfs" | sed 's/..//'`
+do
+        echo "processing $mnt/$i"
+
+        #check whether the file is already archived
+        getfattr -n trusted.glusterfs.cs.remote $i &> /dev/null
+        if [ $? -eq 0 ]
+        then
+                echo "file $mnt/$i is already archived"
+        else
+                #upload to cloud
+                aws s3 cp $mnt/$i s3://gluster-bucket/
+                mtime=`stat -c "%Y" $mnt/$i`
+
+                #post processing of upload
+                setfattr -n trusted.glusterfs.csou.complete -v $mtime $mnt/$i
+                if [ $? -ne 0 ]
+                then
+                        echo "archiving of file $mnt/$i failed"
+                else
+                        echo "archiving of file $mnt/$i succeeded"
+                fi
+
+        fi
+done
+```
diff --git a/doc/developer-guide/xlator-classification.md b/doc/developer-guide/xlator-classification.md
new file mode 100644
index 00000000000..6073df9375f
--- /dev/null
+++ b/doc/developer-guide/xlator-classification.md
@@ -0,0 +1,221 @@
+# xlator categories and expectations
+
+The purpose of the document is to define a category for various xlators
+and expectations around what each category means from a perspective of
+health and maintenance of a xlator.
+
+The need to do this is to ensure certain categories are kept in good
+health, and helps the community and contributors focus their efforts around the
+same.
+
+This document also provides implementation details for xlator developers to
+declare a category for any xlator.
+
+## Table of contents
+1. Audience
+2. Categories (and expectations of each category)
+3. Implementation and usage details
+
+## Audience
+
+This document is intended for the following community participants,
+- New xlator contributors
+- Existing xlator maintainers
+- Packaging and gluster management stack maintainers
+
+For a more user facing understanding it is recommended to read section (TBD)
+in the gluster documentation.
+
+## Categories
+1. Experimental (E)
+2. TechPreview  (TP)
+3. Maintained (M)
+4. Deprecated (D)
+5. Obsolete (O)
+
+### Experimental (E)
+
+Developed in the experimental branch, for exploring new features. These xlators
+are NEVER packaged as a part of releases, interested users and contributors can
+build and work with these from sources. In the future, these maybe available as
+an package based on a weekly build of the same.
+
+#### Quality expectations
+- Compiles or passes smoke tests
+- Does not break nightly experimental regressions
+  - NOTE: If a nightly is broken, then all patches that were merged are reverted
+  till the errant patch is found and subsequently fixed
+
+### TechPreview (TP)
+
+Xlators in master or release branches that are not deemed fit to be in
+production deployments, but are feature complete to invite feedback and host
+user data.
+
+These xlators will be worked upon with priority by maintainers/authors who are
+involved in making them more stable than xlators in the Experimental/Deprecated/
+Obsolete categories.
+
+There is no guarantee that these xlators will move to the Maintained state, and
+may just get Obsoleted based on feedback, or other project goals or technical
+alternatives.
+
+#### Quality expectations
+- Same as Maintained, minus
+  - Performance, Scale, other(?)
+  - *TBD* *NOTE* Need inputs, Intention is all quality goals as in Maintained,
+  other than the list above (which for now has scale and performance)
+
+### Maintained (M)
+
+These xltors are part of the core Gluster functionality and are maintained
+actively. These are part of master and release branches and are higher in
+priority of maintainers and other interested contributors.
+
+#### Quality expectations
+
+NOTE: A short note on what each of these mean are added here, details to follow.
+
+NOTE: Out of the gate all of the following are not mandated, consider the
+following a desirable state to reach as we progress on each
+
+- Bug backlog: Actively address bug backlog
+- Enhancement backlog: Actively maintain outstanding enhancement backlog (need
+        not be acted on, but should be visible to all)
+- Review backlog: Actively keep this below desired counts and states
+- Static code health:  Actively meet near-zero issues in this regard
+  - Coverity, spellcheck and other checks
+- Runtime code health: Actively meet defined coverage levels in this regard
+  - Coverage, others?
+  - Per-patch regressions
+  - Glusto runs
+  - Performance
+  - Scalability
+- Technical specifications: Implementation details should be documented and
+        updated at regular cadence (even per patch that change assumptions in
+        here)
+- User documentation: User facing details should be maintained to current
+        status in the documentation
+- Debuggability: Steps, tools, procedures should be documented and maintained
+        each release/patch as applicable
+- Troubleshooting: Steps, tools, procedures should be documented and maintained
+        each release/patch as applicable
+  - Steps/guides for self service
+  - Knowledge base for problems
+- Other common criteria that will apply: Required metrics/desired states to be
+        defined per criteria
+  - Monitoring, usability, statedump, and other such xlator expectations
+
+### Deprecated (D)
+
+Xlators on master or release branches that would be obsoleted and/or replaced
+with similar or other functionality in the next major release.
+
+#### Quality expectations
+- Retain status-quo when moved to this state, till it is moved to obsoleted
+- Provide migration steps if feature provided by the xlator is replaced with
+other xlators
+
+### Obsolete (O)
+
+Xlator/code still in tree, but not packaged or shipped or maintained in any
+form. This is noted as a category till the code is removed from the tree.
+
+These xlators and their corresponding code and test health will not be executed.
+
+#### Quality expectations
+- None
+
+## Implementation and usage details
+
+### How to specify an xlators category
+
+While defining 'xlator_api_t' structure for the corresponding xlator, add a
+flag like below:
+
+```
+diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c
+index 0f0e53bac2..8267d6897c 100644
+--- a/xlators/performance/nl-cache/src/nl-cache.c
++++ b/xlators/performance/nl-cache/src/nl-cache.c
+@@ -869,4 +869,5 @@ xlator_api_t xlator_api = {
+         .cbks          = &nlc_cbks,
+         .options       = nlc_options,
+         .identifier    = "nl-cache",
++        .category      = GF_TECH_PREVIEW,
+ };
+diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
+index 8d39720e7f..235de27c19 100644
+--- a/xlators/performance/quick-read/src/quick-read.c
++++ b/xlators/performance/quick-read/src/quick-read.c
+@@ -1702,4 +1702,5 @@ xlator_api_t xlator_api = {
+         .cbks          = &qr_cbks,
+         .options       = qr_options,
+         .identifier    = "quick-read",
++        .category      = GF_MAINTAINED,
+ };
+```
+
+Similarly, if a particular option is in different state other than
+the xlator state, one can add the same flag in options structure too.
+
+```
+diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
+index 0e86e33d03..81996743d1 100644
+--- a/xlators/cluster/afr/src/afr.c
++++ b/xlators/cluster/afr/src/afr.c
+@@ -772,6 +772,7 @@ struct volume_options options[] = {
+           .description = "Maximum latency for shd halo replication in msec."
+         },
+         { .key   = {"halo-enabled"},
++          .category = GF_TECH_PREVIEW,
+           .type  = GF_OPTION_TYPE_BOOL,
+           .default_value = "False",
+
+```
+
+
+### User experience using the categories
+
+#### Ability to use a category
+
+This section details which category of xlators can be used when and specifics
+around when each category is enabled.
+
+1. Maintained category xlators can be used by default, this implies, volumes
+created with these xlators enabled will throw no warnings, or need no user
+intervention to use the xlator.
+
+2. Tech Preview category xlators needs cluster configuration changes to allow
+these xlatorss to be used in volumes, further, logs will contain a message
+stating TP xlators are in use. Without the cluster configured to allow TP
+xlators, volumes created or edited to use such xlators would result in errors.
+  - (TBD) Cluster configuration option
+  - (TBD) Warning message
+  - (TBD) Code mechanics on how this is achieved
+
+3. Deprecated category xlators can be used by default, but will throw a warning
+in the logs that such are in use and will be deprecated in the future.
+  - (TBD) Warning message
+
+4. Obsolete category xlators will not be packaged and hence cannot be used from
+release builds.
+
+5. Experimental category xlators will not be packaged and hence cannot be used
+from release builds, if running experimental (weekly or other such) builds,
+these will throw a warning in the logs stating experimental xlators are in use.
+  - (TBD) Warning message
+
+#### Ability to query xlator category
+
+(TBD) Need to provide the ability to query xlator categories, or list xlators
+and their respective categories.
+
+#### User facing changes
+
+User facing changes that are expected due to this change include the following,
+- Cluster wide option to enable TP xlators, or more generically a category
+level of xlators
+- Errors in commands that fail due to invalid categories
+- Warning messages in logs to denote certain categories of xlators are in use
+- (TBD) Ability to query xlators and their respective categories
diff --git a/doc/features/ctime.md b/doc/features/ctime.md
new file mode 100644
index 00000000000..74a77abed4b
--- /dev/null
+++ b/doc/features/ctime.md
@@ -0,0 +1,68 @@
+# Consistent time attributes in gluster across replica/distribute
+
+
+#### Problem:
+Traditionally gluster has been using time attributes (ctime, atime, mtime) of files/dirs from bricks. The problem with this approach is that, it is not consisteant  across replica and distribute bricks. And applications which depend on it breaks as replica  might not  always return time attributes from same brick.
+
+Tar especially gives "file changed as we read it" whenever it detects ctime differences when stat is served from  different bricks. The way we have been trying to solve it is to serve  the stat structures from same brick in afr, max-time in dht. But it doesn't avoid the problem completely. Because there is no way to change ctime at the moment(lutimes() only allows mtime, atime), there is little we can do to make sure ctimes match after self-heals/xattr updates/rebalance.
+
+#### Solution Proposed:
+Store time attribues (ctime, mtime, atime) as an xattr of the file. The xattr is updated based
+on the fop. If a filesystem fop changes only mtime and ctime, update only those in xattr for
+that file.
+
+#### Design Overview:
+1) As part of each fop, top layer will generate a time stamp and pass it to the down along
+ with other information
+  - This will bring a dependency for NTP synced clients along with servers
+  - There can be a diff in time if the fop stuck in the xlator for various reason,
+for ex: because of locks.
+
+ 2)  On the server, posix layer stores the value in the memory (inode ctx) and will sync the data periodically to the disk as an extended attr
+  -  Of course sync call also will force it. And fop comes for an inode which is not linked, we do the sync immediately.
+
+ 3)  Each time when inodes are created or initialized it read the data from disk and store in inode ctx.
+
+ 4)  Before setting to inode_ctx we compare the timestamp stored and the timestamp received, and only store if the stored value is lesser than the current value.
+
+ 5)  So in best case data will be stored and retrieved from the memory. We replace the values in iatt with the values in inode_ctx.
+
+ 6)  File ops that changes the parent directory attr time need to be consistent across all the distributed directories across the subvolumes. (for eg: a create call will change ctime and mtime of parent dir)
+
+  - This has to handle separately because we only send the fop to the hashed subvolume.
+  - We can asynchronously send the timeupdate setattr fop to the other subvoumes and change the values for parent directory if the file fops is successful on hashed subvolume.
+  -  This will have a window where the times are inconsistent across dht subvolume (Please provide your suggestions)
+
+7)  Currently we have couple of mount options for time attributes like noatime, relatime , nodiratime etc. But we are not explicitly handled those options even if it is given as mount option when gluster mount.
+
+
+#### Implementation Overview:
+This features involves changes in following xlators.
+ - utime xlator
+ - posix xlator
+
+##### utime xlator:
+This is a new client side xlator which does following tasks.
+
+1. It will generate a time stamp and passes it down in frame->root->ctime  and over the network.
+2.  Based on fop, it also decides the time attributes to be updated and this passed using "frame->root->flags"
+
+    Patches:
+    1. https://review.gluster.org/#/c/19857/
+
+##### posix xlator:
+Following tasks are done in posix xlator:
+
+1. Provides APIs to set and get the xattr from backend. It also caches the xattr in inode context. During get, it updates time attributes stored in xattr into iatt structure.
+2. Based on the flags from utime xlator, relevant fops update the time attributes in the xattr.
+
+    Patches:
+    1. https://review.gluster.org/#/c/19267/
+    2. https://review.gluster.org/#/c/19795/
+    3. https://review.gluster.org/#/c/19796/
+
+#### Pending Work:
+1. Handling of time related mount options (noatime, realatime,etc)
+2. flag based create (depending on flags in open, create behaviour might change)
+3. Changes in dht for direcotory sync acrosss multiple subvolumes
+4. readdirp stat need to be worked on.
diff --git a/doc/gluster.8 b/doc/gluster.8
index c9a9d500a29..ba595edca15 100644
--- a/doc/gluster.8
+++ b/doc/gluster.8
@@ -16,15 +16,14 @@ gluster - Gluster Console Manager (command line utility)
 .PP
 To run the program and display gluster prompt:
 .PP
-.B gluster [--xml]
+.B gluster [--remote-host=<gluster_node>] [--mode=script] [--xml]
 .PP
 (or)
 .PP
 To specify a command directly:
 .PP
 .B gluster
-.I [commands] [options] [--xml]
-
+.I  [commands] [options] [--remote-host=<gluster_node>] [--mode=script] [--xml]
 .SH DESCRIPTION
 The Gluster Console Manager is a command line utility for elastic volume management. You can run the gluster command on any export server. The command enables administrators to perform cloud operations, such as creating, expanding, shrinking, rebalancing, and migrating volumes without needing to schedule server downtime.
 .SH COMMANDS
@@ -36,7 +35,13 @@ The Gluster Console Manager is a command line utility for elastic volume managem
 \fB\ volume info [all|<VOLNAME>] \fR
 Display information about all volumes, or the specified volume.
 .TP
-\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [disperse [<COUNT>]] [redundancy <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ... \fR
+\fB\ volume list \fR
+List all volumes in cluster
+.TP
+\fB\ volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]] [detail|clients|mem|inode|fd|callpool|tasks|client-list] \fR
+Display status of all or specified volume(s)/brick
+.TP
+\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] [disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ... <TA-BRICK> \fR
 Create a new volume of the specified type using the specified bricks and transport type (the default transport type is tcp).
 To create a volume with both transports (tcp and rdma), give 'transport tcp,rdma' as an option.
 .TP
@@ -52,8 +57,17 @@ Stop the specified volume.
 \fB\ volume set <VOLNAME> <OPTION> <PARAMETER> [<OPTION> <PARAMETER>] ... \fR
 Set the volume options.
 .TP
-\fB\ volume get <VOLNAME> <OPTION/all>\fR
-Get the volume options.
+\fB\ volume get <VOLNAME/all> <OPTION/all> \fR
+Get the value of the all options or given option for volume <VOLNAME> or all option. gluster volume get all all is to get all global options
+.TP
+\fB\ volume reset <VOLNAME> [option] [force] \fR
+Reset all the reconfigured options
+.TP
+\fB\ volume barrier <VOLNAME> {enable|disable} \fR
+Barrier/unbarrier file operations on a volume
+.TP
+\fB\ volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}{inode [range]|entry [basename]|posix [range]} \fR
+Clear locks held on path
 .TP
 \fB\ volume help \fR
 Display help for the volume command.
@@ -71,6 +85,9 @@ If you remove the brick, the data stored in that brick will not be available. Yo
 .B replace-brick
 option.
 .TP
+\fB\ volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} | {<NEW-BRICK> commit}} \fR
+Brings down or replaces the specified source brick with the new brick.
+.TP
 \fB\ volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> commit force \fR
 Replace the specified source brick with a new brick.
 .TP
@@ -92,6 +109,18 @@ Locate the log file for corresponding volume/brick.
 .TP
 \fB\ volume log rotate <VOLNAME> [BRICK] \fB
 Rotate the log file for corresponding volume/brick.
+.TP
+\fB\ volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs] \fR
+Profile operations on the volume. Once started, volume profile <volname> info provides cumulative statistics of the FOPs performed.
+.TP
+\fB\ volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] | {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>] \fR
+Generates a profile of a volume representing the performance and bottlenecks/hotspots of each brick.
+.TP
+\fB\ volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|priv|fd|inode|history]... | [client <hostname:process-id>]] \fR
+Dumps the in memory state of the specified process or the bricks of the volume.
+.TP
+\fB\ volume sync <HOSTNAME> [all|<VOLNAME>] \fR
+Sync the volume information from a peer
 .SS "Peer Commands"
 .TP
 \fB\ peer probe <HOSTNAME> \fR
@@ -103,27 +132,58 @@ Detach the specified peer.
 \fB\ peer status \fR
 Display the status of peers.
 .TP
+\fB\ pool list \fR
+List all the nodes in the pool (including localhost)
+.TP
 \fB\ peer help \fR
 Display help for the peer command.
-.SS "Tier Commands"
+.SS "Quota Commands"
+.TP
+\fB\ volume quota <VOLNAME> enable \fR
+Enable quota on the specified volume. This will cause all the directories in the filesystem hierarchy to be accounted and updated thereafter on each operation in the the filesystem. To kick start this accounting, a crawl is done over the hierarchy with an auxiliary client.
 .TP
-\fB\ volume tier <VOLNAME> attach [<replica COUNT>] <NEW-BRICK>... \fR
-Attach to an existing volume a tier of specified type using the specified bricks.
+\fB\ volume quota <VOLNAME> disable \fR
+Disable quota on the volume. This will disable enforcement and accounting in the filesystem. Any configured limits will be lost.
 .TP
-\fB\ volume tier <VOLNAME> status \fR
-Display statistics on data migration between the hot and cold tiers.
+\fB\ volume quota <VOLNAME> limit-usage <PATH> <SIZE> [<PERCENT>] \fR
+Set a usage  limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit.
 .TP
-\fB\ volume tier <VOLNAME> detach start\fR
-Begin detaching the hot tier from the volume. Data will be moved from the hot tier to the cold tier.
+\fB\ volume quota <VOLNAME> limit-objects <PATH> <SIZE> [<PERCENT>] \fR
+Set an inode limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit.
 .TP
-\fB\ volume tier <VOLNAME> detach commit [force]\fR
-Commit detaching the hot tier from the volume. The volume will revert to its original state before the hot tier was attached.
+NOTE: valid units of SIZE are : B, KB, MB, GB, TB, PB. If no unit is specified, the unit defaults to bytes.
 .TP
-\fB\ volume tier <VOLNAME> detach status\fR
-Check status of data movement from the hot to cold tier.
+\fB\ volume quota <VOLNAME> remove <PATH> \fR
+Remove any usage limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced.
 .TP
-\fB\ volume tier <VOLNAME> detach stop\fR
-Stop detaching the hot tier from the volume.
+\fB\ volume quota <VOLNAME> remove-objects <PATH> \fR
+Remove any inode limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced.
+.TP
+\fB\ volume quota <VOLNAME> list <PATH> \fR
+Lists the  usage and limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's usage. If no path is given, usage and limits are displayed for all directories that has limits configured.
+.TP
+\fB\ volume quota <VOLNAME> list-objects <PATH> \fR
+Lists the inode usage and inode limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's inode usage. If no path is given, usage and limits are displayed for all directories that has limits configured.
+.TP
+\fB\ volume quota <VOLNAME> default-soft-limit <PERCENT> \fR
+Set the percentage value for default soft limit for the volume.
+.TP
+\fB\ volume quota <VOLNAME> soft-timeout <TIME> \fR
+Set the soft timeout for the volume. The interval in which limits are retested before the soft limit is breached.
+.TP
+\fB\ volume quota <VOLNAME> hard-timeout <TIME> \fR
+Set the hard timeout for the volume. The interval in which limits are retested after the soft limit is breached.
+.TP
+\fB\ volume quota <VOLNAME> alert-time <TIME> \fR
+Set the frequency in which warning messages need to be logged (in the brick logs) once soft limit is breached.
+.TP
+\fB\ volume inode-quota <VOLNAME> enable/disable \fR
+Enable/disable inode-quota for <VOLNAME>
+.TP
+\fB\ volume quota help \fR
+Display help for volume quota commands
+.TP
+NOTE: valid units of time and their symbols are : hours(h/hr), minutes(m/min), seconds(s/sec), weeks(w/wk), Days(d/days).
 .SS "Geo-replication Commands"
 .TP
 \fI\ Note\fR: password-less ssh, from the master node (where these commands are executed) to the slave node <SLAVE_HOST>, is a prerequisite for the geo-replication commands.
@@ -131,8 +191,10 @@ Stop detaching the hot tier from the volume.
 \fB\ system:: execute gsec_create\fR
 Generates pem keys which are required for push-pem
 .TP
-\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> create [push-pem] [force]\fR
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> create [[ssh-port n][[no-verify]|[push-pem]]] [force]\fR
 Create a new geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+Use ssh-port n if custom SSH port is configured in slave nodes.
+Use no-verify if the rsa-keys of nodes in master volume is distributed to slave nodes through an external agent.
 Use push-pem to push the keys automatically.
 .TP
 \fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> {start|stop} [force] \fR
@@ -156,17 +218,25 @@ Use "!<OPTION>" to reset option <OPTION> to default value.
 \fB\ volume bitrot <VOLNAME> {enable|disable} \fR
 Enable/disable bitrot for volume <VOLNAME>
 .TP
+\fB\ volume bitrot <VOLNAME> signing-time <time-in-secs> \fR
+Waiting time for an object after last fd is closed to start signing process.
+.TP
+\fB\ volume bitrot <VOLNAME> signer-threads <count> \fR
+Number of signing process threads. Usually set to number of available cores.
+.TP
 \fB\ volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive} \fR
 Scrub-throttle value is a measure of how fast or slow the scrubber scrubs the filesystem for volume <VOLNAME>
 .TP
-\fB\ volume bitrot <VOLNAME> scrub-frequency {daily|weekly|biweekly|monthly} \fR
+\fB\ volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly|monthly} \fR
 Scrub frequency for volume <VOLNAME>
 .TP
 \fB\ volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand} \fR
 Pause/Resume scrub. Upon resume, scrubber continues where it left off. status option shows the statistics of scrubber. ondemand option starts the scrubbing immediately if the scrubber is not paused or already running.
 .TP
+\fB\ volume bitrot help \fR
+Display help for volume bitrot commands
+.TP
 .SS "Snapshot Commands"
-.PP
 .TP
 \fB\ snapshot create <snapname> <volname> [no-timestamp] [description <description>] [force] \fR
 Creates a snapshot of a GlusterFS volume. User can provide a snap-name and a description to identify the snap. Snap will be created by appending timestamp in GMT. User can override this behaviour using "no-timestamp" option. The description cannot be more than 1024 characters. To be able to take a snapshot, volume should be present and it should be in started state.
@@ -269,7 +339,7 @@ Selects <HOSTNAME:BRICKNAME> as the source for all the files that are in split-b
 Selects the split-brained <FILE> present in <HOSTNAME:BRICKNAME> as source and completes heal.
 .SS "Other Commands"
 .TP
-\fB\ get-state [<daemon>] [odir </path/to/output/dir/>] [file <filename>] \fR
+\fB\ get-state [<daemon>] [[odir </path/to/output/dir/>] [file <filename>]] [detail|volumeoptions] \fR
 Get local state representation of mentioned daemon and store data in provided path information
 .TP
 \fB\ help \fR
diff --git a/doc/glusterd.8 b/doc/glusterd.8
index 04a43481eec..e3768c78761 100644
--- a/doc/glusterd.8
+++ b/doc/glusterd.8
@@ -30,6 +30,9 @@ File to use for logging.
 \fB\-L <LOGLEVEL>, \fB\-\-log\-level=<LOGLEVEL>\fR
 Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
 .TP
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
+.TP
 \fB\-\-debug\fR
 Run the program in debug mode. This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
 and \fB\-\-log\-file\fR to console.
diff --git a/doc/glusterfs.8 b/doc/glusterfs.8
index fc28ef68be6..3d359ea85e4 100644
--- a/doc/glusterfs.8
+++ b/doc/glusterfs.8
@@ -53,6 +53,9 @@ Maximum number of connect attempts to server. This option should be provided wit
 \fB\-\-acl\fR
 Mount the filesystem with POSIX ACL support.
 .TP
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
+.TP
 \fB\-\-debug\fR
 Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG,
 and \fB\-\-log\-file\fR to console.
@@ -60,8 +63,8 @@ and \fB\-\-log\-file\fR to console.
 \fB\-\-enable\-ino32=BOOL\fR
 Use 32-bit inodes when mounting to workaround application that doesn't support 64-bit inodes.
 .TP
-\fB\-\-fopen\-keep\-cache\fR
-Do not purge the cache on file open.
+\fB\-\-fopen\-keep\-cache[=BOOL]\fR
+Do not purge the cache on file open (default: false).
 .TP
 \fB\-\-mac\-compat=BOOL\fR
 Provide stubs for attributes needed for seamless operation on Macs (the default is off).
@@ -98,11 +101,17 @@ Mount the filesystem in 'worm' mode.
 .TP
 \fB\-\-xlator\-option=VOLUME\-NAME.OPTION=VALUE\fR
 Add/Override a translator option for a volume with the specified value.
+.TP
+\fB\-\-subdir\-mount=SUBDIR\-MOUNT\-PATH\fR
+Mount subdirectory instead of the '/' of volume.
 
 .SS "Fuse options"
 .PP
 .TP
 
+\fB\-\-attr\-times\-granularity=NANOSECONDS\fR
+Declare supported granularity of file attribute times (default is 0 which kernel handles as unspecified; valid real values are between 1 and 1000000000).
+.TP
 \fB\-\-attribute\-timeout=SECONDS\fR
 Set attribute timeout to SECONDS for inodes in fuse kernel module (the default is 1).
 .TP
@@ -112,8 +121,8 @@ Set fuse module's background queue length to N (the default is 64).
 \fB\-\-congestion\-threshold=N\fR
 Set fuse module's congestion threshold to N (the default is 48).
 .TP
-\fB\-\-direct\-io\-mode=BOOL\fR
-Enable/Disable the direct-I/O mode in fuse module (the default is enable).
+\fB\-\-direct\-io\-mode=BOOL|auto\fR
+Specify fuse direct I/O strategy (the default is auto).
 .TP
 \fB\-\-dump-fuse=PATH\f\R
 Dump fuse traffic to PATH
@@ -124,9 +133,17 @@ Set entry timeout to SECONDS in fuse kernel module (the default is 1).
 \fB\-\-gid\-timeout=SECONDS\fR
 Set auxiliary group list timeout to SECONDS for fuse translator (the default is 0).
 .TP
+\fB\-\-kernel-writeback-cache=BOOL\fR
+Enable fuse in-kernel writeback cache.
+.TP
 \fB\-\-negative\-timeout=SECONDS\fR
 Set negative timeout to SECONDS in fuse kernel module (the default is 0).
 .TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on].
+.TP
 \fB\-\-volfile-check\fR
 Enable strict volume file checking.
 
diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
index 956cb24bca3..bc1de2a8c80 100644
--- a/doc/glusterfsd.8
+++ b/doc/glusterfsd.8
@@ -51,6 +51,9 @@ Server to get the volume from.  This option overrides \fB\-\-volfile option
 .PP
 .TP
 
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
+.TP
 \fB\-\-debug\fR
 Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
 and \fB\-\-log\-file\fR to console
@@ -104,6 +107,11 @@ Enable/Disable direct-io mode in fuse module [default: enable]
 .TP
 \fB\-\-resolve-gids\fR
 Resolve all auxiliary groups in fuse translator (max 32 otherwise)
+.TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
 
 .SS "Miscellaneous Options"
 .PP
diff --git a/doc/mount.glusterfs.8 b/doc/mount.glusterfs.8
index 4e82c2fd57d..ce16e9e40b7 100644
--- a/doc/mount.glusterfs.8
+++ b/doc/mount.glusterfs.8
@@ -12,11 +12,11 @@
 .SH NAME
 .B mount.glusterfs - script to mount native GlusterFS volume
 .SH SYNOPSIS
-.B mount -t glusterfs [-o <options>] <volumeserver>:/<volume>
+.B mount -t glusterfs [-o <options>] <volumeserver>:/<volume>[/<subdir>]
 .B         <mountpoint>
 .TP
 .B mount -t glusterfs [-o <options>] <server1>,<server2>,
-.B        <server3>,..<serverN>:/<volname> <mount_point>
+.B        <server3>,..<serverN>:/<volname>[/<subdir>] <mount_point>
 .TP
 .TP
 .B mount -t glusterfs [-o <options>] <path/to/volumefile> <mountpoint>
@@ -44,8 +44,8 @@ INFO and NONE [default: INFO]
 \fBacl
 Mount the filesystem with POSIX ACL support
 .TP
-\fBfopen\-keep\-cache
-Do not purge the cache on file open
+\fBfopen\-keep\-cache[=BOOL]
+Do not purge the cache on file open (default: false)
 .TP
 \fBworm
 Mount the filesystem in 'worm' mode
@@ -65,6 +65,9 @@ Enable internal memory accounting
 .TP
 \fBcapability
 Enable file capability setting and retrival
+.TP
+\fBthin-client
+Enables thin mount and connects via gfproxyd daemon
 
 .PP
 .SS "Advanced options"
@@ -89,12 +92,15 @@ Set negative timeout to SECONDS in fuse kernel module [default: 0]
 Volume name to be used for MOUNT-POINT [default: top most volume in
 VOLUME-FILE]
 .TP
-\fBdirect\-io\-mode=\fRdisable
-Disable direct I/O mode in fuse kernel module
+\fBdirect\-io\-mode=\fRBOOL|auto
+Specify fuse direct I/O strategy [default: auto]
 .TP
 \fBcongestion\-threshold=\fRN
 Set fuse module's congestion threshold to N [default: 48]
 .TP
+\fsubdir\-mount=\fRN
+Set the subdirectory mount option [default: NULL, ie, no subdirectory mount]
+.TP
 .TP
 \fBbackup\-volfile\-servers=\fRSERVERLIST
 Provide list of backup volfile servers in the following format [default: None]
@@ -116,6 +122,15 @@ Provide list of backup volfile servers in the following format [default: None]
 \fBDeprecated\fR option - placed here for backward compatibility [default: 1]
 .TP
 .TP
+\fBlru-limit=\fRN
+Set fuse module's limit for number of inodes kept in LRU list to N [default: 65536]
+.TP
+.TP
+\fBinvalidate-limit=\fRN
+Suspend fuse invalidations implied by 'lru-limit' if  number of outstanding
+invalidations reaches N
+.TP
+.TP
 \fBbackground-qlen=\fRN
 Set fuse module's background queue length to N [default: 64]
 .TP
@@ -127,6 +142,20 @@ enable root squashing for the trusted client [default: on]
 .TP
 \fBuse\-readdirp=\fRBOOL
 Use readdirp() mode in fuse kernel module [default: on]
+.TP
+\fBdump\-fuse=\fRPATH
+Dump fuse traffic to PATH
+.TP
+\fBkernel\-writeback\-cache=\fRBOOL
+Enable fuse in-kernel writeback cache [default: off]
+.TP
+\fBattr\-times\-granularity=\fRNS
+Declare supported granularity of file attribute [default: 0]
+.TP
+\fBauto\-invalidation=\fRBOOL
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
 .PP
 .SH FILES
 .TP
diff --git a/events/eventskeygen.py b/events/eventskeygen.py
index 57c66ce6e18..e28ebe9b7e6 100644
--- a/events/eventskeygen.py
+++ b/events/eventskeygen.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -22,7 +22,7 @@ gen_header_type = sys.argv[1]
 # When adding new keys add it to the END
 keys = (
     # user driven events
-    #peer and volume managment events
+    #peer and volume management events
     "EVENT_PEER_ATTACH",
     "EVENT_PEER_DETACH",
     "EVENT_VOLUME_CREATE",
@@ -51,6 +51,7 @@ keys = (
     "EVENT_BITROT_SCRUB_THROTTLE",
     "EVENT_BITROT_SCRUB_FREQ",
     "EVENT_BITROT_SCRUB_OPTION",
+    "EVENT_BITROT_SCRUB_ONDEMAND",
 
     #quota events
     "EVENT_QUOTA_ENABLE",
@@ -111,7 +112,6 @@ keys = (
     "EVENT_BRICK_STOP_FAILED",
     "EVENT_BRICK_DISCONNECTED",
     "EVENT_BRICK_CONNECTED",
-    "EVENT_BRICKS_START_FAILED",
     "EVENT_BRICKPATH_RESOLVE_FAILED",
     "EVENT_NOTIFY_UNKNOWN_OP",
     "EVENT_QUORUM_LOST",
@@ -128,6 +128,10 @@ keys = (
     "EVENT_EC_MIN_BRICKS_UP",
     #georep async events
     "EVENT_GEOREP_FAULTY",
+    "EVENT_GEOREP_CHECKPOINT_COMPLETED",
+    "EVENT_GEOREP_ACTIVE",
+    "EVENT_GEOREP_PASSIVE",
+
     #quota async events
     "EVENT_QUOTA_CROSSED_SOFT_LIMIT",
     #bitrot async events
@@ -184,6 +188,13 @@ keys = (
     "EVENT_VOLUME_REBALANCE_FAILED",
     "EVENT_VOLUME_REBALANCE_COMPLETE",
 
+    #tier events
+    "EVENT_TIER_START",
+    "EVENT_TIER_START_FORCE",
+
+    #brick/inodes events
+    "EVENT_DHT_DISK_USAGE",
+    "EVENT_DHT_INODES_USAGE",
 )
 
 LAST_EVENT = "EVENT_LAST"
diff --git a/events/src/Makefile.am b/events/src/Makefile.am
index 87282c6c6f7..3b229691897 100644
--- a/events/src/Makefile.am
+++ b/events/src/Makefile.am
@@ -5,15 +5,19 @@ EXTRA_DIST = glustereventsd.py __init__.py  eventsapiconf.py.in \
 BUILT_SOURCES = eventtypes.py
 CLEANFILES = eventtypes.py
 
-eventsdir = $(libexecdir)/glusterfs/events
+eventsdir = $(GLUSTERFS_LIBEXECDIR)/gfevents
+if BUILD_EVENTS
 events_PYTHON = __init__.py gf_event.py eventsapiconf.py eventtypes.py \
 	utils.py
+endif
+# this does not work, see the Makefile.am in the root for a workaround
+#nodist_events_PYTHON = eventtypes.py
 
 eventtypes.py: $(top_srcdir)/events/eventskeygen.py
 	$(PYTHON) $(top_srcdir)/events/eventskeygen.py PY_HEADER
 
 if BUILD_EVENTS
-eventspeerscriptdir = $(libexecdir)/glusterfs
+eventspeerscriptdir = $(GLUSTERFS_LIBEXECDIR)
 eventsconfdir = $(sysconfdir)/glusterfs
 eventsconf_DATA = eventsconfig.json
 
@@ -24,10 +28,10 @@ eventspeerscript_SCRIPTS = peer_eventsapi.py
 install-exec-hook:
 	$(mkdir_p) $(DESTDIR)$(sbindir)
 	rm -f $(DESTDIR)$(sbindir)/glustereventsd
-	ln -s $(libexecdir)/glusterfs/events/glustereventsd.py \
+	ln -s $(GLUSTERFS_LIBEXECDIR)/gfevents/glustereventsd.py \
 		$(DESTDIR)$(sbindir)/glustereventsd
 	rm -f $(DESTDIR)$(sbindir)/gluster-eventsapi
-	ln -s $(libexecdir)/glusterfs/peer_eventsapi.py \
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_eventsapi.py \
 		$(DESTDIR)$(sbindir)/gluster-eventsapi
 
 uninstall-hook:
diff --git a/events/src/eventsapiconf.py.in b/events/src/eventsapiconf.py.in
index ecccd3dbda4..700093bee60 100644
--- a/events/src/eventsapiconf.py.in
+++ b/events/src/eventsapiconf.py.in
@@ -9,18 +9,51 @@
 #  cases as published by the Free Software Foundation.
 #
 
+import subprocess
+glusterd_workdir = None
+
+# Methods
+def get_glusterd_workdir():
+    global glusterd_workdir
+    if glusterd_workdir is not None:
+        return glusterd_workdir
+    proc = subprocess.Popen(["gluster", "system::", "getwd"],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            universal_newlines = True)
+    out, err = proc.communicate()
+    if proc.returncode == 0:
+        glusterd_workdir = out.strip()
+    else:
+        glusterd_workdir = "@GLUSTERD_WORKDIR@"
+    return glusterd_workdir
+
 SERVER_ADDRESS = "0.0.0.0"
+SERVER_ADDRESSv4 = "0.0.0.0"
+SERVER_ADDRESSv6 = "::1"
 DEFAULT_CONFIG_FILE = "@SYSCONF_DIR@/glusterfs/eventsconfig.json"
 CUSTOM_CONFIG_FILE_TO_SYNC = "/events/config.json"
-CUSTOM_CONFIG_FILE = "@GLUSTERD_WORKDIR@" + CUSTOM_CONFIG_FILE_TO_SYNC
+CUSTOM_CONFIG_FILE = get_glusterd_workdir() + CUSTOM_CONFIG_FILE_TO_SYNC
 WEBHOOKS_FILE_TO_SYNC = "/events/webhooks.json"
-WEBHOOKS_FILE = "@GLUSTERD_WORKDIR@" + WEBHOOKS_FILE_TO_SYNC
+WEBHOOKS_FILE = get_glusterd_workdir() + WEBHOOKS_FILE_TO_SYNC
 LOG_FILE = "@localstatedir@/log/glusterfs/events.log"
 EVENTSD = "glustereventsd"
-CONFIG_KEYS = ["log_level", "port"]
-BOOL_CONFIGS = []
+CONFIG_KEYS = ["log-level", "port", "disable-events-log"]
+BOOL_CONFIGS = ["disable-events-log"]
 INT_CONFIGS = ["port"]
 RESTART_CONFIGS = ["port"]
 EVENTS_ENABLED = @EVENTS_ENABLED@
-UUID_FILE = "@GLUSTERD_WORKDIR@/glusterd.info"
+UUID_FILE = get_glusterd_workdir() + "/glusterd.info"
 PID_FILE = "@localstatedir@/run/glustereventsd.pid"
+AUTO_BOOL_ATTRIBUTES = ["force", "push-pem", "no-verify"]
+AUTO_INT_ATTRIBUTES = ["ssh-port"]
+CERTS_DIR = get_glusterd_workdir() + "/events"
+
+# Errors
+ERROR_SAME_CONFIG = 2
+ERROR_ALL_NODES_STATUS_NOT_OK = 3
+ERROR_PARTIAL_SUCCESS = 4
+ERROR_WEBHOOK_ALREADY_EXISTS = 5
+ERROR_WEBHOOK_NOT_EXISTS = 6
+ERROR_INVALID_CONFIG = 7
+ERROR_WEBHOOK_SYNC_FAILED = 8
+ERROR_CONFIG_SYNC_FAILED = 9
diff --git a/events/src/eventsconfig.json b/events/src/eventsconfig.json
index 45730f9bb83..89e5b9c1d68 100644
--- a/events/src/eventsconfig.json
+++ b/events/src/eventsconfig.json
@@ -1,4 +1,5 @@
 {
-    "log_level": "INFO",
-    "port": 24009
+    "log-level": "INFO",
+    "port": 24009,
+    "disable-events-log": false
 }
diff --git a/events/src/gf_event.py b/events/src/gf_event.py
index f9ece6adc28..260b0d9aa48 100644
--- a/events/src/gf_event.py
+++ b/events/src/gf_event.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -13,10 +12,10 @@
 import socket
 import time
 
-from eventsapiconf import SERVER_ADDRESS, EVENTS_ENABLED
-from eventtypes import all_events
+from gfevents.eventsapiconf import SERVER_ADDRESS, EVENTS_ENABLED
+from gfevents.eventtypes import all_events
 
-from utils import logger, setup_logger, get_config
+from gfevents.utils import logger, setup_logger, get_config
 
 # Run this when this lib loads
 setup_logger()
@@ -36,18 +35,18 @@ def gf_event(event_type, **kwargs):
         logger.error("Unable to connect to events Server: {0}".format(e))
         return
 
+    port = get_config("port")
+    if port is None:
+        logger.error("Unable to get eventsd port details")
+        return
+
     # Convert key value args into KEY1=VALUE1;KEY2=VALUE2;..
     msg = ""
     for k, v in kwargs.items():
         msg += "{0}={1};".format(k, v)
 
     # <TIMESTAMP> <EVENT_TYPE> <MSG>
-    msg = "{0} {1} {2}".format(int(time.time()), event_type, msg.strip(";"))
-
-    port = get_config("port")
-    if port is None:
-        logger.error("Unable to get eventsd port details")
-        return
+    msg = "{0} {1} {2}".format(int(time.time()), event_type, msg.strip(";")).encode()
 
     try:
         sent = client.sendto(msg, (SERVER_ADDRESS, port))
diff --git a/events/src/glustereventsd.py b/events/src/glustereventsd.py
index d057e097c97..341a3b60947 100644
--- a/events/src/glustereventsd.py
+++ b/events/src/glustereventsd.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -13,21 +13,36 @@
 from __future__ import print_function
 import sys
 import signal
-import SocketServer
+import threading
+try:
+    import socketserver
+except ImportError:
+    import SocketServer as socketserver
 import socket
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 
 from eventtypes import all_events
 import handlers
 import utils
-from eventsapiconf import SERVER_ADDRESS, PID_FILE
-from utils import logger, PidFile, PidFileLockFailed
+from eventsapiconf import SERVER_ADDRESSv4, SERVER_ADDRESSv6, PID_FILE
+from eventsapiconf import AUTO_BOOL_ATTRIBUTES, AUTO_INT_ATTRIBUTES
+from utils import logger, PidFile, PidFileLockFailed, boolify
 
+# Subclass so that specifically IPv4 packets are captured
+class UDPServerv4(socketserver.ThreadingUDPServer):
+    address_family = socket.AF_INET
 
-class GlusterEventsRequestHandler(SocketServer.BaseRequestHandler):
+# Subclass so that specifically IPv6 packets are captured
+class UDPServerv6(socketserver.ThreadingUDPServer):
+    address_family = socket.AF_INET6
+
+class GlusterEventsRequestHandler(socketserver.BaseRequestHandler):
 
     def handle(self):
         data = self.request[0].strip()
+        if sys.version_info >= (3,):
+            data = self.request[0].strip().decode("utf-8")
+
         logger.debug("EVENT: {0} from {1}".format(repr(data),
                                                   self.client_address[0]))
         try:
@@ -45,10 +60,20 @@ class GlusterEventsRequestHandler(SocketServer.BaseRequestHandler):
             logger.warn("Unable to parse Event {0}".format(data))
             return
 
+        for k, v in data_dict.items():
+            try:
+                if k in AUTO_BOOL_ATTRIBUTES:
+                    data_dict[k] = boolify(v)
+                if k in AUTO_INT_ATTRIBUTES:
+                    data_dict[k] = int(v)
+            except ValueError:
+                # Auto Conversion failed, Retain the old value
+                continue
+
         try:
-            # Event Type to Function Map, Recieved event data will be in
+            # Event Type to Function Map, Received event data will be in
             # the form <TIMESTAMP> <TYPE> <DETAIL>, Get Event name for the
-            # recieved Type/Key and construct a function name starting with
+            # received Type/Key and construct a function name starting with
             # handle_ For example: handle_event_volume_create
             func_name = "handle_" + all_events[int(key)].lower()
         except IndexError:
@@ -69,26 +94,43 @@ class GlusterEventsRequestHandler(SocketServer.BaseRequestHandler):
 
 def signal_handler_sigusr2(sig, frame):
     utils.load_all()
+    utils.restart_webhook_pool()
+
+
+def UDP_server_thread(sock):
+    sock.serve_forever()
 
 
 def init_event_server():
     utils.setup_logger()
     utils.load_all()
+    utils.init_webhook_pool()
 
     port = utils.get_config("port")
     if port is None:
         sys.stderr.write("Unable to get Port details from Config\n")
         sys.exit(1)
 
-    # Start the Eventing Server, UDP Server
+    # Creating the Eventing Server, UDP Server for IPv4 packets
+    try:
+        serverv4 = UDPServerv4((SERVER_ADDRESSv4, port),
+                   GlusterEventsRequestHandler)
+    except socket.error as e:
+        sys.stderr.write("Failed to start Eventsd for IPv4: {0}\n".format(e))
+        sys.exit(1)
+    # Creating the Eventing Server, UDP Server for IPv6 packets
     try:
-        server = SocketServer.ThreadingUDPServer(
-            (SERVER_ADDRESS, port),
-            GlusterEventsRequestHandler)
+        serverv6 = UDPServerv6((SERVER_ADDRESSv6, port),
+                   GlusterEventsRequestHandler)
     except socket.error as e:
-        sys.stderr.write("Failed to start Eventsd: {0}\n".format(e))
+        sys.stderr.write("Failed to start Eventsd for IPv6: {0}\n".format(e))
         sys.exit(1)
-    server.serve_forever()
+    server_thread1 = threading.Thread(target=UDP_server_thread,
+                     args=(serverv4,))
+    server_thread2 = threading.Thread(target=UDP_server_thread,
+                     args=(serverv6,))
+    server_thread1.start()
+    server_thread2.start()
 
 
 def get_args():
diff --git a/events/src/handlers.py b/events/src/handlers.py
index 21d3e83de54..7746d488bf3 100644
--- a/events/src/handlers.py
+++ b/events/src/handlers.py
@@ -23,7 +23,7 @@ def generic_handler(ts, key, data):
 
 def handle_event_volume_set(ts, key, data):
     """
-    Recieved data will have all the options as one string, split into
+    Received data will have all the options as one string, split into
     list of options. "key1,value1,key2,value2" into
     [[key1, value1], [key2, value2]]
     """
diff --git a/events/src/peer_eventsapi.py b/events/src/peer_eventsapi.py
index 7f80f791c4d..4d2e5f35b1c 100644
--- a/events/src/peer_eventsapi.py
+++ b/events/src/peer_eventsapi.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -17,26 +17,49 @@ from errno import EEXIST
 import fcntl
 from errno import EACCES, EAGAIN
 import signal
+import sys
+import time
 
 import requests
 from prettytable import PrettyTable
 
-from gluster.cliutils import (Cmd, execute, node_output_ok, node_output_notok,
+from gluster.cliutils import (Cmd, node_output_ok, node_output_notok,
                               sync_file_to_peers, GlusterCmdException,
-                              output_error, execute_in_peers, runcli)
-from events.utils import LockedOpen
-
-from events.eventsapiconf import (WEBHOOKS_FILE_TO_SYNC,
-                                  WEBHOOKS_FILE,
-                                  DEFAULT_CONFIG_FILE,
-                                  CUSTOM_CONFIG_FILE,
-                                  CUSTOM_CONFIG_FILE_TO_SYNC,
-                                  EVENTSD,
-                                  CONFIG_KEYS,
-                                  BOOL_CONFIGS,
-                                  INT_CONFIGS,
-                                  PID_FILE,
-                                  RESTART_CONFIGS)
+                              output_error, execute_in_peers, runcli,
+                              set_common_args_func)
+from gfevents.utils import LockedOpen, get_jwt_token, save_https_cert
+
+from gfevents.eventsapiconf import (WEBHOOKS_FILE_TO_SYNC,
+                                    WEBHOOKS_FILE,
+                                    DEFAULT_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE_TO_SYNC,
+                                    EVENTSD,
+                                    CONFIG_KEYS,
+                                    BOOL_CONFIGS,
+                                    INT_CONFIGS,
+                                    PID_FILE,
+                                    RESTART_CONFIGS,
+                                    ERROR_INVALID_CONFIG,
+                                    ERROR_WEBHOOK_NOT_EXISTS,
+                                    ERROR_CONFIG_SYNC_FAILED,
+                                    ERROR_WEBHOOK_ALREADY_EXISTS,
+                                    ERROR_PARTIAL_SUCCESS,
+                                    ERROR_ALL_NODES_STATUS_NOT_OK,
+                                    ERROR_SAME_CONFIG,
+                                    ERROR_WEBHOOK_SYNC_FAILED,
+                                    CERTS_DIR)
+
+
+def handle_output_error(err, errcode=1, json_output=False):
+    if json_output:
+        print (json.dumps({
+            "output": "",
+            "error": err
+            }))
+        sys.exit(errcode)
+    else:
+        output_error(err, errcode)
 
 
 def file_content_overwrite(fname, data):
@@ -46,15 +69,27 @@ def file_content_overwrite(fname, data):
     os.rename(fname + ".tmp", fname)
 
 
-def create_custom_config_file_if_not_exists():
-    mkdirp(os.path.dirname(CUSTOM_CONFIG_FILE))
+def create_custom_config_file_if_not_exists(args):
+    try:
+        config_dir = os.path.dirname(CUSTOM_CONFIG_FILE)
+        mkdirp(config_dir)
+    except OSError as e:
+        handle_output_error("Failed to create dir %s: %s" % (config_dir, e),
+                            json_output=args.json)
+
     if not os.path.exists(CUSTOM_CONFIG_FILE):
         with open(CUSTOM_CONFIG_FILE, "w") as f:
             f.write("{}")
 
 
-def create_webhooks_file_if_not_exists():
-    mkdirp(os.path.dirname(WEBHOOKS_FILE))
+def create_webhooks_file_if_not_exists(args):
+    try:
+        webhooks_dir = os.path.dirname(WEBHOOKS_FILE)
+        mkdirp(webhooks_dir)
+    except OSError as e:
+        handle_output_error("Failed to create dir %s: %s" % (webhooks_dir, e),
+                            json_output=args.json)
+
     if not os.path.exists(WEBHOOKS_FILE):
         with open(WEBHOOKS_FILE, "w") as f:
             f.write("{}")
@@ -75,11 +110,9 @@ def mkdirp(path, exit_on_err=False, logger=None):
     """
     try:
         os.makedirs(path)
-    except (OSError, IOError) as e:
-        if e.errno == EEXIST and os.path.isdir(path):
-            pass
-        else:
-            output_error("Fail to create dir %s: %s" % (path, e))
+    except OSError as e:
+        if e.errno != EEXIST or not os.path.isdir(path):
+            raise
 
 
 def is_active():
@@ -111,33 +144,81 @@ def reload_service():
     return (0, "", "")
 
 
-def sync_to_peers():
+def rows_to_json(json_out, column_name, rows):
+    num_ok_rows = 0
+    for row in rows:
+        num_ok_rows += 1 if row.ok else 0
+        json_out.append({
+            "node": row.hostname,
+            "node_status": "UP" if row.node_up else "DOWN",
+            column_name: "OK" if row.ok else "NOT OK",
+            "error": row.error
+        })
+    return num_ok_rows
+
+
+def rows_to_table(table, rows):
+    num_ok_rows = 0
+    for row in rows:
+        num_ok_rows += 1 if row.ok else 0
+        table.add_row([row.hostname,
+                       "UP" if row.node_up else "DOWN",
+                       "OK" if row.ok else "NOT OK: {0}".format(
+                           row.error)])
+    return num_ok_rows
+
+
+def sync_to_peers(args):
     if os.path.exists(WEBHOOKS_FILE):
         try:
             sync_file_to_peers(WEBHOOKS_FILE_TO_SYNC)
         except GlusterCmdException as e:
-            output_error("Failed to sync Webhooks file: [Error: {0}]"
-                         "{1}".format(e[0], e[2]))
+            # Print stdout if stderr is empty
+            errmsg = e.message[2] if e.message[2] else e.message[1]
+            handle_output_error("Failed to sync Webhooks file: [Error: {0}]"
+                                "{1}".format(e.message[0], errmsg),
+                                errcode=ERROR_WEBHOOK_SYNC_FAILED,
+                                json_output=args.json)
 
     if os.path.exists(CUSTOM_CONFIG_FILE):
         try:
             sync_file_to_peers(CUSTOM_CONFIG_FILE_TO_SYNC)
         except GlusterCmdException as e:
-            output_error("Failed to sync Config file: [Error: {0}]"
-                         "{1}".format(e[0], e[2]))
+            # Print stdout if stderr is empty
+            errmsg = e.message[2] if e.message[2] else e.message[1]
+            handle_output_error("Failed to sync Config file: [Error: {0}]"
+                                "{1}".format(e.message[0], errmsg),
+                                errcode=ERROR_CONFIG_SYNC_FAILED,
+                                json_output=args.json)
 
     out = execute_in_peers("node-reload")
-    table = PrettyTable(["NODE", "NODE STATUS", "SYNC STATUS"])
-    table.align["NODE STATUS"] = "r"
-    table.align["SYNC STATUS"] = "r"
+    if not args.json:
+        table = PrettyTable(["NODE", "NODE STATUS", "SYNC STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["SYNC STATUS"] = "r"
 
-    for p in out:
-        table.add_row([p.hostname,
-                       "UP" if p.node_up else "DOWN",
-                       "OK" if p.ok else "NOT OK: {0}".format(
-                           p.error)])
+    json_out = []
+    if args.json:
+        num_ok_rows = rows_to_json(json_out, "sync_status", out)
+    else:
+        num_ok_rows = rows_to_table(table, out)
+
+    ret = 0
+    if num_ok_rows == 0:
+        ret = ERROR_ALL_NODES_STATUS_NOT_OK
+    elif num_ok_rows != len(out):
+        ret = ERROR_PARTIAL_SUCCESS
+
+    if args.json:
+        print (json.dumps({
+            "output": json_out,
+            "error": ""
+        }))
+    else:
+        print (table)
 
-    print (table)
+    # If sync status is not ok for any node set error code as partial success
+    sys.exit(ret)
 
 
 def node_output_handle(resp):
@@ -148,29 +229,24 @@ def node_output_handle(resp):
         node_output_notok(err)
 
 
-def action_handle(action):
+def action_handle(action, json_output=False):
     out = execute_in_peers("node-" + action)
     column_name = action.upper()
     if action == "status":
         column_name = EVENTSD.upper()
 
-    table = PrettyTable(["NODE", "NODE STATUS", column_name + " STATUS"])
-    table.align["NODE STATUS"] = "r"
-    table.align[column_name + " STATUS"] = "r"
-
-    for p in out:
-        status_col_val = "OK" if p.ok else "NOT OK: {0}".format(
-            p.error)
-        if action == "status":
-            status_col_val = "DOWN"
-            if p.ok:
-                status_col_val = p.output
+    if not json_output:
+        table = PrettyTable(["NODE", "NODE STATUS", column_name + " STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align[column_name + " STATUS"] = "r"
 
-        table.add_row([p.hostname,
-                       "UP" if p.node_up else "DOWN",
-                       status_col_val])
+    json_out = []
+    if json_output:
+        rows_to_json(json_out, column_name.lower() + "_status", out)
+    else:
+        rows_to_table(table, out)
 
-    print (table)
+    return json_out if json_output else table
 
 
 class NodeReload(Cmd):
@@ -184,7 +260,14 @@ class ReloadCmd(Cmd):
     name = "reload"
 
     def run(self, args):
-        action_handle("reload")
+        out = action_handle("reload", args.json)
+        if args.json:
+            print (json.dumps({
+                "output": out,
+                "error": ""
+            }))
+        else:
+            print (out)
 
 
 class NodeStatus(Cmd):
@@ -202,12 +285,25 @@ class StatusCmd(Cmd):
         if os.path.exists(WEBHOOKS_FILE):
             webhooks = json.load(open(WEBHOOKS_FILE))
 
-        print ("Webhooks: " + ("" if webhooks else "None"))
-        for w in webhooks:
-            print (w)
-
-        print ()
-        action_handle("status")
+        json_out = {"webhooks": [], "data": []}
+        if args.json:
+            json_out["webhooks"] = webhooks.keys()
+        else:
+            print ("Webhooks: " + ("" if webhooks else "None"))
+            for w in webhooks:
+                print (w)
+
+            print ()
+
+        out = action_handle("status", args.json)
+        if args.json:
+            json_out["data"] = out
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
+        else:
+            print (out)
 
 
 class WebhookAddCmd(Cmd):
@@ -217,19 +313,24 @@ class WebhookAddCmd(Cmd):
         parser.add_argument("url", help="URL of Webhook")
         parser.add_argument("--bearer_token", "-t", help="Bearer Token",
                             default="")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to add JWT Bearer Token", default="")
 
     def run(self, args):
-        create_webhooks_file_if_not_exists()
+        create_webhooks_file_if_not_exists(args)
 
         with LockedOpen(WEBHOOKS_FILE, 'r+'):
             data = json.load(open(WEBHOOKS_FILE))
             if data.get(args.url, None) is not None:
-                output_error("Webhook already exists")
+                handle_output_error("Webhook already exists",
+                                    errcode=ERROR_WEBHOOK_ALREADY_EXISTS,
+                                    json_output=args.json)
 
-            data[args.url] = args.bearer_token
+            data[args.url] = {"token": args.bearer_token,
+                              "secret": args.secret}
             file_content_overwrite(WEBHOOKS_FILE, data)
 
-        sync_to_peers()
+        sync_to_peers(args)
 
 
 class WebhookModCmd(Cmd):
@@ -239,19 +340,31 @@ class WebhookModCmd(Cmd):
         parser.add_argument("url", help="URL of Webhook")
         parser.add_argument("--bearer_token", "-t", help="Bearer Token",
                             default="")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to add JWT Bearer Token", default="")
 
     def run(self, args):
-        create_webhooks_file_if_not_exists()
+        create_webhooks_file_if_not_exists(args)
 
         with LockedOpen(WEBHOOKS_FILE, 'r+'):
             data = json.load(open(WEBHOOKS_FILE))
             if data.get(args.url, None) is None:
-                output_error("Webhook does not exists")
+                handle_output_error("Webhook does not exists",
+                                    errcode=ERROR_WEBHOOK_NOT_EXISTS,
+                                    json_output=args.json)
+
+            if isinstance(data[args.url], str):
+                data[args.url]["token"] = data[args.url]
+
+            if args.bearer_token != "":
+                data[args.url]["token"] = args.bearer_token
+
+            if args.secret != "":
+                data[args.url]["secret"] = args.secret
 
-            data[args.url] = args.bearer_token
             file_content_overwrite(WEBHOOKS_FILE, data)
 
-        sync_to_peers()
+        sync_to_peers(args)
 
 
 class WebhookDelCmd(Cmd):
@@ -261,17 +374,19 @@ class WebhookDelCmd(Cmd):
         parser.add_argument("url", help="URL of Webhook")
 
     def run(self, args):
-        create_webhooks_file_if_not_exists()
+        create_webhooks_file_if_not_exists(args)
 
         with LockedOpen(WEBHOOKS_FILE, 'r+'):
             data = json.load(open(WEBHOOKS_FILE))
             if data.get(args.url, None) is None:
-                output_error("Webhook does not exists")
+                handle_output_error("Webhook does not exists",
+                                    errcode=ERROR_WEBHOOK_NOT_EXISTS,
+                                    json_output=args.json)
 
             del data[args.url]
             file_content_overwrite(WEBHOOKS_FILE, data)
 
-        sync_to_peers()
+        sync_to_peers(args)
 
 
 class NodeWebhookTestCmd(Cmd):
@@ -280,16 +395,57 @@ class NodeWebhookTestCmd(Cmd):
     def args(self, parser):
         parser.add_argument("url")
         parser.add_argument("bearer_token")
+        parser.add_argument("secret")
 
     def run(self, args):
         http_headers = {}
+        hashval = ""
         if args.bearer_token != ".":
-            http_headers["Authorization"] = "Bearer " + args.bearer_token
+            hashval = args.bearer_token
 
-        try:
-            resp = requests.post(args.url, headers=http_headers)
-        except requests.ConnectionError as e:
-            node_output_notok("{0}".format(e))
+        if args.secret != ".":
+            hashval = get_jwt_token(args.secret, "TEST", int(time.time()))
+
+        if hashval:
+            http_headers["Authorization"] = "Bearer " + hashval
+
+        urldata = requests.utils.urlparse(args.url)
+        parts = urldata.netloc.split(":")
+        domain = parts[0]
+        # Default https port if not specified
+        port = 443
+        if len(parts) == 2:
+            port = int(parts[1])
+
+        cert_path = os.path.join(CERTS_DIR, args.url.replace("/", "_").strip())
+        verify = True
+        while True:
+            try:
+                resp = requests.post(args.url, headers=http_headers,
+                                     verify=verify)
+                # Successful webhook push
+                break
+            except requests.exceptions.SSLError as e:
+                # If verify is equal to cert path, but still failed with
+                # SSLError, Looks like some issue with custom downloaded
+                # certificate, Try with verify = false
+                if verify == cert_path:
+                    verify = False
+                    continue
+
+                # If verify is instance of bool and True, then custom cert
+                # is required, download the cert and retry
+                try:
+                    save_https_cert(domain, port, cert_path)
+                    verify = cert_path
+                except Exception:
+                    verify = False
+
+                # Done with collecting cert, continue
+                continue
+            except Exception as e:
+                node_output_notok("{0}".format(e))
+                break
 
         if resp.status_code != 200:
             node_output_notok("{0}".format(resp.status_code))
@@ -303,28 +459,51 @@ class WebhookTestCmd(Cmd):
     def args(self, parser):
         parser.add_argument("url", help="URL of Webhook")
         parser.add_argument("--bearer_token", "-t", help="Bearer Token")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to generate Bearer Token")
 
     def run(self, args):
         url = args.url
         bearer_token = args.bearer_token
+        secret = args.secret
+
         if not args.url:
             url = "."
         if not args.bearer_token:
             bearer_token = "."
+        if not args.secret:
+            secret = "."
 
-        out = execute_in_peers("node-webhook-test", [url, bearer_token])
+        out = execute_in_peers("node-webhook-test", [url, bearer_token,
+                                                     secret])
 
-        table = PrettyTable(["NODE", "NODE STATUS", "WEBHOOK STATUS"])
-        table.align["NODE STATUS"] = "r"
-        table.align["WEBHOOK STATUS"] = "r"
+        if not args.json:
+            table = PrettyTable(["NODE", "NODE STATUS", "WEBHOOK STATUS"])
+            table.align["NODE STATUS"] = "r"
+            table.align["WEBHOOK STATUS"] = "r"
 
-        for p in out:
-            table.add_row([p.hostname,
-                           "UP" if p.node_up else "DOWN",
-                           "OK" if p.ok else "NOT OK: {0}".format(
-                               p.error)])
+        num_ok_rows = 0
+        json_out = []
+        if args.json:
+            num_ok_rows = rows_to_json(json_out, "webhook_status", out)
+        else:
+            num_ok_rows = rows_to_table(table, out)
+
+        ret = 0
+        if num_ok_rows == 0:
+            ret = ERROR_ALL_NODES_STATUS_NOT_OK
+        elif num_ok_rows != len(out):
+            ret = ERROR_PARTIAL_SUCCESS
+
+        if args.json:
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
+        else:
+            print (table)
 
-        print (table)
+        sys.exit(ret)
 
 
 class ConfigGetCmd(Cmd):
@@ -339,16 +518,30 @@ class ConfigGetCmd(Cmd):
             data.update(json.load(open(CUSTOM_CONFIG_FILE)))
 
         if args.name is not None and args.name not in CONFIG_KEYS:
-            output_error("Invalid Config item")
+            handle_output_error("Invalid Config item",
+                                errcode=ERROR_INVALID_CONFIG,
+                                json_output=args.json)
+
+        if args.json:
+            json_out = {}
+            if args.name is None:
+                json_out = data
+            else:
+                json_out[args.name] = data[args.name]
 
-        table = PrettyTable(["NAME", "VALUE"])
-        if args.name is None:
-            for k, v in data.items():
-                table.add_row([k, v])
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
         else:
-            table.add_row([args.name, data[args.name]])
+            table = PrettyTable(["NAME", "VALUE"])
+            if args.name is None:
+                for k, v in data.items():
+                    table.add_row([k, v])
+            else:
+                table.add_row([args.name, data[args.name]])
 
-        print (table)
+            print (table)
 
 
 def read_file_content_json(fname):
@@ -370,9 +563,11 @@ class ConfigSetCmd(Cmd):
 
     def run(self, args):
         if args.name not in CONFIG_KEYS:
-            output_error("Invalid Config item")
+            handle_output_error("Invalid Config item",
+                                errcode=ERROR_INVALID_CONFIG,
+                                json_output=args.json)
 
-        create_custom_config_file_if_not_exists()
+        create_custom_config_file_if_not_exists(args)
 
         with LockedOpen(CUSTOM_CONFIG_FILE, 'r+'):
             data = json.load(open(DEFAULT_CONFIG_FILE))
@@ -382,7 +577,9 @@ class ConfigSetCmd(Cmd):
 
             # Do Nothing if same as previous value
             if data[args.name] == args.value:
-                return
+                handle_output_error("Config value not changed. Same config",
+                                    errcode=ERROR_SAME_CONFIG,
+                                    json_output=args.json)
 
             # TODO: Validate Value
             new_data = read_file_content_json(CUSTOM_CONFIG_FILE)
@@ -402,10 +599,11 @@ class ConfigSetCmd(Cmd):
             if args.name in RESTART_CONFIGS:
                 restart = True
 
-            sync_to_peers()
             if restart:
                 print ("\nRestart glustereventsd in all nodes")
 
+            sync_to_peers(args)
+
 
 class ConfigResetCmd(Cmd):
     name = "config-reset"
@@ -414,7 +612,7 @@ class ConfigResetCmd(Cmd):
         parser.add_argument("name", help="Config Name or all")
 
     def run(self, args):
-        create_custom_config_file_if_not_exists()
+        create_custom_config_file_if_not_exists(args)
 
         with LockedOpen(CUSTOM_CONFIG_FILE, 'r+'):
             changed_keys = []
@@ -422,8 +620,14 @@ class ConfigResetCmd(Cmd):
             if os.path.exists(CUSTOM_CONFIG_FILE):
                 data = read_file_content_json(CUSTOM_CONFIG_FILE)
 
-            if not data:
-                return
+            # If No data available in custom config or, the specific config
+            # item is not available in custom config
+            if not data or \
+               (args.name != "all" and data.get(args.name, None) is None):
+                handle_output_error("Config value not reset. Already "
+                                    "set to default value",
+                                    errcode=ERROR_SAME_CONFIG,
+                                    json_output=args.json)
 
             if args.name.lower() == "all":
                 for k, v in data.items():
@@ -443,17 +647,23 @@ class ConfigResetCmd(Cmd):
                     restart = True
                     break
 
-            sync_to_peers()
             if restart:
                 print ("\nRestart glustereventsd in all nodes")
 
+            sync_to_peers(args)
+
 
 class SyncCmd(Cmd):
     name = "sync"
 
     def run(self, args):
-        sync_to_peers()
+        sync_to_peers(args)
+
+
+def common_args(parser):
+    parser.add_argument("--json", help="JSON Output", action="store_true")
 
 
 if __name__ == "__main__":
+    set_common_args_func(common_args)
     runcli()
diff --git a/events/src/utils.py b/events/src/utils.py
index dadd9ae3332..6d4e0791a2b 100644
--- a/events/src/utils.py
+++ b/events/src/utils.py
@@ -9,22 +9,39 @@
 #  cases as published by the Free Software Foundation.
 #
 
+import sys
 import json
 import os
 import logging
+import logging.handlers
 import fcntl
-from errno import ESRCH, EBADF
-
-from eventsapiconf import (LOG_FILE,
-                           WEBHOOKS_FILE,
-                           DEFAULT_CONFIG_FILE,
-                           CUSTOM_CONFIG_FILE,
-                           UUID_FILE)
-import eventtypes
+from errno import EBADF
+from threading import Thread
+import multiprocessing
+try:
+    from queue import Queue
+except ImportError:
+    from Queue import Queue
+from datetime import datetime, timedelta
+import base64
+import hmac
+from hashlib import sha256
+from calendar import timegm
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from gfevents.eventsapiconf import (LOG_FILE,
+                                    WEBHOOKS_FILE,
+                                    DEFAULT_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE,
+                                    UUID_FILE,
+                                    CERTS_DIR)
+from gfevents import eventtypes
 
 
 # Webhooks list
 _webhooks = {}
+_webhooks_file_mtime = 0
 # Default Log Level
 _log_level = "INFO"
 # Config Object
@@ -33,6 +50,21 @@ _config = {}
 # Init Logger instance
 logger = logging.getLogger(__name__)
 NodeID = None
+webhooks_pool = None
+
+
+def boolify(value):
+    value = str(value)
+    if value.lower() in ["1", "on", "true", "yes"]:
+        return True
+    else:
+        return False
+
+
+def log_event(data):
+    # Log all published events unless it is disabled
+    if not _config.get("disable-events-log", False):
+        logger.info(repr(data))
 
 
 def get_node_uuid():
@@ -45,10 +77,10 @@ def get_node_uuid():
     return val
 
 
-def get_config(key):
+def get_config(key, default_value=None):
     if not _config:
         load_config()
-    return _config.get(key, None)
+    return _config.get(key, default_value)
 
 
 def get_event_type_name(idx):
@@ -67,7 +99,7 @@ def setup_logger():
     logger.setLevel(logging.INFO)
 
     # create the logging file handler
-    fh = logging.FileHandler(LOG_FILE)
+    fh = logging.handlers.WatchedFileHandler(LOG_FILE)
 
     formatter = logging.Formatter("[%(asctime)s] %(levelname)s "
                                   "[%(module)s - %(lineno)s:%(funcName)s] "
@@ -98,7 +130,7 @@ def load_log_level():
     be triggered during init and when SIGUSR2.
     """
     global logger, _log_level
-    new_log_level = _config.get("log_level", "INFO")
+    new_log_level = _config.get("log-level", "INFO")
     if _log_level != new_log_level:
         logger.setLevel(getattr(logging, new_log_level.upper()))
         _log_level = new_log_level.upper()
@@ -109,10 +141,12 @@ def load_webhooks():
     Load/Reload the webhooks list. This function will
     be triggered during init and when SIGUSR2.
     """
-    global _webhooks
+    global _webhooks, _webhooks_file_mtime
     _webhooks = {}
     if os.path.exists(WEBHOOKS_FILE):
         _webhooks = json.load(open(WEBHOOKS_FILE))
+        st = os.lstat(WEBHOOKS_FILE)
+        _webhooks_file_mtime = st.st_mtime
 
 
 def load_all():
@@ -130,12 +164,17 @@ def publish(ts, event_key, data):
     if NodeID is None:
         NodeID = get_node_uuid()
 
+    autoload_webhooks()
+
     message = {
         "nodeid": NodeID,
         "ts": int(ts),
         "event": get_event_type_name(event_key),
         "message": data
     }
+
+    log_event(message)
+
     if _webhooks:
         plugin_webhook(message)
     else:
@@ -143,35 +182,135 @@ def publish(ts, event_key, data):
         pass
 
 
-def plugin_webhook(message):
+def autoload_webhooks():
+    global _webhooks_file_mtime
+    try:
+        st = os.lstat(WEBHOOKS_FILE)
+    except OSError:
+        st = None
+
+    if st is not None:
+        # If Stat is available and mtime is not matching with
+        # previously recorded mtime, reload the webhooks file
+        if st.st_mtime != _webhooks_file_mtime:
+            load_webhooks()
+
+
+def base64_urlencode(inp):
+    return base64.urlsafe_b64encode(inp).replace("=", "").strip()
+
+
+def get_jwt_token(secret, event_type, event_ts, jwt_expiry_time_seconds=60):
+    exp = datetime.utcnow() + timedelta(seconds=jwt_expiry_time_seconds)
+    payload = {
+        "exp": timegm(exp.utctimetuple()),
+        "iss": "gluster",
+        "sub": event_type,
+        "iat": event_ts
+    }
+    header = '{"alg":"HS256","typ":"JWT"}'
+    payload = json.dumps(payload, separators=(',', ':'), sort_keys=True)
+    msg = base64_urlencode(header) + "." + base64_urlencode(payload)
+    return "%s.%s" % (
+        msg,
+        base64_urlencode(hmac.HMAC(str(secret), msg, sha256).digest())
+    )
+
+
+def save_https_cert(domain, port, cert_path):
+    import ssl
+
+    # Cert file already available for this URL
+    if os.path.exists(cert_path):
+        return
+
+    cert_data = ssl.get_server_certificate((domain, port))
+    with open(cert_path, "w") as f:
+        f.write(cert_data)
+
+
+def publish_to_webhook(url, token, secret, message_queue):
     # Import requests here since not used in any other place
     import requests
 
-    message_json = json.dumps(message, sort_keys=True)
-    logger.debug("EVENT: {0}".format(message_json))
-    for url, token in _webhooks.items():
-        http_headers = {"Content-Type": "application/json"}
+    http_headers = {"Content-Type": "application/json"}
+    urldata = requests.utils.urlparse(url)
+    parts = urldata.netloc.split(":")
+    domain = parts[0]
+    # Default https port if not specified
+    port = 443
+    if len(parts) == 2:
+        port = int(parts[1])
+
+    cert_path = os.path.join(CERTS_DIR, url.replace("/", "_").strip())
+
+    while True:
+        hashval = ""
+        event_type, event_ts, message_json = message_queue.get()
         if token != "" and token is not None:
-            http_headers["Authorization"] = "Bearer " + token
+            hashval = token
 
-        try:
-            resp = requests.post(url, headers=http_headers, data=message_json)
-        except requests.ConnectionError as e:
-            logger.warn("Event push failed to URL: {url}, "
-                        "Event: {event}, "
-                        "Status: {error}".format(
-                            url=url,
-                            event=message_json,
-                            error=e))
-            continue
-
-        if resp.status_code != 200:
-            logger.warn("Event push failed to URL: {url}, "
-                        "Event: {event}, "
-                        "Status Code: {status_code}".format(
-                            url=url,
-                            event=message_json,
-                            status_code=resp.status_code))
+        if secret != "" and secret is not None:
+            hashval = get_jwt_token(secret, event_type, event_ts)
+
+        if hashval:
+            http_headers["Authorization"] = "Bearer " + hashval
+
+        verify = True
+        while True:
+            try:
+                resp = requests.post(url, headers=http_headers,
+                                     data=message_json,
+                                     verify=verify)
+                # Successful webhook push
+                message_queue.task_done()
+                if resp.status_code != 200:
+                    logger.warn("Event push failed to URL: {url}, "
+                                "Event: {event}, "
+                                "Status Code: {status_code}".format(
+                                    url=url,
+                                    event=message_json,
+                                    status_code=resp.status_code))
+                break
+            except requests.exceptions.SSLError as e:
+                # If verify is equal to cert path, but still failed with
+                # SSLError, Looks like some issue with custom downloaded
+                # certificate, Try with verify = false
+                if verify == cert_path:
+                    logger.warn("Event push failed with certificate, "
+                                "ignoring verification url={0} "
+                                "Error={1}".format(url, e))
+                    verify = False
+                    continue
+
+                # If verify is instance of bool and True, then custom cert
+                # is required, download the cert and retry
+                try:
+                    save_https_cert(domain, port, cert_path)
+                    verify = cert_path
+                except Exception as ex:
+                    verify = False
+                    logger.warn("Unable to get Server certificate, "
+                                "ignoring verification url={0} "
+                                "Error={1}".format(url, ex))
+
+                # Done with collecting cert, continue
+                continue
+            except Exception as e:
+                logger.warn("Event push failed to URL: {url}, "
+                            "Event: {event}, "
+                            "Status: {error}".format(
+                                url=url,
+                                event=message_json,
+                                error=e))
+                message_queue.task_done()
+                break
+
+
+def plugin_webhook(message):
+    message_json = json.dumps(message, sort_keys=True)
+    logger.debug("EVENT: {0}".format(message_json))
+    webhooks_pool.send(message["event"], message["ts"], message_json)
 
 
 class LockedOpen(object):
@@ -247,3 +386,60 @@ class PidFile(object):
 
     def __exit__(self, _exc_type, _exc_value, _traceback):
         self.cleanup()
+
+
+def webhook_monitor(proc_queue, webhooks):
+    queues = {}
+    for url, data in webhooks.items():
+        if isinstance(data, str):
+            token = data
+            secret = None
+        else:
+            token = data["token"]
+            secret = data["secret"]
+
+        queues[url] = Queue()
+        t = Thread(target=publish_to_webhook, args=(url, token, secret,
+                                                    queues[url]))
+        t.start()
+
+    # Get the message sent to Process queue and distribute to all thread queues
+    while True:
+        message = proc_queue.get()
+        for _, q in queues.items():
+            q.put(message)
+
+
+class WebhookThreadPool(object):
+    def start(self):
+        # Separate process to emit messages to webhooks
+        # which maintains one thread per webhook. Separate
+        # process is required since on reload we need to stop
+        # and start the thread pool. In Python Threads can't be stopped
+        # so terminate the process and start again. Note: In transit
+        # events will be missed during reload
+        self.queue = multiprocessing.Queue()
+        self.proc = multiprocessing.Process(target=webhook_monitor,
+                                            args=(self.queue, _webhooks))
+        self.proc.start()
+
+    def restart(self):
+        # In transit messages are skipped, since webhooks monitor process
+        # is terminated.
+        self.proc.terminate()
+        self.start()
+
+    def send(self, event_type, event_ts, message):
+        self.queue.put((event_type, event_ts, message))
+
+
+def init_webhook_pool():
+    global webhooks_pool
+    webhooks_pool = WebhookThreadPool()
+    webhooks_pool.start()
+
+
+def restart_webhook_pool():
+    global webhooks_pool
+    if webhooks_pool is not None:
+        webhooks_pool.restart()
diff --git a/events/tools/eventsdash.py b/events/tools/eventsdash.py
index 47fc56dda6e..6479ea59da6 100644
--- a/events/tools/eventsdash.py
+++ b/events/tools/eventsdash.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -10,6 +10,7 @@
 #  cases as published by the Free Software Foundation.
 #
 
+from __future__ import print_function
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 import logging
 from datetime import datetime
@@ -41,11 +42,11 @@ def listen():
     for k, v in data.get("message", {}).items():
         message.append("{0}={1}".format(k, v))
 
-    print ("{0:20s} {1:20s} {2:36} {3}".format(
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
         human_time(data.get("ts")),
         data.get("event"),
         data.get("nodeid"),
-        " ".join(message)))
+        " ".join(message))))
 
     return "OK"
 
@@ -58,12 +59,12 @@ def main():
                         action="store_true")
     args = parser.parse_args()
 
-    print ("{0:20s} {1:20s} {2:36} {3}".format(
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
         "TIMESTAMP", "EVENT", "NODE ID", "MESSAGE"
-    ))
-    print ("{0:20s} {1:20s} {2:36} {3}".format(
+    )))
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
         "-"*20, "-"*20, "-"*36, "-"*20
-    ))
+    )))
     if args.debug:
         app.debug = True
 
diff --git a/extras/LinuxRPM/Makefile.am b/extras/LinuxRPM/Makefile.am
index 61fd6da74c8..f02853798c0 100644
--- a/extras/LinuxRPM/Makefile.am
+++ b/extras/LinuxRPM/Makefile.am
@@ -18,7 +18,7 @@ autogen:
 	cd ../.. && \
 	rm -rf autom4te.cache && \
 	./autogen.sh && \
-	./configure --with-previous-options
+	./configure --enable-gnfs --with-previous-options
 
 prep:
 	$(MAKE) -C ../.. dist;
@@ -36,7 +36,7 @@ srcrpm:
 	mv rpmbuild/SRPMS/* .
 
 rpms:
-	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bb rpmbuild/SPECS/glusterfs.spec
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' --with gnfs -bb rpmbuild/SPECS/glusterfs.spec
 	mv rpmbuild/RPMS/*/* .
 
 # EPEL-5 does not like new versions of rpmbuild and requires some
diff --git a/extras/Makefile.am b/extras/Makefile.am
index 091d7a9df36..983f014cca6 100644
--- a/extras/Makefile.am
+++ b/extras/Makefile.am
@@ -1,29 +1,58 @@
-addonexecdir = $(libexecdir)/glusterfs
-addonexec_SCRIPTS = peer_add_secret_pub
+addonexecdir = $(GLUSTERFS_LIBEXECDIR)
+addonexec_SCRIPTS =
+if WITH_SERVER
+addonexec_SCRIPTS += peer_add_secret_pub
+if USE_SYSTEMD
+addonexec_SCRIPTS += mount-shared-storage.sh
+endif
+endif
 
 EditorModedir = $(docdir)
 EditorMode_DATA = glusterfs-mode.el glusterfs.vim
 
 SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM \
-          $(GEOREP_EXTRAS_SUBDIR) ganesha snap_scheduler firewalld cliutils
+          $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils python \
+		  ganesha
 
 confdir = $(sysconfdir)/glusterfs
+if WITH_SERVER
 conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
-	logger.conf.example glusterfs-georep-logrotate group-virt.example
+	    logger.conf.example glusterfs-georep-logrotate group-virt.example \
+	    group-metadata-cache group-gluster-block group-nl-cache \
+	    group-db-workload group-distributed-virt group-samba
+endif
 
 voldir = $(sysconfdir)/glusterfs
-vol_DATA = glusterd.vol
+vol_DATA = thin-arbiter/thin-arbiter.vol
+if WITH_SERVER
+vol_DATA += glusterd.vol
+endif
+
 scriptsdir = $(datadir)/glusterfs/scripts
-scripts_SCRIPTS = post-upgrade-script-for-quota.sh \
+scripts_SCRIPTS = thin-arbiter/setup-thin-arbiter.sh
+if WITH_SERVER
+scripts_SCRIPTS += post-upgrade-script-for-quota.sh \
 	pre-upgrade-script-for-quota.sh stop-all-gluster-processes.sh
+if USE_SYSTEMD
+scripts_SCRIPTS += control-cpu-load.sh
+scripts_SCRIPTS += control-mem.sh
+endif
+endif
 
-EXTRA_DIST = $(conf_DATA) specgen.scm glusterfs-mode.el glusterfs.vim \
-	migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh \
-	disk_usage_sync.sh clear_xattrs.sh glusterd-sysconfig glusterd.vol \
-	post-upgrade-script-for-quota.sh pre-upgrade-script-for-quota.sh \
-	command-completion/gluster.bash command-completion/Makefile \
-	command-completion/README stop-all-gluster-processes.sh clang-checker.sh
+EXTRA_DIST = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
+	logger.conf.example glusterfs-georep-logrotate group-virt.example \
+	group-metadata-cache group-gluster-block group-nl-cache \
+	group-db-workload group-samba specgen.scm glusterfs-mode.el glusterfs.vim \
+	migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \
+	backend-cleanup.sh disk_usage_sync.sh clear_xattrs.sh \
+	glusterd-sysconfig glusterd.vol post-upgrade-script-for-quota.sh \
+	pre-upgrade-script-for-quota.sh command-completion/gluster.bash \
+	command-completion/Makefile command-completion/README \
+	stop-all-gluster-processes.sh clang-checker.sh mount-shared-storage.sh \
+	control-cpu-load.sh control-mem.sh group-distributed-virt \
+	thin-arbiter/thin-arbiter.vol thin-arbiter/setup-thin-arbiter.sh
 
+if WITH_SERVER
 install-data-local:
 	if [ -n "$(tmpfilesdir)" ]; then \
 		$(mkdir_p) $(DESTDIR)$(tmpfilesdir); \
@@ -33,3 +62,16 @@ install-data-local:
 	$(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)/groups
 	$(INSTALL_DATA) $(top_srcdir)/extras/group-virt.example \
 		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/virt
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-metadata-cache \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/metadata-cache
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-gluster-block \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/gluster-block
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-nl-cache \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/nl-cache
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-db-workload \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/db-workload
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-distributed-virt \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/distributed-virt
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-samba \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/samba
+endif
diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c
index dc717f33c16..f7f5873f84d 100644
--- a/extras/benchmarking/glfs-bm.c
+++ b/extras/benchmarking/glfs-bm.c
@@ -25,365 +25,338 @@
 #include <sys/time.h>
 
 struct state {
-        char need_op_write:1;
-        char need_op_read:1;
+    char need_op_write : 1;
+    char need_op_read : 1;
 
-        char need_iface_fileio:1;
-        char need_iface_xattr:1;
+    char need_iface_fileio : 1;
+    char need_iface_xattr : 1;
 
-        char need_mode_posix:1;
+    char need_mode_posix : 1;
 
-        char prefix[512];
-        long int count;
+    char prefix[512];
+    long int count;
 
-        size_t block_size;
+    size_t block_size;
 
-        char *specfile;
+    char *specfile;
 
-        long int io_size;
+    long int io_size;
 };
 
-
-#define MEASURE(func, arg) measure (func, #func, arg)
-
+#define MEASURE(func, arg) measure(func, #func, arg)
 
 void
-tv_difference (struct timeval *tv_stop,
-               struct timeval *tv_start,
-               struct timeval *tv_diff)
+tv_difference(struct timeval *tv_stop, struct timeval *tv_start,
+              struct timeval *tv_diff)
 {
-        if (tv_stop->tv_usec < tv_start->tv_usec) {
-                tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec;
-                tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec);
-        } else {
-                tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec;
-                tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec;
-        }
+    if (tv_stop->tv_usec < tv_start->tv_usec) {
+        tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec;
+        tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec);
+    } else {
+        tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec;
+        tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec;
+    }
 }
 
-
 void
-measure (int (*func)(struct state *state),
-         char *func_name, struct state *state)
+measure(int (*func)(struct state *state), char *func_name, struct state *state)
 {
-        struct timeval tv_start, tv_stop, tv_diff;
-        state->io_size = 0;
-        long int count;
+    struct timeval tv_start, tv_stop, tv_diff;
+    state->io_size = 0;
+    long int count;
 
-        gettimeofday (&tv_start, NULL);
-        count = func (state);
-        gettimeofday (&tv_stop, NULL);
+    gettimeofday(&tv_start, NULL);
+    count = func(state);
+    gettimeofday(&tv_stop, NULL);
 
-        tv_difference (&tv_stop, &tv_start, &tv_diff);
+    tv_difference(&tv_stop, &tv_start, &tv_diff);
 
-        fprintf (stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n",
-                 func_name, count, state->io_size,
-                 tv_diff.tv_sec, tv_diff.tv_usec);
+    fprintf(stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", func_name, count,
+            state->io_size, tv_diff.tv_sec, tv_diff.tv_usec);
 }
 
-
 static error_t
-parse_opts (int key, char *arg,
-            struct argp_state *_state)
+parse_opts(int key, char *arg, struct argp_state *_state)
 {
-        struct state *state = _state->input;
+    struct state *state = _state->input;
 
-        switch (key)
-        {
+    switch (key) {
         case 'o':
-                if (strcasecmp (arg, "read") == 0) {
-                        state->need_op_write = 0;
-                        state->need_op_read = 1;
-                } else if (strcasecmp (arg, "write") == 0) {
-                        state->need_op_write = 1;
-                        state->need_op_read = 0;
-                } else if (strcasecmp (arg, "both") == 0) {
-                        state->need_op_write = 1;
-                        state->need_op_read = 1;
-                } else {
-                        fprintf (stderr, "unknown op: %s\n", arg);
-                        return -1;
-                }
-                break;
+            if (strcasecmp(arg, "read") == 0) {
+                state->need_op_write = 0;
+                state->need_op_read = 1;
+            } else if (strcasecmp(arg, "write") == 0) {
+                state->need_op_write = 1;
+                state->need_op_read = 0;
+            } else if (strcasecmp(arg, "both") == 0) {
+                state->need_op_write = 1;
+                state->need_op_read = 1;
+            } else {
+                fprintf(stderr, "unknown op: %s\n", arg);
+                return -1;
+            }
+            break;
         case 'i':
-                if (strcasecmp (arg, "fileio") == 0) {
-                        state->need_iface_fileio = 1;
-                        state->need_iface_xattr = 0;
-                } else if (strcasecmp (arg, "xattr") == 0) {
-                        state->need_iface_fileio = 0;
-                        state->need_iface_xattr = 1;
-                } else if (strcasecmp (arg, "both") == 0) {
-                        state->need_iface_fileio = 1;
-                        state->need_iface_xattr = 1;
-                } else {
-                        fprintf (stderr, "unknown interface: %s\n", arg);
-                        return -1;
-                }
-                break;
-        case 'b':
-        {
-                size_t block_size = atoi (arg);
-                if (!block_size) {
-                        fprintf (stderr, "incorrect size: %s\n", arg);
-                        return -1;
-                }
-                state->block_size = block_size;
-        }
-        break;
+            if (strcasecmp(arg, "fileio") == 0) {
+                state->need_iface_fileio = 1;
+                state->need_iface_xattr = 0;
+            } else if (strcasecmp(arg, "xattr") == 0) {
+                state->need_iface_fileio = 0;
+                state->need_iface_xattr = 1;
+            } else if (strcasecmp(arg, "both") == 0) {
+                state->need_iface_fileio = 1;
+                state->need_iface_xattr = 1;
+            } else {
+                fprintf(stderr, "unknown interface: %s\n", arg);
+                return -1;
+            }
+            break;
+        case 'b': {
+            size_t block_size = atoi(arg);
+            if (!block_size) {
+                fprintf(stderr, "incorrect size: %s\n", arg);
+                return -1;
+            }
+            state->block_size = block_size;
+        } break;
         case 's':
-                state->specfile = strdup (arg);
-                break;
+            state->specfile = strdup(arg);
+            break;
         case 'p':
-                fprintf (stderr, "using prefix: %s\n", arg);
-                strncpy (state->prefix, arg, 512);
-                break;
-        case 'c':
-        {
-                long count = atol (arg);
-                if (!count) {
-                        fprintf (stderr, "incorrect count: %s\n", arg);
-                        return -1;
-                }
-                state->count = count;
-        }
-        break;
+            fprintf(stderr, "using prefix: %s\n", arg);
+            strncpy(state->prefix, arg, 512);
+            break;
+        case 'c': {
+            long count = atol(arg);
+            if (!count) {
+                fprintf(stderr, "incorrect count: %s\n", arg);
+                return -1;
+            }
+            state->count = count;
+        } break;
         case ARGP_KEY_NO_ARGS:
-                break;
+            break;
         case ARGP_KEY_ARG:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-do_mode_posix_iface_fileio_write (struct state *state)
+do_mode_posix_iface_fileio_write(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-
-        for (i=0; i<state->count; i++) {
-                int fd = -1;
-                char filename[512];
-
-                sprintf (filename, "%s.%06ld", state->prefix, i);
-
-                fd = open (filename, O_CREAT|O_WRONLY, 00600);
-                if (fd == -1) {
-                        fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno));
-                        break;
-                }
-                ret = write (fd, block, state->block_size);
-                if (ret != state->block_size) {
-                        fprintf (stderr, "write (%s) => %d/%s\n", filename, ret,
-                                 strerror (errno));
-                        close (fd);
-                        break;
-                }
-                close (fd);
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+
+    for (i = 0; i < state->count; i++) {
+        int fd = -1;
+        char filename[512];
+
+        sprintf(filename, "%s.%06ld", state->prefix, i);
+
+        fd = open(filename, O_CREAT | O_WRONLY, 00600);
+        if (fd == -1) {
+            fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno));
+            break;
+        }
+        ret = write(fd, block, state->block_size);
+        if (ret != state->block_size) {
+            fprintf(stderr, "write (%s) => %d/%s\n", filename, ret,
+                    strerror(errno));
+            close(fd);
+            break;
         }
+        close(fd);
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_fileio_read (struct state *state)
+do_mode_posix_iface_fileio_read(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-
-        for (i=0; i<state->count; i++) {
-                int fd = -1;
-                char filename[512];
-
-                sprintf (filename, "%s.%06ld", state->prefix, i);
-
-                fd = open (filename, O_RDONLY);
-                if (fd == -1) {
-                        fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno));
-                        break;
-                }
-                ret = read (fd, block, state->block_size);
-                if (ret == -1) {
-                        fprintf (stderr, "read(%s) => %d/%s\n", filename, ret, strerror (errno));
-                        close (fd);
-                        break;
-                }
-                close (fd);
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+
+    for (i = 0; i < state->count; i++) {
+        int fd = -1;
+        char filename[512];
+
+        sprintf(filename, "%s.%06ld", state->prefix, i);
+
+        fd = open(filename, O_RDONLY);
+        if (fd == -1) {
+            fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno));
+            break;
         }
+        ret = read(fd, block, state->block_size);
+        if (ret == -1) {
+            fprintf(stderr, "read(%s) => %d/%s\n", filename, ret,
+                    strerror(errno));
+            close(fd);
+            break;
+        }
+        close(fd);
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_fileio (struct state *state)
+do_mode_posix_iface_fileio(struct state *state)
 {
-        if (state->need_op_write)
-                MEASURE (do_mode_posix_iface_fileio_write, state);
+    if (state->need_op_write)
+        MEASURE(do_mode_posix_iface_fileio_write, state);
 
-        if (state->need_op_read)
-                MEASURE (do_mode_posix_iface_fileio_read, state);
+    if (state->need_op_read)
+        MEASURE(do_mode_posix_iface_fileio_read, state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-do_mode_posix_iface_xattr_write (struct state *state)
+do_mode_posix_iface_xattr_write(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-        char *dname = NULL, *dirc = NULL;
-        char *bname = NULL, *basec = NULL;
-
-        dirc = strdup (state->prefix);
-        basec = strdup (state->prefix);
-        dname = dirname (dirc);
-        bname = basename (basec);
-
-        for (i=0; i<state->count; i++) {
-                char key[512];
-
-                sprintf (key, "glusterfs.file.%s.%06ld", bname, i);
-
-                ret = lsetxattr (dname, key, block, state->block_size, 0);
-
-                if (ret != 0) {
-                        fprintf (stderr, "lsetxattr (%s, %s, %p) => %s\n",
-                                 dname, key, block, strerror (errno));
-                        break;
-                }
-                state->io_size += state->block_size;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+    char *dname = NULL, *dirc = NULL;
+    char *bname = NULL, *basec = NULL;
+
+    dirc = strdup(state->prefix);
+    basec = strdup(state->prefix);
+    dname = dirname(dirc);
+    bname = basename(basec);
+
+    for (i = 0; i < state->count; i++) {
+        char key[512];
+
+        sprintf(key, "glusterfs.file.%s.%06ld", bname, i);
+
+        ret = lsetxattr(dname, key, block, state->block_size, 0);
+
+        if (ret != 0) {
+            fprintf(stderr, "lsetxattr (%s, %s, %p) => %s\n", dname, key, block,
+                    strerror(errno));
+            break;
         }
+        state->io_size += state->block_size;
+    }
 
-        free (dirc);
-        free (basec);
+    free(dirc);
+    free(basec);
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_xattr_read (struct state *state)
+do_mode_posix_iface_xattr_read(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-        char *dname = NULL, *dirc = NULL;
-        char *bname = NULL, *basec = NULL;
-
-        dirc = strdup (state->prefix);
-        basec = strdup (state->prefix);
-        dname = dirname (dirc);
-        bname = basename (basec);
-
-        for (i=0; i<state->count; i++) {
-                char key[512];
-
-                sprintf (key, "glusterfs.file.%s.%06ld", bname, i);
-
-                ret = lgetxattr (dname, key, block, state->block_size);
-
-                if (ret < 0) {
-                        fprintf (stderr, "lgetxattr (%s, %s, %p) => %s\n",
-                                 dname, key, block, strerror (errno));
-                        break;
-                }
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+    char *dname = NULL, *dirc = NULL;
+    char *bname = NULL, *basec = NULL;
+
+    dirc = strdup(state->prefix);
+    basec = strdup(state->prefix);
+    dname = dirname(dirc);
+    bname = basename(basec);
+
+    for (i = 0; i < state->count; i++) {
+        char key[512];
+
+        sprintf(key, "glusterfs.file.%s.%06ld", bname, i);
+
+        ret = lgetxattr(dname, key, block, state->block_size);
+
+        if (ret < 0) {
+            fprintf(stderr, "lgetxattr (%s, %s, %p) => %s\n", dname, key, block,
+                    strerror(errno));
+            break;
         }
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_xattr (struct state *state)
+do_mode_posix_iface_xattr(struct state *state)
 {
-        if (state->need_op_write)
-                MEASURE (do_mode_posix_iface_xattr_write, state);
+    if (state->need_op_write)
+        MEASURE(do_mode_posix_iface_xattr_write, state);
 
-        if (state->need_op_read)
-                MEASURE (do_mode_posix_iface_xattr_read, state);
+    if (state->need_op_read)
+        MEASURE(do_mode_posix_iface_xattr_read, state);
 
-        return 0;
+    return 0;
 }
 
 int
-do_mode_posix (struct state *state)
+do_mode_posix(struct state *state)
 {
-        if (state->need_iface_fileio)
-                do_mode_posix_iface_fileio (state);
+    if (state->need_iface_fileio)
+        do_mode_posix_iface_fileio(state);
 
-        if (state->need_iface_xattr)
-                do_mode_posix_iface_xattr (state);
+    if (state->need_iface_xattr)
+        do_mode_posix_iface_xattr(state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-do_actions (struct state *state)
+do_actions(struct state *state)
 {
-        if (state->need_mode_posix)
-                do_mode_posix (state);
+    if (state->need_mode_posix)
+        do_mode_posix(state);
 
-        return 0;
+    return 0;
 }
 
 static struct argp_option options[] = {
-        {"op", 'o', "OPERATIONS", 0,
-         "WRITE|READ|BOTH - defaults to BOTH"},
-        {"iface", 'i', "INTERFACE", 0,
-         "FILEIO|XATTR|BOTH - defaults to FILEIO"},
-        {"block", 'b', "BLOCKSIZE", 0,
-         "<NUM> - defaults to 4096"},
-        {"specfile", 's', "SPECFILE", 0,
-         "absolute path to specfile"},
-        {"prefix", 'p', "PREFIX", 0,
-         "filename prefix"},
-        {"count", 'c', "COUNT", 0,
-         "number of files"},
-        {0, 0, 0, 0, 0}
-};
+    {"op", 'o', "OPERATIONS", 0, "WRITE|READ|BOTH - defaults to BOTH"},
+    {"iface", 'i', "INTERFACE", 0, "FILEIO|XATTR|BOTH - defaults to FILEIO"},
+    {"block", 'b', "BLOCKSIZE", 0, "<NUM> - defaults to 4096"},
+    {"specfile", 's', "SPECFILE", 0, "absolute path to specfile"},
+    {"prefix", 'p', "PREFIX", 0, "filename prefix"},
+    {"count", 'c', "COUNT", 0, "number of files"},
+    {0, 0, 0, 0, 0}};
 
-static struct argp argp = {
-        options,
-        parse_opts,
-        "tool",
-        "tool to benchmark small file performance"
-};
+static struct argp argp = {options, parse_opts, "tool",
+                           "tool to benchmark small file performance"};
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        struct state state = {0, };
+    struct state state = {
+        0,
+    };
 
-        state.need_op_write = 1;
-        state.need_op_read = 1;
+    state.need_op_write = 1;
+    state.need_op_read = 1;
 
-        state.need_iface_fileio = 1;
-        state.need_iface_xattr = 0;
+    state.need_iface_fileio = 1;
+    state.need_iface_xattr = 0;
 
-        state.need_mode_posix = 1;
+    state.need_mode_posix = 1;
 
-        state.block_size = 4096;
+    state.block_size = 4096;
 
-        strcpy (state.prefix, "tmpfile");
-        state.count = 1048576;
+    strcpy(state.prefix, "tmpfile");
+    state.count = 1048576;
 
-        if (argp_parse (&argp, argc, argv, 0, 0, &state) != 0) {
-                fprintf (stderr, "argp_parse() failed\n");
-                return 1;
-        }
+    if (argp_parse(&argp, argc, argv, 0, 0, &state) != 0) {
+        fprintf(stderr, "argp_parse() failed\n");
+        return 1;
+    }
 
-        do_actions (&state);
+    do_actions(&state);
 
-        return 0;
+    return 0;
 }
diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c
index a667c6a1d65..efc9d342a37 100644
--- a/extras/benchmarking/rdd.c
+++ b/extras/benchmarking/rdd.c
@@ -20,633 +20,586 @@
 
 #define TWO_POWER(power) (2UL << (power))
 
-#define RDD_INTEGER_VALUE ((TWO_POWER ((sizeof (int) * 8))) - 1)
+#define RDD_INTEGER_VALUE ((TWO_POWER((sizeof(int) * 8))) - 1)
 
 #ifndef UNIX_PATH_MAX
 #define UNIX_PATH_MAX 108
 #endif
 
 #define UNIT_KB 1024ULL
-#define UNIT_MB UNIT_KB*1024ULL
-#define UNIT_GB UNIT_MB*1024ULL
-#define UNIT_TB UNIT_GB*1024ULL
-#define UNIT_PB UNIT_TB*1024ULL
+#define UNIT_MB UNIT_KB * 1024ULL
+#define UNIT_GB UNIT_MB * 1024ULL
+#define UNIT_TB UNIT_GB * 1024ULL
+#define UNIT_PB UNIT_TB * 1024ULL
 
-#define UNIT_KB_STRING    "KB"
-#define UNIT_MB_STRING    "MB"
-#define UNIT_GB_STRING    "GB"
-#define UNIT_TB_STRING    "TB"
-#define UNIT_PB_STRING    "PB"
+#define UNIT_KB_STRING "KB"
+#define UNIT_MB_STRING "MB"
+#define UNIT_GB_STRING "GB"
+#define UNIT_TB_STRING "TB"
+#define UNIT_PB_STRING "PB"
 
 struct rdd_file {
-        char path[UNIX_PATH_MAX];
-        struct stat st;
-        int fd;
+    char path[UNIX_PATH_MAX];
+    struct stat st;
+    int fd;
 };
 
 struct rdd_config {
-        long iters;
-        long max_ops_per_seq;
-        size_t max_bs;
-        size_t min_bs;
-        int thread_count;
-        pthread_t *threads;
-        pthread_barrier_t barrier;
-        pthread_mutex_t lock;
-        struct rdd_file in_file;
-        struct rdd_file out_file;
-        ssize_t file_size;
+    long iters;
+    long max_ops_per_seq;
+    size_t max_bs;
+    size_t min_bs;
+    int thread_count;
+    pthread_t *threads;
+    pthread_barrier_t barrier;
+    pthread_mutex_t lock;
+    struct rdd_file in_file;
+    struct rdd_file out_file;
+    ssize_t file_size;
 };
 static struct rdd_config rdd_config;
 
 enum rdd_keys {
-        RDD_MIN_BS_KEY = 1,
-        RDD_MAX_BS_KEY,
+    RDD_MIN_BS_KEY = 1,
+    RDD_MAX_BS_KEY,
 };
 
 static error_t
-rdd_parse_opts (int key, char *arg,
-                struct argp_state *_state)
+rdd_parse_opts(int key, char *arg, struct argp_state *_state)
 {
-        switch (key) {
-        case 'o':
-        {
-                int len = 0;
-                len = strlen (arg);
-                if (len > UNIX_PATH_MAX) {
-                        fprintf (stderr, "output file name too long (%s)\n",
-                                 arg);
-                        return -1;
-                }
-
-                strncpy (rdd_config.out_file.path, arg, len);
-        }
-        break;
-
-        case 'i':
-        {
-                int len = 0;
-                len = strlen (arg);
-                if (len > UNIX_PATH_MAX) {
-                        fprintf (stderr, "input file name too long (%s)\n",
-                                 arg);
-                        return -1;
-                }
-
-                strncpy (rdd_config.in_file.path, arg, len);
-                rdd_config.in_file.path[len] = '\0';
-        }
-        break;
-
-        case 'f':
-        {
-                char *tmp = NULL;
-                unsigned long long fs = 0;
-                if (string2bytesize (arg, &fs) == -1) {
-                        fprintf (stderr, "invalid argument for file size "
-                                 "(%s)\n", arg);
-                        return -1;
-                }
-
-                rdd_config.file_size = fs;
-        }
-        break;
-
-        case RDD_MIN_BS_KEY:
-        {
-                char *tmp = NULL;
-                long bs = 0;
-                bs = strtol (arg, &tmp, 10);
-                if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
-                        fprintf (stderr, "invalid argument for minimum block"
-                                 "size (%s)\n", arg);
-                        return -1;
-                }
-
-                rdd_config.min_bs = bs;
-        }
-        break;
-
-        case RDD_MAX_BS_KEY:
-        {
-                char *tmp = NULL;
-                long bs = 0;
-                bs = strtol (arg, &tmp, 10);
-                if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
-                        fprintf (stderr, "invalid argument for maximum block"
-                                 "size (%s)\n", arg);
-                        return -1;
-                }
-
-                rdd_config.max_bs = bs;
-        }
-        break;
-
-        case 'r':
-        {
-                char *tmp = NULL;
-                long iters = 0;
-                iters = strtol (arg, &tmp, 10);
-                if ((iters == LONG_MAX) ||
-                    (iters == LONG_MIN) ||
-                    (tmp && *tmp)) {
-                        fprintf (stderr, "invalid argument for iterations"
-                                 "(%s)\n", arg);
-                        return -1;
-                }
-
-                rdd_config.iters = iters;
-        }
-        break;
-
-        case 'm':
-        {
-                char *tmp = NULL;
-                long max_ops = 0;
-                max_ops = strtol (arg, &tmp, 10);
-                if ((max_ops == LONG_MAX) ||
-                    (max_ops == LONG_MIN) ||
-                    (tmp && *tmp)) {
-                        fprintf (stderr, "invalid argument for max-ops"
-                                 "(%s)\n", arg);
-                        return -1;
-                }
+    switch (key) {
+        case 'o': {
+            int len = 0;
+            len = strlen(arg);
+            if (len > UNIX_PATH_MAX) {
+                fprintf(stderr, "output file name too long (%s)\n", arg);
+                return -1;
+            }
 
-                rdd_config.max_ops_per_seq = max_ops;
-        }
-        break;
+            strncpy(rdd_config.out_file.path, arg, len);
+        } break;
 
-        case 't':
-        {
-                char *tmp = NULL;
-                long threads = 0;
-                threads = strtol (arg, &tmp, 10);
-                if ((threads == LONG_MAX) ||
-                    (threads == LONG_MIN) ||
-                    (tmp && *tmp)) {
-                        fprintf (stderr, "invalid argument for thread count"
-                                 "(%s)\n", arg);
-                        return -1;
-                }
+        case 'i': {
+            int len = 0;
+            len = strlen(arg);
+            if (len > UNIX_PATH_MAX) {
+                fprintf(stderr, "input file name too long (%s)\n", arg);
+                return -1;
+            }
+
+            strncpy(rdd_config.in_file.path, arg, len);
+            rdd_config.in_file.path[len] = '\0';
+        } break;
+
+        case 'f': {
+            char *tmp = NULL;
+            unsigned long long fs = 0;
+            if (string2bytesize(arg, &fs) == -1) {
+                fprintf(stderr,
+                        "invalid argument for file size "
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.file_size = fs;
+        } break;
+
+        case RDD_MIN_BS_KEY: {
+            char *tmp = NULL;
+            long bs = 0;
+            bs = strtol(arg, &tmp, 10);
+            if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for minimum block"
+                        "size (%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.min_bs = bs;
+        } break;
+
+        case RDD_MAX_BS_KEY: {
+            char *tmp = NULL;
+            long bs = 0;
+            bs = strtol(arg, &tmp, 10);
+            if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for maximum block"
+                        "size (%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.max_bs = bs;
+        } break;
+
+        case 'r': {
+            char *tmp = NULL;
+            long iters = 0;
+            iters = strtol(arg, &tmp, 10);
+            if ((iters == LONG_MAX) || (iters == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for iterations"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.iters = iters;
+        } break;
+
+        case 'm': {
+            char *tmp = NULL;
+            long max_ops = 0;
+            max_ops = strtol(arg, &tmp, 10);
+            if ((max_ops == LONG_MAX) || (max_ops == LONG_MIN) ||
+                (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for max-ops"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.max_ops_per_seq = max_ops;
+        } break;
+
+        case 't': {
+            char *tmp = NULL;
+            long threads = 0;
+            threads = strtol(arg, &tmp, 10);
+            if ((threads == LONG_MAX) || (threads == LONG_MIN) ||
+                (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for thread count"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
 
-                rdd_config.thread_count = threads;
-        }
-        break;
+            rdd_config.thread_count = threads;
+        } break;
 
         case ARGP_KEY_NO_ARGS:
-                break;
+            break;
         case ARGP_KEY_ARG:
-                break;
+            break;
         case ARGP_KEY_END:
-                if (_state->argc == 1) {
-                        argp_usage (_state);
-                }
+            if (_state->argc == 1) {
+                argp_usage(_state);
+            }
+    }
 
-        }
-
-        return 0;
+    return 0;
 }
 
 int
-string2bytesize (const char *str, unsigned long long *n)
+string2bytesize(const char *str, unsigned long long *n)
 {
-        unsigned long long value = 0ULL;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL)
-        {
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++)
-        {
-                if (isspace (*s))
-                {
-                        continue;
-                }
-                if (*s == '-')
-                {
-                        return -1;
-                }
-                break;
+    unsigned long long value = 0ULL;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s)) {
+            continue;
         }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtoull (str, &tail, 10);
-
-        if (errno == ERANGE || errno == EINVAL)
-        {
-                return -1;
+        if (*s == '-') {
+            return -1;
         }
-
-        if (errno == 0)
-        {
-                errno = old_errno;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoull(str, &tail, 10);
+
+    if (errno == ERANGE || errno == EINVAL) {
+        return -1;
+    }
+
+    if (errno == 0) {
+        errno = old_errno;
+    }
+
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, UNIT_KB_STRING) == 0) {
+            value *= UNIT_KB;
+        } else if (strcasecmp(tail, UNIT_MB_STRING) == 0) {
+            value *= UNIT_MB;
+        } else if (strcasecmp(tail, UNIT_GB_STRING) == 0) {
+            value *= UNIT_GB;
+        } else if (strcasecmp(tail, UNIT_TB_STRING) == 0) {
+            value *= UNIT_TB;
+        } else if (strcasecmp(tail, UNIT_PB_STRING) == 0) {
+            value *= UNIT_PB;
         }
 
-        if (tail[0] != '\0')
-        {
-                if (strcasecmp (tail, UNIT_KB_STRING) == 0)
-                {
-                        value *= UNIT_KB;
-                }
-                else if (strcasecmp (tail, UNIT_MB_STRING) == 0)
-                {
-                        value *= UNIT_MB;
-                }
-                else if (strcasecmp (tail, UNIT_GB_STRING) == 0)
-                {
-                        value *= UNIT_GB;
-                }
-                else if (strcasecmp (tail, UNIT_TB_STRING) == 0)
-                {
-                        value *= UNIT_TB;
-                }
-                else if (strcasecmp (tail, UNIT_PB_STRING) == 0)
-                {
-                        value *= UNIT_PB;
-                }
-
-                else
-                {
-                        return -1;
-                }
+        else {
+            return -1;
         }
+    }
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 static struct argp_option rdd_options[] = {
-        {"if", 'i', "INPUT_FILE", 0, "input-file"},
-        {"of", 'o', "OUTPUT_FILE", 0, "output-file"},
-        {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"},
-        {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0,
-         "Minimum block size in bytes (defaults to 1024)"},
-        {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0,
-         "Maximum block size in bytes (defaults to 4096)"},
-        {"iters", 'r', "ITERS", 0,
-         "Number of read-write sequences (defaults to 1000000)"},
-        {"max-ops", 'm', "MAXOPS", 0,
-         "maximum number of read-writes to be performed in a sequence (defaults to 1)"},
-        {"file-size", 'f', "FILESIZE", 0,
-         "the size of the file which will be created and upon it I/O will be done"
-         " (defaults to 100MB"},
-        {0, 0, 0, 0, 0}
-};
+    {"if", 'i', "INPUT_FILE", 0, "input-file"},
+    {"of", 'o', "OUTPUT_FILE", 0, "output-file"},
+    {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"},
+    {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0,
+     "Minimum block size in bytes (defaults to 1024)"},
+    {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0,
+     "Maximum block size in bytes (defaults to 4096)"},
+    {"iters", 'r', "ITERS", 0,
+     "Number of read-write sequences (defaults to 1000000)"},
+    {"max-ops", 'm', "MAXOPS", 0,
+     "maximum number of read-writes to be performed in a sequence (defaults to "
+     "1)"},
+    {"file-size", 'f', "FILESIZE", 0,
+     "the size of the file which will be created and upon it I/O will be done"
+     " (defaults to 100MB"},
+    {0, 0, 0, 0, 0}};
 
 static struct argp argp = {
-  rdd_options,
-  rdd_parse_opts,
-  "",
-  "random dd - tool to do a sequence of random block-sized continuous"
-  "read writes starting at a random offset"
-};
-
+    rdd_options, rdd_parse_opts, "",
+    "random dd - tool to do a sequence of random block-sized continuous"
+    "read writes starting at a random offset"};
 
 static void
-rdd_default_config (void)
+rdd_default_config(void)
 {
-        char *tmp_path = "rdd.in";
-
-        rdd_config.thread_count = 2;
-        rdd_config.iters = 1000000;
-        rdd_config.max_bs = 4096;
-        rdd_config.min_bs = 1024;
-        rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-        rdd_config.max_ops_per_seq = 1;
-        strncpy (rdd_config.in_file.path, tmp_path, strlen (tmp_path));
-        rdd_config.file_size = 104857600;
-
-        return;
+    char *tmp_path = "rdd.in";
+
+    rdd_config.thread_count = 2;
+    rdd_config.iters = 1000000;
+    rdd_config.max_bs = 4096;
+    rdd_config.min_bs = 1024;
+    rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
+    rdd_config.max_ops_per_seq = 1;
+    strncpy(rdd_config.in_file.path, tmp_path, strlen(tmp_path));
+    rdd_config.file_size = 104857600;
+
+    return;
 }
 
-
 static char
-rdd_valid_config (void)
+rdd_valid_config(void)
 {
-        char ret = 1;
-        int fd = -1;
+    char ret = 1;
+    int fd = -1;
 
-        fd = open (rdd_config.in_file.path, O_RDONLY);
-        if (fd == -1 && (errno != ENOENT)) {
-                fprintf (stderr, "open: (%s)", strerror (errno));
-                ret = 0;
-                goto out;
-        }
-        close (fd);
-
-        if (rdd_config.min_bs > rdd_config.max_bs) {
-                fprintf (stderr, "minimum blocksize %ld is greater than the "
-                         "maximum blocksize %ld", rdd_config.min_bs,
-                         rdd_config.max_bs);
-                ret = 0;
-                goto out;
-        }
+    fd = open(rdd_config.in_file.path, O_RDONLY);
+    if (fd == -1 && (errno != ENOENT)) {
+        fprintf(stderr, "open: (%s)", strerror(errno));
+        ret = 0;
+        goto out;
+    }
+    close(fd);
+
+    if (rdd_config.min_bs > rdd_config.max_bs) {
+        fprintf(stderr,
+                "minimum blocksize %ld is greater than the "
+                "maximum blocksize %ld",
+                rdd_config.min_bs, rdd_config.max_bs);
+        ret = 0;
+        goto out;
+    }
 
-        if (strlen (rdd_config.out_file.path) == 0) {
-                sprintf (rdd_config.out_file.path, "%s.rddout",
-                         rdd_config.in_file.path);
-        }
+    if (strlen(rdd_config.out_file.path) == 0) {
+        sprintf(rdd_config.out_file.path, "%s.rddout", rdd_config.in_file.path);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 static void *
-rdd_read_write (void *arg)
+rdd_read_write(void *arg)
 {
-        int i = 0, ret = 0;
-        size_t bs = 0;
-        off_t offset = 0;
-        long rand = 0;
-        long max_ops = 0;
-        char *buf = NULL;
-
-        buf = calloc (1, rdd_config.max_bs);
-        if (!buf) {
-                fprintf (stderr, "calloc failed (%s)\n", strerror (errno));
+    int i = 0, ret = 0;
+    size_t bs = 0;
+    off_t offset = 0;
+    long rand = 0;
+    long max_ops = 0;
+    char *buf = NULL;
+
+    buf = calloc(1, rdd_config.max_bs);
+    if (!buf) {
+        fprintf(stderr, "calloc failed (%s)\n", strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < rdd_config.iters; i++) {
+        pthread_mutex_lock(&rdd_config.lock);
+        {
+            int bytes = 0;
+            rand = random();
+
+            if (rdd_config.min_bs == rdd_config.max_bs) {
+                bs = rdd_config.max_bs;
+            } else {
+                bs = rdd_config.min_bs +
+                     (rand % (rdd_config.max_bs - rdd_config.min_bs));
+            }
+
+            offset = rand % rdd_config.in_file.st.st_size;
+            max_ops = rand % rdd_config.max_ops_per_seq;
+            if (!max_ops) {
+                max_ops++;
+            }
+
+            ret = lseek(rdd_config.in_file.fd, offset, SEEK_SET);
+            if (ret != offset) {
+                fprintf(stderr, "lseek failed (%s)\n", strerror(errno));
                 ret = -1;
-                goto out;
-        }
+                goto unlock;
+            }
 
-        for (i = 0; i < rdd_config.iters; i++)
-        {
-                pthread_mutex_lock (&rdd_config.lock);
-                {
-                        int bytes = 0;
-                        rand = random ();
-
-                        if (rdd_config.min_bs == rdd_config.max_bs) {
-                                bs = rdd_config.max_bs;
-                        } else {
-                                bs = rdd_config.min_bs +
-                                        (rand %
-                                         (rdd_config.max_bs -
-                                          rdd_config.min_bs));
-                        }
-
-                        offset = rand % rdd_config.in_file.st.st_size;
-                        max_ops = rand % rdd_config.max_ops_per_seq;
-                        if (!max_ops) {
-                                max_ops ++;
-                        }
-
-                        ret = lseek (rdd_config.in_file.fd, offset, SEEK_SET);
-                        if (ret != offset) {
-                                fprintf (stderr, "lseek failed (%s)\n",
-                                         strerror (errno));
-                                ret = -1;
-                                goto unlock;
-                        }
-
-                        ret = lseek (rdd_config.out_file.fd, offset, SEEK_SET);
-                        if (ret != offset) {
-                                fprintf (stderr, "lseek failed (%s)\n",
-                                         strerror (errno));
-                                ret = -1;
-                                goto unlock;
-                        }
-
-                        while (max_ops--)
-                        {
-                                bytes = read (rdd_config.in_file.fd, buf, bs);
-                                if (!bytes) {
-                                        break;
-                                }
-
-                                if (bytes == -1) {
-                                        fprintf (stderr, "read failed (%s)\n",
-                                                 strerror (errno));
-                                        ret = -1;
-                                        goto unlock;
-                                }
-
-                                if (write (rdd_config.out_file.fd, buf, bytes)
-                                    != bytes) {
-                                        fprintf (stderr, "write failed (%s)\n",
-                                                 strerror (errno));
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                        }
+            ret = lseek(rdd_config.out_file.fd, offset, SEEK_SET);
+            if (ret != offset) {
+                fprintf(stderr, "lseek failed (%s)\n", strerror(errno));
+                ret = -1;
+                goto unlock;
+            }
+
+            while (max_ops--) {
+                bytes = read(rdd_config.in_file.fd, buf, bs);
+                if (!bytes) {
+                    break;
                 }
-        unlock:
-                pthread_mutex_unlock (&rdd_config.lock);
-                if (ret == -1) {
-                        goto out;
+
+                if (bytes == -1) {
+                    fprintf(stderr, "read failed (%s)\n", strerror(errno));
+                    ret = -1;
+                    goto unlock;
+                }
+
+                if (write(rdd_config.out_file.fd, buf, bytes) != bytes) {
+                    fprintf(stderr, "write failed (%s)\n", strerror(errno));
+                    ret = -1;
+                    goto unlock;
                 }
-                ret = 0;
+            }
         }
+    unlock:
+        pthread_mutex_unlock(&rdd_config.lock);
+        if (ret == -1) {
+            goto out;
+        }
+        ret = 0;
+    }
 out:
-        free (buf);
-        pthread_barrier_wait (&rdd_config.barrier);
+    free(buf);
+    pthread_barrier_wait(&rdd_config.barrier);
 
-        return NULL;
+    return NULL;
 }
 
 static void
-cleanup (void)
+cleanup(void)
 {
-        close (rdd_config.in_file.fd);
-        close (rdd_config.out_file.fd);
-        rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
+    close(rdd_config.in_file.fd);
+    close(rdd_config.out_file.fd);
+    rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
 }
 
 static int
-check_and_create (void)
+check_and_create(void)
 {
-        int ret = -1;
-        char buf[4096] = {0,};
-        struct stat stbuf = {0,};
-        int  fd[2] = {-1,};
-        size_t  total_size = -1;
-
-        total_size = rdd_config.file_size;
-
-        ret = stat (rdd_config.in_file.path, &stbuf);
-        if (ret == -1 && (errno != ENOENT))
+    int ret = -1;
+    char buf[4096] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int fd[2] = {
+        -1,
+    };
+    size_t total_size = -1;
+
+    total_size = rdd_config.file_size;
+
+    ret = stat(rdd_config.in_file.path, &stbuf);
+    if (ret == -1 && (errno != ENOENT))
+        goto out;
+
+    fd[1] = open(rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC);
+    if (fd[1] == -1)
+        goto out;
+
+    fd[0] = open("/dev/urandom", O_RDONLY);
+    if (fd[0] == -1)
+        goto out;
+
+    while (total_size > 0) {
+        if (total_size >= 4096) {
+            ret = read(fd[0], buf, 4096);
+            if (ret == -1)
                 goto out;
-
-        fd[1] = open (rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC);
-        if (fd[1] == -1)
+            ret = write(fd[1], buf, 4096);
+            if (ret == -1)
                 goto out;
-
-        fd[0] = open ("/dev/urandom", O_RDONLY);
-        if (fd[0] == -1)
+            total_size = total_size - 4096;
+        } else {
+            ret = read(fd[0], buf, total_size);
+            if (ret == -1)
                 goto out;
-
-        while (total_size > 0) {
-                if (total_size >= 4096) {
-                        ret = read (fd[0], buf, 4096);
-                        if (ret == -1)
-                                goto out;
-                        ret = write (fd[1], buf, 4096);
-                        if (ret == -1)
-                                goto out;
-                        total_size = total_size - 4096;
-                } else {
-                        ret = read (fd[0], buf, total_size);
-                        if (ret == -1)
-                                goto out;
-                        ret = write (fd[1], buf, total_size);
-                        if (ret == -1)
-                                goto out;
-                        total_size = total_size - total_size;
-                }
-
+            ret = write(fd[1], buf, total_size);
+            if (ret == -1)
+                goto out;
+            total_size = total_size - total_size;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (fd[0] > 0)
-                close (fd[0]);
-        if (fd[1] > 0)
-                close (fd[1]);
-        return ret;
+    if (fd[0] > 0)
+        close(fd[0]);
+    if (fd[1] > 0)
+        close(fd[1]);
+    return ret;
 }
 
 static int
-rdd_spawn_threads (void)
+rdd_spawn_threads(void)
 {
-        int i = 0, ret = -1, fd = -1;
-        char buf[4096];
-
-        ret = check_and_create ();
-        if (ret == -1)
-                goto out;
-
-        fd = open (rdd_config.in_file.path, O_RDONLY);
-        if (fd < 0) {
-                fprintf (stderr, "cannot open %s (%s)\n",
-                         rdd_config.in_file.path, strerror (errno));
-                ret = -1;
-                goto out;
-        }
-        ret = fstat (fd, &rdd_config.in_file.st);
-        if (ret != 0) {
-                close (fd);
-                fprintf (stderr, "cannot stat %s (%s)\n",
-                         rdd_config.in_file.path, strerror (errno));
-                ret = -1;
-                goto out;
-        }
-        rdd_config.in_file.fd = fd;
-
-        fd = open (rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC,
-                   S_IRWXU | S_IROTH);
-        if (fd < 0) {
-                close (rdd_config.in_file.fd);
-                rdd_config.in_file.fd = -1;
-                fprintf (stderr, "cannot open %s (%s)\n",
-                         rdd_config.out_file.path, strerror (errno));
-                ret = -1;
-                goto out;
-        }
-        rdd_config.out_file.fd = fd;
-
-        while ((ret = read (rdd_config.in_file.fd, buf, 4096)) > 0) {
-                if (write (rdd_config.out_file.fd, buf, ret) != ret) {
-                        fprintf (stderr, "write failed (%s)\n",
-                                 strerror (errno));
-                        cleanup ();
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        rdd_config.threads = calloc (rdd_config.thread_count,
-                                     sizeof (pthread_t));
-        if (rdd_config.threads == NULL) {
-                fprintf (stderr, "calloc() failed (%s)\n", strerror (errno));
-
-                ret = -1;
-                cleanup ();
-                goto out;
-        }
-
-        ret = pthread_barrier_init (&rdd_config.barrier, NULL,
-                                    rdd_config.thread_count + 1);
-        if (ret != 0) {
-                fprintf (stderr, "pthread_barrier_init() failed (%s)\n",
-                         strerror (ret));
-
-                free (rdd_config.threads);
-                cleanup ();
-                ret = -1;
-                goto out;
+    int i = 0, ret = -1, fd = -1;
+    char buf[4096];
+
+    ret = check_and_create();
+    if (ret == -1)
+        goto out;
+
+    fd = open(rdd_config.in_file.path, O_RDONLY);
+    if (fd < 0) {
+        fprintf(stderr, "cannot open %s (%s)\n", rdd_config.in_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    ret = fstat(fd, &rdd_config.in_file.st);
+    if (ret != 0) {
+        close(fd);
+        fprintf(stderr, "cannot stat %s (%s)\n", rdd_config.in_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    rdd_config.in_file.fd = fd;
+
+    fd = open(rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC,
+              S_IRWXU | S_IROTH);
+    if (fd < 0) {
+        close(rdd_config.in_file.fd);
+        rdd_config.in_file.fd = -1;
+        fprintf(stderr, "cannot open %s (%s)\n", rdd_config.out_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    rdd_config.out_file.fd = fd;
+
+    while ((ret = read(rdd_config.in_file.fd, buf, 4096)) > 0) {
+        if (write(rdd_config.out_file.fd, buf, ret) != ret) {
+            fprintf(stderr, "write failed (%s)\n", strerror(errno));
+            cleanup();
+            ret = -1;
+            goto out;
         }
-
-        ret = pthread_mutex_init (&rdd_config.lock, NULL);
+    }
+
+    rdd_config.threads = calloc(rdd_config.thread_count, sizeof(pthread_t));
+    if (rdd_config.threads == NULL) {
+        fprintf(stderr, "calloc() failed (%s)\n", strerror(errno));
+
+        ret = -1;
+        cleanup();
+        goto out;
+    }
+
+    ret = pthread_barrier_init(&rdd_config.barrier, NULL,
+                               rdd_config.thread_count + 1);
+    if (ret != 0) {
+        fprintf(stderr, "pthread_barrier_init() failed (%s)\n", strerror(ret));
+
+        free(rdd_config.threads);
+        cleanup();
+        ret = -1;
+        goto out;
+    }
+
+    ret = pthread_mutex_init(&rdd_config.lock, NULL);
+    if (ret != 0) {
+        fprintf(stderr, "pthread_mutex_init() failed (%s)\n", strerror(ret));
+
+        free(rdd_config.threads);
+        pthread_barrier_destroy(&rdd_config.barrier);
+        cleanup();
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < rdd_config.thread_count; i++) {
+        ret = pthread_create(&rdd_config.threads[i], NULL, rdd_read_write,
+                             NULL);
         if (ret != 0) {
-                fprintf (stderr, "pthread_mutex_init() failed (%s)\n",
-                         strerror (ret));
-
-                free (rdd_config.threads);
-                pthread_barrier_destroy (&rdd_config.barrier);
-                cleanup ();
-                ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < rdd_config.thread_count; i++)
-        {
-                ret = pthread_create (&rdd_config.threads[i], NULL,
-                                      rdd_read_write, NULL);
-                if (ret != 0) {
-                        fprintf (stderr, "pthread_create failed (%s)\n",
-                                 strerror (errno));
-                        exit (1);
-                }
+            fprintf(stderr, "pthread_create failed (%s)\n", strerror(errno));
+            exit(1);
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static void
-rdd_wait_for_completion (void)
+rdd_wait_for_completion(void)
 {
-        pthread_barrier_wait (&rdd_config.barrier);
+    pthread_barrier_wait(&rdd_config.barrier);
 }
 
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        int ret = -1;
+    int ret = -1;
 
-        rdd_default_config ();
+    rdd_default_config();
 
-        ret = argp_parse (&argp, argc, argv, 0, 0, NULL);
-        if (ret != 0) {
-                ret = -1;
-                fprintf (stderr, "%s: argp_parse() failed\n", argv[0]);
-                goto err;
-        }
+    ret = argp_parse(&argp, argc, argv, 0, 0, NULL);
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "%s: argp_parse() failed\n", argv[0]);
+        goto err;
+    }
 
-        if (!rdd_valid_config ()) {
-                ret = -1;
-                fprintf (stderr, "%s: configuration validation failed\n",
-                         argv[0]);
-                goto err;
-        }
+    if (!rdd_valid_config()) {
+        ret = -1;
+        fprintf(stderr, "%s: configuration validation failed\n", argv[0]);
+        goto err;
+    }
 
-        ret = rdd_spawn_threads ();
-        if (ret != 0) {
-                fprintf (stderr, "%s: spawning threads failed\n", argv[0]);
-                goto err;
-        }
+    ret = rdd_spawn_threads();
+    if (ret != 0) {
+        fprintf(stderr, "%s: spawning threads failed\n", argv[0]);
+        goto err;
+    }
 
-        rdd_wait_for_completion ();
+    rdd_wait_for_completion();
 
 err:
-        return ret;
+    return ret;
 }
diff --git a/extras/cliutils/README.md b/extras/cliutils/README.md
index ccb60802c3d..309beb1ca25 100644
--- a/extras/cliutils/README.md
+++ b/extras/cliutils/README.md
@@ -81,7 +81,7 @@ to address the following issues
 Create a file in `$LIBEXEC/glusterfs/peer_message.py` with following
 content.
 
-    #!/usr/bin/env python
+    #!/usr/bin/python3
     from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok
 
     class NodeHello(Cmd):
@@ -149,7 +149,7 @@ Now users can use `gluster-message` instead of calling
 Following example uses prettytable library, which can be installed
 using `pip install prettytable` or `dnf install python-prettytable`
 
-    #!/usr/bin/env python
+    #!/usr/bin/python3
     from prettytable import PrettyTable
     from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok
 
@@ -221,7 +221,7 @@ required.(Under `%files` section)
 - gluster-mountbroker   http://review.gluster.org/14544
 - gluster-eventsapi     http://review.gluster.org/14248
 - gluster-georep-sshkey http://review.gluster.org/14732
-- gluster-restapi       https://github.com/aravindavk/glusterfs-restapi
+- gluster-restapi       https://github.com/gluster/restapi
 
 ## Limitations/TODOs
 - Not yet possible to create CLI without any subcommand, For example
diff --git a/extras/cliutils/__init__.py b/extras/cliutils/__init__.py
index 4bb8395bb46..8765cc85099 100644
--- a/extras/cliutils/__init__.py
+++ b/extras/cliutils/__init__.py
@@ -1,17 +1,18 @@
 # -*- coding: utf-8 -*-
 # Reexporting the utility funcs and classes
-from cliutils import (runcli,
-                      sync_file_to_peers,
-                      execute_in_peers,
-                      execute,
-                      node_output_ok,
-                      node_output_notok,
-                      output_error,
-                      oknotok,
-                      yesno,
-                      get_node_uuid,
-                      Cmd,
-                      GlusterCmdException)
+from .cliutils import (runcli,
+                       sync_file_to_peers,
+                       execute_in_peers,
+                       execute,
+                       node_output_ok,
+                       node_output_notok,
+                       output_error,
+                       oknotok,
+                       yesno,
+                       get_node_uuid,
+                       Cmd,
+                       GlusterCmdException,
+                       set_common_args_func)
 
 
 # This will be useful when `from cliutils import *`
@@ -26,4 +27,5 @@ __all__ = ["runcli",
            "yesno",
            "get_node_uuid",
            "Cmd",
-           "GlusterCmdException"]
+           "GlusterCmdException",
+           "set_common_args_func"]
diff --git a/extras/cliutils/cliutils.py b/extras/cliutils/cliutils.py
index 4e035d7ff5c..55fbaf56704 100644
--- a/extras/cliutils/cliutils.py
+++ b/extras/cliutils/cliutils.py
@@ -16,10 +16,18 @@ subparsers = parser.add_subparsers(dest="mode")
 subcommands = {}
 cache_data = {}
 ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError
+_common_args_func = lambda p: True
 
 
 class GlusterCmdException(Exception):
-    pass
+    def __init__(self, message):
+        self.message = message
+        try:
+            # Python 3
+            super().__init__(message)
+        except TypeError:
+            # Python 2
+            super(GlusterCmdException, self).__init__(message)
 
 
 def get_node_uuid():
@@ -50,9 +58,9 @@ def oknotok(flag):
     return "OK" if flag else "NOT OK"
 
 
-def output_error(message):
+def output_error(message, errcode=1):
     print (message, file=sys.stderr)
-    sys.exit(1)
+    sys.exit(errcode)
 
 
 def node_output_ok(message=""):
@@ -70,7 +78,8 @@ def node_output_notok(message):
 
 
 def execute(cmd):
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         universal_newlines=True)
     out, err = p.communicate()
     return p.returncode, out, err
 
@@ -186,6 +195,7 @@ def runcli():
     # a subcommand as specified in the Class name. Call the args
     # method by passing subcommand parser, Derived class can add
     # arguments to the subcommand parser.
+    metavar_data = []
     for c in Cmd.__subclasses__():
         cls = c()
         if getattr(cls, "name", "") == "":
@@ -193,14 +203,24 @@ def runcli():
                                       "to \"{0}\"".format(
                                           cls.__class__.__name__))
 
+        # Do not show in help message if subcommand starts with node-
+        if not cls.name.startswith("node-"):
+            metavar_data.append(cls.name)
+
         p = subparsers.add_parser(cls.name)
         args_func = getattr(cls, "args", None)
         if args_func is not None:
             args_func(p)
 
+        # Apply common args if any
+        _common_args_func(p)
+
         # A dict to save subcommands, key is name of the subcommand
         subcommands[cls.name] = cls
 
+    # Hide node commands in Help message
+    subparsers.metavar = "{" + ",".join(metavar_data) + "}"
+
     # Get all parsed arguments
     args = parser.parse_args()
 
@@ -210,3 +230,8 @@ def runcli():
     # Run
     if cls is not None:
         cls.run(args)
+
+
+def set_common_args_func(func):
+    global _common_args_func
+    _common_args_func = func
diff --git a/extras/collect-system-stats.sh b/extras/collect-system-stats.sh
new file mode 100755
index 00000000000..865e70bbc11
--- /dev/null
+++ b/extras/collect-system-stats.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+################################################################################
+# Usage: collect-system-stats.sh <delay-in-seconds>
+# This script starts sar/top/iostat/vmstat processes which collect system stats
+# with the interval <delay-in-seconds> given as argument to the script. When
+# the script is stopped either by entering any input or Ctrl+C the list of
+# files where output is captured will be printed on the screen which can be
+# observed to find any problems/bottlenecks.
+###############################################################################
+
+function stop_processes {
+        echo "Stopping the monitoring processes"
+        echo "sar pid:$sar_pid", "top pid: $top_pid", "iostat pid: $iostat_pid", "vmstat pid: $vmstat_pid"
+        kill "$sar_pid" "$top_pid" "$iostat_pid" "$vmstat_pid"
+        echo "Files created: ${timestamp}-network.out, ${timestamp}-top.out, ${timestamp}-iostat.out, ${timestamp}-vmstat.out"
+}
+
+function check_dependent_commands_exist()
+{
+        declare -a arr=("sar" "top" "iostat" "vmstat")
+        for i in "${arr[@]}"
+        do
+                if ! command -v "$i" > /dev/null 2>&1
+                then
+                        echo "ERROR: '$i' command is not found"
+                        exit 1
+                fi
+        done
+
+}
+
+case "$1" in
+    ''|*[!0-9]*) echo "Usage: $0 <delay-between-successive-metrics-collection-in-seconds>"; exit 1 ;;
+    *) interval="$1" ;;
+esac
+
+timestamp=$(date +"%s")
+
+check_dependent_commands_exist
+sar -n DEV "$interval" > "${timestamp}"-network.out &
+sar_pid="$!"
+top -bHd "$interval" > "${timestamp}"-top.out &
+top_pid="$!"
+iostat -Ntkdx "$interval" > "${timestamp}"-iostat.out &
+iostat_pid="$!"
+vmstat -t "$interval" > "${timestamp}"-vmstat.out &
+vmstat_pid="$!"
+echo "Started sar, vmstat, iostat, top for collecting stats"
+
+
+trap stop_processes EXIT
+read -r -p "Press anything and ENTER to exit";
diff --git a/extras/command-completion/gluster.bash b/extras/command-completion/gluster.bash
index 680ecd964d5..73d16098875 100644
--- a/extras/command-completion/gluster.bash
+++ b/extras/command-completion/gluster.bash
@@ -26,28 +26,28 @@ GLUSTER_TOP_SUBOPTIONS2="
 "
 GLUSTER_TOP_OPTIONS="
         {open
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {read
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {write
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {opendir
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {readdir
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {clear
-                [ $TOP_SUBOPTIONS1 ]
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
         },
         {read-perf
-                [ $TOP_SUBOPTIONS2 ]
+                [ $GLUSTER_TOP_SUBOPTIONS2 ]
         },
         {write-perf
-                [ $TOP_SUBOPTIONS2 ]
+                [ $GLUSTER_TOP_SUBOPTIONS2 ]
         }
 "
 
@@ -282,16 +282,16 @@ _gluster_throw () {
         exit
 }
 
-declare FINAL_LIST=''
-declare LIST=''
-declare -i TOP=0
+declare GLUSTER_FINAL_LIST=''
+declare GLUSTER_LIST=''
+declare -i GLUSTER_TOP=0
 _gluster_push () {
-        TOP=$((TOP + 1))
-        return $TOP
+        GLUSTER_TOP=$((GLUSTER_TOP + 1))
+        return $GLUSTER_TOP
 }
 _gluster_pop () {
-        TOP=$((TOP - 1))
-        return $TOP
+        GLUSTER_TOP=$((GLUSTER_TOP - 1))
+        return $GLUSTER_TOP
 }
 
 _gluster_goto_end ()
@@ -333,7 +333,7 @@ _gluster_form_list ()
                 top=$?
                 read -r key
                 if [ "X$cur_word" == "X" -o "${cur_word:0:1}" == "${key:0:1}" -o "${key:0:1}" == "_" ]; then
-                        LIST="$LIST $key"
+                        GLUSTER_LIST="$GLUSTER_LIST $key"
                 fi
 
                 _gluster_goto_end $top
@@ -452,10 +452,10 @@ _gluster_parse ()
 
         elif [ "$token" == '{' ]; then
                 read -r tmp_token
-                LIST="$tmp_token"
+                GLUSTER_LIST="$tmp_token"
         fi
 
-        echo $LIST
+        echo $GLUSTER_LIST
 }
 
 _gluster_handle_list ()
@@ -479,12 +479,12 @@ _gluster_handle_list ()
 
 _gluster_completion ()
 {
-        FINAL_LIST=`echo $GLUSTER_COMMAND_TREE |                              \
+        GLUSTER_FINAL_LIST=`echo $GLUSTER_COMMAND_TREE |                      \
                 egrep -ao --color=never "([A-Za-z0-9_.-]+)|[[:space:]]+|." |  \
                         egrep -v --color=never "^[[:space:]]*$" |             \
                                 _gluster_parse`
 
-        ARG="FINAL_LIST"
+        ARG="GLUSTER_FINAL_LIST"
         _gluster_handle_list $ARG ${COMP_WORDS[COMP_CWORD]}
         return
 }
diff --git a/extras/control-cpu-load.sh b/extras/control-cpu-load.sh
new file mode 100755
index 00000000000..52dcf62fd9f
--- /dev/null
+++ b/extras/control-cpu-load.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+USAGE="This script provides a utility to control CPU utilization for any
+gluster daemon.In this, we use cgroup framework to configure CPU quota
+for a process(like selfheal daemon). Before running this script, make
+sure that daemon is running.Every time daemon restarts, it is required
+to rerun this command to set CPU quota on new daemon process id.
+User can enter any value between 10 to 100 for CPU quota.
+Recommended value of quota period is 25. 25 means, kernel will allocate
+25 ms period to this group of tasks in every 100 ms period. This 25ms
+could be considered as the maximum percentage of CPU quota daemon can take.
+This value will be reflected on CPU usage of "top" command.If provided pid
+is the only process and no other process is in competition to get CPU, more
+ than 25% could be allocated to daemon to speed up the process."
+
+if [  $# -ge 1 ]; then
+  case $1 in
+    -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g'
+               exit 0;
+               ;;
+  *) echo "Please Provide correct input for script."
+     echo "For help correct options are -h or --help."
+     exit 1;
+               ;;
+  esac
+fi
+
+DIR_EXIST=0
+LOC="/sys/fs/cgroup/cpu,cpuacct/system.slice/glusterd.service"
+echo "Enter gluster daemon pid for which you want to control CPU."
+read daemon_pid
+
+if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then
+  CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid})
+  if [ -z "${CHECK_PID}" ]; then
+    echo "No daemon is running or pid ${daemon_pid} does not match."
+    echo "with running gluster processes."
+    exit 1
+  fi
+else
+  echo "Entered daemon_pid is not numeric so Rerun the script."
+  exit 1
+fi
+
+
+if [ -f ${LOC}/tasks ];then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    echo "pid ${daemon_pid} is attached with glusterd.service cgroup."
+  fi
+fi
+
+cgroup_name=cgroup_gluster_${daemon_pid}
+if [ -f ${LOC}/${cgroup_name}/tasks ]; then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    val=`cat ${LOC}/${cgroup_name}/cpu.cfs_quota_us`
+    qval=$((val / 1000))
+    echo "pid ${daemon_pid} is already attached ${cgroup_name} with quota value ${qval}."
+    echo "Press n if you don't want to reassign ${daemon_pid} with new quota value."
+    DIR_EXIST=1
+  else
+    echo "pid ${daemon_pid} is not attached with ${cgroup_name}."
+  fi
+fi
+
+read -p "If you want to continue the script to attach ${daemon_pid} with new ${cgroup_name} cgroup Press (y/n)?" choice
+case "$choice" in
+  y|Y ) echo "yes";;
+  n|N ) echo "no";exit;;
+  * ) echo "invalid";exit;;
+esac
+
+systemctl set-property glusterd.service CPUShares=1024
+
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service."
+  mkdir -p ${LOC}/${cgroup_name}
+  if [ ! -f ${LOC}/${cgroup_name}/tasks ];then
+    echo "Not able to create ${cgroup_name} directory so exit."
+    exit 1
+  fi
+fi
+
+echo "Enter quota value in range [10,100]:  "
+
+read quota_value
+if expr ${quota_value} + 0 > /dev/null 2>&1 ;then
+  if [ ${quota_value} -lt 10 ] || [ ${quota_value} -gt 100 ]; then
+    echo "Entered quota value is not correct,it should be in the range ."
+    echo "10-100. Ideal value is 25."
+    echo "Rerun the sript with correct value."
+    exit 1
+  else
+    echo "Entered quota value is $quota_value"
+  fi
+else
+  echo "Entered quota value is not numeric so Rerun the script."
+  exit 1
+fi
+
+quota_value=$((quota_value * 1000))
+echo "Setting $quota_value to cpu.cfs_quota_us for gluster_cgroup."
+echo ${quota_value} > ${LOC}/${cgroup_name}/cpu.cfs_quota_us
+
+if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then
+  for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`;
+    do
+      echo ${thid} > ${LOC}/${cgroup_name}/tasks ;
+    done
+  if cat /proc/${daemon_pid}/cgroup | grep -w ${cgroup_name} > /dev/null; then
+    echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}."
+  else
+    echo "Tasks are not attached successfully."
+  fi
+fi
diff --git a/extras/control-mem.sh b/extras/control-mem.sh
new file mode 100755
index 00000000000..91b36f8107a
--- /dev/null
+++ b/extras/control-mem.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+USAGE="This commands provides a utility to control MEMORY utilization for any
+gluster daemon.In this, we use cgroup framework to configure MEMORY limit for
+a process. Before running this script, make sure that daemon is running.Every
+time daemon restarts, it is required to rerun this command to set memory limit
+(in bytes) on new daemon process id.User can enter any value between 100
+(in Mega bytes) to 8000000000000 for Memory limit in Mega bytes.
+Memory limit value is depends on how much maximum memory user wants to restrict
+for specific daemon process.If a process will try to consume memore more than
+configured value then cgroup will hang/sleep this task and to resume the task
+rerun the script with new increase memory limit value ."
+
+if [  $# -ge 1 ]; then
+  case $1 in
+    -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g'
+               exit 0;
+               ;;
+    *) echo "Please Provide correct input for script."
+       echo "For help correct options are -h of --help."
+       exit 1;
+               ;;
+  esac
+fi
+
+DIR_EXIST=0
+LOC="/sys/fs/cgroup/memory/system.slice/glusterd.service"
+echo "Enter Any gluster daemon pid for that you want to control MEMORY."
+read daemon_pid
+
+if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then
+  CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid})
+  if [ -z "${CHECK_PID}" ]; then
+    echo "No daemon is running or pid ${daemon_pid} does not match."
+    echo "with running gluster processes."
+    exit 1
+  fi
+else
+  echo "Entered daemon_pid is not numeric so Rerun the script."
+  exit 1
+fi
+
+
+if [ -f ${LOC}/tasks ]; then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks)
+  if [ ${CHECK_CGROUP} ] ;then
+    echo "pid ${daemon_pid} is attached with default glusterd.service cgroup."
+  fi
+fi
+
+cgroup_name=cgroup_gluster_${daemon_pid}
+if [ -f ${LOC}/${cgroup_name}/tasks ];then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    val=`cat ${LOC}/${cgroup_name}/memory.limit_in_bytes`
+    mval=$((val / 1024 / 1024))
+    echo "pid ${daemon_pid} is already attached ${cgroup_name} with mem value ${mval}."
+    echo "Press n if you don't want to reassign ${daemon_pid} with new mem value."
+    DIR_EXIST=1
+  else
+    echo "pid ${daemon_pid} is not attached with ${cgroup_name}."
+  fi
+fi
+
+read -p "If you want to continue the script to attach daeomon with new cgroup. Press (y/n)?" choice
+case "$choice" in
+  y|Y ) echo "yes";;
+  n|N ) echo "no";exit;;
+  * ) echo "invalid";exit;;
+esac
+
+systemctl set-property glusterd.service CPUShares=1024
+
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service."
+  mkdir -p ${LOC}/${cgroup_name}
+  if [ ! -f ${LOC}/${cgroup_name}/tasks ];then
+    echo "Not able to create ${LOC}/${cgroup_name} directory so exit."
+    exit 1
+  fi
+fi
+
+echo "Enter Memory value in Mega bytes [100,8000000000000]:  "
+
+read mem_value
+if expr ${mem_value} + 0 > /dev/null 2>&1 ;then
+  if [ ${mem_value} -lt 100 ] || [ ${mem_value} -gt 8000000000000 ]; then
+    echo "Entered memory value is not correct,it should be in the range ."
+    echo "100-8000000000000, Rerun the script with correct value ."
+    exit 1
+  else
+    echo "Entered memory limit value is ${mem_value}."
+  fi
+else
+  echo "Entered memory value is not numeric so Rerun the script."
+  exit 1
+fi
+
+mem_value=$(($mem_value * 1024 * 1024))
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Setting ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}."
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes
+  #Set memory value to memory.memsw.limit_in_bytes
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes
+  # disable oom_control so that kernel will not send kill signal to the
+  # task once limit has reached
+  echo 1 > ${LOC}/${cgroup_name}/memory.oom_control
+else
+  #Increase mem_value to memory.memsw.limit_in_bytes
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes
+  echo "Increase ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}."
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes
+  # disable oom_control so that kernel will not send kill signal to the
+  # task once limit has reached
+  echo 1 > ${LOC}/${cgroup_name}/memory.oom_control
+fi
+
+if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then
+  for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`;
+    do
+      echo ${thid} > ${LOC}/${cgroup_name}/tasks ;
+    done
+  if cat /proc/${daemon_pid}/cgroup | grep -iw ${cgroup_name} > /dev/null; then
+    echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}."
+  else
+    echo "Tasks are not attached successfully."
+  fi
+fi
diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py
index dd45b1ef55e..983868c04db 100755
--- a/extras/create_new_xlator/generate_xlator.py
+++ b/extras/create_new_xlator/generate_xlator.py
@@ -1,4 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+from __future__ import print_function
 import os
 import re
 import sys
@@ -11,11 +13,12 @@ from generator import ops, xlator_cbks, xlator_dumpops
 MAKEFILE_FMT = """
 xlator_LTLIBRARIES = @XL_NAME@.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/@XL_TYPE@
-@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module -avoid-version
+@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 @XL_NAME_NO_HYPHEN@_la_SOURCES = @XL_NAME@.c
 @XL_NAME_NO_HYPHEN@_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 noinst_HEADERS = @XL_NAME@.h @XL_NAME@-mem-types.h @XL_NAME@-messages.h
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+           -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 CLEANFILES =
 """
@@ -33,11 +36,11 @@ def get_error_arg(type_str):
 
 def get_param(names, types):
         # Convert two separate tuples to one of (name, type) sub-tuples.
-        as_tuples = zip(types, names)
+        as_tuples = list(zip(types, names))
         # Convert each sub-tuple into a "type name" string.
-        as_strings = map(string.join, as_tuples)
+        as_strings = [' '.join(item) for item in as_tuples]
         # Join all of those into one big string.
-        return string.join(as_strings, ",\n\t")
+        return ',\n\t'.join(as_strings)
 
 
 def generate(tmpl, name, table):
@@ -53,18 +56,18 @@ def generate(tmpl, name, table):
         sdict = {}
         #Parameters are (t1, var1), (t2, var2)...
         #Args are (var1, var2,...)
-        sdict["@WIND_ARGS@"] = string.join(w_arg_names, ", ")
-        sdict["@UNWIND_ARGS@"] = string.join(u_arg_names, ", ")
-        sdict["@ERROR_ARGS@"] = string.join(map(get_error_arg, u_arg_types), ", ")
+        sdict["@WIND_ARGS@"] = ', '.join(w_arg_names)
+        sdict["@UNWIND_ARGS@"] = ', '.join(u_arg_names)
+        sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, u_arg_types)))
         sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types)
         sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types)
         sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types)
         sdict["@NAME@"] = name
         sdict["@FOP_PREFIX@"] = fop_prefix
-        sdict["@RET_TYPE@"] = string.join(ret_type, "")
-        sdict["@RET_VAR@"] = string.join(ret_var, "")
+        sdict["@RET_TYPE@"] = ''.join(ret_type)
+        sdict["@RET_VAR@"] = ''.join(ret_var)
 
-        for old, new in sdict.iteritems():
+        for old, new in sdict.items():
                 tmpl = tmpl.replace(old, new)
         # TBD: reindent/reformat the result for maximum readability.
         return tmpl
@@ -73,42 +76,44 @@ def generate(tmpl, name, table):
 def gen_xlator():
         xl = open(src_dir_path+"/"+xl_name+".c", 'w+')
 
-        print >> xl, COPYRIGHT
-        print >> xl, fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@",
-                                                              xl_name)
+        print(COPYRIGHT, file=xl)
+        print(fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@",
+                                                              xl_name), file=xl)
 
         #Generate cbks and fops
         for fop in ops:
-                print >> xl, generate(fragments["CBK_TEMPLATE"], fop, ops)
-                print >> xl, generate(fragments["FOP_TEMPLATE"], fop, ops)
+                print(generate(fragments["CBK_TEMPLATE"], fop, ops), file=xl)
+                print(generate(fragments["FOP_TEMPLATE"], fop, ops), file=xl)
 
         for cbk in xlator_cbks:
-                print >> xl, generate(fragments["FUNC_TEMPLATE"], cbk,
-                                      xlator_cbks)
+                print(generate(fragments["FUNC_TEMPLATE"], cbk,
+                                      xlator_cbks), file=xl)
 
         for dops in xlator_dumpops:
-                print >> xl, generate(fragments["FUNC_TEMPLATE"], dops,
-                                      xlator_dumpops)
-
-        print >> xl, fragments["XLATOR_METHODS"]
+                print(generate(fragments["FUNC_TEMPLATE"], dops,
+                                      xlator_dumpops), file=xl)
 
         #Generate fop table
-        print >> xl, "struct xlator_fops fops = {"
+        print("struct xlator_fops fops = {", file=xl)
         for fop in ops:
-                print >> xl, "        .{0:20} = {1}_{2},".format(fop, fop_prefix, fop)
-        print >> xl, "};"
+                print("        .{0:20} = {1}_{2},".format(fop, fop_prefix, fop), file=xl)
+        print("};", file=xl)
 
         #Generate xlator_cbks table
-        print >> xl, "struct xlator_cbks cbks = {"
+        print("struct xlator_cbks cbks = {", file=xl)
         for cbk in xlator_cbks:
-                print >> xl, "        .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk)
-        print >> xl, "};"
+                print("        .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk), file=xl)
+        print("};", file=xl)
 
         #Generate xlator_dumpops table
-        print >> xl, "struct xlator_dumpops dumpops = {"
+        print("struct xlator_dumpops dumpops = {", file=xl)
         for dops in xlator_dumpops:
-                print >> xl, "        .{0:20} = {1}_{2},".format(dops, fop_prefix, dops)
-        print >> xl, "};"
+                print("        .{0:20} = {1}_{2},".format(dops, fop_prefix, dops), file=xl)
+        print("};", file=xl)
+
+        xlator_methods = fragments["XLATOR_METHODS"].replace("@XL_NAME@", xl_name)
+        xlator_methods = xlator_methods.replace("@FOP_PREFIX@", fop_prefix)
+        print(xlator_methods, file=xl)
 
         xl.close()
 
@@ -121,38 +126,37 @@ def create_dir_struct():
 def gen_header_files():
         upname = xl_name_no_hyphen.upper()
         h = open(src_dir_path+"/"+xl_name+".h", 'w+')
-        print >> h, COPYRIGHT
+        print(COPYRIGHT, file=h)
         txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname)
-        txt2 = fragments["INCLUDE_IN_HEADER_FILE"].replace("@XL_NAME@", xl_name)
-        txt = txt.replace("@INCLUDE_SECT@",txt2)
-        print >> h, txt
+        txt = txt.replace("@XL_NAME@", xl_name)
+        print(txt, file=h)
         h.close()
 
         h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+')
-        print >> h, COPYRIGHT
-        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES")
-        txt = txt.replace("@INCLUDE_SECT@", '#include "mem-types.h"')
-        print >> h, txt
+        print(COPYRIGHT, file=h)
+        txt = fragments["MEM_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES")
+        txt = txt.replace("@FOP_PREFIX@", fop_prefix)
+        print(txt, file=h)
         h.close()
 
         h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+')
-        print >> h, COPYRIGHT
-        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES")
-        txt = txt.replace("@INCLUDE_SECT@", '')
-        print >> h, txt
+        print(COPYRIGHT, file=h)
+        txt = fragments["MSG_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES")
+        txt = txt.replace("@FOP_PREFIX@", fop_prefix.upper())
+        print(txt, file=h)
         h.close()
 
 
 def gen_makefiles():
         m = open(dir_path+"/Makefile.am", 'w+')
-        print >> m, "SUBDIRS = src\n\nCLEANFILES ="
+        print("SUBDIRS = src\n\nCLEANFILES =", file=m)
         m.close()
 
         m = open(src_dir_path+"/Makefile.am", 'w+')
         txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name)
         txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen)
-        txt = txt.replace("@XL_TYPE@",xlator_type)
-        print >> m, txt
+        txt = txt.replace("@XL_TYPE@", xlator_type)
+        print(txt, file=m)
         m.close()
 
 def get_copyright ():
@@ -165,8 +169,8 @@ def load_fragments ():
         cur_value = ""
         result = {}
         basepath = os.path.abspath(os.path.dirname(__file__))
-        fragpath = basepath + "/new-xlator-tmpl.c"
-        for line in open(fragpath,"r").readlines():
+        fragpath = basepath + "/new-xlator.c.tmpl"
+        for line in open(fragpath, "r").readlines():
                 m = pragma_re.search(line)
                 if m:
                         if cur_symbol:
@@ -182,7 +186,7 @@ def load_fragments ():
 if __name__ == '__main__':
 
         if len(sys.argv) < 3:
-                print "USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>"
+                print("USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>")
                 sys.exit(0)
 
         xl_name = sys.argv[2]
diff --git a/extras/create_new_xlator/new-xlator-tmpl.c b/extras/create_new_xlator/new-xlator-tmpl.c
deleted file mode 100644
index ac08f3732a7..00000000000
--- a/extras/create_new_xlator/new-xlator-tmpl.c
+++ /dev/null
@@ -1,89 +0,0 @@
-#pragma fragment CBK_TEMPLATE
-int32_t
-@FOP_PREFIX@_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, @UNWIND_PARAMS@)
-{
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                             @UNWIND_ARGS@);
-        return 0;
-}
-
-#pragma fragment COMMENT
-If you are generating the leaf xlators, remove the STACK_WIND
-and replace the @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary
-
-#pragma fragment FOP_TEMPLATE
-int32_t
-@FOP_PREFIX@_@NAME@ (call_frame_t *frame, xlator_t *this,
-        @WIND_PARAMS@)
-{
-        STACK_WIND (frame, @FOP_PREFIX@_@NAME@_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                    @WIND_ARGS@);
-        return 0;
-err:
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, errno,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-#pragma fragment FUNC_TEMPLATE
-@RET_TYPE@
-@FOP_PREFIX@_@NAME@ (@FUNC_PARAMS@)
-{
-        return @RET_VAR@;
-}
-
-#pragma fragment CP
-/*
- *   Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com>
- *   This file is part of GlusterFS.
- *
- *   This file is licensed to you under your choice of the GNU Lesser
- *   General Public License, version 3 or any later version (LGPLv3 or
- *   later), or the GNU General Public License, version 2 (GPLv2), in all
- *   cases as published by the Free Software Foundation.
- */
-
-#pragma fragment INCLUDE_IN_SRC_FILE
-#include "@XL_NAME@.h"
-
-#pragma fragment INCLUDE_IN_HEADER_FILE
-#include "@XL_NAME@-mem-types.h"
-#include "@XL_NAME@-messages.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
-
-#pragma fragment XLATOR_METHODS
-int32_t
-init (xlator_t *this)
-{
-        return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-int32_t
-reconfigure (xlator_t *this, dict_t *dict)
-{
-        return 0;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        return default_notify (this, event, data);
-}
-
-#pragma fragment HEADER_FMT
-#ifndef __@HFL_NAME@_H__
-#define __@HFL_NAME@_H__
-
-@INCLUDE_SECT@
-
-#endif /* __@HFL_NAME@_H__ */
diff --git a/extras/create_new_xlator/new-xlator.c.tmpl b/extras/create_new_xlator/new-xlator.c.tmpl
new file mode 100644
index 00000000000..fe9735bfcf1
--- /dev/null
+++ b/extras/create_new_xlator/new-xlator.c.tmpl
@@ -0,0 +1,151 @@
+#pragma fragment CBK_TEMPLATE
+int32_t @FOP_PREFIX@_@NAME@_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, @UNWIND_PARAMS@)
+{
+    STACK_UNWIND_STRICT(@NAME@, frame, op_ret, op_errno, @UNWIND_ARGS@);
+    return 0;
+}
+
+#pragma fragment COMMENT
+If you are generating the leaf xlators, remove the STACK_WIND and replace the
+    @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary
+
+#pragma fragment FOP_TEMPLATE
+        int32_t @FOP_PREFIX@_@NAME@(call_frame_t *frame, xlator_t *this, @WIND_PARAMS@)
+{
+    STACK_WIND(frame, @FOP_PREFIX@_@NAME@_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->@NAME@, @WIND_ARGS@);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(@NAME@, frame, -1, errno, @ERROR_ARGS@);
+    return 0;
+}
+
+#pragma fragment FUNC_TEMPLATE
+@RET_TYPE@ @FOP_PREFIX@_@NAME@(@FUNC_PARAMS@)
+{
+    return @RET_VAR@;
+}
+
+#pragma fragment CP
+/*
+ *   Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#pragma fragment INCLUDE_IN_SRC_FILE
+#include "@XL_NAME@.h"
+
+#pragma fragment XLATOR_METHODS
+
+static int32_t @FOP_PREFIX@_init(xlator_t *this)
+{
+    return 0;
+}
+
+static void @FOP_PREFIX@_fini(xlator_t *this)
+{
+    return;
+}
+
+static int32_t @FOP_PREFIX@_reconfigure(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+static int @FOP_PREFIX@_notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+static int32_t @FOP_PREFIX@_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_@FOP_PREFIX@_mt_end + 1);
+    return ret;
+}
+
+static int32_t @FOP_PREFIX@_dump_metrics(xlator_t *this, int fd)
+{
+    return 0;
+}
+
+struct volume_options @FOP_PREFIX@_options[] = {
+    /*{ .key  = {""},
+      .type = GF_OPTION_TYPE_BOOL,
+      .default_value = "",
+      .op_version = {GD_OP_VERSION_},
+      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+      .tags = {""},
+      .description = "",
+      .category = GF_EXPERIMENTAL,
+    },
+    { .key = {NULL} },
+    */
+};
+
+xlator_api_t xlator_api = {
+    .init = @FOP_PREFIX@_init,
+    .fini = @FOP_PREFIX@_fini,
+    .notify = @FOP_PREFIX@_notify,
+    .reconfigure = @FOP_PREFIX@_reconfigure,
+    .mem_acct_init = @FOP_PREFIX@_mem_acct_init,
+    .dump_metrics = @FOP_PREFIX@_dump_metrics,
+    .op_version = {GD_OP_VERSION_},
+    .dumpops = &@FOP_PREFIX@_dumpops,
+    .fops = &@FOP_PREFIX@_fops,
+    .cbks = &@FOP_PREFIX @_cbks,
+    .options = @FOP_PREFIX@_options,
+    .identifier = "@XL_NAME@",
+    .category = GF_EXPERIMENTAL,
+};
+#pragma fragment HEADER_FMT
+#ifndef __ @HFL_NAME@_H__
+#define __ @HFL_NAME@_H__
+
+#include "@XL_NAME@-mem-types.h"
+#include "@XL_NAME@-messages.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+#endif /* __@HFL_NAME@_H__ */
+
+#pragma fragment MEM_HEADER_FMT
+#ifndef __ @HFL_NAME@_H__
+#define __ @HFL_NAME@_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_mdc_mem_types_ {
+    gf_@FOP_PREFIX@_mt_ = gf_common_mt_end + 1,
+    gf_@FOP_PREFIX@_mt_end
+};
+
+#endif /* __@HFL_NAME@_H__ */
+
+#pragma fragment MSG_HEADER_FMT
+#ifndef __@HFL_NAME@_H__
+#define __@HFL_NAME@_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(@FOP_PREFIX@, @FOP_PREFIX@_MSG_NO_MEMORY);
+
+#endif /* __@HFL_NAME@_H__ */
diff --git a/extras/devel-tools/print-backtrace.sh b/extras/devel-tools/print-backtrace.sh
new file mode 100755
index 00000000000..33fbae288bc
--- /dev/null
+++ b/extras/devel-tools/print-backtrace.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+# sample unresolved backtrace lines picked up from a brick log that should go
+# into a backtrace file eg. bt-file.txt:
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323]
+#
+# following is the output of the script for the above backtrace lines:
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] __afr_selfheal_data_finalize_source inlined at /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:684 in __afr_selfheal_data_prepare /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:603
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] __afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:740
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:883
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] afr_selfheal_do /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:1968
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] afr_selfheal /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:2015
+#
+# Usage with debuginfo RPM:
+# print-backtrace.sh $HOME/Downloads/glusterfs-debuginfo-3.8.4-10.el7.x86_64.rpm bt-file.txt
+#
+# Usage with source install:
+# print-packtrace.sh none bt-file.txt
+
+function version_compare() { test $(echo $1|awk -F '.' '{print $1 $2 $3}') -gt $(echo $2|awk -F '.' '{print $1 $2 $3}'); }
+
+function Usage()
+{
+        echo -e "Usage:\n\t$0 { none | <debuginfo-rpm> } <backtrace-file>"
+        echo "none: implies we don't have a debuginfo rpm but want to resolve"
+        echo "      against a source install which already has the debuginfo"
+        echo "      NOTE: in this case you should have configured the build"
+        echo "            with --enable-debug and the linker options should"
+        echo "            have the option -rdynamic"
+}
+
+debuginfo_rpm=$1
+backtrace_file=$2
+
+if [ ! $debuginfo_rpm ] || [ ! $backtrace_file ]; then
+        Usage
+        exit 1
+fi
+
+if [ $debuginfo_rpm != "none" ]; then
+        if [ ! -f $debuginfo_rpm ]; then
+                echo "no such rpm file: $debuginfo_rpm"
+                exit 1
+        fi
+fi
+
+if [ ! -f $backtrace_file ]; then
+        echo "no such backtrace file: $backtrace_file"
+        exit 1
+fi
+
+if [ "$debuginfo_rpm" != "none" ]; then
+        if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then
+                echo "file does not look like an rpm: $debuginfo_rpm"
+                exit 1
+        fi
+fi
+
+cpio_version=$(cpio --version|grep cpio|cut -f 2 -d ')'|sed -e 's/^[[:space:]]*//')
+rpm_name=""
+debuginfo_path=""
+debuginfo_extension=""
+
+if [ $debuginfo_rpm != "none" ]; then
+        # extract the gluster debuginfo rpm to resolve the symbols against
+        rpm_name=$(basename $debuginfo_rpm '.rpm')
+        if [ -d $rpm_name ]; then
+                echo "directory already exists: $rpm_name"
+                echo "please remove/move it and reattempt"
+                exit 1
+        fi
+        mkdir -p $rpm_name
+        if version_compare $cpio_version "2.11"; then
+                rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name
+                ret=$?
+        else
+                current_dir="$PWD"
+                cd $rpm_name
+                rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time
+                ret=$?
+                cd $current_dir
+        fi
+        if [ $ret -eq 1 ]; then
+                echo "failed to extract rpm $debuginfo_rpm to $PWD/$rpm_name directory"
+                rm -rf $rpm_name
+                exit 1
+        fi
+        debuginfo_path="$PWD/$rpm_name/usr/lib/debug"
+        debuginfo_extension=".debug"
+else
+        debuginfo_path=""
+        debuginfo_extension=""
+fi
+
+# NOTE: backtrace file should contain only the lines which need to be resolved
+for bt in $(cat $backtrace_file)
+do
+        libname=$(echo $bt | cut -f 1 -d '(')
+        addr=$(echo $bt | cut -f 2 -d '(' | cut -f 1 -d ')')
+        libpath=${debuginfo_path}${libname}${debuginfo_extension}
+        if [ ! -f $libpath ]; then
+                continue
+        fi
+        newbt=( $(eu-addr2line --functions --exe=$libpath $addr) )
+        echo "$bt ${newbt[*]}"
+done
+
+# remove the temporary directory
+if [ -d $rpm_name ]; then
+        rm -rf $rpm_name
+fi
+
diff --git a/extras/devel-tools/strace-brick.sh b/extras/devel-tools/strace-brick.sh
new file mode 100755
index 00000000000..a140729111c
--- /dev/null
+++ b/extras/devel-tools/strace-brick.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+# Usage:
+# nice -n -19 strace-brick.sh glusterfsd 50
+
+brick_process_name=$1
+min_watch_cpu=$2
+if [ ! $brick_process_name ]; then
+        brick_process_name=glusterfsd
+fi
+
+if [ ! $min_watch_cpu ]; then
+        min_watch_cpu=50
+fi
+
+echo "min_watch_cpu: $min_watch_cpu"
+
+break=false
+
+while ! $break;
+do
+        mypids=( $(pgrep $brick_process_name) )
+        echo "mypids: ${mypids[*]}"
+
+        pid_args=$(echo ${mypids[*]} | sed -e 's/ / -p /g;s/^/-p /')
+        echo "pid_args: $pid_args"
+
+        pcpu=( $(ps $pid_args -o pcpu -h ) )
+        echo "pcpu: ${pcpu[*]}"
+
+        wait_longer=false
+
+        for i in $( seq 0 $((${#pcpu[*]} - 1)) )
+        do
+                echo "i: $i"
+                echo "mypids[$i]: ${mypids[$i]}"
+
+                int_pcpu=$(echo ${pcpu[$i]} | cut -f 1 -d '.')
+                echo "int_pcpu: $int_pcpu"
+                if [ ! $int_pcpu ] || [ ! $min_watch_cpu ]; then
+                        break=true
+                        echo "breaking"
+                fi
+                if [ $int_pcpu -ge $min_watch_cpu ]; then
+                        wait_longer=true
+                        mydirname="${brick_process_name}-${mypids[$i]}-$(date --utc +'%Y%m%d-%H%M%S.%N')"
+                        $(mkdir $mydirname && cd $mydirname && timeout --kill-after=5 --signal=KILL 60 nice -n -19 strace -p ${mypids[$i]} -ff -tt -T -o $brick_process_name) &
+                fi
+        done
+
+        if $wait_longer; then
+                sleep 90
+        else
+                sleep 15
+        fi
+done
diff --git a/extras/distributed-testing/README b/extras/distributed-testing/README
new file mode 100644
index 00000000000..928d943f211
--- /dev/null
+++ b/extras/distributed-testing/README
@@ -0,0 +1,28 @@
+PROBLEM
+
+The testing methodology of Gluster is extremely slow. It takes a very long time (6+ hrs) to run the basic tests on a single machine. It takes about 20+ hours to run code analysis version of tests like valgrind, asan, tsan etc.
+
+SOLUTION
+
+The fundamental problem is that the tests cannot be parallelized on a single machine. The natural solution is to run these tests on a cluster of machines. In a nutshell, apply map-reduce to run unit tests.
+
+WORK @ Facebook
+
+At Facebook we have applied the map-reduce approach to testing and have observed 10X improvements.
+
+The solution supports the following
+
+Distribute tests across machines, collect results/logs
+Share worker pool across different testers
+Try failure 3 times on 3 different machines before calling it a failure
+Support running asan, valgrind, asan-noleaks
+Self management of worker pools. The clients will manage the worker pool including version update, no manual maintenance required
+WORK
+
+Port the code from gluster-fb-3.8 to gluster master
+
+HOW TO RUN
+
+./extras/distributed-testing/distributed-test.sh --hosts '<h1> <h2> <h3>'
+
+All hosts should have no password for ssh via root. This can be achieved with keys setup on the client and the server machines.
diff --git a/extras/distributed-testing/distributed-test-build-env b/extras/distributed-testing/distributed-test-build-env
new file mode 100644
index 00000000000..cd68ff717da
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-build-env
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+GF_CONF_OPTS="--localstatedir=/var --sysconfdir /var/lib --prefix /usr --libdir /usr/lib64 \
+            --enable-bd-xlator=yes --enable-debug --enable-gnfs"
+
+if [ -x /usr/lib/rpm/redhat/dist.sh ]; then
+  REDHAT_MAJOR=$(/usr/lib/rpm/redhat/dist.sh --distnum)
+else
+  REDHAT_MAJOR=0
+fi
+
+ASAN_ENABLED=${ASAN_ENABLED:=0}
+if [ "$ASAN_ENABLED" -eq "1" ]; then
+    GF_CONF_OPTS="$GF_CONF_OPTS --with-asan"
+fi
+
+GF_CONF_OPTS="$GF_CONF_OPTS --with-systemd"
+export GF_CONF_OPTS
+
+export CFLAGS="-O0 -ggdb -fPIC -Wall"
diff --git a/extras/distributed-testing/distributed-test-build.sh b/extras/distributed-testing/distributed-test-build.sh
new file mode 100755
index 00000000000..e8910d8425c
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+EXTRA_CONFIGURE_ARGS="$@"
+ASAN_REQUESTED=false
+for arg in $EXTRA_CONFIGURE_ARGS; do
+  if [ $arg == "--with-asan" ]; then
+    echo "Requested ASAN, cleaning build first."
+    make -j distclean || true
+    touch .with_asan
+    ASAN_REQUESTED=true
+  fi
+done
+
+if [ $ASAN_REQUESTED == false ]; then
+  if [ -f .with_asan ]; then
+    echo "Previous build was with ASAN, cleaning build first."
+    make -j distclean || true
+    rm -v .with_asan
+  fi
+fi
+
+source extras/distributed-testing/distributed-test-build-env
+./autogen.sh
+./configure $GF_CONF_OPTS $EXTRA_CONFIGURE_ARGS
+make -j
diff --git a/extras/distributed-testing/distributed-test-env b/extras/distributed-testing/distributed-test-env
new file mode 100644
index 00000000000..36fdd82e5dd
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-env
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+SMOKE_TESTS="\
+    tests/basic/*.t\
+    tests/basic/afr/*.t\
+    tests/basic/distribute/*.t\
+    tests/bugs/fb*.t\
+    tests/features/brick-min-free-space.t\
+"
+
+KNOWN_FLAKY_TESTS="\
+"
+
+BROKEN_TESTS="\
+  tests/features/lock_revocation.t\
+  tests/features/recon.t\
+  tests/features/fdl-overflow.t\
+  tests/features/fdl.t\
+  tests/features/ipc.t\
+  tests/bugs/distribute/bug-1247563.t\
+  tests/bugs/distribute/bug-1543279.t\
+  tests/bugs/distribute/bug-1066798.t\
+  tests/bugs/ec/bug-1304988.t\
+  tests/bugs/unclassified/bug-1357397.t\
+  tests/bugs/quota/bug-1235182.t\
+  tests/bugs/fuse/bug-1309462.t\
+  tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\
+  tests/bugs/stripe/bug-1002207.t\
+  tests/bugs/stripe/bug-1111454.t\
+  tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
+  tests/bugs/write-behind/bug-1279730.t\
+  tests/bugs/gfapi/bug-1093594.t\
+  tests/bugs/replicate/bug-1473026.t\
+  tests/bugs/replicate/bug-802417.t\
+  tests/basic/inode-leak.t\
+  tests/basic/distribute/force-migration.t\
+  tests/basic/ec/heal-info.t\
+  tests/basic/ec/ec-seek.t\
+  tests/basic/jbr/jbr-volgen.t\
+  tests/basic/jbr/jbr.t\
+  tests/basic/afr/tarissue.t\
+  tests/basic/tier/tierd_check.t\
+  tests/basic/gfapi/bug1291259.t\
+"
+
+SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ')
+KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ')
+BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ')
diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py
new file mode 100755
index 00000000000..5a07e2feab1
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-runner.py
@@ -0,0 +1,859 @@
+#!/usr/bin/python3
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+import re
+import sys
+import fcntl
+import base64
+import threading
+import socket
+import os
+import shlex
+import argparse
+import subprocess
+import time
+import SimpleXMLRPCServer
+import xmlrpclib
+import md5
+import httplib
+import uuid
+
+DEFAULT_PORT = 9999
+TEST_TIMEOUT_S = 15 * 60
+CLIENT_CONNECT_TIMEOUT_S = 10
+CLIENT_TIMEOUT_S = 60
+PATCH_FILE_UID = str(uuid.uuid4())
+SSH_TIMEOUT_S = 10
+MAX_ATTEMPTS = 3
+ADDRESS_FAMILY = 'IPv4'
+
+
+def socket_instance(address_family):
+    if address_family.upper() == 'ipv4'.upper():
+        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    elif address_family.upper() == 'ipv6'.upper():
+        return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        Log.error("Invalid IP address family")
+        sys.exit(1)
+
+
+def patch_file():
+    return "/tmp/%s-patch.tar.gz" % PATCH_FILE_UID
+
+# ..............................................................................
+# SimpleXMLRPCServer IPvX Wrapper
+# ..............................................................................
+
+
+class GeneralXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
+    def __init__(self, addr):
+        SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr)
+
+    def server_bind(self):
+        if self.socket:
+            self.socket.close()
+        self.socket = socket_instance(args.address_family)
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)
+
+
+class HTTPConnection(httplib.HTTPConnection):
+    def __init__(self, host):
+        self.host = host
+        httplib.HTTPConnection.__init__(self, host)
+
+    def connect(self):
+        old_timeout = socket.getdefaulttimeout()
+        self.sock = socket.create_connection((self.host, self.port),
+                                             timeout=CLIENT_CONNECT_TIMEOUT_S)
+        self.sock.settimeout(old_timeout)
+
+
+class IPTransport(xmlrpclib.Transport):
+    def __init__(self, *args, **kwargs):
+        xmlrpclib.Transport.__init__(self, *args, **kwargs)
+
+    def make_connection(self, host):
+        return HTTPConnection(host)
+
+
+# ..............................................................................
+# Common
+# ..............................................................................
+
+
+class Timer:
+    def __init__(self):
+        self.start = time.time()
+
+    def elapsed_s(self):
+        return int(time.time() - self.start)
+
+    def reset(self):
+        ret = self.elapsed_s()
+        self.start = time.time()
+        return ret
+
+
+def encode(buf):
+    return base64.b16encode(buf)
+
+
+def decode(buf):
+    return base64.b16decode(buf)
+
+
+def get_file_content(path):
+    with open(path, "r") as f:
+        return f.read()
+
+
+def write_to_file(path, data):
+    with open(path, "w") as f:
+        f.write(data)
+
+
+def failsafe(fn, args=()):
+    try:
+        return (True, fn(*args))
+    except (xmlrpclib.Fault, xmlrpclib.ProtocolError, xmlrpclib.ResponseError,
+            Exception) as err:
+        Log.debug(str(err))
+    return (False, None)
+
+
+class LogLevel:
+    DEBUG = 2
+    ERROR = 1
+    CLI = 0
+
+
+class Log:
+    LOGLEVEL = LogLevel.ERROR
+
+    @staticmethod
+    def _normalize(msg):
+        return msg[:100]
+
+    @staticmethod
+    def debug(msg):
+        if Log.LOGLEVEL >= LogLevel.DEBUG:
+            sys.stdout.write("<debug> %s\n" % Log._normalize(msg))
+            sys.stdout.flush()
+
+    @staticmethod
+    def error(msg):
+        sys.stderr.write("<error> %s\n" % Log._normalize(msg))
+
+    @staticmethod
+    def header(msg):
+        sys.stderr.write("* %s *\n" % Log._normalize(msg))
+
+    @staticmethod
+    def cli(msg):
+        sys.stderr.write("%s\n" % msg)
+
+
+class Shell:
+    def __init__(self, cwd=None, logpath=None):
+        self.cwd = cwd
+        self.shell = True
+        self.redirect = open(os.devnull if not logpath else logpath, "wr+")
+
+    def __del__(self):
+        self.redirect.close()
+
+    def cd(self, cwd):
+        Log.debug("cd %s" % cwd)
+        self.cwd = cwd
+
+    def truncate(self):
+        self.redirect.truncate(0)
+
+    def read_logs(self):
+        self.redirect.seek(0)
+        return self.redirect.read()
+
+    def check_call(self, cmd):
+        status = self.call(cmd)
+        if status:
+            raise Exception("Error running command %s. status=%s"
+                            % (cmd, status))
+
+    def call(self, cmd):
+        if isinstance(cmd, list):
+            return self._calls(cmd)
+
+        return self._call(cmd)
+
+    def ssh(self, hostname, cmd, id_rsa=None):
+        flags = "" if not id_rsa else "-i " + id_rsa
+        return self.call("timeout %s ssh %s root@%s \"%s\"" %
+                            (SSH_TIMEOUT_S, flags, hostname, cmd))
+
+    def scp(self, hostname, src, dest, id_rsa=None):
+        flags = "" if not id_rsa else "-i " + id_rsa
+        return self.call("timeout %s scp %s %s root@%s:%s" %
+                            (SSH_TIMEOUT_S, flags, src, hostname, dest))
+
+    def output(self, cmd, cwd=None):
+        Log.debug("%s> %s" % (cwd, cmd))
+        return subprocess.check_output(shlex.split(cmd), cwd=self.cwd)
+
+    def _calls(self, cmds):
+        Log.debug("Running commands. %s" % cmds)
+        for c in cmds:
+            status = self.call(c)
+            if status:
+                Log.error("Commands failed with %s" % status)
+                return status
+        return 0
+
+    def _call(self, cmd):
+        if not self.shell:
+            cmd = shlex.split(cmd)
+
+        Log.debug("%s> %s" % (self.cwd, cmd))
+
+        status = subprocess.call(cmd, cwd=self.cwd, shell=self.shell,
+                                 stdout=self.redirect, stderr=self.redirect)
+
+        Log.debug("return %s" % status)
+        return status
+
+
+# ..............................................................................
+# Server role
+# ..............................................................................
+
+class TestServer:
+    def __init__(self, port, scratchdir):
+        self.port = port
+        self.scratchdir = scratchdir
+        self.shell = Shell()
+        self.rpc = None
+        self.pidf = None
+
+        self.shell.check_call("mkdir -p %s" % self.scratchdir)
+        self._process_lock()
+
+    def __del__(self):
+        if self.pidf:
+            self.pidf.close()
+
+    def init(self):
+        Log.debug("Starting xmlrpc server on port %s" % self.port)
+        self.rpc = GeneralXMLRPCServer(("", self.port))
+        self.rpc.register_instance(Handlers(self.scratchdir))
+
+    def serve(self):
+        (status, _) = failsafe(self.rpc.serve_forever)
+        Log.cli("== End ==")
+
+    def _process_lock(self):
+        pid_filename = os.path.basename(__file__).replace("/", "-")
+        pid_filepath = "%s/%s.pid" % (self.scratchdir, pid_filename)
+        self.pidf = open(pid_filepath, "w")
+        try:
+            fcntl.lockf(self.pidf, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # We have the lock, kick anybody listening on this port
+            self.shell.call("kill $(lsof -t -i:%s)" % self.port)
+        except IOError:
+            Log.error("Another process instance is running")
+            sys.exit(0)
+
+#
+# Server Handler
+#
+
+
+handler_lock = threading.Lock()
+handler_serving_since = Timer()
+
+
+def synchronized(func):
+    def decorator(*args, **kws):
+        handler_lock.acquire()
+        h = args[0]
+        try:
+            h.shell.truncate()
+            ret = func(*args, **kws)
+            return ret
+        except Exception() as err:
+            Log.error(str(err))
+            Log.error(decode(h._log_content()))
+            raise
+        finally:
+            handler_lock.release()
+            handler_serving_since.reset()
+
+    return decorator
+
+
+class Handlers:
+    def __init__(self, scratchdir):
+        self.client_id = None
+        self.scratchdir = scratchdir
+        self.gluster_root = "%s/glusterfs" % self.scratchdir
+        self.shell = Shell(logpath="%s/test-handlers.log" % self.scratchdir)
+
+    def hello(self, id):
+        if not handler_lock.acquire(False):
+            return False
+        try:
+            return self._hello_locked(id)
+        finally:
+            handler_lock.release()
+
+    def _hello_locked(self, id):
+        if handler_serving_since.elapsed_s() > CLIENT_TIMEOUT_S:
+            Log.debug("Disconnected client %s" % self.client_id)
+            self.client_id = None
+
+        if not self.client_id:
+            self.client_id = id
+            handler_serving_since.reset()
+            return True
+
+        return (id == self.client_id)
+
+    @synchronized
+    def ping(self, id=None):
+        if id:
+            return id == self.client_id
+        return True
+
+    @synchronized
+    def bye(self, id):
+        assert id == self.client_id
+        self.client_id = None
+        handler_serving_since.reset()
+        return True
+
+    @synchronized
+    def cleanup(self, id):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        self.shell.check_call("PATH=.:$PATH; sudo ./clean_gfs_devserver.sh")
+        return True
+
+    @synchronized
+    def copy(self, id, name, content):
+        with open("%s/%s" % (self.scratchdir, name), "w+") as f:
+            f.write(decode(content))
+        return True
+
+    @synchronized
+    def copygzip(self, id, content):
+        assert id == self.client_id
+        gzipfile = "%s/tmp.tar.gz" % self.scratchdir
+        tarfile = "%s/tmp.tar" % self.scratchdir
+        self.shell.check_call("rm -f %s" % gzipfile)
+        self.shell.check_call("rm -f %s" % tarfile)
+        write_to_file(gzipfile, decode(content))
+
+        self.shell.cd(self.scratchdir)
+        self.shell.check_call("rm -r -f %s" % self.gluster_root)
+        self.shell.check_call("mkdir -p %s" % self.gluster_root)
+
+        self.shell.cd(self.gluster_root)
+        cmds = [
+            "gunzip -f -q %s" % gzipfile,
+            "tar -xvf %s" % tarfile
+        ]
+        return self.shell.call(cmds) == 0
+
+    @synchronized
+    def build(self, id, asan=False):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        self.shell.call("make clean")
+        env = "ASAN_ENABLED=1" if asan else ""
+        return self.shell.call(
+		"%s ./extras/distributed-testing/distributed-test-build.sh" % env) == 0
+
+    @synchronized
+    def install(self, id):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        return self.shell.call("make install") == 0
+
+    @synchronized
+    def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        env = "DEBUG=1 "
+        if valgrind == "memcheck" or valgrind == "yes":
+            cmd = "valgrind"
+            cmd += " --tool=memcheck --leak-check=full --track-origins=yes"
+            cmd += " --show-leak-kinds=all -v prove -v"
+        elif valgrind == "drd":
+            cmd = "valgrind"
+            cmd += " --tool=drd -v prove -v"
+        elif asan_noleaks:
+            cmd = "prove -v"
+            env += "ASAN_OPTIONS=detect_leaks=0 "
+        else:
+            cmd = "prove -v"
+
+        status = self.shell.call(
+		"%s timeout %s %s %s" % (env, timeout, cmd, test))
+
+        if status != 0:
+            return (False, self._log_content())
+        return (True, "")
+
+    def _log_content(self):
+        return encode(self.shell.read_logs())
+
+# ..............................................................................
+# Cli role
+# ..............................................................................
+
+
+class RPCConnection((threading.Thread)):
+    def __init__(self, host, port, path, cb):
+        threading.Thread.__init__(self)
+        self.host = host
+        self.port = port
+        self.path = path
+        self.shell = Shell()
+        self.cb = cb
+        self.stop = False
+        self.proxy = None
+        self.logid = "%s:%s" % (self.host, self.port)
+
+    def connect(self):
+        (status, ret) = failsafe(self._connect)
+        return (status and ret)
+
+    def _connect(self):
+        url = "http://%s:%s" % (self.host, self.port)
+        self.proxy = xmlrpclib.ServerProxy(url, transport=IPTransport())
+        return self.proxy.hello(self.cb.id)
+
+    def disconnect(self):
+        self.stop = True
+
+    def ping(self):
+        return self.proxy.ping()
+
+    def init(self):
+        return self._copy() and self._compile_and_install()
+
+    def run(self):
+        (status, ret) = failsafe(self.init)
+        if not status:
+            self.cb.note_lost_connection(self)
+            return
+        elif not ret:
+            self.cb.note_setup_failed(self)
+            return
+
+        while not self.stop:
+            (status, ret) = failsafe(self._run)
+            if not status or not ret:
+                self.cb.note_lost_connection(self)
+                break
+            time.sleep(0)
+
+        failsafe(self.proxy.bye, (self.cb.id,))
+        Log.debug("%s connection thread stopped" % self.host)
+
+    def _run(self):
+        test = self.cb.next_test()
+        (status, _) = failsafe(self._execute_next, (test,))
+        if not status:
+            self.cb.note_retry(test)
+            return False
+        return True
+
+    def _execute_next(self, test):
+        if not test:
+            time.sleep(1)
+            return
+
+        (status, error) = self.proxy.prove(self.cb.id, test,
+                                           self.cb.test_timeout,
+                                           self.cb.valgrind,
+                                           self.cb.asan_noleaks)
+        if status:
+            self.cb.note_done(test)
+        else:
+            self.cb.note_error(test, error)
+
+    def _compile_and_install(self):
+        Log.debug("<%s> Build " % self.logid)
+        asan = self.cb.asan or self.cb.asan_noleaks
+        return (self.proxy.build(self.cb.id, asan) and
+                self.proxy.install(self.cb.id))
+
+    def _copy(self):
+        return self._copy_gzip()
+
+    def _copy_gzip(self):
+        Log.cli("<%s> copying and compiling %s to remote" %
+                 (self.logid, self.path))
+        data = encode(get_file_content(patch_file()))
+        Log.debug("GZIP size = %s B" % len(data))
+        return self.proxy.copygzip(self.cb.id, data)
+
+
+class RPCConnectionPool:
+    def __init__(self, gluster_path, hosts, n, id_rsa):
+        self.gluster_path = gluster_path
+        self.hosts = hosts
+        self.conns = []
+        self.faulty = []
+        self.n = int(len(hosts) / 2) + 1 if not n else n
+        self.id_rsa = id_rsa
+        self.stop = False
+        self.scanner = threading.Thread(target=self._scan_hosts_loop)
+        self.kicker = threading.Thread(target=self._kick_hosts_loop)
+        self.shell = Shell()
+        self.since_start = Timer()
+
+        self.shell.check_call("rm -f %s" % patch_file())
+        self.shell.check_call("tar -zcvf %s ." % patch_file())
+        self.id = md5.new(get_file_content(patch_file())).hexdigest()
+        Log.cli("client UID %s" % self.id)
+        Log.cli("patch UID %s" % PATCH_FILE_UID)
+
+    def __del__(self):
+        self.shell.check_call("rm -f %s" % patch_file())
+
+    def pool_status(self):
+        elapsed_m = int(self.since_start.elapsed_s() / 60)
+        return "%s/%s connected, %smin elapsed" % (len(self.conns), self.n,
+                                                   elapsed_m)
+
+    def connect(self):
+        Log.debug("Starting scanner")
+        self.scanner.start()
+        self.kicker.start()
+
+    def disconnect(self):
+        self.stop = True
+        for conn in self.conns:
+            conn.disconnect()
+
+    def note_lost_connection(self, conn):
+        Log.cli("lost connection to %s" % conn.host)
+        self.conns.remove(conn)
+        self.hosts.append((conn.host, conn.port))
+
+    def note_setup_failed(self, conn):
+        Log.error("Setup failed on %s:%s" % (conn.host, conn.port))
+        self.conns.remove(conn)
+        self.faulty.append((conn.host, conn.port))
+
+    def _scan_hosts_loop(self):
+        Log.debug("Scanner thread started")
+        while not self.stop:
+            failsafe(self._scan_hosts)
+            time.sleep(5)
+
+    def _scan_hosts(self):
+        if len(self.hosts) == 0 and len(self.conns) == 0:
+            Log.error("no more hosts available to loadbalance")
+            sys.exit(1)
+
+        for (host, port) in self.hosts:
+            if (len(self.conns) >= self.n) or self.stop:
+                break
+            self._scan_host(host, port)
+
+    def _scan_host(self, host, port):
+        Log.debug("scanning %s:%s" % (host, port))
+        c = RPCConnection(host, port, self.gluster_path, self)
+        (status, result) = failsafe(c.connect)
+        if status and result:
+            self.hosts.remove((host, port))
+            Log.debug("Connected to %s:%s" % (host, port))
+            self.conns.append(c)
+            c.start()
+            Log.debug("%s / %s connected" % (len(self.conns), self.n))
+        else:
+            Log.debug("Failed to connect to %s:%s" % (host, port))
+
+    def _kick_hosts_loop(self):
+        Log.debug("Kick thread started")
+        while not self.stop:
+            time.sleep(10)
+            failsafe(self._kick_hosts)
+
+        Log.debug("Kick thread stopped")
+
+    def _is_pingable(self, host, port):
+        c = RPCConnection(host, port, self.gluster_path, self)
+        failsafe(c.connect)
+        (status, result) = failsafe(c.ping)
+        return status and result
+
+    def _kick_hosts(self):
+        # Do not kick hosts if we have the optimal number of connections
+        if (len(self.conns) >= self.n) or self.stop:
+            Log.debug("Skip kicking hosts")
+            return
+
+        # Check and if dead kick all hosts
+        for (host, port) in self.hosts:
+            if self.stop:
+                Log.debug("Break kicking hosts")
+                break
+
+            if self._is_pingable(host, port):
+                Log.debug("Host=%s is alive. Won't kick" % host)
+                continue
+
+            Log.debug("Kicking %s" % host)
+            mypath = sys.argv[0]
+            myname = os.path.basename(mypath)
+            destpath = "/tmp/%s" % myname
+            sh = Shell()
+            sh.scp(host, mypath, destpath, self.id_rsa)
+            sh.ssh(host, "nohup %s --server &>> %s.log &" %
+                         (destpath, destpath), self.id_rsa)
+
+    def join(self):
+        self.scanner.join()
+        self.kicker.join()
+        for c in self.conns:
+            c.join()
+
+
+# ..............................................................................
+# test role
+# ..............................................................................
+
+class TestRunner(RPCConnectionPool):
+    def __init__(self, gluster_path, hosts, n, tests, flaky_tests, valgrind,
+                 asan, asan_noleaks, id_rsa, test_timeout):
+        RPCConnectionPool.__init__(self, gluster_path, self._parse_hosts(hosts),
+                                   n, id_rsa)
+        self.flaky_tests = flaky_tests.split(" ")
+        self.pending = []
+        self.done = []
+        self.error = []
+        self.retry = {}
+        self.error_logs = []
+        self.stats_timer = Timer()
+        self.valgrind = valgrind
+        self.asan = asan
+        self.asan_noleaks = asan_noleaks
+        self.test_timeout = test_timeout
+
+        self.tests = self._get_tests(tests)
+
+        Log.debug("tests: %s" % self.tests)
+
+    def _get_tests(self, tests):
+        if not tests or tests == "all":
+            return self._not_flaky(self._all())
+        elif tests == "flaky":
+            return self.flaky_tests
+        else:
+            return self._not_flaky(tests.strip().split(" "))
+
+    def run(self):
+        self.connect()
+        self.join()
+        return len(self.error)
+
+    def _pretty_print(self, data):
+        if isinstance(data, list):
+            str = ""
+            for i in data:
+                str = "%s %s" % (str, i)
+            return str
+        return "%s" % data
+
+    def print_result(self):
+        Log.cli("== RESULTS ==")
+        Log.cli("SUCCESS  : %s" % len(self.done))
+        Log.cli("ERRORS   : %s" % len(self.error))
+        Log.cli("== ERRORS ==")
+        Log.cli(self._pretty_print(self.error))
+        Log.cli("== LOGS ==")
+        Log.cli(self._pretty_print(self.error_logs))
+        Log.cli("== END ==")
+
+    def next_test(self):
+        if len(self.tests):
+            test = self.tests.pop()
+            self.pending.append(test)
+            return test
+
+        if not len(self.pending):
+            self.disconnect()
+
+        return None
+
+    def _pct_completed(self):
+        total = len(self.tests) + len(self.pending) + len(self.done)
+        total += len(self.error)
+        completed = len(self.done) + len(self.error)
+        return 0 if not total else int(completed / total * 100)
+
+    def note_done(self, test):
+        Log.cli("%s PASS (%s%% done) (%s)" % (test, self._pct_completed(),
+                                              self.pool_status()))
+        self.pending.remove(test)
+        self.done.append(test)
+        if test in self.retry:
+            del self.retry[test]
+
+    def note_error(self, test, errstr):
+        Log.cli("%s FAIL" % test)
+        self.pending.remove(test)
+        if test not in self.retry:
+            self.retry[test] = 1
+
+        if errstr:
+            path = "%s/%s-%s.log" % ("/tmp", test.replace("/", "-"),
+                                     self.retry[test])
+            failsafe(write_to_file, (path, decode(errstr),))
+            self.error_logs.append(path)
+
+        if self.retry[test] < MAX_ATTEMPTS:
+            self.retry[test] += 1
+            Log.debug("retry test %s attempt %s" % (test, self.retry[test]))
+            self.tests.append(test)
+        else:
+            Log.debug("giveup attempt test %s" % test)
+            del self.retry[test]
+            self.error.append(test)
+
+    def note_retry(self, test):
+        Log.cli("retry %s on another host" % test)
+        self.pending.remove(test)
+        self.tests.append(test)
+
+    #
+    # test classifications
+    #
+    def _all(self):
+        return self._list_tests(["tests"], recursive=True)
+
+    def _not_flaky(self, tests):
+        for t in self.flaky_tests:
+            if t in tests:
+                tests.remove(t)
+        return tests
+
+    def _list_tests(self, prefixes, recursive=False, ignore_ifnotexist=False):
+        tests = []
+        for prefix in prefixes:
+            real_path = "%s/%s" % (self.gluster_path, prefix)
+            if not os.path.exists(real_path) and ignore_ifnotexist:
+                continue
+            for f in os.listdir(real_path):
+                if os.path.isdir(real_path + "/" + f):
+                    if recursive:
+                        tests += self._list_tests([prefix + "/" + f], recursive)
+                else:
+                    if re.match(r".*\.t$", f):
+                        tests += [prefix + "/" + f]
+        return tests
+
+    def _parse_hosts(self, hosts):
+        ret = []
+        for h in args.hosts.split(" "):
+            ret.append((h, DEFAULT_PORT))
+        Log.debug(ret)
+        return ret
+
+# ..............................................................................
+# Roles entry point
+# ..............................................................................
+
+
+def run_as_server(args):
+    if not args.server_path:
+        Log.error("please provide server path")
+        return 1
+
+    server = TestServer(args.port, args.server_path)
+    server.init()
+    server.serve()
+    return 0
+
+
+def run_as_tester(args):
+    Log.header("GLUSTER TEST CLI")
+
+    Log.debug("args = %s" % args)
+
+    tests = TestRunner(args.gluster_path, args.hosts, args.n, args.tests,
+                       args.flaky_tests, valgrind=args.valgrind,
+                       asan=args.asan, asan_noleaks=args.asan_noleaks,
+                       id_rsa=args.id_rsa, test_timeout=args.test_timeout)
+    result = tests.run()
+    tests.print_result()
+    return result
+
+# ..............................................................................
+# main
+# ..............................................................................
+
+
+def main(args):
+    if args.v:
+        Log.LOGLEVEL = LogLevel.DEBUG
+
+    if args.server and args.tester:
+        Log.error("Invalid arguments. More than one role specified")
+        sys.exit(1)
+
+    if args.server:
+        sys.exit(run_as_server(args))
+    elif args.tester:
+        sys.exit(run_as_tester(args))
+    else:
+        Log.error("please specify a mode for CI")
+        parser.print_help()
+        sys.exit(1)
+
+
+parser = argparse.ArgumentParser(description="Gluster CI")
+
+# server role
+parser.add_argument("--server", help="start server", action="store_true")
+parser.add_argument("--server_path", help="server scratch space",
+                    default="/tmp/gluster-test")
+parser.add_argument("--host", help="server address to listen", default="")
+parser.add_argument("--port", help="server port to listen",
+                    type=int, default=DEFAULT_PORT)
+# test role
+parser.add_argument("--tester", help="start tester", action="store_true")
+parser.add_argument("--valgrind[=memcheck,drd]",
+                    help="run tests with valgrind tool 'memcheck' or 'drd'",
+                    default="no")
+parser.add_argument("--asan", help="test with asan enabled",
+                    action="store_true")
+parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks",
+                    action="store_true")
+parser.add_argument("--tests", help="all/flaky/list of tests", default=None)
+parser.add_argument("--flaky_tests", help="list of flaky tests", default=None)
+parser.add_argument("--n", help="max number of machines to use", type=int,
+                    default=0)
+parser.add_argument("--hosts", help="list of worker machines")
+parser.add_argument("--gluster_path", help="gluster path to test",
+                    default=os.getcwd())
+parser.add_argument("--id-rsa", help="private key to use for ssh",
+                    default=None)
+parser.add_argument("--test-timeout",
+                    help="test timeout in sec (default 15min)",
+                    default=TEST_TIMEOUT_S)
+# general
+parser.add_argument("-v", help="verbose", action="store_true")
+parser.add_argument("--address_family", help="IPv6 or IPv4 to use",
+                    default=ADDRESS_FAMILY)
+
+args = parser.parse_args()
+
+main(args)
diff --git a/extras/distributed-testing/distributed-test.sh b/extras/distributed-testing/distributed-test.sh
new file mode 100755
index 00000000000..8f1e0310f33
--- /dev/null
+++ b/extras/distributed-testing/distributed-test.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+source ./extras/distributed-testing/distributed-test-env
+
+N=0
+TESTS='all'
+FLAKY=$KNOWN_FLAKY_TESTS
+BROKEN=$BROKEN_TESTS
+TEST_TIMEOUT_S=900
+ADDRESS_FAMILY='IPv4'
+
+FLAGS=""
+
+function print_env {
+    echo "Settings:"
+    echo "N=$N"
+    echo -e "-------\nHOSTS\n$HOSTS\n-------"
+    echo -e "TESTS\n$TESTS\n-------"
+    echo -e "SKIP\n$FLAKY $BROKEN\n-------"
+    echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n"
+}
+
+function cleanup {
+    rm -f /tmp/test*.log
+}
+
+function usage {
+    echo "Usage: $0 [-h or --help] [-v or --verbose]
+             [--all] [--flaky] [--smoke] [--broken]
+             [--valgrind] [--asan] [--asan-noleaks]
+             [--hosts <hosts>] [-n <parallelism>]
+             [--tests <tests>]
+             [--id-rsa <ssh private key>]
+             [--address_family <IPv4 or IPv6>]
+    "
+}
+
+function parse_args () {
+    args=`getopt \
+            -o hvn: \
+            --long help,verbose,address_family:,valgrind,asan,asan-noleaks,all,\
+smoke,flaky,broken,hosts:,tests:,id-rsa:,test-timeout: \
+            -n 'fb-remote-test.sh' --  "$@"`
+
+    if [ $? != 0 ]; then
+        echo "Error parsing getopt"
+        exit 1
+    fi
+
+    eval set -- "$args"
+
+    while true; do
+        case "$1" in
+            -h | --help) usage ; exit 1 ;;
+            -v | --verbose) FLAGS="$FLAGS -v" ; shift ;;
+            --address_family) ADDRESS_FAMILY=$2; shift 2 ;;
+            --valgrind) FLAGS="$FLAGS --valgrind" ; shift ;;
+            --asan-noleaks) FLAGS="$FLAGS --asan-noleaks"; shift ;;
+            --asan) FLAGS="$FLAGS --asan" ; shift ;;
+            --hosts) HOSTS=$2; shift 2 ;;
+            --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;;
+            --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;;
+            --all) TESTS='all' ; shift 1 ;;
+            --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;;
+            --smoke) TESTS=$SMOKE_TESTS; shift 1 ;;
+            --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;;
+            --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;;
+            -n) N=$2; shift 2 ;;
+            *) break ;;
+            esac
+        done
+        run_tests_args="$@"
+}
+
+function main {
+    parse_args "$@"
+
+    if [ -z "$HOSTS" ]; then
+        echo "Please provide hosts to run the tests in"
+	exit -1
+    fi
+
+    print_env
+
+    cleanup
+
+    "extras/distributed-testing/distributed-test-runner.py" $FLAGS --tester \
+        --n "$N" --hosts "$HOSTS" --tests "$TESTS" \
+        --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S" \
+        --address_family "$ADDRESS_FAMILY"
+
+    exit $?
+}
+
+main "$@"
diff --git a/extras/ec-heal-script/README.md b/extras/ec-heal-script/README.md
new file mode 100644
index 00000000000..aaefd6681f6
--- /dev/null
+++ b/extras/ec-heal-script/README.md
@@ -0,0 +1,69 @@
+# gluster-heal-scripts
+Scripts to correct extended attributes of fragments of files to make them healble.
+
+Following are the guidelines/suggestions to use these scripts.
+
+1 - Passwordless ssh should be setup for all the nodes of the cluster.
+
+2 - Scripts should be executed from one of these nodes.
+
+3 - Make sure NO "IO" is going on for the files for which we are running
+these two scripts.
+
+4 - There should be no heal going on for the file for which xattrs are being
+set by correct_pending_heals.sh. Disable the self heal while running this script.
+
+5 - All the bricks of the volume should be UP to identify good and bad fragments
+and to decide if an entry is healble or not.
+
+6 - If correct_pending_heals.sh is stopped in the middle while it was processing
+healble entries, it is suggested to re-run gfid_needing_heal_parallel.sh to create
+latest list of healble and non healble entries and "potential_heal" "can_not_heal" files.
+
+7 - Based on the number of entries, these files might take time to get and set the
+stats and xattrs of entries.
+
+8 - A backup of the fragments will be taken on <brick path>/.glusterfs/correct_pending_heals
+    directory with a file name same as gfid.
+
+9 - Once the correctness of the file gets verified by user, these backup should be removed.
+
+10 - Make sure we have enough space on bricks to take these backups.
+
+11 - At the end this will create two files -
+     1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed.
+     2 - can_not_heal - Contains list of files which can not be healed.
+
+12 - It is suggested that the integrity of the data of files, which were modified and healed,
+     should be checked by the user.
+
+
+Usage:
+
+Following are the sequence of steps to use these scripts -
+
+1 - ./gfid_needing_heal_parallel.sh <volume name>
+
+    Execute gfid_needing_heal_parallel.sh with volume name to create list of files which could
+    be healed and can not be healed. It creates "potential_heal" and "can_not_heal" files.
+    During execution, it also displays the list of files on consol with the verdict.
+
+2 - ./correct_pending_heals.sh
+
+    Execute correct_pending_heals.sh without any argument. This script processes entries present
+    in "heal" file. It asks user to enter how many files we want to process in one attempt.
+    Once the count is provided, this script will fetch the entries one by one from "potential_heal" file and takes necessary action.
+    If at this point also a file can not be healed, it will be pushed to "can_not_heal" file.
+    If a file can be healed, this script will modify the xattrs of that file fragments and create an entry in "modified_and_backedup_files" file
+
+3 - At the end, all the entries of "potential_heal" will be processed and based on the processing only two files will be left.
+
+     1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed.
+     2 - can_not_heal - Contains list of files which can not be healed.
+
+Logs and other files -
+
+1 - modified_and_backedup_files  - It contains all the files which could be healed and the location of backup of each fragments.
+2 - can_not_heal - It contains all the files which can not be healed.
+3 - potential_heal - List of files which could be healed and should be processed by "correct_pending_heals.sh"
+4 - /var/log/glusterfs/ec-heal-script.log - It contains logs of both the files.
diff --git a/extras/ec-heal-script/correct_pending_heals.sh b/extras/ec-heal-script/correct_pending_heals.sh
new file mode 100755
index 00000000000..c9f19dd7c89
--- /dev/null
+++ b/extras/ec-heal-script/correct_pending_heals.sh
@@ -0,0 +1,415 @@
+#!/bin/bash
+#  Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+# This script finally resets the xattrs of all the fragments of a file
+# which can be healed as per gfid_needing_heal_parallel.sh.
+# gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal.
+# This script takes potential_heal as input and resets xattrs of all the fragments
+# of those files present in this file and which could be healed as per
+# trusted.ec.size xattar of the file else it will place the entry in can_not_heal
+# file.  Those entries which must be healed will be place in must_heal file
+# after setting xattrs so that user can track those files.
+
+
+MOD_BACKUP_FILES="modified_and_backedup_files"
+CAN_NOT_HEAL="can_not_heal"
+LOG_DIR="/var/log/glusterfs"
+LOG_FILE="$LOG_DIR/ec-heal-script.log"
+LINE_SEP="==================================================="
+
+function heal_log()
+{
+    echo "$1" >> "$LOG_FILE"
+}
+
+function desc ()
+{
+    echo ""
+    echo "This script finally resets the xattrs of all the fragments of a file
+which can be healed as per gfid_needing_heal_parallel.sh.
+gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal.
+This script takes potential_heal as input and resets xattrs of all the fragments
+of those files present in this file and which could be healed as per
+trusted.ec.size xattar of the file else it will place the entry in can_not_heal
+file.  Those entries which must be healed will be place in must_heal file
+after setting xattrs so that user can track those files."
+}
+
+function _init ()
+{
+    if [ $# -ne 0 ]
+    then
+        echo "usage: $0"
+        desc
+        exit 2
+    fi
+
+    if [ ! -f "potential_heal" ]
+    then
+        echo "Nothing to correct. File "potential_heal" does not exist"
+        echo ""
+        desc
+        exit 2
+    fi
+}
+
+function total_file_size_in_hex()
+{
+    local frag_size=$1
+    local size=0
+    local hex_size=""
+
+    size=$((frag_size * 4))
+    hex_size=$(printf '0x%016x' $size)
+    echo "$hex_size"
+}
+
+function backup_file_fragment()
+{
+    local file_host=$1
+    local file_entry=$2
+    local gfid_actual_paths=$3
+    local brick_root=""
+    local temp=""
+    local backup_dir=""
+    local cmd=""
+    local gfid=""
+
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+    temp=$(echo "$(basename "$BASH_SOURCE")" | cut -d '.' -f 1)
+    backup_dir=$(echo "${brick_root}/.glusterfs/${temp}")
+    file_entry=${file_entry//#}
+
+    gfid=$(echo "${gfid_actual_paths}" | cut -d '|' -f 1 | cut -d '/' -f 5)
+    echo "${file_host}:${backup_dir}/${gfid}" >> "$MOD_BACKUP_FILES"
+
+    cmd="mkdir -p ${backup_dir} && yes | cp -af ${file_entry} ${backup_dir}/${gfid} 2>/dev/null"
+    ssh -n "${file_host}" "${cmd}"
+}
+
+function set_frag_xattr ()
+{
+    local file_host=$1
+    local file_entry=$2
+    local good=$3
+    local cmd1=""
+    local cmd2=""
+    local cmd=""
+    local version="0x00000000000000010000000000000001"
+    local dirty="0x00000000000000010000000000000001"
+
+    if [[ $good -eq 0 ]]
+    then
+            version="0x00000000000000000000000000000000"
+    fi
+
+    cmd1=" setfattr -n trusted.ec.version -v ${version} ${file_entry} &&"
+    cmd2=" setfattr -n trusted.ec.dirty -v ${dirty} ${file_entry}"
+    cmd=${cmd1}${cmd2}
+    ssh -n "${file_host}" "${cmd}"
+}
+
+function set_version_dirty_xattr ()
+{
+    local file_paths=$1
+    local good=$2
+    local gfid_actual_paths=$3
+    local file_entry=""
+    local file_host=""
+    local bpath=""
+
+    for bpath in ${file_paths//,/ }
+    do
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        backup_file_fragment "$file_host" "$file_entry" "$gfid_actual_paths"
+        file_entry=${file_entry//#}
+        set_frag_xattr "$file_host" "$file_entry" "$good"
+    done
+}
+
+function match_size_xattr_quorum ()
+{
+    local file_paths=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local size_xattr=""
+    local bpath=""
+    declare -A xattr_count
+
+    for bpath in ${file_paths//,/ }
+    do
+        size_xattr=""
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        file_entry=${file_entry//#}
+
+        cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2"
+        size_xattr=$(ssh -n "${file_host}" "${cmd}")
+        if [[ -n $size_xattr ]]
+        then
+            count=$((xattr_count["$size_xattr"] + 1))
+            xattr_count["$size_xattr"]=${count}
+            if [[ $count -ge 4 ]]
+            then
+                echo "${size_xattr}"
+                return
+            fi
+        fi
+    done
+    echo "False"
+}
+
+function match_version_xattr ()
+{
+    local file_paths=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local version=""
+    local bpath=""
+    declare -A ver_count
+
+    for bpath in ${file_paths//,/ }
+    do
+        version=""
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        file_entry=${file_entry//#}
+
+        cmd="getfattr -n trusted.ec.version -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.version" | cut -d '=' -f 2"
+        version=$(ssh -n "${file_host}" "${cmd}")
+        ver_count["$version"]=$((ver_count["$version"] + 1))
+    done
+    for key in "${ver_count[@]}"
+    do
+        if [[ $key -ge 4 ]]
+        then
+            echo "True"
+            return
+        else
+            echo "False"
+            return
+        fi
+    done
+}
+
+function match_stat_size_with_xattr ()
+{
+    local bpath=$1
+    local size=$2
+    local file_stat=$3
+    local xattr=$4
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local stat_output=""
+    local hex_size=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+
+    file_entry=${file_entry//#}
+    cmd="stat --format=%F:%B:%s $file_entry 2>/dev/null"
+    stat_output=$(ssh -n "${file_host}" "${cmd}")
+    echo "$stat_output" | grep -w "${file_stat}" > /dev/null
+
+    if [[ $? -eq 0 ]]
+    then
+        cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2"
+        hex_size=$(ssh -n "${file_host}" "${cmd}")
+
+        if [[ -z $hex_size || "$hex_size" != "$xattr" ]]
+        then
+            echo "False"
+            return
+        fi
+        size_diff=$(printf '%d' $(( size - hex_size )))
+        if [[ $size_diff -gt 2047 ]]
+        then
+            echo "False"
+            return
+        else
+            echo "True"
+            return
+        fi
+    else
+        echo "False"
+        return
+    fi
+}
+
+function find_file_paths ()
+{
+    local bpath=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local brick_root=""
+    local gfid=""
+    local actual_path=""
+    local gfid_path=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+
+    gfid=$(echo "${file_entry}" | grep ".glusterfs")
+    if [[ -n "$gfid" ]]
+    then
+        gfid_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' "
+        actual_path=$(ssh -n "${file_host}" "${cmd}")
+        #removing absolute path so that user can refer this from mount point
+        actual_path=${actual_path#"$brick_root"}
+    else
+        actual_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' "
+        gfid_path=$(ssh -n "${file_host}" "${cmd}")
+        gfid_path=${gfid_path#"$brick_root"}
+    fi
+
+    echo "${gfid_path}|${actual_path}"
+}
+
+function log_can_not_heal ()
+{
+    local gfid_actual_paths=$1
+    local file_paths=$2
+    file_paths=${file_paths//#}
+
+    echo "${LINE_SEP}" >> "$CAN_NOT_HEAL"
+    echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL"
+    for bpath in ${file_paths//,/ }
+    do
+        echo "${bpath}" >> "$CAN_NOT_HEAL"
+    done
+}
+
+function check_all_frag_and_set_xattr ()
+{
+    local file_paths=$1
+    local total_size=$2
+    local file_stat=$3
+    local bpath=""
+    local healthy_count=0
+    local match="False"
+    local matching_bricks=""
+    local bad_bricks=""
+    local gfid_actual_paths=""
+
+    for bpath in ${file_paths//,/ }
+    do
+        if [[ -n "$gfid_actual_paths" ]]
+        then
+            break
+        fi
+        gfid_actual_paths=$(find_file_paths "$bpath")
+    done
+
+   match=$(match_size_xattr_quorum "$file_paths")
+
+#   echo "${match} : $bpath" >> "$MOD_BACKUP_FILES"
+
+    if [[ "$match" != "False" ]]
+    then
+        xattr="$match"
+        for bpath in ${file_paths//,/ }
+        do
+            match="False"
+            match=$(match_stat_size_with_xattr "$bpath" "$total_size" "$file_stat" "$xattr")
+            if [[ "$match" == "True" ]]
+            then
+                matching_bricks="${bpath},${matching_bricks}"
+                healthy_count=$((healthy_count + 1))
+            else
+                bad_bricks="${bpath},${bad_bricks}"
+            fi
+        done
+    fi
+
+    if [[ $healthy_count -ge 4 ]]
+    then
+        match="True"
+        echo "${LINE_SEP}" >> "$MOD_BACKUP_FILES"
+        echo "Modified : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$MOD_BACKUP_FILES"
+        set_version_dirty_xattr  "$matching_bricks" 1 "$gfid_actual_paths"
+        set_version_dirty_xattr  "$bad_bricks" 0 "$gfid_actual_paths"
+    else
+        log_can_not_heal "$gfid_actual_paths" "${file_paths}"
+    fi
+
+    echo "$match"
+}
+function set_xattr()
+{
+    local count=$1
+    local heal_entry=""
+    local file_stat=""
+    local frag_size=""
+    local total_size=""
+    local file_paths=""
+    local num=""
+    local can_heal_count=0
+
+    heal_log "Started $(basename $BASH_SOURCE) on $(date) "
+
+    while read -r heal_entry
+    do
+        heal_log "$LINE_SEP"
+        heal_log "${heal_entry}"
+
+        file_stat=$(echo "$heal_entry" | cut -d "|" -f 1)
+        frag_size=$(echo "$file_stat" | rev | cut -d ":" -f 1 | rev)
+        total_size="$(total_file_size_in_hex "$frag_size")"
+        file_paths=$(echo "$heal_entry" | cut -d "|" -f 2)
+        match=$(check_all_frag_and_set_xattr "$file_paths" "$total_size" "$file_stat")
+        if [[ "$match" == "True" ]]
+        then
+            can_heal_count=$((can_heal_count + 1))
+        fi
+
+        sed -i '1d' potential_heal
+        count=$((count - 1))
+        if [ $count == 0 ]
+        then
+            num=$(cat potential_heal | wc -l)
+            heal_log "$LINE_SEP"
+            heal_log "${1} : Processed"
+            heal_log "${can_heal_count} : Modified to Heal"
+            heal_log "$((${1} - can_heal_count)) : Moved to can_not_heal."
+            heal_log "${num} : Pending as Potential Heal"
+            exit 0
+        fi
+
+    done < potential_heal
+}
+
+function main ()
+{
+    local count=0
+
+    read -p "Number of files to correct: [choose between 1-1000] (0 for All):" count
+    if [[ $count -lt 0 || $count -gt 1000 ]]
+    then
+        echo "Provide correct value:"
+        exit 2
+    fi
+
+    if [[ $count -eq 0 ]]
+    then
+        count=$(cat potential_heal | wc -l)
+    fi
+    set_xattr "$count"
+}
+
+_init "$@" && main "$@"
diff --git a/extras/ec-heal-script/gfid_needing_heal_parallel.sh b/extras/ec-heal-script/gfid_needing_heal_parallel.sh
new file mode 100755
index 00000000000..d7f53c97c33
--- /dev/null
+++ b/extras/ec-heal-script/gfid_needing_heal_parallel.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+#  Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+# This script provides a list of all the files which can be healed or not healed.
+# It also generates two files, potential_heal and can_not_heal, which contains the information
+# of all theose files. These files could be used by correct_pending_heals.sh to correct
+# the fragmnets so that files could be healed by shd.
+
+CAN_NOT_HEAL="can_not_heal"
+CAN_HEAL="potential_heal"
+LINE_SEP="==================================================="
+LOG_DIR="/var/log/glusterfs"
+LOG_FILE="$LOG_DIR/ec-heal-script.log"
+
+function heal_log()
+{
+    echo "$1" >> "$LOG_FILE"
+}
+
+function _init ()
+{
+    if [ $# -ne 1 ]; then
+    echo "usage: $0 <gluster volume name>";
+    echo "This script provides a list of all the files which can be healed or not healed.
+It also generates two files, potential_heal and can_not_heal, which contains the information
+of all theose files. These files could be used by correct_pending_heals.sh to correct
+the fragmnets so that files could be healed by shd."
+    exit 2;
+    fi
+
+    volume=$1;
+}
+
+function get_pending_entries ()
+{
+    local volume_name=$1
+
+    gluster volume heal "$volume_name" info | grep -v ":/" | grep -v "Number of entries" | grep -v "Status:" | sort -u | sed '/^$/d'
+}
+
+function get_entry_path_on_brick()
+{
+    local path="$1"
+    local gfid_string=""
+    if [[ "${path:0:1}" == "/" ]];
+    then
+        echo "$path"
+    else
+        gfid_string="$(echo "$path" | cut -f2 -d':' | cut -f1 -d '>')"
+        echo "/.glusterfs/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string"
+    fi
+}
+
+function run_command_on_server()
+{
+	local subvolume="$1"
+	local host="$2"
+	local cmd="$3"
+	local output
+        output=$(ssh -n "${host}" "${cmd}")
+	if [ -n "$output" ]
+	then
+            echo "$subvolume:$output"
+	fi
+}
+
+function get_entry_path_all_bricks ()
+{
+    local entry="$1"
+    local bricks="$2"
+    local cmd=""
+    for brick in $bricks
+    do
+	    echo "${brick}#$(get_entry_path_on_brick "$entry")"
+    done | tr '\n' ','
+}
+
+function get_stat_for_entry_from_all_bricks ()
+{
+    local entry="$1"
+    local bricks="$2"
+    local subvolume=0
+    local host=""
+    local bpath=""
+    local cmd=""
+
+    for brick in $bricks
+    do
+        if [[ "$((subvolume % 6))" == "0" ]]
+        then
+            subvolume=$((subvolume+1))
+        fi
+        host=$(echo "$brick" | cut -f1 -d':')
+        bpath=$(echo "$brick" | cut -f2 -d':')
+
+	cmd="stat --format=%F:%B:%s $bpath$(get_entry_path_on_brick "$entry") 2>/dev/null"
+    run_command_on_server "$subvolume" "${host}" "${cmd}" &
+    done | sort | uniq -c | sort -rnk1
+}
+
+function get_bricks_from_volume()
+{
+    local v=$1
+    gluster volume info "$v" | grep -E "^Brick[0-9][0-9]*:" | cut -f2- -d':'
+}
+
+function print_entry_gfid()
+{
+    local host="$1"
+    local dirpath="$2"
+    local entry="$3"
+    local gfid
+    gfid="$(ssh -n "${host}" "getfattr -d -m. -e hex $dirpath/$entry 2>/dev/null | grep trusted.gfid=|cut -f2 -d'='")"
+    echo "$entry" - "$gfid"
+}
+
+function print_brick_directory_info()
+{
+    local h="$1"
+    local dirpath="$2"
+    while read -r e
+    do
+        print_entry_gfid "${h}" "${dirpath}" "${e}"
+    done < <(ssh -n "${h}" "ls $dirpath 2>/dev/null")
+}
+
+function print_directory_info()
+{
+    local entry="$1"
+    local bricks="$2"
+    local h
+    local b
+    local gfid
+    for brick in $bricks;
+    do
+        h="$(echo "$brick" | cut -f1 -d':')"
+        b="$(echo "$brick" | cut -f2 -d':')"
+        dirpath="$b$(get_entry_path_on_brick "$entry")"
+	print_brick_directory_info "${h}" "${dirpath}" &
+    done | sort | uniq -c
+}
+
+function print_entries_needing_heal()
+{
+    local quorum=0
+    local entry="$1"
+    local bricks="$2"
+    while read -r line
+    do
+        quorum=$(echo "$line" | awk '{print $1}')
+        if [[ "$quorum" -lt 4 ]]
+        then
+            echo "$line - Not in Quorum"
+        else
+            echo "$line - In Quorum"
+        fi
+    done < <(print_directory_info "$entry" "$bricks")
+}
+
+function find_file_paths ()
+{
+    local bpath=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local brick_root=""
+    local gfid=""
+    local actual_path=""
+    local gfid_path=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+
+    gfid=$(echo "${file_entry}" | grep ".glusterfs")
+
+    if [[ -n "$gfid" ]]
+    then
+        gfid_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' "
+        actual_path=$(ssh -n "${file_host}" "${cmd}")
+        #removing absolute path so that user can refer this from mount point
+        actual_path=${actual_path#"$brick_root"}
+    else
+        actual_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' "
+        gfid_path=$(ssh -n "${file_host}" "${cmd}")
+        gfid_path=${gfid_path#"$brick_root"}
+    fi
+
+    echo "${gfid_path}|${actual_path}"
+}
+
+function log_can_not_heal ()
+{
+    local gfid_actual_paths=$1
+    local file_paths=$2
+    file_paths=${file_paths//#}
+
+    echo "${LINE_SEP}" >> "$CAN_NOT_HEAL"
+    echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL"
+    for bpath in ${file_paths//,/ }
+    do
+        echo "${bpath}" >> "$CAN_NOT_HEAL"
+    done
+}
+
+function main ()
+{
+    local bricks=""
+    local quorum=0
+    local stat_info=""
+    local file_type=""
+    local gfid_actual_paths=""
+    local bpath=""
+    local file_paths=""
+    local good=0
+    local bad=0
+    bricks=$(get_bricks_from_volume "$volume")
+    rm -f "$CAN_HEAL"
+    rm -f "$CAN_NOT_HEAL"
+    mkdir "$LOG_DIR" -p
+
+    heal_log "Started $(basename "$BASH_SOURCE") on $(date) "
+    while read -r heal_entry
+    do
+        heal_log "------------------------------------------------------------------"
+        heal_log "$heal_entry"
+
+        gfid_actual_paths=""
+        file_paths="$(get_entry_path_all_bricks "$heal_entry" "$bricks")"
+        stat_info="$(get_stat_for_entry_from_all_bricks "$heal_entry" "$bricks")"
+        heal_log "$stat_info"
+
+        quorum=$(echo "$stat_info" | head -1 | awk '{print $1}')
+        good_stat=$(echo "$stat_info" | head -1 | awk '{print $3}')
+        file_type="$(echo "$stat_info" | head -1 | cut -f2 -d':')"
+        if [[ "$file_type" == "directory" ]]
+        then
+            print_entries_needing_heal "$heal_entry" "$bricks"
+        else
+            if [[ "$quorum" -ge 4 ]]
+            then
+                good=$((good + 1))
+                heal_log "Verdict: Healable"
+
+                echo "${good_stat}|$file_paths" >> "$CAN_HEAL"
+            else
+                bad=$((bad + 1))
+                heal_log "Verdict: Not Healable"
+                for bpath in ${file_paths//,/ }
+                do
+                    if [[ -z "$gfid_actual_paths" ]]
+                    then
+                        gfid_actual_paths=$(find_file_paths "$bpath")
+                    else
+                        break
+                    fi
+                done
+                log_can_not_heal "$gfid_actual_paths" "${file_paths}"
+            fi
+        fi
+    done < <(get_pending_entries "$volume")
+    heal_log "========================================="
+    heal_log "Total number of  potential heal : ${good}"
+    heal_log "Total number of can not heal    : ${bad}"
+    heal_log "========================================="
+}
+
+_init "$@" && main "$@"
diff --git a/extras/failed-tests.py b/extras/failed-tests.py
index 476e24331e5..f7f110246b5 100755
--- a/extras/failed-tests.py
+++ b/extras/failed-tests.py
@@ -1,5 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import blessings
 import requests
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
@@ -25,7 +26,7 @@ def process_failure(url, node):
         if t.find("Result: FAIL") != -1:
             for t2 in accum:
                 if VERBOSE:
-                    print t2.encode('utf-8')
+                    print(t2.encode('utf-8'))
                 if t2.find("Wstat") != -1:
                     test_case = re.search('\./tests/.*\.t', t2)
                     if test_case:
@@ -69,26 +70,26 @@ def print_summary(failed_builds, total_builds, html=False):
     template = 0
     if html:
         template = 1
-    print render(
+    print(render(
             count[template],
             {'failed': failed_builds, 'total': total_builds}
-    )
-    for k, v in summary.iteritems():
+    ))
+    for k, v in summary.items():
         if k == 'core':
-            print ''.join([TERM.red, "Found cores:", TERM.normal])
+            print(''.join([TERM.red, "Found cores:", TERM.normal]))
             for comp, link in zip(v[::2], v[1::2]):
-                print render(component[template], {'comp': comp})
-                print render(
+                print(render(component[template], {'comp': comp}))
+                print(render(
                         regression_link[template],
                         {'link': link[0], 'node': link[1]}
-                )
+                ))
         else:
-            print render(failure_count[template], {'test': k, 'count': len(v)})
+            print(render(failure_count[template], {'test': k, 'count': len(v)}))
             for link in v:
-                print render(
+                print(render(
                         regression_link[template],
                         {'link': link[0], 'node': link[1]}
-                )
+                ))
 
 
 def get_summary(cut_off_date, reg_link):
@@ -97,7 +98,7 @@ def get_summary(cut_off_date, reg_link):
     '''
     success_count = 0
     failure_count = 0
-    for page in xrange(0, MAX_BUILDS, 100):
+    for page in range(0, MAX_BUILDS, 100):
         build_info = requests.get(''.join([
                 BASE,
                 reg_link,
@@ -114,11 +115,11 @@ def get_summary(cut_off_date, reg_link):
                 success_count += 1
                 continue
             if VERBOSE:
-                print ''.join([
+                print(''.join([
                     TERM.red,
                     'FAILURE on {0}'.format(build['url']),
                     TERM.normal
-                ])
+                ]))
             url = ''.join([build['url'], 'consoleText'])
             failure_count += 1
             process_failure(url, build['builtOn'])
diff --git a/extras/firewalld/Makefile.am b/extras/firewalld/Makefile.am
index a5c11b0b783..530881fb8eb 100644
--- a/extras/firewalld/Makefile.am
+++ b/extras/firewalld/Makefile.am
@@ -1,6 +1,8 @@
 EXTRA_DIST = glusterfs.xml
 
 if USE_FIREWALLD
+if WITH_SERVER
 staticdir = /usr/lib/firewalld/services/
 static_DATA = glusterfs.xml
 endif
+endif
diff --git a/extras/ganesha/ocf/Makefile.am b/extras/ganesha/ocf/Makefile.am
index 6aed9548a0f..990a609f254 100644
--- a/extras/ganesha/ocf/Makefile.am
+++ b/extras/ganesha/ocf/Makefile.am
@@ -9,4 +9,3 @@ ocfdir = $(prefix)/lib/ocf
 radir = $(ocfdir)/resource.d/heartbeat
 
 ra_SCRIPTS = ganesha_grace ganesha_mon ganesha_nfsd
-
diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace
index 84202df9aa2..825f7164597 100644
--- a/extras/ganesha/ocf/ganesha_grace
+++ b/extras/ganesha/ocf/ganesha_grace
@@ -164,7 +164,6 @@ ganesha_grace_notify()
 
 ganesha_grace_monitor()
 {
-	local rc=${OCF_ERR_GENERIC}
 	local host=$(hostname -s)
 
 	ocf_log debug "monitor"
@@ -187,10 +186,10 @@ ganesha_grace_monitor()
 	fi
 
 	if [[ "${attr}" = *"value=1" ]]; then
-		rc=${OCF_SUCCESS}
+		return ${OCF_SUCCESS}
 	fi
 
-	return ${rc}
+	return ${OCF_NOT_RUNNING}
 }
 
 ganesha_grace_validate()
@@ -220,4 +219,3 @@ rc=$?
 # The resource agent may optionally log a debug message
 ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
 exit $rc
-
diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon
index 7d2c268d412..2b4a9d6da84 100644
--- a/extras/ganesha/ocf/ganesha_mon
+++ b/extras/ganesha/ocf/ganesha_mon
@@ -232,4 +232,3 @@ rc=$?
 # The resource agent may optionally log a debug message
 ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
 exit $rc
-
diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd
index 29e333ca903..f91e8b6b8f7 100644
--- a/extras/ganesha/ocf/ganesha_nfsd
+++ b/extras/ganesha/ocf/ganesha_nfsd
@@ -36,7 +36,7 @@ else
 	. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
 fi
 
-OCF_RESKEY_ha_vol_mnt_default="/var/run/gluster/shared_storage"
+OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage"
 : ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}}
 
 ganesha_meta_data() {
@@ -165,4 +165,3 @@ rc=$?
 # The resource agent may optionally log a debug message
 ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
 exit $rc
-
diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am
index 224ed26e75b..7e345fd5f19 100644
--- a/extras/ganesha/scripts/Makefile.am
+++ b/extras/ganesha/scripts/Makefile.am
@@ -1,6 +1,6 @@
-EXTRA_DIST= ganesha-ha.sh dbus-send.sh create-export-ganesha.sh \
-            generate-epoch.py
+EXTRA_DIST= create-export-ganesha.sh generate-epoch.py dbus-send.sh \
+			ganesha-ha.sh
 
 scriptsdir = $(libexecdir)/ganesha
-scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh ganesha-ha.sh \
-                  generate-epoch.py
+scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh generate-epoch.py \
+				  ganesha-ha.sh
diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh
index 1ffba427457..3040e8138b0 100755
--- a/extras/ganesha/scripts/create-export-ganesha.sh
+++ b/extras/ganesha/scripts/create-export-ganesha.sh
@@ -64,6 +64,7 @@ echo "      Pseudo=\"/$VOL\";"
 echo '      Protocols = "3", "4" ;'
 echo '      Transports = "UDP","TCP";'
 echo '      SecType = "sys";'
+echo '      Security_Label = False;'
 echo "     }"
 }
 if [ "$OPTION" = "on" ];
diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh
index c071d03480c..9d613a0e7ad 100755
--- a/extras/ganesha/scripts/dbus-send.sh
+++ b/extras/ganesha/scripts/dbus-send.sh
@@ -41,8 +41,18 @@ string:"EXPORT(Path=/$VOL)"
 #This function removes an export dynamically(uses the export_id of the export)
 function dynamic_export_remove()
 {
-        removed_id=`cat $GANESHA_DIR/exports/export.$VOL.conf |\
-grep Export_Id | awk -F"[=,;]" '{print$2}'| tr -d '[[:space:]]'`
+        # Below bash fetch all the export from ShowExport command and search
+        # export entry based on path and then get its export entry.
+        # There are two possiblities for path, either entire volume will be
+        # exported or subdir. It handles both cases. But it remove only first
+        # entry from the list based on assumption that entry exported via cli
+        # has lowest export id value
+	removed_id=$(dbus-send --type=method_call --print-reply --system \
+                    --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
+                    org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \
+                    "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \
+		    | head -1)
+
         dbus-send --print-reply --system \
 --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
 org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id
@@ -58,4 +68,3 @@ if [ "$OPTION" = "off" ];
 then
         dynamic_export_remove $@
 fi
-
diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh
index 5162eb5e4da..9790a719e10 100644
--- a/extras/ganesha/scripts/ganesha-ha.sh
+++ b/extras/ganesha/scripts/ganesha-ha.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2015 Red Hat Inc.  All Rights Reserved
+# Copyright 2015-2016 Red Hat Inc.  All Rights Reserved
 #
 # Pacemaker+Corosync High Availability for NFS-Ganesha
 #
@@ -20,16 +20,26 @@
 # ensure that the NFS GRACE DBUS signal is sent after the VIP moves to
 # the new host.
 
+GANESHA_HA_SH=$(realpath $0)
 HA_NUM_SERVERS=0
 HA_SERVERS=""
 HA_VOL_NAME="gluster_shared_storage"
-HA_VOL_MNT="/var/run/gluster/shared_storage"
+HA_VOL_MNT="/run/gluster/shared_storage"
 HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha"
 SERVICE_MAN="DISTRO_NOT_FOUND"
 
-RHEL6_PCS_CNAME_OPTION="--name"
+# rhel, fedora id, version
+ID=""
+VERSION_ID=""
+
+PCS9OR10_PCS_CNAME_OPTION=""
+PCS9OR10_PCS_CLONE_OPTION="clone"
 SECRET_PEM="/var/lib/glusterd/nfs/secret.pem"
 
+# UNBLOCK RA uses shared_storage which may become unavailable
+# during any of the nodes reboot. Hence increase timeout value.
+PORTBLOCK_UNBLOCK_TIMEOUT="60s"
+
 # Try loading the config from any of the distro
 # specific configuration locations
 if [ -f /etc/sysconfig/ganesha ]
@@ -64,9 +74,9 @@ function find_rhel7_conf
          done
 }
 
-if [ -z $CONFFILE ]
+if [ -z ${CONFFILE} ]
         then
-        find_rhel7_conf $OPTIONS
+        find_rhel7_conf ${OPTIONS}
 
 fi
 
@@ -74,20 +84,21 @@ GANESHA_CONF=${CONFFILE:-/etc/ganesha/ganesha.conf}
 
 usage() {
 
-        echo "Usage      : add|delete|status"
-        echo "Add-node   : ganesha-ha.sh --add <HA_CONF_DIR>  \
+        echo "Usage      : add|delete|refresh-config|status"
+        echo "Add-node   : ganesha-ha.sh --add <HA_CONF_DIR> \
 <NODE-HOSTNAME>  <NODE-VIP>"
-        echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR>  \
+        echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR> \
 <NODE-HOSTNAME>"
-        echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR>\
- <volume>"
+        echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR> \
+<volume>"
+        echo "Status : ganesha-ha.sh --status <HA_CONFDIR>"
 }
 
 determine_service_manager () {
 
-        if [ -e "/usr/bin/systemctl" ];
+        if [ -e "/bin/systemctl" ];
         then
-                SERVICE_MAN="/usr/bin/systemctl"
+                SERVICE_MAN="/bin/systemctl"
         elif [ -e "/sbin/invoke-rc.d" ];
         then
                 SERVICE_MAN="/sbin/invoke-rc.d"
@@ -95,9 +106,9 @@ determine_service_manager () {
         then
                 SERVICE_MAN="/sbin/service"
         fi
-        if [ "$SERVICE_MAN" == "DISTRO_NOT_FOUND" ]
+        if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]]
         then
-                echo "Service manager not recognized, exiting"
+                logger "Service manager not recognized, exiting"
                 exit 1
         fi
 }
@@ -108,21 +119,21 @@ manage_service ()
         local new_node=${2}
         local option=
 
-        if [ "$action" == "start" ]; then
+        if [[ "${action}" == "start" ]]; then
                 option="yes"
         else
                 option="no"
         fi
         ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
-${SECRET_PEM} root@${new_node} "/usr/libexec/ganesha/ganesha-ha.sh --setup-ganesha-conf-files $HA_CONFDIR $option"
+${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option"
 
-        if [ "$SERVICE_MAN" == "/usr/bin/systemctl" ]
+        if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]
         then
                 ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
-${SECRET_PEM} root@${new_node} "$SERVICE_MAN  ${action} nfs-ganesha"
+${SECRET_PEM} root@${new_node} "${SERVICE_MAN}  ${action} nfs-ganesha"
         else
                 ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
-${SECRET_PEM} root@${new_node} "$SERVICE_MAN nfs-ganesha ${action}"
+${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}"
         fi
 }
 
@@ -134,7 +145,7 @@ check_cluster_exists()
 
     if [ -e /var/run/corosync.pid ]; then
         cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3)
-        if [ ${cluster_name} -a ${cluster_name} = ${name} ]; then
+        if [[ "${cluster_name}X" == "${name}X" ]]; then
             logger "$name already exists, exiting"
             exit 0
         fi
@@ -149,7 +160,7 @@ determine_servers()
     local tmp_ifs=${IFS}
     local ha_servers=""
 
-    if [[ "X${cmd}X" != "XsetupX" ]]; then
+    if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then
         ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//')
         IFS=$' '
         for server in ${ha_servers} ; do
@@ -169,6 +180,13 @@ determine_servers()
     fi
 }
 
+stop_ganesha_all()
+{
+    local serverlist=${1}
+    for node in ${serverlist} ; do
+        manage_service "stop" ${node}
+    done
+}
 
 setup_cluster()
 {
@@ -176,16 +194,25 @@ setup_cluster()
     local num_servers=${2}
     local servers=${3}
     local unclean=""
+    local quorum_policy="stop"
 
     logger "setting up cluster ${name} with the following ${servers}"
 
-    pcs cluster auth ${servers}
-    # pcs cluster setup --name ${name} ${servers}
-    pcs cluster setup ${RHEL6_PCS_CNAME_OPTION} ${name} ${servers}
+    # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers}
+    pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers}
     if [ $? -ne 0 ]; then
-        logger "pcs cluster setup ${RHEL6_PCS_CNAME_OPTION} ${name} ${servers} failed"
+        logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out"
+        #set up failed stop all ganesha process and clean up symlinks in cluster
+        stop_ganesha_all "${servers}"
         exit 1;
     fi
+
+    # pcs cluster auth ${servers}
+    pcs cluster auth
+    if [ $? -ne 0 ]; then
+        logger "pcs cluster auth failed"
+    fi
+
     pcs cluster start --all
     if [ $? -ne 0 ]; then
         logger "pcs cluster start failed"
@@ -201,17 +228,18 @@ setup_cluster()
     done
 
     unclean=$(pcs status | grep -u "UNCLEAN")
-    while [[ "${unclean}X" = "UNCLEANX" ]]; do
+    while [[ "${unclean}X" == "UNCLEANX" ]]; do
          sleep 1
          unclean=$(pcs status | grep -u "UNCLEAN")
     done
     sleep 1
 
     if [ ${num_servers} -lt 3 ]; then
-        pcs property set no-quorum-policy=ignore
-        if [ $? -ne 0 ]; then
-            logger "warning: pcs property set no-quorum-policy=ignore failed"
-        fi
+        quorum_policy="ignore"
+    fi
+    pcs property set no-quorum-policy=${quorum_policy}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed"
     fi
 
     pcs property set stonith-enabled=false
@@ -227,7 +255,7 @@ setup_finalize_ha()
     local stopped=""
 
     stopped=$(pcs status | grep -u "Stopped")
-    while [[ "${stopped}X" = "StoppedX" ]]; do
+    while [[ "${stopped}X" == "StoppedX" ]]; do
          sleep 1
          stopped=$(pcs status | grep -u "Stopped")
     done
@@ -241,36 +269,24 @@ refresh_config ()
         local HA_CONFDIR=${2}
         local short_host=$(hostname -s)
 
-        removed_id=`cat $HA_CONFDIR/exports/export.$VOL.conf |\
-grep Export_Id | awk -F"[=,;]" '{print$2}' | tr -d '[[:space:]]'`
+        local export_id=$(grep ^[[:space:]]*Export_Id $HA_CONFDIR/exports/export.$VOL.conf |\
+                          awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]')
+
 
         if [ -e ${SECRET_PEM} ]; then
         while [[ ${3} ]]; do
             current_host=`echo ${3} | cut -d "." -f 1`
-            if [ ${short_host} != ${current_host} ]; then
-                output=$(ssh -oPasswordAuthentication=no \
--oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \
-"dbus-send --print-reply --system --dest=org.ganesha.nfsd \
-/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.RemoveExport \
-uint16:$removed_id 2>&1")
-                ret=$?
-                logger <<< "${output}"
-                if [ ${ret} -ne 0 ]; then
-                       echo "Error: refresh-config failed on ${current_host}."
-                       exit 1
-                fi
-                sleep 1
+            if [[ ${short_host} != ${current_host} ]]; then
                 output=$(ssh -oPasswordAuthentication=no \
 -oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \
 "dbus-send --print-reply --system --dest=org.ganesha.nfsd \
-/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
+/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \
 string:$HA_CONFDIR/exports/export.$VOL.conf \
-string:\"EXPORT(Path=/$VOL)\" 2>&1")
+string:\"EXPORT(Export_Id=$export_id)\" 2>&1")
                 ret=$?
                 logger <<< "${output}"
                 if [ ${ret} -ne 0 ]; then
-                        echo "Error: refresh-config failed on ${current_host}."
-                        exit 1
+                        echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}"
                 else
                         echo "Refresh-config completed on ${current_host}."
                 fi
@@ -285,24 +301,13 @@ string:\"EXPORT(Path=/$VOL)\" 2>&1")
 
     # Run the same command on the localhost,
         output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \
-/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.RemoveExport \
-uint16:$removed_id 2>&1)
-        ret=$?
-        logger <<< "${output}"
-        if [ ${ret} -ne 0 ]; then
-                echo "Error: refresh-config failed on localhost."
-                exit 1
-        fi
-        sleep 1
-        output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \
-/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
+/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \
 string:$HA_CONFDIR/exports/export.$VOL.conf \
-string:"EXPORT(Path=/$VOL)" 2>&1)
+string:"EXPORT(Export_Id=$export_id)" 2>&1)
         ret=$?
         logger <<< "${output}"
         if [ ${ret} -ne 0 ] ; then
-                echo "Error: refresh-config failed on localhost."
-                exit 1
+                echo "Refresh-config failed on localhost."
         else
                 echo "Success: refresh-config completed."
         fi
@@ -352,8 +357,10 @@ teardown_cluster()
 
 cleanup_ganesha_config ()
 {
-       rm -rf /etc/cluster/cluster.conf*
-       rm -rf /var/lib/pacemaker/cib/*
+    rm -f /etc/corosync/corosync.conf
+    rm -rf /etc/cluster/cluster.conf*
+    rm -rf /var/lib/pacemaker/cib/*
+    sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d'  $HA_CONFDIR/ganesha.conf
 }
 
 do_create_virt_ip_constraints()
@@ -402,7 +409,7 @@ wrap_create_virt_ip_constraints()
     # the result is "node2 node3 node4"; for node2, "node3 node4 node1"
     # and so on.
     while [[ ${1} ]]; do
-        if [ "${1}" = "${primary}" ]; then
+        if [[ ${1} == ${primary} ]]; then
             shift
             while [[ ${1} ]]; do
                 tail=${tail}" "${1}
@@ -433,15 +440,15 @@ setup_create_resources()
     local cibfile=$(mktemp -u)
 
     # fixup /var/lib/nfs
-    logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone"
-    pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone
+    logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}"
+    pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}
     if [ $? -ne 0 ]; then
-        logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone failed"
+        logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed"
     fi
 
-    pcs resource create nfs-mon ocf:heartbeat:ganesha_mon --clone
+    pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION}
     if [ $? -ne 0 ]; then
-        logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon --clone failed"
+        logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed"
     fi
 
     # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace
@@ -449,9 +456,9 @@ setup_create_resources()
     # ganesha-active crm_attribute
     sleep 5
 
-    pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone meta notify=true
+    pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true
     if [ $? -ne 0 ]; then
-        logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone failed"
+        logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed"
     fi
 
     pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1
@@ -500,7 +507,9 @@ setup_create_resources()
 
         pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \
         portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \
-        tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1
+        tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \
+        op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \
+        op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
         if [ $? -ne 0 ]; then
             logger "warning pcs resource create ${1}-nfs_unblock failed"
         fi
@@ -569,9 +578,16 @@ recreate_resources()
         eval tmp_ipaddr=\$${clean_name}
         ipaddr=${tmp_ipaddr//_/.}
 
-        pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} cidr_netmask=32 op monitor interval=15s
+        pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=block ip=${ipaddr} --group ${1}-group
         if [ $? -ne 0 ]; then
-            logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} cidr_netmask=32 op monitor interval=10s failed"
+            logger "warning pcs resource create ${1}-nfs_block failed"
+        fi
+        pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+        cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+            cidr_netmask=32 op monitor interval=15s failed"
         fi
 
         pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1
@@ -579,6 +595,15 @@ recreate_resources()
             logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed"
         fi
 
+        pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \
+        tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \
+        op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \
+        op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-nfs_unblock failed"
+        fi
+
         shift
     done
 }
@@ -592,15 +617,32 @@ addnode_recreate_resources()
 
     recreate_resources ${cibfile} ${HA_SERVERS}
 
-    pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${add_vip} cidr_netmask=32 op monitor interval=15s
+    pcs -f ${cibfile} resource create ${add_node}-nfs_block ocf:heartbeat:portblock \
+    protocol=tcp portno=2049 action=block ip=${add_vip} --group ${add_node}-group
+    if [ $? -ne 0 ]; then
+        logger "warning pcs resource create ${add_node}-nfs_block failed"
+    fi
+    pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \
+    ip=${add_vip} cidr_netmask=32 op monitor interval=15s --group ${add_node}-group \
+    --after ${add_node}-nfs_block
     if [ $? -ne 0 ]; then
-        logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${add_vip} cidr_netmask=32 op monitor interval=10s failed"
+        logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \
+        ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed"
     fi
 
     pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1
     if [ $? -ne 0 ]; then
         logger "warning: pcs constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 failed"
     fi
+    pcs -f ${cibfile} resource create ${add_node}-nfs_unblock ocf:heartbeat:portblock \
+    protocol=tcp portno=2049 action=unblock ip=${add_vip} reset_local_on_unblock_stop=true \
+    tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${add_node}-group --after \
+    ${add_node}-cluster_ip-1 op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start \
+    timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op monitor interval=10s \
+    timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
+    if [ $? -ne 0 ]; then
+        logger "warning pcs resource create ${add_node}-nfs_unblock failed"
+    fi
 }
 
 
@@ -609,9 +651,9 @@ clear_resources()
     local cibfile=${1}; shift
 
     while [[ ${1} ]]; do
-        pcs -f ${cibfile} resource delete ${1}-cluster_ip-1
+        pcs -f ${cibfile} resource delete ${1}-group
         if [ $? -ne 0 ]; then
-            logger "warning: pcs -f ${cibfile} resource delete ${1}-cluster_ip-1"
+            logger "warning: pcs -f ${cibfile} resource delete ${1}-group"
         fi
 
         shift
@@ -688,7 +730,7 @@ deletenode_update_haconfig()
     local clean_name=${name//[-.]/_}
 
     ha_servers=$(echo ${HA_SERVERS} | sed -e "s/ /,/")
-    sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${clean_name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf
+    sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf
 }
 
 
@@ -711,7 +753,6 @@ setup_state_volume()
             dirname=${1}${dname}
         fi
 
-
         if [ ! -d ${mnt}/nfs-ganesha/tickle_dir ]; then
             mkdir ${mnt}/nfs-ganesha/tickle_dir
         fi
@@ -726,9 +767,11 @@ setup_state_volume()
         fi
         if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then
             mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd
         fi
         if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then
             touch ${mnt}/nfs-ganesha/${dirname}/nfs/state
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state
         fi
         if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then
             mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov
@@ -738,15 +781,17 @@ setup_state_volume()
         fi
         if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then
             mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
         fi
         if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then
             mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
         fi
         if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then
             touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state
         fi
         for server in ${HA_SERVERS} ; do
-            if [ ${server} != ${dirname} ]; then
+            if [[ ${server} != ${dirname} ]]; then
                 ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server}
                 ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server}
             fi
@@ -757,32 +802,180 @@ setup_state_volume()
 }
 
 
+enable_pacemaker()
+{
+    while [[ ${1} ]]; do
+        if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then
+            ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker"
+        else
+            ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${1} "${SERVICE_MAN} pacemaker enable"
+        fi
+        shift
+    done
+}
+
+
+addnode_state_volume()
+{
+    local newnode=${1}; shift
+    local mnt=${HA_VOL_MNT}
+    local longname=""
+    local dname=""
+    local dirname=""
+
+    longname=$(hostname)
+    dname=${longname#$(hostname -s)}
+
+    if [[ ${newnode} == *${dname} ]]; then
+        dirname=${newnode}
+    else
+        dirname=${newnode}${dname}
+    fi
+
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+    fi
+    if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then
+        touch ${mnt}/nfs-ganesha/${dirname}/nfs/state
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+    fi
+    if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then
+        touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state
+    fi
+
+    for server in ${HA_SERVERS} ; do
+
+        if [[ ${server} != ${dirname} ]]; then
+            ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server}
+            ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server}
+
+            ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname}
+            ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/statd ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname}
+        fi
+    done
+
+}
+
+
+delnode_state_volume()
+{
+    local delnode=${1}; shift
+    local mnt=${HA_VOL_MNT}
+    local longname=""
+    local dname=""
+    local dirname=""
+
+    longname=$(hostname)
+    dname=${longname#$(hostname -s)}
+
+    if [[ ${delnode} == *${dname} ]]; then
+        dirname=${delnode}
+    else
+        dirname=${delnode}${dname}
+    fi
+
+    rm -rf ${mnt}/nfs-ganesha/${dirname}
+
+    for server in ${HA_SERVERS} ; do
+        if [[ ${server} != ${dirname} ]]; then
+            rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname}
+            rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname}
+        fi
+    done
+}
+
+
 status()
 {
-    local regex_str="^ ${1}"; shift
-    local status_file=$(mktemp)
+    local scratch=$(mktemp)
+    local regex_str="^${1}-cluster_ip-1"
+    local healthy=0
+    local index=1
+    local nodes
 
-    while [[ ${1} ]]; do
+    # change tabs to spaces, strip leading spaces, including any 
+    # new '*' at the beginning of a line introduced in pcs-0.10.x
+    pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch}
+
+    nodes[0]=${1}; shift
 
-        regex_str="${regex_str}|^ ${1}"
+    # make a regex of the configured nodes
+    # and initalize the nodes array for later
+    while [[ ${1} ]]; do
 
+        regex_str="${regex_str}|^${1}-cluster_ip-1"
+        nodes[${index}]=${1}
+        ((index++))
         shift
     done
 
-    pcs status | egrep "^Online:" > ${status_file}
+    # print the nodes that are expected to be online
+    grep -E "Online:" ${scratch}
+
+    echo
+
+    # print the VIPs and which node they are on
+    grep -E "${regex_str}" < ${scratch} | cut -d ' ' -f 1,4
 
-    echo >> ${status_file}
+    echo
 
-    pcs status | egrep "${regex_str}" | sed -e "s/\t/ /" | cut -d ' ' -f 2,4 >> ${status_file}
+    # check if the VIP and port block/unblock RAs are on the expected nodes
+    for n in ${nodes[*]}; do
 
-    cat ${status_file}
+        grep -E -x "${n}-nfs_block \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+        grep -E -x "${n}-cluster_ip-1 \(ocf::heartbeat:IPaddr\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+        grep -E -x "${n}-nfs_unblock \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+    done
+
+    grep -E "\):\ Stopped|FAILED" > /dev/null 2>&1 ${scratch}
+    result=$?
+
+    if [ ${result} -eq 0 ]; then
+        echo "Cluster HA Status: BAD"
+    elif [ ${healthy} -eq 0 ]; then
+        echo "Cluster HA Status: HEALTHY"
+    else
+        echo "Cluster HA Status: FAILOVER"
+    fi
 
-    rm -f ${status_file}
+    rm -f ${scratch}
 }
 
 create_ganesha_conf_file()
 {
-        if [ $1 == "yes" ];
+        if [[ "$1" == "yes" ]];
         then
                 if [  -e $GANESHA_CONF ];
                 then
@@ -799,11 +992,25 @@ create_ganesha_conf_file()
         else
         # Restoring previous file
                 rm -rf $GANESHA_CONF
-                sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf
                 cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+                sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $GANESHA_CONF
         fi
 }
 
+set_quorum_policy()
+{
+    local quorum_policy="stop"
+    local num_servers=${1}
+
+    if [ ${num_servers} -lt 3 ]; then
+        quorum_policy="ignore"
+    fi
+    pcs property set no-quorum-policy=${quorum_policy}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed"
+    fi
+}
+
 main()
 {
 
@@ -812,19 +1019,29 @@ main()
         usage
         exit 0
     fi
-    if [[ ${cmd} != *status ]]; then
-        HA_CONFDIR=${1%/}; shift
-        local ha_conf=${HA_CONFDIR}/ganesha-ha.conf
-        local node=""
-        local vip=""
 
-        # ignore any comment lines
-        cfgline=$(grep  ^HA_NAME= ${ha_conf})
-        eval $(echo ${cfgline} | grep -F HA_NAME=)
-        cfgline=$(grep  ^HA_CLUSTER_NODES= ${ha_conf})
-        eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=)
+    if (selinuxenabled) ;then
+     semanage boolean -m gluster_use_execmem --on
     fi
 
+    local osid=""
+
+    osid=$(grep ^ID= /etc/os-release)
+    eval $(echo ${osid} | grep -F ID=)
+    osid=$(grep ^VERSION_ID= /etc/os-release)
+    eval $(echo ${osid} | grep -F VERSION_ID=)
+
+    HA_CONFDIR=${1%/}; shift
+    local ha_conf=${HA_CONFDIR}/ganesha-ha.conf
+    local node=""
+    local vip=""
+
+    # ignore any comment lines
+    cfgline=$(grep  ^HA_NAME= ${ha_conf})
+    eval $(echo ${cfgline} | grep -F HA_NAME=)
+    cfgline=$(grep  ^HA_CLUSTER_NODES= ${ha_conf})
+    eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=)
+
     case "${cmd}" in
 
     setup | --setup)
@@ -834,7 +1051,19 @@ main()
 
         determine_servers "setup"
 
-        if [ "X${HA_NUM_SERVERS}X" != "X1X" ]; then
+        # Fedora 29+ and rhel/centos 8 has PCS-0.10.x
+        # default is pcs-0.10.x options but check for
+        # rhel/centos 7 (pcs-0.9.x) and adjust accordingly
+        if [[ ! ${ID} =~ {rhel,centos} ]]; then
+            if [[ ${VERSION_ID} == 7.* ]]; then
+                PCS9OR10_PCS_CNAME_OPTION="--name"
+                PCS9OR10_PCS_CLONE_OPTION="--clone"
+            fi
+        fi
+
+        if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then
+
+            determine_service_manager
 
             setup_cluster ${HA_NAME} ${HA_NUM_SERVERS} "${HA_SERVERS}"
 
@@ -844,6 +1073,8 @@ main()
 
             setup_state_volume ${HA_SERVERS}
 
+            enable_pacemaker ${HA_SERVERS}
+
         else
 
             logger "insufficient servers for HA, aborting"
@@ -858,6 +1089,8 @@ main()
         teardown_resources ${HA_SERVERS}
 
         teardown_cluster ${HA_NAME}
+
+        cleanup_ganesha_config ${HA_CONFDIR}
         ;;
 
     cleanup | --cleanup)
@@ -888,13 +1121,20 @@ main()
         # newly added node to the file so that the resources specfic
         # to this node is correctly recreated in the future.
         clean_node=${node//[-.]/_}
-        echo "VIP_$clean_node=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf
+        echo "VIP_${node}=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf
 
         NEW_NODES="$HA_CLUSTER_NODES,${node}"
 
         sed -i s/HA_CLUSTER_NODES.*/"HA_CLUSTER_NODES=\"$NEW_NODES\""/ \
 $HA_CONFDIR/ganesha-ha.conf
-        HA_SERVERS="${HA_SERVERS} ${node}"
+
+        addnode_state_volume ${node}
+
+        # addnode_create_resources() already appended ${node} to
+        # HA_SERVERS, so only need to increment HA_NUM_SERVERS
+        # and set quorum policy
+        HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} + 1)
+        set_quorum_policy ${HA_NUM_SERVERS}
         ;;
 
     delete | --delete)
@@ -913,11 +1153,14 @@ $HA_CONFDIR/ganesha-ha.conf
 
         deletenode_update_haconfig ${node}
 
-        rm -rf ${HA_VOL_MNT}/nfs-ganesha/${node}
+        delnode_state_volume ${node}
 
         determine_service_manager
 
         manage_service "stop" ${node}
+
+        HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} - 1)
+        set_quorum_policy ${HA_NUM_SERVERS}
         ;;
 
     status | --status)
@@ -947,7 +1190,10 @@ $HA_CONFDIR/ganesha-ha.conf
         ;;
 
     esac
+
+    if (selinuxenabled) ;then
+     semanage boolean -m gluster_use_execmem --off
+    fi
 }
 
 main $*
-
diff --git a/extras/ganesha/scripts/generate-epoch.py b/extras/ganesha/scripts/generate-epoch.py
index 5db5e56b480..77af014bab9 100755
--- a/extras/ganesha/scripts/generate-epoch.py
+++ b/extras/ganesha/scripts/generate-epoch.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 #
 # Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
@@ -36,13 +36,13 @@ def epoch_uuid():
 
         uuid_bin = binascii.unhexlify(glusterd_uuid.replace("-",""))
 
-        epoch_uuid = int(uuid_bin.encode('hex'), 32) & 0xFFFF0000
+        epoch_uuid = int(binascii.hexlify(uuid_bin), 32) & 0xFFFF0000
         return epoch_uuid
 
 # Construct epoch as follows -
 #        first 32-bit contains the now() time
 #        rest 32-bit value contains the local glusterd node uuid
 epoch = (epoch_now() | epoch_uuid())
-print str(epoch)
+print((str(epoch)))
 
 exit(0)
diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am
index e4603ae80b8..09eff308ac4 100644
--- a/extras/geo-rep/Makefile.am
+++ b/extras/geo-rep/Makefile.am
@@ -1,4 +1,4 @@
-scriptsdir = $(datadir)/glusterfs/scripts
+scriptsdir = $(libexecdir)/glusterfs/scripts
 scripts_SCRIPTS =  gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \
 	slave-upgrade.sh schedule_georep.py
 
diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c
index e9b9e633402..47dca0413e9 100644
--- a/extras/geo-rep/gsync-sync-gfid.c
+++ b/extras/geo-rep/gsync-sync-gfid.c
@@ -7,103 +7,103 @@
 #include <libgen.h>
 #include <ctype.h>
 #include <stdlib.h>
-#include "glusterfs.h"
-#include "syscall.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
 
 #ifndef UUID_CANONICAL_FORM_LEN
 #define UUID_CANONICAL_FORM_LEN 36
 #endif
 
 #ifndef GF_FUSE_AUX_GFID_HEAL
-#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal"
 #endif
 
-#define GLFS_LINE_MAX           (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN))
+#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN))
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        char *file                = NULL;
-        char *tmp = NULL;
-        char *tmp1 = NULL;
-        char *parent_dir = NULL;
-        char *gfid                = NULL;
-        char *bname = NULL;
-        int   ret                 = -1;
-        int len = 0;
-        FILE *fp                  = NULL;
-        char  line[GLFS_LINE_MAX] = {0,};
-        char *path = NULL;
-        void  *blob               = NULL;
-        void  *tmp_blob               = NULL;
-
-        if (argc != 2) {
-                /* each line in the file has the following format
-                 * uuid-in-canonical-form path-relative-to-gluster-mount.
-                 * Both uuid and relative path are from master mount.
-                 */
-                fprintf (stderr, "usage: %s <file-of-paths-to-be-synced>\n",
-                         argv[0]);
-                goto out;
+    char *file = NULL;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    char *parent_dir = NULL;
+    char *gfid = NULL;
+    char *bname = NULL;
+    int ret = -1;
+    int len = 0;
+    FILE *fp = NULL;
+    char line[GLFS_LINE_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    void *blob = NULL;
+    void *tmp_blob = NULL;
+
+    if (argc != 2) {
+        /* each line in the file has the following format
+         * uuid-in-canonical-form path-relative-to-gluster-mount.
+         * Both uuid and relative path are from master mount.
+         */
+        fprintf(stderr, "usage: %s <file-of-paths-to-be-synced>\n", argv[0]);
+        goto out;
+    }
+
+    file = argv[1];
+
+    fp = fopen(file, "r");
+    if (fp == NULL) {
+        fprintf(stderr, "cannot open %s for reading (%s)\n", file,
+                strerror(errno));
+        goto out;
+    }
+
+    while (fgets(line, GLFS_LINE_MAX, fp) != NULL) {
+        tmp = line;
+        path = gfid = line;
+
+        path += UUID_CANONICAL_FORM_LEN + 1;
+
+        while (isspace(*path))
+            path++;
+
+        len = strlen(line);
+        if ((len < GLFS_LINE_MAX) && (line[len - 1] == '\n'))
+            line[len - 1] = '\0';
+
+        line[UUID_CANONICAL_FORM_LEN] = '\0';
+
+        tmp = strdup(path);
+        tmp1 = strdup(path);
+        parent_dir = dirname(tmp);
+        bname = basename(tmp1);
+
+        /* gfid + '\0' + bname + '\0' */
+        len = UUID_CANONICAL_FORM_LEN + 1 + strlen(bname) + 1;
+
+        blob = malloc(len);
+
+        memcpy(blob, gfid, UUID_CANONICAL_FORM_LEN);
+
+        tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1;
+
+        memcpy(tmp_blob, bname, strlen(bname));
+
+        ret = sys_lsetxattr(parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len, 0);
+        if (ret < 0) {
+            fprintf(stderr, "setxattr on %s/%s failed (%s)\n", parent_dir,
+                    bname, strerror(errno));
         }
+        memset(line, 0, GLFS_LINE_MAX);
 
-        file = argv[1];
+        free(blob);
+        free(tmp);
+        free(tmp1);
+        blob = NULL;
+    }
 
-        fp = fopen (file, "r");
-        if (fp == NULL) {
-                fprintf (stderr, "cannot open %s for reading (%s)\n",
-                         file, strerror (errno));
-                goto out;
-        }
-
-        while (fgets (line, GLFS_LINE_MAX, fp) != NULL) {
-                tmp = line;
-                path = gfid = line;
-
-                path += UUID_CANONICAL_FORM_LEN + 1;
-
-                while(isspace (*path))
-                        path++;
-
-                if ((strlen (line) < GLFS_LINE_MAX) &&
-                    (line[strlen (line) - 1] == '\n'))
-                        line[strlen (line) - 1] = '\0';
-
-                line[UUID_CANONICAL_FORM_LEN] = '\0';
-
-                tmp = strdup (path);
-                tmp1 = strdup (path);
-                parent_dir = dirname (tmp);
-                bname = basename (tmp1);
-
-                /* gfid + '\0' + bname + '\0' */
-                len = UUID_CANONICAL_FORM_LEN + 1 + strlen (bname) + 1;
-
-                blob = calloc (1, len);
-
-                memcpy (blob, gfid, UUID_CANONICAL_FORM_LEN);
-
-                tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1;
-
-                memcpy (tmp_blob, bname, strlen (bname));
-
-                ret = sys_lsetxattr (parent_dir, GF_FUSE_AUX_GFID_HEAL, 
-                                blob, len, 0);
-                if (ret < 0) {
-                        fprintf (stderr, "setxattr on %s/%s failed (%s)\n",
-                                 parent_dir, bname, strerror (errno));
-                }
-                memset (line, 0, GLFS_LINE_MAX);
-
-                free (blob);
-                free (tmp); free (tmp1);
-                blob = NULL;
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        if (fp)
-                fclose(fp);
-        return ret;
+    if (fp)
+        fclose(fp);
+    return ret;
 }
-
diff --git a/extras/geo-rep/schedule_georep.py.in b/extras/geo-rep/schedule_georep.py.in
index c931111b365..48b2b507060 100644
--- a/extras/geo-rep/schedule_georep.py.in
+++ b/extras/geo-rep/schedule_georep.py.in
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 """
 Schedule Geo-replication
 ------------------------
@@ -43,7 +43,7 @@ SESSION_MOUNT_LOG_FILE = ("/var/log/glusterfs/geo-replication"
                           "/schedule_georep.mount.log")
 
 USE_CLI_COLOR = True
-
+mnt_list = []
 
 class GlusterBadXmlFormat(Exception):
     """
@@ -83,13 +83,15 @@ def execute(cmd, success_msg="", failure_msg="", exitcode=-1):
     On success it can print message in stdout if specified.
     On failure, exits after writing to stderr.
     """
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
     out, err = p.communicate()
     if p.returncode == 0:
         if success_msg:
             output_ok(success_msg)
         return out
     else:
+        if exitcode == 0:
+            return
         err_msg = err if err else out
         output_notok(failure_msg, err=err_msg, exitcode=exitcode)
 
@@ -112,12 +114,12 @@ def cleanup(hostname, volname, mnt):
     """
     Unmount the Volume and Remove the temporary directory
     """
-    execute(["umount", mnt],
+    execute(["umount", "-l", mnt],
             failure_msg="Unable to Unmount Gluster Volume "
             "{0}:{1}(Mounted at {2})".format(hostname, volname, mnt))
     execute(["rmdir", mnt],
             failure_msg="Unable to Remove temp directory "
-            "{0}".format(mnt))
+            "{0}".format(mnt), exitcode=0)
 
 
 @contextmanager
@@ -130,6 +132,7 @@ def glustermount(hostname, volname):
     Automatically unmounts it in case of Exceptions/out of context
     """
     mnt = tempfile.mkdtemp(prefix="georepsetup_")
+    mnt_list.append(mnt)
     execute(["@SBIN_DIR@/glusterfs",
              "--volfile-server", hostname,
              "--volfile-id", volname,
@@ -297,6 +300,7 @@ def get_summary(mastervol, slave_url):
     status_data = get(mastervol, slave_url)
 
     for session in status_data:
+        session_name = ""
         summary = {
             "active": 0,
             "passive": 0,
@@ -339,7 +343,8 @@ def get_summary(mastervol, slave_url):
         if summary["faulty"] == 0 and summary["offline"] == 0:
             summary["ok"] = True
 
-        out.append([session_name, summary, faulty_rows, down_rows])
+        if session_name != "":
+                out.append([session_name, summary, faulty_rows, down_rows])
 
     return out
 
@@ -347,7 +352,7 @@ def get_summary(mastervol, slave_url):
 def touch_mount_root(mastervol):
     # Create a Mount and Touch the Mount point root,
     # Hack to make sure some event available after
-    # setting Checkpoint. Without this their is a chance of
+    # setting Checkpoint. Without this there is a chance of
     # Checkpoint never completes.
     with glustermount("localhost", mastervol) as mnt:
         execute(["touch", mnt])
@@ -376,14 +381,14 @@ def main(args):
     output_ok("Started Geo-replication and watching Status for "
               "Checkpoint completion")
 
-    touch_mount_root(args.mastervol)
-
     start_time = int(time.time())
     duration = 0
 
     # Sleep till Geo-rep initializes
     time.sleep(60)
 
+    touch_mount_root(args.mastervol)
+
     slave_url = "{0}::{1}".format(args.slave, args.slavevol)
 
     # Loop to Check the Geo-replication Status and Checkpoint
@@ -397,41 +402,39 @@ def main(args):
             # or any other error. Gluster cmd still produces XML output
             # with different message
             output_warning("Unable to get Geo-replication Status")
-            time.sleep(1)
-            continue
-
-        session_name, summary, faulty_rows, down_rows = session_summary[0]
-        chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \
-                       "NOT COMPLETE"
-        ok_status = "OK" if summary["ok"] else "NOT OK"
-
-        if summary["ok"]:
-            output_ok("All Checkpoints {1}, "
-                      "All status {2} (Turns {0:>3})".format(
-                          turns, chkpt_status, ok_status))
-        else:
-            output_warning("All Checkpoints {1}, "
-                           "All status {2} (Turns {0:>3})".format(
-                               turns, chkpt_status, ok_status))
-
-            output_warning("Geo-rep workers Faulty/Offline, "
-                           "Faulty: {0} Offline: {1}".format(
-                               repr(faulty_rows),
-                               repr(down_rows)))
-
-        if summary["checkpoints_ok"]:
-            output_ok("Stopping Geo-replication session now")
-            cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
-                   args.mastervol,
-                   "%s::%s" % (args.slave, args.slavevol), "stop"]
-            execute(cmd)
-            break
         else:
-            # If Checkpoint is not complete after a iteration means brick
-            # was down and came online now. SETATTR on mount is not
-            # recorded, So again issue touch on mount root So that
-            # Stime will increase and Checkpoint will complete.
-            touch_mount_root(args.mastervol)
+            session_name, summary, faulty_rows, down_rows = session_summary[0]
+            chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \
+                           "NOT COMPLETE"
+            ok_status = "OK" if summary["ok"] else "NOT OK"
+
+            if summary["ok"]:
+                output_ok("All Checkpoints {1}, "
+                          "All status {2} (Turns {0:>3})".format(
+                              turns, chkpt_status, ok_status))
+            else:
+                output_warning("All Checkpoints {1}, "
+                               "All status {2} (Turns {0:>3})".format(
+                                   turns, chkpt_status, ok_status))
+
+                output_warning("Geo-rep workers Faulty/Offline, "
+                               "Faulty: {0} Offline: {1}".format(
+                                   repr(faulty_rows),
+                                   repr(down_rows)))
+
+            if summary["checkpoints_ok"]:
+                output_ok("Stopping Geo-replication session now")
+                cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
+                       args.mastervol,
+                       "%s::%s" % (args.slave, args.slavevol), "stop"]
+                execute(cmd)
+                break
+            else:
+                # If Checkpoint is not complete after a iteration means brick
+                # was down and came online now. SETATTR on mount is not
+                # recorded, So again issue touch on mount root So that
+                # Stime will increase and Checkpoint will complete.
+                touch_mount_root(args.mastervol)
 
         # Increment the turns and Sleep for 10 sec
         turns += 1
@@ -446,13 +449,18 @@ def main(args):
 
         time.sleep(args.interval)
 
+    for mnt in mnt_list:
+        execute(["rmdir", mnt],
+                failure_msg="Unable to Remove temp directory "
+                "{0}".format(mnt), exitcode=0)
+
 if __name__ == "__main__":
     parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
                             description=__doc__)
     parser.add_argument("mastervol", help="Master Volume Name")
     parser.add_argument("slave",
-                        help="SLAVEHOST or root@SLAVEHOST "
-                        "or user@SLAVEHOST",
+                        help="Slave hostname "
+                        "(<username>@SLAVEHOST or SLAVEHOST)",
                         metavar="SLAVE")
     parser.add_argument("slavevol", help="Slave Volume Name")
     parser.add_argument("--interval", help="Interval in Seconds. "
@@ -462,12 +470,23 @@ if __name__ == "__main__":
                         "stop Geo-replication if Checkpoint is not complete "
                         "in the specified timeout time", type=int,
                         default=0)
-    parser.add_argument("--no-color", help="Use Color in CLI output",
+    parser.add_argument("--no-color", help="Don't use Color in CLI output",
                         action="store_true")
     args = parser.parse_args()
     if args.no_color:
         USE_CLI_COLOR = False
     try:
+        # Check for session existence
+        cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
+               args.mastervol, "%s::%s" % (args.slave, args.slavevol), "status"]
+        execute(cmd)
         main(args)
     except KeyboardInterrupt:
+        for mnt in mnt_list:
+            execute(["umount", "-l", mnt],
+                    failure_msg="Unable to Unmount Gluster Volume "
+                    "Mounted at {0}".format(mnt), exitcode=0)
+            execute(["rmdir", mnt],
+                    failure_msg="Unable to Remove temp directory "
+                    "{0}".format(mnt), exitcode=0)
         output_notok("Exiting...")
diff --git a/extras/git-branch-diff.py b/extras/git-branch-diff.py
index c8d74ec9f31..382513e069e 100755
--- a/extras/git-branch-diff.py
+++ b/extras/git-branch-diff.py
@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/bin/python2
 
 """
   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -75,6 +75,7 @@
   Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
 """
 
+from __future__ import print_function
 import os
 import sys
 import argparse
@@ -118,16 +119,16 @@ class GitBranchDiff:
         status_tbr, op = commands.getstatusoutput('git log ' +
                                                   self.t_pattern)
         if status_sbr != 0:
-            print "Error: --source=" + self.s_pattern + " doesn't exit\n"
+            print("Error: --source=" + self.s_pattern + " doesn't exit\n")
             self.parser.print_help()
             exit(status_sbr)
         elif status_tbr != 0:
-            print "Error: --target=" + self.t_pattern + " doesn't exit\n"
+            print("Error: --target=" + self.t_pattern + " doesn't exit\n")
             self.parser.print_help()
             exit(status_tbr)
 
     def check_author_exist (self):
-        " defend to check given author exist, format incase of multiple"
+        " defend to check given author exist, format in case of multiple"
         contrib_list = ['', '*', 'all', 'All', 'ALL', 'null', 'Null', 'NULL']
         if self.g_author in contrib_list:
             self.g_author = ""
@@ -137,8 +138,8 @@ class GitBranchDiff:
                 cmd4 = 'git log ' + self.s_pattern + ' --author=' + ide
                 c_list = subprocess.check_output(cmd4, shell = True)
                 if len(c_list) is 0:
-                    print "Error: --author=%s doesn't exit" %self.g_author
-                    print "see '%s --help'" %__file__
+                    print("Error: --author=%s doesn't exit" %self.g_author)
+                    print("see '%s --help'" %__file__)
                     exit(1)
             if len(ide_list) > 1:
                 self.g_author = "\|".join(ide_list)
@@ -150,16 +151,16 @@ class GitBranchDiff:
             return True
         except requests.Timeout as err:
             " request timed out"
-            print "Warning: failed to get list of open review commits on " \
+            print("Warning: failed to get list of open review commits on " \
                             "gerrit.\n" \
                   "hint: Request timed out! gerrit server could possibly " \
-                  "slow ...\n"
+                  "slow ...\n")
             return False
         except requests.RequestException as err:
             " handle other errors"
-            print "Warning: failed to get list of open review commits on " \
+            print("Warning: failed to get list of open review commits on " \
                             "gerrit\n" \
-                  "hint: check with internet connection ...\n"
+                  "hint: check with internet connection ...\n")
             return False
 
     def parse_cmd_args (self):
@@ -189,7 +190,7 @@ class GitBranchDiff:
                                  '--author',
                                  help = 'default: git config name/email, '
                                         'to provide multiple specify comma'
-                                        ' seperated values',
+                                        ' separated values',
                                  default = author,
                                  dest = 'author')
         self.parser.add_argument('-p',
@@ -212,39 +213,39 @@ class GitBranchDiff:
 
     def print_output (self):
         " display the result list"
-        print "\n------------------------------------------------------------\n"
-        print self.tick + " Successfully Backported changes:"
-        print '      {' + 'from: ' + self.s_pattern + \
-              '  to: '+ self.t_pattern + '}\n'
-        for key, value in self.s_dict.iteritems():
+        print("\n------------------------------------------------------------\n")
+        print(self.tick + " Successfully Backported changes:")
+        print('      {' + 'from: ' + self.s_pattern + \
+              '  to: '+ self.t_pattern + '}\n')
+        for key, value in self.s_dict.items():
             if value in self.t_dict.itervalues():
-                print "[%s%s%s] %s" %(self.yello_set,
+                print("[%s%s%s] %s" %(self.yello_set,
                                       key,
                                       self.color_unset,
-                                      value)
-        print "\n------------------------------------------------------------\n"
-        print self.cross + " Missing patches in " + self.t_pattern + ':\n'
+                                      value))
+        print("\n------------------------------------------------------------\n")
+        print(self.cross + " Missing patches in " + self.t_pattern + ':\n')
         if self.connected_to_gerrit():
             cmd3 = "git review -r origin -l"
             review_list = subprocess.check_output(cmd3, shell = True).split('\n')
         else:
             review_list = []
 
-        for key, value in self.s_dict.iteritems():
+        for key, value in self.s_dict.items():
             if value not in self.t_dict.itervalues():
                 if any(value in s for s in review_list):
-                    print "[%s%s%s] %s %s(under review)%s" %(self.yello_set,
+                    print("[%s%s%s] %s %s(under review)%s" %(self.yello_set,
                                                             key,
                                                             self.color_unset,
                                                             value,
                                                             self.green_set,
-                                                            self.color_unset)
+                                                            self.color_unset))
                 else:
-                    print "[%s%s%s] %s" %(self.yello_set,
+                    print("[%s%s%s] %s" %(self.yello_set,
                                           key,
                                           self.color_unset,
-                                          value)
-        print "\n------------------------------------------------------------\n"
+                                          value))
+        print("\n------------------------------------------------------------\n")
 
     def main (self):
         self.check_pattern_exist()
@@ -262,8 +263,8 @@ class GitBranchDiff:
         t_list = subprocess.check_output(cmd2, shell = True)
 
         if len(t_list) is 0:
-            print "No commits in the target: %s" %self.t_pattern
-            print "see '%s --help'" %__file__
+            print("No commits in the target: %s" %self.t_pattern)
+            print("see '%s --help'" %__file__)
             exit()
         else:
             t_list = t_list.split('\n')
diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in
index 957b277801c..5d7bad0e4c8 100644
--- a/extras/glusterd.vol.in
+++ b/extras/glusterd.vol.in
@@ -1,12 +1,15 @@
 volume management
     type mgmt/glusterd
     option working-directory @GLUSTERD_WORKDIR@
-    option transport-type socket,rdma
+    option transport-type socket
     option transport.socket.keepalive-time 10
     option transport.socket.keepalive-interval 2
     option transport.socket.read-fail-log off
+    option transport.socket.listen-port 24007
     option ping-timeout 0
     option event-threads 1
+#   option lock-timer 180
 #   option transport.address-family inet6
 #   option base-port 49152
+    option max-port  60999
 end-volume
diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate
index 6fdb8c65aaf..3e7ecf373a1 100644
--- a/extras/glusterfs-georep-logrotate
+++ b/extras/glusterfs-georep-logrotate
@@ -1,6 +1,12 @@
 /var/log/glusterfs/geo-replication/*/*.log {
     sharedscripts
-    rotate 52
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
     missingok
     compress
     delaycompress
@@ -15,7 +21,13 @@
 
 /var/log/glusterfs/geo-replication-slaves/*.log {
     sharedscripts
-    rotate 52
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
     missingok
     compress
     delaycompress
@@ -30,7 +42,13 @@
 
 /var/log/glusterfs/geo-replication-slaves/*/*.log {
     sharedscripts
-    rotate 52
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
     missingok
     compress
     delaycompress
diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py
new file mode 100755
index 00000000000..634576058d6
--- /dev/null
+++ b/extras/glusterfs-georep-upgrade.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+"""
+
+Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com>
+This file is part of GlusterFS.
+
+This file is licensed to you under your choice of the GNU Lesser
+General Public License, version 3 or any later version (LGPLv3 or
+later), or the GNU General Public License, version 2 (GPLv2), in all
+cases as published by the Free Software Foundation.
+
+"""
+
+import argparse
+import errno
+import os, sys
+import shutil
+from datetime import datetime
+
+def find_htime_path(brick_path):
+    dirs = []
+    htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime')
+    for file in os.listdir(htime_dir):
+        if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"):
+            dirs.append(os.path.join(htime_dir, file))
+        else:
+            raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file)))
+    return dirs
+
+def modify_htime_file(brick_path):
+    htime_file_path_list = find_htime_path(brick_path)
+
+    for htime_file_path in htime_file_path_list:
+        changelog_path = os.path.join(brick_path, '.glusterfs/changelogs')
+        temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file')
+        with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file:
+            #extract epoch times from htime file
+            paths = htime_file.read().split("\x00")
+
+            for pth in paths:
+                epoch_no = pth.split(".")[-1]
+                changelog = os.path.basename(pth)
+                #convert epoch time to year, month and day
+                if epoch_no != '':
+                    date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d"))
+                    #update paths in temp htime file
+                    temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog))
+                    #create directory in the format year/month/days
+                    path = os.path.join(changelog_path, date)
+
+                if changelog.startswith("CHANGELOG."):
+                    try:
+                        os.makedirs(path, mode = 0o600);
+                    except OSError as exc:
+                        if exc.errno == errno.EEXIST:
+                            pass
+                        else:
+                            raise
+
+                    #copy existing changelogs to new directory structure, delete old changelog files
+                    shutil.copyfile(pth, os.path.join(path, changelog))
+                    os.remove(pth)
+
+        #rename temp_htime_file with htime file
+        os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path))
+        os.rename(temp_htime_path, htime_file_path)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('brick_path', help="This upgrade script, which is to be run on\
+                         server side, takes brick path as the argument, \
+                         updates paths inside htime file and alters the directory structure \
+                         above the changelog files inorder to support new optimised format \
+                         of the directory structure as per \
+                         https://review.gluster.org/#/c/glusterfs/+/23733/")
+    args = parser.parse_args()
+    modify_htime_file(args.brick_path)
diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate
index 575c0eee771..6ba6ef18e9f 100644
--- a/extras/glusterfs-logrotate
+++ b/extras/glusterfs-logrotate
@@ -2,7 +2,12 @@
 /var/log/glusterfs/*.log {
   sharedscripts
   weekly
-  rotate 52
+  maxsize 10M
+  minsize 100k
+
+# 6 months of logs are good enough
+  rotate 26
+
   missingok
   compress
   delaycompress
@@ -17,7 +22,12 @@
 /var/log/glusterfs/bricks/*.log {
   sharedscripts
   weekly
-  rotate 52
+  maxsize 10M
+  minsize 100k
+
+# 6 months of logs are good enough
+  rotate 26
+
   missingok
   compress
   delaycompress
@@ -35,3 +45,24 @@
   compress
   delaycompress
 }
+
+# Rotate snapd log
+/var/log/glusterfs/snaps/*/*.log {
+    sharedscripts
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
+    missingok
+    compress
+    delaycompress
+    notifempty
+    postrotate
+    for pid in `ps -aef | grep glusterfs | egrep "snapd" | awk '{print $2}'`; do
+        /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true
+    done
+    endscript
+}
diff --git a/extras/glusterfs-mode.el b/extras/glusterfs-mode.el
index d4f6dc568b6..a9ed2335ab3 100644
--- a/extras/glusterfs-mode.el
+++ b/extras/glusterfs-mode.el
@@ -1,112 +1,113 @@
-;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com>
-;;;  
-;;; This program is free software; you can redistribute it and/or modify
-;;; it under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 2 of the License, or
-;;; (at your option) any later version.
-;;;  
-;;; This program is distributed in the hope that it will be useful,
-;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;;; GNU General Public License for more details.
-;;;  
-;;; You should have received a copy of the GNU General Public License
-;;; along with this program; if not, write to the Free Software
-;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-;;;  
-
-(defvar glusterfs-mode-hook nil)
-
-;; (defvar glusterfs-mode-map
-;;   (let ((glusterfs-mode-map (make-keymap)))
-;;     (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
-;;     glusterfs-mode-map)
-;;   "Keymap for WPDL major mode")
-
-(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
-
-(defconst glusterfs-font-lock-keywords-1
-  (list
-					; "cluster/{unify,afr,stripe}" 
-					; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
-					; "protocol/{client/server}"
-					; "features/{trash,posix-locks,fixed-id,filter}"
-					; "stroage/posix"
-					; "encryption/rot-13"
-					; "debug/trace"
-    '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
-"Additional Keywords to highlight in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-2
-  (append glusterfs-font-lock-keywords-1
-	  (list
-      ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host" 
-      ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
-      ; "limits.min-free-disk" "directory"
-	; TODO: add all the keys here.
-	   '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
-  "option keys in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-3
-  (append glusterfs-font-lock-keywords-2
-	  (list
-					; "option" "volume" "end-volume" "subvolumes" "type"
-	   '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
-					;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
-  "Minimal highlighting expressions for GlusterFS mode.")
-
-
-(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
-  "Default highlighting expressions for GlusterFS mode.")
-
-(defvar glusterfs-mode-syntax-table
-  (let ((glusterfs-mode-syntax-table (make-syntax-table)))
-    (modify-syntax-entry ?\# "<"  glusterfs-mode-syntax-table)
-    (modify-syntax-entry ?* ". 23"  glusterfs-mode-syntax-table)
-    (modify-syntax-entry ?\n ">#"  glusterfs-mode-syntax-table)
-    glusterfs-mode-syntax-table)
-  "Syntax table for glusterfs-mode")
-
-;; TODO: add an indentation table
-
-(defun glusterfs-indent-line ()
-  "Indent current line as GlusterFS code"
-  (interactive)
-  (beginning-of-line)
-  (if (bobp)
-      (indent-line-to 0)   ; First line is always non-indented
-    (let ((not-indented t) cur-indent)
-      (if (looking-at "^[ \t]*volume\ ")
-	  (progn
-	    (save-excursion
-	      (forward-line -1)
-	      (setq not-indented nil)
-	      (setq cur-indent 0))))
-      (if (looking-at "^[ \t]*end-volume")
-	  (progn
-	    (save-excursion
-	      (forward-line -1)
-	      (setq cur-indent 0))
-	    (if (< cur-indent 0) ; We can't indent past the left margin
-		(setq cur-indent 0)))
-	(save-excursion
-	  (while not-indented ; Iterate backwards until we find an indentation hint
-	    (progn
-	      (setq cur-indent 2) ; Do the actual indenting
-	      (setq not-indented nil)))))
-      (if cur-indent
-	  (indent-line-to cur-indent)
-	(indent-line-to 0)))))
-
-(defun glusterfs-mode ()
-  (interactive)
-  (kill-all-local-variables)
-  ;; (use-local-map glusterfs-mode-map)
-  (set-syntax-table glusterfs-mode-syntax-table)
-  (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)  
-  (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
-  (setq major-mode 'glusterfs-mode)
-  (setq mode-name "GlusterFS")
-  (run-hooks 'glusterfs-mode-hook))
-
-(provide 'glusterfs-mode)
+;;; Copyright (C) 2007-2017 Red Hat, Inc. <http://www.redhat.com>
+;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com>
+;;;
+;;; This program is free software; you can redistribute it and/or
+;;; modify it under the terms of the GNU General Public License
+;;; as published by the Free Software Foundation; either version 2
+;;; of the License, or (at your option) any later version.
+;;;
+;;; This program is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program; if not, write to the Free Software
+;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+;;;
+
+(defvar glusterfs-mode-hook nil)
+
+;; (defvar glusterfs-mode-map
+;;   (let ((glusterfs-mode-map (make-keymap)))
+;;     (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
+;;     glusterfs-mode-map)
+;;   "Keymap for WPDL major mode")
+
+(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
+
+(defconst glusterfs-font-lock-keywords-1
+  (list
+					; "cluster/{unify,afr,stripe}"
+					; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
+					; "protocol/{client/server}"
+					; "features/{trash,posix-locks,fixed-id,filter}"
+					; "storage/posix"
+					; "encryption/rot-13"
+					; "debug/trace"
+    '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
+"Additional Keywords to highlight in GlusterFS mode.")
+
+(defconst glusterfs-font-lock-keywords-2
+  (append glusterfs-font-lock-keywords-1
+	  (list
+      ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host"
+      ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
+      ; "limits.min-free-disk" "directory"
+	; TODO: add all the keys here.
+	   '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
+  "option keys in GlusterFS mode.")
+
+(defconst glusterfs-font-lock-keywords-3
+  (append glusterfs-font-lock-keywords-2
+	  (list
+					; "option" "volume" "end-volume" "subvolumes" "type"
+	   '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
+					;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
+  "Minimal highlighting expressions for GlusterFS mode.")
+
+
+(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
+  "Default highlighting expressions for GlusterFS mode.")
+
+(defvar glusterfs-mode-syntax-table
+  (let ((glusterfs-mode-syntax-table (make-syntax-table)))
+    (modify-syntax-entry ?\# "<"  glusterfs-mode-syntax-table)
+    (modify-syntax-entry ?* ". 23"  glusterfs-mode-syntax-table)
+    (modify-syntax-entry ?\n ">#"  glusterfs-mode-syntax-table)
+    glusterfs-mode-syntax-table)
+  "Syntax table for glusterfs-mode")
+
+;; TODO: add an indentation table
+
+(defun glusterfs-indent-line ()
+  "Indent current line as GlusterFS code"
+  (interactive)
+  (beginning-of-line)
+  (if (bobp)
+      (indent-line-to 0)   ; First line is always non-indented
+    (let ((not-indented t) cur-indent)
+      (if (looking-at "^[ \t]*volume\ ")
+	  (progn
+	    (save-excursion
+	      (forward-line -1)
+	      (setq not-indented nil)
+	      (setq cur-indent 0))))
+      (if (looking-at "^[ \t]*end-volume")
+	  (progn
+	    (save-excursion
+	      (forward-line -1)
+	      (setq cur-indent 0))
+	    (if (< cur-indent 0) ; We can't indent past the left margin
+		(setq cur-indent 0)))
+	(save-excursion
+	  (while not-indented ; Iterate backwards until we find an indentation hint
+	    (progn
+	      (setq cur-indent 2) ; Do the actual indenting
+	      (setq not-indented nil)))))
+      (if cur-indent
+	  (indent-line-to cur-indent)
+	(indent-line-to 0)))))
+
+(defun glusterfs-mode ()
+  (interactive)
+  (kill-all-local-variables)
+  ;; (use-local-map glusterfs-mode-map)
+  (set-syntax-table glusterfs-mode-syntax-table)
+  (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)
+  (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
+  (setq major-mode 'glusterfs-mode)
+  (setq mode-name "GlusterFS")
+  (run-hooks 'glusterfs-mode-hook))
+
+(provide 'glusterfs-mode)
diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py
index 71e79b6be4e..6341d007188 100644..100755
--- a/extras/gnfs-loganalyse.py
+++ b/extras/gnfs-loganalyse.py
@@ -10,6 +10,7 @@
 
 """
 
+from __future__ import print_function
 import os
 import string
 import sys
@@ -72,7 +73,7 @@ class NFSRequest:
                         self.replygfid = tokens [gfididx + 1].strip(",")
 
         def dump (self):
-                print "ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata
+                print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata)
 
 class NFSLogAnalyzer:
 
@@ -149,7 +150,7 @@ class NFSLogAnalyzer:
                         return
                 rcount = len (self.xid_request_map.keys ())
                 orphancount = len (self.orphan_replies.keys ())
-                print "Requests: " + str(rcount) + ", Orphans: " + str(orphancount)
+                print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount))
 
         def dump (self):
                 self.getStats ()
diff --git a/extras/group-db-workload b/extras/group-db-workload
new file mode 100644
index 00000000000..9334d6fb942
--- /dev/null
+++ b/extras/group-db-workload
@@ -0,0 +1,12 @@
+performance.open-behind=on
+performance.write-behind=off
+performance.stat-prefetch=off
+performance.quick-read=off
+performance.strict-o-direct=on
+performance.read-ahead=off
+performance.io-cache=off
+performance.readdir-ahead=off
+performance.client-io-threads=on
+server.event-threads=4
+client.event-threads=4
+performance.read-after-open=yes
diff --git a/extras/group-distributed-virt b/extras/group-distributed-virt
new file mode 100644
index 00000000000..a960b76c694
--- /dev/null
+++ b/extras/group-distributed-virt
@@ -0,0 +1,10 @@
+performance.quick-read=off
+performance.read-ahead=off
+performance.io-cache=off
+performance.low-prio-threads=32
+network.remote-dio=enable
+features.shard=on
+user.cifs=off
+client.event-threads=4
+server.event-threads=4
+performance.client-io-threads=on
diff --git a/extras/group-gluster-block b/extras/group-gluster-block
new file mode 100644
index 00000000000..1e398019e6b
--- /dev/null
+++ b/extras/group-gluster-block
@@ -0,0 +1,27 @@
+performance.quick-read=off
+performance.read-ahead=off
+performance.io-cache=off
+performance.stat-prefetch=off
+performance.open-behind=off
+performance.readdir-ahead=off
+performance.strict-o-direct=on
+performance.client-io-threads=on
+performance.io-thread-count=32
+performance.high-prio-threads=32
+performance.normal-prio-threads=32
+performance.low-prio-threads=32
+performance.least-prio-threads=4
+client.event-threads=8
+server.event-threads=8
+network.remote-dio=disable
+cluster.eager-lock=enable
+cluster.quorum-type=auto
+cluster.data-self-heal-algorithm=full
+cluster.locking-scheme=granular
+cluster.shd-max-threads=8
+cluster.shd-wait-qlength=10000
+features.shard=on
+features.shard-block-size=64MB
+user.cifs=off
+server.allow-insecure=on
+cluster.choose-local=off
diff --git a/extras/group-metadata-cache b/extras/group-metadata-cache
new file mode 100644
index 00000000000..b890b288fc7
--- /dev/null
+++ b/extras/group-metadata-cache
@@ -0,0 +1,6 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.stat-prefetch=on
+performance.cache-invalidation=on
+performance.md-cache-timeout=600
+network.inode-lru-limit=200000
diff --git a/extras/group-nl-cache b/extras/group-nl-cache
new file mode 100644
index 00000000000..897807e8933
--- /dev/null
+++ b/extras/group-nl-cache
@@ -0,0 +1,5 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.nl-cache=on
+performance.nl-cache-timeout=600
+network.inode-lru-limit=200000
diff --git a/extras/group-samba b/extras/group-samba
new file mode 100644
index 00000000000..eeee6e06031
--- /dev/null
+++ b/extras/group-samba
@@ -0,0 +1,11 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.cache-samba-metadata=on
+performance.stat-prefetch=on
+performance.cache-invalidation=on
+performance.md-cache-timeout=600
+network.inode-lru-limit=200000
+performance.nl-cache=on
+performance.nl-cache-timeout=600
+performance.readdir-ahead=on
+performance.parallel-readdir=on
diff --git a/extras/group-virt.example b/extras/group-virt.example
index b699c9e98e2..cc37c98a25c 100644
--- a/extras/group-virt.example
+++ b/extras/group-virt.example
@@ -1,8 +1,24 @@
 performance.quick-read=off
 performance.read-ahead=off
 performance.io-cache=off
-performance.stat-prefetch=off
+performance.low-prio-threads=32
+network.remote-dio=disable
+performance.strict-o-direct=on
 cluster.eager-lock=enable
-network.remote-dio=enable
 cluster.quorum-type=auto
 cluster.server-quorum-type=server
+cluster.data-self-heal-algorithm=full
+cluster.locking-scheme=granular
+cluster.shd-max-threads=8
+cluster.shd-wait-qlength=10000
+features.shard=on
+user.cifs=off
+cluster.choose-local=off
+client.event-threads=4
+server.event-threads=4
+performance.client-io-threads=on
+network.ping-timeout=20
+server.tcp-user-timeout=20
+server.keepalive-time=10
+server.keepalive-interval=2
+server.keepalive-count=5
diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am
index 771b37e3fdf..26059d7dbb9 100644
--- a/extras/hook-scripts/Makefile.am
+++ b/extras/hook-scripts/Makefile.am
@@ -1,5 +1,5 @@
 EXTRA_DIST = S40ufo-stop.py S56glusterd-geo-rep-create-post.sh
-SUBDIRS = add-brick set start stop reset
+SUBDIRS = add-brick create delete set start stop reset
 
 scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/gsync-create/post/
 if USE_GEOREP
diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py
index 107f1968355..2c79eb1d54a 100755
--- a/extras/hook-scripts/S40ufo-stop.py
+++ b/extras/hook-scripts/S40ufo-stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 
 import os
 from optparse import OptionParser
diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
index a5e472e9267..7d6052315bb 100755
--- a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
+++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
@@ -77,18 +77,28 @@ if [ "$val" == "" ]; then
     exit;
 fi
 SSH_PORT=`echo $val`
+SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no"
 
 if [ -f $pub_file ]; then
     # For a non-root user copy the pub file to the user's home directory
     # For a root user copy the pub files to priv_dir->geo-rep.
     if [ "$slave_user" != "root" ]; then
-        slave_user_home_dir=`ssh -p ${SSH_PORT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"`
-        scp -P ${SSH_PORT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp
-        ssh -p ${SSH_PORT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub"
+        slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"`
+        scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp
+        ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub"
     else
-        scp -P ${SSH_PORT} $pub_file $slave_ip:$pub_file_tmp
-        ssh -p ${SSH_PORT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
-        ssh -p ${SSH_PORT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
-        ssh -p ${SSH_PORT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+        if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+            scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on"
+        else
+            scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on"
+        fi
     fi
 fi
diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am
index 5ca5a669de9..9b236df096d 100644
--- a/extras/hook-scripts/add-brick/post/Makefile.am
+++ b/extras/hook-scripts/add-brick/post/Makefile.am
@@ -1,4 +1,6 @@
-EXTRA_DIST = disabled-quota-root-xattr-heal.sh
+EXTRA_DIST = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh
 
 hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/post/
-hook_SCRIPTS = disabled-quota-root-xattr-heal.sh
+if WITH_SERVER
+hook_SCRIPTS = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh
+endif
diff --git a/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh
new file mode 100755
index 00000000000..4a17c993a77
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/add-brick/post
+#
+# Add an SELinux file context for each brick using the glusterd_brick_t type.
+# This ensures that the brick is relabeled correctly on an SELinux restart or
+# restore. Subsequently, run a restore on the brick path to set the selinux
+# labels.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+VOL=
+
+parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+        ;;
+      --gd-workdir)
+          shift
+          GLUSTERD_WORKDIR=$1
+          ;;
+      --version)
+          shift
+          ;;
+      --volume-op)
+          shift
+          ;;
+      *)
+          shift
+          break
+          ;;
+    esac
+    shift
+  done
+}
+
+set_brick_labels()
+{
+  local volname="${1}"
+  local fctx
+  local list=()
+
+  fctx="$(semanage fcontext --list -C)"
+
+  # wait for new brick path to be updated under
+  # ${GLUSTERD_WORKDIR}/vols/${volname}/bricks/
+  sleep 5
+
+  # grab the path for each local brick
+  brickpath="${GLUSTERD_WORKDIR}/vols/${volname}/bricks/"
+  brickdirs=$(
+    find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | \
+    sort -u
+  )
+
+  # create a list of bricks for which custom SELinux
+  # label doesn't exist
+  for b in ${brickdirs}; do
+    pattern="${b}(/.*)?"
+    echo "${fctx}" | grep "^${pattern}\s" >/dev/null
+    if [[ $? -ne 0 ]]; then
+      list+=("${pattern}")
+    fi
+  done
+
+  # Add a file context for each brick path in the list and associate with the
+  # glusterd_brick_t SELinux type.
+  for p in ${list[@]}
+  do
+    semanage fcontext --add -t glusterd_brick_t -r s0 "${p}"
+  done
+
+  # Set the labels for which SELinux label was added above
+  for b in ${brickdirs}
+  do
+    echo "${list[@]}" | grep "${b}" >/dev/null
+    if [[ $? -eq 0 ]]; then
+      restorecon -R "${b}"
+    fi
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+set_brick_labels "${VOL}"
+
+exit 0
diff --git a/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh
new file mode 100755
index 00000000000..1a6923ee7aa
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+##---------------------------------------------------------------------------
+## This script runs the self-heal of the directories which are expected to
+## be present as they are mounted as subdirectory mounts.
+##---------------------------------------------------------------------------
+
+MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`;
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="add-brick-create-subdir"
+VOL_NAME=test
+GLUSTERD_WORKDIR="/var/lib/glusterd"
+
+cleanup_mountpoint ()
+{
+        umount -f $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+
+        rmdir $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+eval set -- "$ARGS"
+
+while true;
+do
+    case $1 in
+        --volname)
+            shift
+            VOL_NAME=$1
+            ;;
+        --gd-workdir)
+            shift
+            GLUSTERD_WORKDIR=$1
+            ;;
+	--version)
+	    shift
+	    ;;
+	--volume-op)
+	    shift
+	    ;;
+	*)
+	    shift
+	    break
+	    ;;
+    esac
+    shift
+done
+
+## See if we have any subdirs to be healed before going further
+subdirs=$(grep 'auth.allow' ${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info | cut -f2 -d'=' | tr ',' '\n' | cut -f1 -d'(');
+
+if [ -z ${subdirs} ]; then
+    rmdir $MOUNT_DIR;
+    exit 0;
+fi
+
+##----------------------------------------
+## Mount the volume in temp directory.
+## -----------------------------------
+glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-50 $MOUNT_DIR;
+if [ 0 -ne $? ]
+then
+    exit $?;
+fi
+
+## -----------------------------------
+# Do the 'stat' on all the directory for now. Ideal fix is to look at subdir
+# list from 'auth.allow' option and only stat them.
+for subdir in ${subdirs}
+do
+    stat ${MOUNT_DIR}/${subdir} > /dev/null;
+done
+
+## Clean up and exit
+cleanup_mountpoint;
diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
index bde7249d429..ca17a903549 100755
--- a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
+++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
@@ -13,123 +13,133 @@
 
 QUOTA_LIMIT_XATTR="trusted.glusterfs.quota.limit-set"
 QUOTA_OBJECT_LIMIT_XATTR="trusted.glusterfs.quota.limit-objects"
-MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`;
+MOUNT_DIR=$(mktemp -d -t "${0##*/}.XXXXXX");
 OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
 PROGNAME="Quota-xattr-heal-add-brick"
 VOL_NAME=
 VERSION=
 VOLUME_OP=
 GLUSTERD_WORKDIR=
-ENABLED_NAME="S28Quota-root-xattr-heal.sh"
+ENABLED_NAME_PREFIX="S28"
+ENABLED_NAME="Quota-root-xattr-heal.sh"
+
+THIS_SCRIPT=$(echo "${0}" | awk -F'/' '{print $NF}')
 
 cleanup_mountpoint ()
 {
-        umount -f $MOUNT_DIR;
-        if [ 0 -ne $? ]
-        then
-                return $?
-        fi
-
-        rmdir $MOUNT_DIR;
-        if [ 0 -ne $? ]
-        then
-                return $?
-        fi
+
+  if umount -f "${MOUNT_DIR}"; then
+    return $?
+  fi
+
+  if rmdir "${MOUNT_DIR}"; then
+    return $?
+  fi
 }
 
 disable_and_exit ()
 {
-        if [ -e "$ENABLED_STATE" ]
-        then
-                unlink $ENABLED_STATE;
-                exit $?
-        fi
+  if [ -e "${ENABLED_STATE}" ]
+  then
+    unlink "${ENABLED_STATE}";
+    exit $?
+  fi
 
-        exit 0
+  exit 0
 }
 
 get_and_set_xattr ()
 {
-        XATTR=$1
-
-        VALUE=$(getfattr -n $XATTR -e hex --absolute-names $MOUNT_DIR 2>&1)
-        RET=$?
-        if [ 0 -eq $RET ]; then
-                VALUE=$(echo $VALUE | grep $XATTR | awk -F'=' '{print $NF}')
-                setfattr -n $XATTR -v $VALUE $MOUNT_DIR;
-                RET=$?
-        else
-                echo $VALUE | grep -iq "No such attribute"
-                if [ 0 -eq $? ]; then
-                        RET=0
-                fi
-        fi
-
-        return $RET;
+  XATTR=$1
+
+  VALUE=$(getfattr -n "${XATTR}" -e hex --absolute-names "${MOUNT_DIR}" 2>&1)
+  RET=$?
+  if [ 0 -eq ${RET} ]; then
+    VALUE=$(echo "${VALUE}" | grep "${XATTR}" | awk -F'=' '{print $NF}')
+    setfattr -n "${XATTR}" -v "${VALUE}" "${MOUNT_DIR}";
+    RET=$?
+  else
+    if echo "${VALUE}" | grep -iq "No such attribute" ; then
+      RET=0
+    fi
+  fi
+
+  return ${RET};
 }
 
 ##------------------------------------------
 ## Parse the arguments
 ##------------------------------------------
-ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
 eval set -- "$ARGS"
 
 while true;
 do
-    case $1 in
-        --volname)
-            shift
-            VOL_NAME=$1
-            ;;
-        --version)
-            shift
-            VERSION=$1
-            ;;
-        --gd-workdir)
-            shift
-            GLUSTERD_WORKDIR=$1
-            ;;
-        --volume-op)
-            shift
-            VOLUME_OP=$1
-            ;;
-        *)
-            shift
-            break
-            ;;
-    esac
-    shift
+  case $1 in
+    --volname)
+      shift
+      VOL_NAME=$1
+    ;;
+    --version)
+      shift
+      VERSION=$1
+    ;;
+    --gd-workdir)
+      shift
+      GLUSTERD_WORKDIR=$1
+    ;;
+    --volume-op)
+      shift
+      VOLUME_OP=$1
+    ;;
+    *)
+      shift
+      break
+    ;;
+  esac
+  shift
 done
 ##----------------------------------------
 
-ENABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/$VOLUME_OP/post/$ENABLED_NAME"
+# Avoid long lines
+ENABLED_STATE_1="${GLUSTERD_WORKDIR}/hooks/${VERSION}/${VOLUME_OP}/"
+ENABLED_STATE_2="post/${ENABLED_NAME_PREFIX}${VOL_NAME}-${ENABLED_NAME}"
+ENABLED_STATE="${ENABLED_STATE_1}${ENABLED_STATE_2}"
+
+if [ "${THIS_SCRIPT}" != *"${VOL_NAME}"* ]; then
+  exit 0
+fi
 
 ## Is quota enabled?
-FLAG=`grep "^features.quota=" $GLUSTERD_WORKDIR/vols/$VOL_NAME/info \
-      | awk -F'=' '{print $NF}'`;
-if [ "$FLAG" != "on" ]
+FLAG=$(grep "^features.quota=" "${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info" \
+| awk -F'=' '{print $NF}');
+if [ "${FLAG}" != "on" ]
 then
-        disable_and_exit
+  disable_and_exit
 fi
 
 ## -----------------------------------
 ## Mount the volume in temp directory.
 ## -----------------------------------
-glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-42 $MOUNT_DIR;
-if [ 0 -ne $? ]
+# Avoid long lines
+CMD_1="glusterfs -s localhost"
+CMD_2="--volfile-id=${VOL_NAME} client-pid=-42 ${MOUNT_DIR}"
+CMD="${CMD_1}${CMD_2}"
+
+if ${CMD}
 then
-        exit $?;
+  exit $?;
 fi
 ## -----------------------------------
 
-RET1=$(get_and_set_xattr $QUOTA_LIMIT_XATTR)
-RET2=$(get_and_set_xattr $QUOTA_OBJECT_LIMIT_XATTR)
+RET1=$(get_and_set_xattr "${QUOTA_LIMIT_XATTR}")
+RET2=$(get_and_set_xattr "${QUOTA_OBJECT_LIMIT_XATTR}")
 
 ## Clean up and exit
 cleanup_mountpoint;
 
-if [ $RET1 -ne 0 -o $RET2 -ne 0 ]; then
-        exit 1
+if [ "${RET1}" -ne 0 ] || [ "${RET2}" -ne 0 ]; then
+  exit 1
 fi
 
 disable_and_exit;
diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am
index 6329ad1d4bd..3288581aa57 100644
--- a/extras/hook-scripts/add-brick/pre/Makefile.am
+++ b/extras/hook-scripts/add-brick/pre/Makefile.am
@@ -1,4 +1,6 @@
 EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh
 
 hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/pre/
+if WITH_SERVER
 hook_SCRIPTS = S28Quota-enable-root-xattr-heal.sh
+endif
diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
index 348f34ec3db..27e85231f45 100755
--- a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
+++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
@@ -26,10 +26,11 @@ VOL_NAME=
 GLUSTERD_WORKDIR=
 VOLUME_OP=
 VERSION=
-ENABLED_NAME="S28Quota-root-xattr-heal.sh"
+ENABLED_NAME_PREFIX="S28"
+ENABLED_NAME="Quota-root-xattr-heal.sh"
 DISABLED_NAME="disabled-quota-root-xattr-heal.sh"
 
-enable ()
+activate ()
 {
         ln -sf $DISABLED_STATE $1;
 }
@@ -37,7 +38,7 @@ enable ()
 ##------------------------------------------
 ## Parse the arguments
 ##------------------------------------------
-ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
 eval set -- "$ARGS"
 
 while true;
@@ -69,8 +70,8 @@ done
 ##----------------------------------------
 
 DISABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME"
-ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/$ENABLED_NAME"
-ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$ENABLED_NAME";
+ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/""$ENABLED_NAME_PREFIX$VOL_NAME""-""$ENABLED_NAME"
+ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/""$ENABLED_NAME_PREFIX""$VOL_NAME""-""$ENABLED_NAME";
 
 ## Why to proceed if the required script itself is not present?
 ls $DISABLED_STATE;
@@ -92,9 +93,9 @@ FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^status=" \
       | awk -F'=' '{print $NF}'`;
 if [ "$FLAG" != "1" ]
 then
-        enable $ENABLED_STATE_START;
+        activate $ENABLED_STATE_START;
         exit $?
 fi
 
-enable $ENABLED_STATE_ADD_BRICK;
+activate $ENABLED_STATE_ADD_BRICK;
 exit $?
diff --git a/extras/hook-scripts/create/Makefile.am b/extras/hook-scripts/create/Makefile.am
new file mode 100644
index 00000000000..b083a9145d6
--- /dev/null
+++ b/extras/hook-scripts/create/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = post
diff --git a/extras/hook-scripts/create/post/Makefile.am b/extras/hook-scripts/create/post/Makefile.am
new file mode 100644
index 00000000000..fd1892e9589
--- /dev/null
+++ b/extras/hook-scripts/create/post/Makefile.am
@@ -0,0 +1,8 @@
+EXTRA_DIST = S10selinux-label-brick.sh
+
+scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/create/post/
+if WITH_SERVER
+if USE_SELINUX
+scripts_SCRIPTS = S10selinux-label-brick.sh
+endif
+endif
diff --git a/extras/hook-scripts/create/post/S10selinux-label-brick.sh b/extras/hook-scripts/create/post/S10selinux-label-brick.sh
new file mode 100755
index 00000000000..f9b4b1a57e3
--- /dev/null
+++ b/extras/hook-scripts/create/post/S10selinux-label-brick.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/create/post
+#
+# Add an SELinux file context for each brick using the glusterd_brick_t type.
+# This ensures that the brick is relabeled correctly on an SELinux restart or
+# restore. Subsequently, run a restore on the brick path to set the selinux
+# labels.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:"
+VOL=
+
+parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+      ;;
+      *)
+        shift
+        break
+      ;;
+    esac
+    shift
+  done
+}
+
+set_brick_labels()
+{
+  volname="${1}"
+
+  # grab the path for each local brick
+  brickpath="/var/lib/glusterd/vols/${volname}/bricks/"
+  brickdirs=$(
+    find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | \
+    sort -u
+  )
+
+  for b in ${brickdirs}; do
+    # Add a file context for each brick path and associate with the
+    # glusterd_brick_t SELinux type.
+    pattern="${b}(/.*)?"
+    semanage fcontext --add -t glusterd_brick_t -r s0 "${pattern}"
+    # Set the labels on the new brick path.
+    restorecon -R "${b}"
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+set_brick_labels "${VOL}"
+
+exit 0
diff --git a/extras/hook-scripts/delete/Makefile.am b/extras/hook-scripts/delete/Makefile.am
new file mode 100644
index 00000000000..c98a05d9205
--- /dev/null
+++ b/extras/hook-scripts/delete/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = pre
diff --git a/extras/hook-scripts/delete/pre/Makefile.am b/extras/hook-scripts/delete/pre/Makefile.am
new file mode 100644
index 00000000000..4fbfbe7311f
--- /dev/null
+++ b/extras/hook-scripts/delete/pre/Makefile.am
@@ -0,0 +1,8 @@
+EXTRA_DIST = S10selinux-del-fcontext.sh
+
+scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/delete/pre/
+if WITH_SERVER
+if USE_SELINUX
+scripts_SCRIPTS = S10selinux-del-fcontext.sh
+endif
+endif
diff --git a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh
new file mode 100755
index 00000000000..056b52afe76
--- /dev/null
+++ b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/delete/pre
+#
+# Delete the file context associated with the brick path on volume deletion. The
+# associated file context was added during volume creation.
+#
+# We do not explicitly relabel the brick, as this could be time consuming and
+# unnecessary.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:"
+VOL=
+
+function parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+      ;;
+      *)
+        shift
+        break
+      ;;
+    esac
+    shift
+  done
+}
+
+function delete_brick_fcontext()
+{
+  local volname=$1
+  local fctx
+  local list=()
+
+  fctx="$(semanage fcontext --list -C)"
+  # grab the path for each local brick
+  brickpath="/var/lib/glusterd/vols/${volname}/bricks/"
+  brickdirs=$(find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | sort -u)
+  for b in ${brickdirs}
+  do
+    pattern="${b}(/.*)?"
+    echo "${fctx}" | grep "^${pattern}\s" >/dev/null
+    if [[ $? -eq 0 ]]; then
+      list+=("${pattern}")
+    fi
+  done
+  if [[ ${#list[@]} -gt 0 ]]; then
+    printf 'fcontext --delete %s\n' "${list[@]}" | semanage -i -
+  fi
+  for b in ${brickdirs}
+  do
+    restorecon -R "${b}"
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+delete_brick_fcontext "${VOL}"
+
+# failure to delete the fcontext is not fatal
+exit 0
diff --git a/extras/hook-scripts/reset/post/Makefile.am b/extras/hook-scripts/reset/post/Makefile.am
index bb28d990596..1b336ac1a85 100644
--- a/extras/hook-scripts/reset/post/Makefile.am
+++ b/extras/hook-scripts/reset/post/Makefile.am
@@ -1,4 +1 @@
-EXTRA_DIST = S31ganesha-reset.sh
-
-hookdir = $(GLUSTERD_WORKDIR)/hooks/1/reset/post/
-hook_SCRIPTS = S31ganesha-reset.sh
+EXTRA_DIST =
diff --git a/extras/hook-scripts/reset/post/S31ganesha-reset.sh b/extras/hook-scripts/reset/post/S31ganesha-reset.sh
deleted file mode 100755
index 9538911a842..00000000000
--- a/extras/hook-scripts/reset/post/S31ganesha-reset.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-
-PROGNAME="Sganesha-reset"
-OPTSPEC="volname:,gd-workdir:"
-VOL=
-GLUSTERD_WORKDIR=
-
-function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  -o "o" -name $PROGNAME $@)
-        eval set -- "$ARGS"
-        while true; do
-            case $1 in
-                --volname)
-                    shift
-                    VOL=$1
-                    ;;
-                --gd-workdir)
-                    shift
-                    GLUSTERD_WORKDIR=$1
-                    ;;
-                 *)
-                    shift
-                    break
-                    ;;
-            esac
-            shift
-        done
-}
-
-function is_volume_started () {
-        volname=$1
-        echo "$(grep status $GLUSTERD_WORKDIR/vols/"$volname"/info |\
-                cut -d"=" -f2)"
-}
-
-parse_args $@
-if ps aux | grep -q "[g]anesha.nfsd"
-        then
-        kill -s TERM `cat /var/run/ganesha.pid`
-        sleep 10
-        rm -rf /var/lib/glusterfs-ganesha/exports
-        rm -rf /var/lib/glusterfs-ganesha/.export_added
-        sed -i /conf/d /var/lib/ganesha/nfs-ganesha.conf
-        if [ "1" = $(is_volume_started "$VOL") ];
-                then
-                gluster volume start $VOL force
-        fi
-fi
diff --git a/extras/hook-scripts/set/post/Makefile.am b/extras/hook-scripts/set/post/Makefile.am
index cea579cb2d9..506a25a8666 100644
--- a/extras/hook-scripts/set/post/Makefile.am
+++ b/extras/hook-scripts/set/post/Makefile.am
@@ -1,4 +1,6 @@
 EXTRA_DIST = S30samba-set.sh S32gluster_enable_shared_storage.sh
 
 hookdir = $(GLUSTERD_WORKDIR)/hooks/1/set/post/
+if WITH_SERVER
 hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/set/post/S30samba-set.sh b/extras/hook-scripts/set/post/S30samba-set.sh
index 97d067fc33f..854f131f6c8 100755
--- a/extras/hook-scripts/set/post/S30samba-set.sh
+++ b/extras/hook-scripts/set/post/S30samba-set.sh
@@ -28,7 +28,7 @@ USERSMB_SET=""
 USERCIFS_SET=""
 
 function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  --name $PROGNAME -o "o:" -- $@)
+        ARGS=$(getopt -o 'o:' -l $OPTSPEC -n $PROGNAME -- "$@")
         eval set -- "$ARGS"
 
         while true; do
@@ -89,7 +89,7 @@ function add_samba_share () {
         STRING+="glusterfs:loglevel = 7\n"
         STRING+="path = /\n"
         STRING+="read only = no\n"
-        STRING+="guest ok = yes\n"
+        STRING+="kernel share modes = no\n"
         printf "$STRING"  >> ${CONFIGFILE}
 }
 
@@ -103,9 +103,9 @@ function sighup_samba () {
         fi
 }
 
-function del_samba_share () {
+function deactivate_samba_share () {
         volname=$1
-        sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE}
+        sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE}
 }
 
 function is_volume_started () {
@@ -123,29 +123,39 @@ function get_smb () {
         usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\
                        cut -d"=" -f2)
 
-        if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then
-                uservalue="disable"
+        if [ -n "$usercifsvalue" ]; then
+                if [ "$usercifsvalue" = "disable" ] || [ "$usercifsvalue" = "off" ]; then
+                        uservalue="disable"
+                fi
         fi
+
+        if [ -n "$usersmbvalue" ]; then
+                if [ "$usersmbvalue" = "disable" ] || [ "$usersmbvalue" = "off" ]; then
+                        uservalue="disable"
+                fi
+        fi
+
         echo "$uservalue"
 }
 
-parse_args $@
-if [ "0" = $(is_volume_started "$VOL") ]; then
+parse_args "$@"
+if [ "0" = "$(is_volume_started "$VOL")" ]; then
     exit 0
 fi
 
 
-if [[ "$USERCIFS_SET" = "YES" || "$USERSMB_SET" = "YES" ]]; then
+if [ "$USERCIFS_SET" = "YES" ] || [ "$USERSMB_SET" = "YES" ]; then
     #Find smb.conf, smbd pid directory and smbd logfile path
     find_config_info
 
-    if [ $(get_smb "$VOL") = "disable" ]; then
-        del_samba_share $VOL
-        sighup_samba
+    if [ "$(get_smb "$VOL")" = "disable" ]; then
+        deactivate_samba_share $VOL
     else
         if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then
             add_samba_share $VOL
-            sighup_samba
+        else
+            sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' ${CONFIGFILE}
         fi
     fi
+    sighup_samba
 fi
diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh
index ad51babd5f7..1f2564b44ff 100755
--- a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh
+++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh
@@ -2,7 +2,7 @@
 
 key=`echo $3 | cut -d '=' -f 1`
 val=`echo $3 | cut -d '=' -f 2`
-if [ ! "$key" -eq "enable-shared-storage" -o "$key" -eq "cluster.enable-shared-storage" ]; then
+if [ "$key" != "cluster.enable-shared-storage" ] && [ "$key" != "enable-shared-storage" ]; then
     exit;
 fi
 if [ "$val" != 'enable' ]; then
@@ -79,9 +79,9 @@ done
 
 if [ "$option" == "disable" ]; then
     # Unmount the volume on all the nodes
-    umount /var/run/gluster/shared_storage
-    cat /etc/fstab  | grep -v "gluster_shared_storage /var/run/gluster/shared_storage/" > /var/run/gluster/fstab.tmp
-    mv /var/run/gluster/fstab.tmp /etc/fstab
+    umount /run/gluster/shared_storage
+    cat /etc/fstab  | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp
+    mv /run/gluster/fstab.tmp /etc/fstab
 fi
 
 if [ "$is_originator" == 1 ]; then
@@ -104,8 +104,15 @@ function check_volume_status()
     echo $status
 }
 
-mount_cmd="mount -t glusterfs "$local_node_hostname":/gluster_shared_storage \
-           /var/run/gluster/shared_storage"
+key=`echo $5 | cut -d '=' -f 1`
+val=`echo $5 | cut -d '=' -f 2`
+if [ "$key" == "transport.address-family" ]; then
+    mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \
+               $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage"
+else
+    mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \
+           /run/gluster/shared_storage"
+fi
 
 if [ "$option" == "enable" ]; then
     retry=0;
@@ -117,13 +124,13 @@ if [ "$option" == "enable" ]; then
         if [ "$retry" == 3 ]; then
             break;
         fi
-        status = check_volume_status;
+        status=$(check_volume_status)
     done
     # Mount the volume on all the nodes
-    umount /var/run/gluster/shared_storage
-    mkdir -p /var/run/gluster/shared_storage
+    umount /run/gluster/shared_storage
+    mkdir -p /run/gluster/shared_storage
     $mount_cmd
-    cp /etc/fstab /var/run/gluster/fstab.tmp
-    echo "$local_node_hostname:/gluster_shared_storage /var/run/gluster/shared_storage/ glusterfs defaults        0 0" >> /var/run/gluster/fstab.tmp
-    mv /var/run/gluster/fstab.tmp /etc/fstab
+    cp /etc/fstab /run/gluster/fstab.tmp
+    echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults        0 0" >> /run/gluster/fstab.tmp
+    mv /run/gluster/fstab.tmp /etc/fstab
 fi
diff --git a/extras/hook-scripts/start/post/Makefile.am b/extras/hook-scripts/start/post/Makefile.am
index 03bb300c5c2..792019d3c9f 100644
--- a/extras/hook-scripts/start/post/Makefile.am
+++ b/extras/hook-scripts/start/post/Makefile.am
@@ -1,4 +1,6 @@
 EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S31ganesha-start.sh
 
 hookdir = $(GLUSTERD_WORKDIR)/hooks/1/start/post/
+if WITH_SERVER
 hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/start/post/S29CTDBsetup.sh b/extras/hook-scripts/start/post/S29CTDBsetup.sh
index 4265cba54ee..69a0d89a3eb 100755
--- a/extras/hook-scripts/start/post/S29CTDBsetup.sh
+++ b/extras/hook-scripts/start/post/S29CTDBsetup.sh
@@ -9,10 +9,14 @@ CTDB_MNT=/gluster/lock
 # Make sure ping-timeout is not default for CTDB volume
 PING_TIMEOUT_SECS=10
 PROGNAME="ctdb"
-OPTSPEC="volname:"
+OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:"
 HOSTNAME=`hostname`
 MNTOPTS="_netdev,transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}"
 VOL=
+GLUSTERD_WORKDIR=
+VERSION=
+VOLUME_OP=
+FIRST=
 # $META is the volume that will be used by CTDB as a shared filesystem.
 # It is not desirable to use this volume for storing 'data' as well.
 # META is set to 'all' (viz. a keyword and hence not a legal volume name)
@@ -21,7 +25,7 @@ VOL=
 META="all"
 
 function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
         eval set -- "$ARGS"
 
         while true; do
@@ -29,13 +33,27 @@ function parse_args () {
                 --volname)
                     shift
                     VOL=$1
-                ;;
-
+                    ;;
+                --gd-workdir)
+                    shift
+                    GLUSTERD_WORKDIR=$1
+                    ;;
+                --version)
+                    shift
+                    VERSION=$1
+                    ;;
+                --volume-op)
+                    shift
+                    VOLUME_OP=$1
+                    ;;
+                --first)
+                    shift
+                    FIRST=$1
+                    ;;
                 *)
-                     shift
-                     break
-                ;;
-
+                    shift
+                    break
+                    ;;
             esac
 
             shift
@@ -55,7 +73,7 @@ function add_fstab_entry () {
         fi
 }
 
-parse_args $@
+parse_args "$@"
 if [ "$META" = "$VOL" ]
 then
         mkdir -p $CTDB_MNT
diff --git a/extras/hook-scripts/start/post/S30samba-start.sh b/extras/hook-scripts/start/post/S30samba-start.sh
index 752eca650d3..cac0cbf1464 100755
--- a/extras/hook-scripts/start/post/S30samba-start.sh
+++ b/extras/hook-scripts/start/post/S30samba-start.sh
@@ -21,15 +21,18 @@
 #volume.
 
 PROGNAME="Ssamba-start"
-OPTSPEC="volname:,gd-workdir:"
+OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:"
 VOL=
 CONFIGFILE=
 LOGFILEBASE=
 PIDDIR=
 GLUSTERD_WORKDIR=
+VERSION=
+VOLUME_OP=
+FIRST=
 
 function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
         eval set -- "$ARGS"
 
         while true; do
@@ -42,24 +45,37 @@ function parse_args () {
                     shift
                     GLUSTERD_WORKDIR=$1
                     ;;
+                --version)
+                    shift
+                    VERSION=$1
+                    ;;
+                --volume-op)
+                    shift
+                    VOLUME_OP=$1
+                    ;;
+                --first)
+                    shift
+                    FIRST=$1
+                    ;;
                 *)
                     shift
                     break
                     ;;
             esac
+
             shift
         done
 }
 
 function find_config_info () {
-        cmdout=`smbd -b | grep smb.conf`
-        if [ $? -ne 0 ];then
+        cmdout=$(smbd -b 2> /dev/null)
+        CONFIGFILE=$(echo "$cmdout" | grep CONFIGFILE | awk '{print $2}')
+        if [ -z "$CONFIGFILE" ]; then
                 echo "Samba is not installed"
                 exit 1
         fi
-        CONFIGFILE=`echo $cmdout | awk {'print $2'}`
-        PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}`
-        LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'`
+        PIDDIR=$(echo "$cmdout" | grep PIDDIR | awk '{print $2}')
+        LOGFILEBASE=$(echo "$cmdout" | grep 'LOGFILEBASE' | awk '{print $2}')
 }
 
 function add_samba_share () {
@@ -72,12 +88,12 @@ function add_samba_share () {
         STRING+="glusterfs:loglevel = 7\n"
         STRING+="path = /\n"
         STRING+="read only = no\n"
-        STRING+="guest ok = yes\n"
-        printf "$STRING"  >> ${CONFIGFILE}
+        STRING+="kernel share modes = no\n"
+        printf "$STRING"  >> "${CONFIGFILE}"
 }
 
 function sighup_samba () {
-        pid=`cat ${PIDDIR}/smbd.pid`
+        pid=$(cat "${PIDDIR}/smbd.pid" 2> /dev/null)
         if [ "x$pid" != "x" ]
         then
                 kill -HUP "$pid";
@@ -90,26 +106,40 @@ function get_smb () {
         volname=$1
         uservalue=
 
-        usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\
+        usercifsvalue=$(grep user.cifs "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\
                         cut -d"=" -f2)
-        usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\
+        usersmbvalue=$(grep user.smb "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\
                        cut -d"=" -f2)
 
-        if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then
-                uservalue="disable"
+        if [ -n "$usercifsvalue" ]; then
+                if [ "$usercifsvalue" = "enable" ] || [ "$usercifsvalue" = "on" ]; then
+                        uservalue="enable"
+                fi
+        fi
+
+        if [ -n "$usersmbvalue" ]; then
+                if [ "$usersmbvalue" = "enable" ] || [ "$usersmbvalue" = "on" ]; then
+                        uservalue="enable"
+                fi
         fi
+
         echo "$uservalue"
 }
 
-parse_args $@
-if [ $(get_smb "$VOL") = "disable" ]; then
+parse_args "$@"
+
+value=$(get_smb "$VOL")
+
+if [ -z "$value" ] || [ "$value" != "enable" ]; then
         exit 0
 fi
 
 #Find smb.conf, smbd pid directory and smbd logfile path
 find_config_info
 
-if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then
-        add_samba_share $VOL
-        sighup_samba
+if ! grep --quiet "\[gluster-$VOL\]" "${CONFIGFILE}" ; then
+        add_samba_share "$VOL"
+else
+        sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' "${CONFIGFILE}"
 fi
+sighup_samba
diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh
index 1b258cc0ae4..7ad6f23ad06 100755
--- a/extras/hook-scripts/start/post/S31ganesha-start.sh
+++ b/extras/hook-scripts/start/post/S31ganesha-start.sh
@@ -4,7 +4,7 @@ OPTSPEC="volname:,gd-workdir:"
 VOL=
 declare -i EXPORT_ID
 ganesha_key="ganesha.enable"
-GANESHA_DIR="/var/run/gluster/shared_storage/nfs-ganesha"
+GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha"
 CONF1="$GANESHA_DIR/ganesha.conf"
 GLUSTERD_WORKDIR=
 
@@ -60,46 +60,12 @@ echo "      SecType = \"sys\";"
 echo "}"
 }
 
-#This function keeps track of export IDs and increments it with every new entry
+#It adds the export dynamically by sending dbus signals
 function export_add()
 {
-        count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l`
-        if [ "$count" = "1" ] ;
-                then
-                EXPORT_ID=2
-        else
-        #if [ -s /var/lib/ganesha/export_removed ];
-        #               then
-        #               EXPORT_ID=`head -1 /var/lib/ganesha/export_removed`
-        #               sed -i -e "1d" /var/lib/ganesha/export_removed
-        #               else
-
-                 EXPORT_ID=`cat $GANESHA_DIR/.export_added`
-                 EXPORT_ID=EXPORT_ID+1
-        #fi
-        fi
-        echo $EXPORT_ID > $GANESHA_DIR/.export_added
-        sed -i s/Export_Id.*/"Export_Id= $EXPORT_ID ;"/ \
-$GANESHA_DIR/exports/export.$VOL.conf
-        echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1
-}
-
-#This function adds a new export dynamically by sending dbus signals
-function dynamic_export_add()
-{
         dbus-send --print-reply --system --dest=org.ganesha.nfsd \
 /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
-string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Path=/$VOL)"
-
-}
-
-function start_ganesha()
-{
-        #Remove export entry from nfs-ganesha.conf
-        sed -i /$VOL.conf/d  $CONF1
-        #Create a new export entry
-        export_add $VOL
-        dynamic_export_add $VOL
+string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Export_Id=$EXPORT_ID)"
 
 }
 
@@ -137,9 +103,20 @@ if ganesha_enabled ${VOL} && ! is_exported ${VOL}
 then
         if [ ! -e ${GANESHA_DIR}/exports/export.${VOL}.conf ]
         then
+                #Remove export entry from nfs-ganesha.conf
+                sed -i /$VOL.conf/d  $CONF1
                 write_conf ${VOL} > ${GANESHA_DIR}/exports/export.${VOL}.conf
+                EXPORT_ID=`cat $GANESHA_DIR/.export_added`
+                EXPORT_ID=EXPORT_ID+1
+                echo $EXPORT_ID > $GANESHA_DIR/.export_added
+                sed -i s/Export_Id.*/"Export_Id=$EXPORT_ID;"/ \
+                        $GANESHA_DIR/exports/export.$VOL.conf
+                echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1
+        else
+                EXPORT_ID=$(grep ^[[:space:]]*Export_Id $GANESHA_DIR/exports/export.$VOL.conf |\
+                          awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]')
         fi
-        start_ganesha ${VOL}
+        export_add $VOL
 fi
 
 exit 0
diff --git a/extras/hook-scripts/stop/pre/Makefile.am b/extras/hook-scripts/stop/pre/Makefile.am
index bf63e7393d3..9e8d1565e93 100644
--- a/extras/hook-scripts/stop/pre/Makefile.am
+++ b/extras/hook-scripts/stop/pre/Makefile.am
@@ -1,4 +1,6 @@
 EXTRA_DIST =  S29CTDB-teardown.sh S30samba-stop.sh
 
 hookdir = $(GLUSTERD_WORKDIR)/hooks/1/stop/pre/
+if WITH_SERVER
 hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh
index 5fb49bd9e97..0975a00f18d 100755
--- a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh
+++ b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh
@@ -2,8 +2,9 @@
 
 CTDB_MNT=/gluster/lock
 PROGNAME="ctdb"
-OPTSPEC="volname:"
+OPTSPEC="volname:,last:"
 VOL=
+LAST=
 # $META is the volume that will be used by CTDB as a shared filesystem.
 # It is not desirable to use this volume for storing 'data' as well.
 # META is set to 'all' (viz. a keyword and hence not a legal volume name)
@@ -12,7 +13,7 @@ VOL=
 META="all"
 
 function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
         eval set -- "$ARGS"
 
         while true; do
@@ -20,15 +21,16 @@ function parse_args () {
                 --volname)
                     shift
                     VOL=$1
-                ;;
-
+                    ;;
+                --last)
+                    shift
+                    LAST=$1
+                    ;;
                 *)
-                     shift
-                     break
-                ;;
-
+                    shift
+                    break
+                    ;;
             esac
-
             shift
         done
 }
@@ -51,7 +53,7 @@ function remove_fstab_entry () {
         fi
 }
 
-parse_args $@
+parse_args "$@"
 if [ "$META" = "$VOL" ]
 then
         umount "$CTDB_MNT"
diff --git a/extras/hook-scripts/stop/pre/S30samba-stop.sh b/extras/hook-scripts/stop/pre/S30samba-stop.sh
index 62cf7d1e0d2..ea799381d62 100755
--- a/extras/hook-scripts/stop/pre/S30samba-stop.sh
+++ b/extras/hook-scripts/stop/pre/S30samba-stop.sh
@@ -16,27 +16,33 @@
 #event by removing the volume related entries(if any) in smb.conf file.
 
 PROGNAME="Ssamba-stop"
-OPTSPEC="volname:"
+OPTSPEC="volname:,last:"
 VOL=
 CONFIGFILE=
 PIDDIR=
+LAST=
 
 function parse_args () {
-        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
         eval set -- "$ARGS"
 
         while true; do
-        case $1 in
-        --volname)
-         shift
-         VOL=$1
-         ;;
-        *)
-         shift
-         break
-         ;;
-        esac
-        shift
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --last)
+                    shift
+                    LAST=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+
+            shift
         done
 }
 
@@ -46,13 +52,13 @@ function find_config_info () {
                 echo "Samba is not installed"
                 exit 1
         fi
-        CONFIGFILE=`echo $cmdout | awk {'print $2'}`
-        PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}`
+        CONFIGFILE=`echo $cmdout | awk '{print $2}'`
+        PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'`
 }
 
-function del_samba_share () {
+function deactivate_samba_share () {
         volname=$1
-        sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE}
+        sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE}
 }
 
 function sighup_samba () {
@@ -65,7 +71,7 @@ function sighup_samba () {
         fi
 }
 
-parse_args $@
+parse_args "$@"
 find_config_info
-del_samba_share $VOL
+deactivate_samba_share $VOL
 sighup_samba
diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh
new file mode 100755
index 00000000000..ebc6bf144aa
--- /dev/null
+++ b/extras/identify-hangs.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+function get_statedump_fnames_without_timestamps
+{
+    ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u
+}
+
+function get_non_uniq_fields
+{
+    local statedump_fname_prefix=$1
+    print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}'
+}
+
+function print_stack_lkowner_unique_in_one_line
+{
+    local statedump_fname_prefix=$1
+    sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - -
+}
+
+function get_stacks_that_appear_in_multiple_statedumps
+{
+    #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps
+    #print the stack
+    local statedump_fname_prefix=$1
+    while read -r non_uniq_stack;
+    do
+        if [ -z "$printed" ];
+        then
+            printed="1"
+        fi
+        echo "$statedump_fname_prefix" "$non_uniq_stack"
+    done < <(get_non_uniq_fields "$statedump_fname_prefix")
+}
+
+statedumpdir=${1}
+if [ -z "$statedumpdir" ];
+then
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+if [ ! -d "$statedumpdir" ];
+then
+    echo "$statedumpdir: Is not a directory"
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+cd "$statedumpdir" || exit 1
+for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps);
+do
+    get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix"
+done | column -t
+echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2
diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am
index bd8837bd9f7..8d8cc69571a 100644
--- a/extras/init.d/Makefile.am
+++ b/extras/init.d/Makefile.am
@@ -1,7 +1,7 @@
 
-EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat glusterd-SuSE \
-	glusterd.plist rhel5-load-fuse.modules \
-	glustereventsd-FreeBSD glustereventsd-Redhat
+EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat \
+	glusterd-SuSE glusterd.plist glustereventsd-FreeBSD \
+	glustereventsd-Redhat glustereventsd-Debian
 
 CLEANFILES =
 
@@ -10,11 +10,12 @@ SYSTEMD_DIR = @systemddir@
 LAUNCHD_DIR = @launchddir@
 
 $(GF_DISTRIBUTION):
+if WITH_SERVER
 	@if [ ! -d $(SYSTEMD_DIR) ]; then \
 		$(mkdir_p) $(DESTDIR)$(INIT_DIR); \
 		$(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \
 	fi
-
+endif
 if BUILD_EVENTS
 	@if [ ! -d $(SYSTEMD_DIR) ]; then \
 		$(mkdir_p) $(DESTDIR)$(INIT_DIR); \
diff --git a/extras/init.d/glustereventsd-Debian.in b/extras/init.d/glustereventsd-Debian.in
new file mode 100644
index 00000000000..6eebdb2b8d8
--- /dev/null
+++ b/extras/init.d/glustereventsd-Debian.in
@@ -0,0 +1,91 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          glustereventsd
+# Required-Start:    $local_fs $network
+# Required-Stop:     $local_fs $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Gluster Events Server
+# Description:       Gluster Events Server
+### END INIT INFO
+
+# Author: Chris AtLee <chris@atlee.ca>
+# Patched by: Matthias Albert < matthias@linux4experts.de>
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+NAME=glustereventsd
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON=@prefix@/sbin/$NAME
+PIDFILE=/var/run/$NAME.pid
+GLUSTEREVENTSD_OPTS=""
+PID=`test -f $PIDFILE && cat $PIDFILE`
+
+
+# Gracefully exit if the package has been removed.
+test -x $DAEMON || exit 0
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+
+do_start()
+{
+    pidofproc -p $PIDFILE $DAEMON >/dev/null
+    status=$?
+    if [ $status -eq 0 ]; then
+      log_success_msg "glustereventsd service is already running with pid $PID"
+    else
+      log_daemon_msg "Starting glustereventsd service" "glustereventsd"
+      start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTEREVENTSD_OPTS
+      log_end_msg $?
+      start_daemon -p $PIDFILE $DAEMON -f $CONFIGFILE
+      return $?
+    fi
+}
+
+do_stop()
+{
+    log_daemon_msg "Stopping glustereventsd service" "glustereventsd"
+    start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+    log_end_msg $?
+    rm -f $PIDFILE
+    killproc -p $PIDFILE $DAEMON
+    return $?
+}
+
+do_status()
+{
+     pidofproc -p $PIDFILE $DAEMON >/dev/null
+     status=$?
+     if [ $status -eq 0 ]; then
+       log_success_msg "glustereventsd service is running with pid $PID"
+     else
+       log_failure_msg "glustereventsd service is not running."
+     fi
+     exit $status
+}
+
+case "$1" in
+  start)
+        do_start
+        ;;
+  stop)
+        do_stop
+        ;;
+  status)
+        do_status;
+        ;;
+  restart|force-reload)
+        do_stop
+        sleep 2
+        do_start
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac
+
diff --git a/extras/init.d/rhel5-load-fuse.modules b/extras/init.d/rhel5-load-fuse.modules
deleted file mode 100755
index ee194db99b8..00000000000
--- a/extras/init.d/rhel5-load-fuse.modules
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-#
-# fusermount-glusterfs requires the /dev/fuse character device. The fuse module
-# provides this and is loaded on demand in newer Linux distributions.
-#
-
-[ -c /dev/fuse ] || /sbin/modprobe fuse
diff --git a/extras/mount-shared-storage.sh b/extras/mount-shared-storage.sh
new file mode 100755
index 00000000000..cc40e13c3e3
--- /dev/null
+++ b/extras/mount-shared-storage.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#Post reboot there is a chance in which mounting of shared storage will fail
+#This will impact starting of features like NFS-Ganesha. So this script will
+#try to mount the shared storage if it fails
+
+exitStatus=0
+
+while IFS= read -r glm
+do
+	IFS=$' \t' read -r -a arr <<< "$glm"
+
+	#Validate storage type is glusterfs
+	if [ "${arr[2]}" == "glusterfs" ]
+	then
+
+		#check whether shared storage is mounted
+		#if it is mounted then mountpoint -q will return a 0 success code
+		if mountpoint -q "${arr[1]}"
+		then
+			echo "${arr[1]} is already mounted"
+			continue
+		fi
+
+		mount -t glusterfs -o "${arr[3]}" "${arr[0]}" "${arr[1]}"
+		#wait for few seconds
+		sleep 10
+
+		#recheck mount got succeed
+		if mountpoint -q "${arr[1]}"
+		then
+			echo "${arr[1]} has been mounted"
+			continue
+		else
+			echo "${arr[1]} failed to mount"
+			exitStatus=1
+		fi
+	fi
+done <<< "$(sed '/^#/ d' </etc/fstab | grep 'glusterfs')"
+exit $exitStatus
diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in
index 72fd1213af2..76cc649e55f 100755
--- a/extras/ocf/volume.in
+++ b/extras/ocf/volume.in
@@ -6,6 +6,7 @@
 #               HA resource
 #
 # Authors:      Florian Haas (hastexo Professional Services GmbH)
+#               Jiri Lunacek (Hosting90 Systems s.r.o.)
 #
 # License:      GNU General Public License (GPL)
 
@@ -54,6 +55,14 @@ must have clone ordering enabled.
       <shortdesc lang="en">gluster executable</shortdesc>
       <content type="string" default="$OCF_RESKEY_binary_default"/>
     </parameter>
+    <parameter name="peer_map">
+      <longdesc lang="en">
+      Mapping of hostname - peer name in the gluster cluster
+      in format hostname1:peername1,hostname2:peername2,...
+      </longdesc>
+      <shortdesc lang="en">gluster peer map</shortdesc>
+      <content type="string" default=""/>
+    </parameter>
   </parameters>
   <actions>
     <action name="start"        timeout="20" />
@@ -68,9 +77,13 @@ EOF
 
 }
 
+if [ -n "${OCF_RESKEY_peer_map}" ]; then
+    SHORTHOSTNAME=`echo "${OCF_RESKEY_peer_map}" | egrep -o "$SHORTHOSTNAME\:[^,]+" | awk -F: '{print $2}'`
+fi
+
 volume_getdir() {
     local voldir
-    voldir="@sysconfdir@/glusterd/vols/${OCF_RESKEY_volname}"
+    voldir="@GLUSTERD_WORKDIR@/vols/${OCF_RESKEY_volname}"
 
     [ -d ${voldir} ] || return 1
 
@@ -78,6 +91,16 @@ volume_getdir() {
     return 0
 }
 
+volume_getpid_dir() {
+    local volpid_dir
+    volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}"
+
+    [ -d ${volpid_dir} ] || return 1
+
+    echo "${volpid_dir}"
+    return 0
+}
+
 volume_getbricks() {
     local infofile
     local voldir
@@ -92,17 +115,19 @@ volume_getbricks() {
 
 volume_getpids() {
     local bricks
-    local piddir
     local pidfile
     local infofile
-    local voldir
+    local volpid_dir
 
-    voldir=`volume_getdir`
+    volpid_dir=`volume_getpid_dir`
     bricks=`volume_getbricks`
-    piddir="${voldir}/run"
+    
+    if [ -z "$bricks" ]; then
+        return 1
+    fi
 
     for brick in ${bricks}; do
-	pidfile="${piddir}/${SHORTHOSTNAME}${brick}.pid"
+	pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid"
 	[ -e $pidfile ] || return 1
 	cat $pidfile
     done
@@ -206,6 +231,11 @@ volume_validate_all() {
 
     # Test for required binaries
     check_binary $OCF_RESKEY_binary
+	
+	if [ -z "$SHORTHOSTNAME" ]; then
+		ocf_log err 'Unable to get host in node map'
+		return $OCF_ERR_CONFIGURED
+	fi
 
     return $OCF_SUCCESS
 }
diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler
index 65d445864aa..aaafd088648 100755
--- a/extras/profiler/glusterfs-profiler
+++ b/extras/profiler/glusterfs-profiler
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 
 #  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
 #  This file is part of GlusterFS.
@@ -291,7 +291,7 @@ class Texttable:
         s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()],
             horiz)
         # build the line
-        l = string.join([horiz*n for n in self._width], s)
+        l = s.join([horiz*n for n in self._width])
         # add border if needed
         if self._has_border():
             l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz,
diff --git a/extras/prot_filter.py b/extras/prot_filter.py
deleted file mode 100755
index 7dccacf155e..00000000000
--- a/extras/prot_filter.py
+++ /dev/null
@@ -1,144 +0,0 @@
-#!/usr/bin/python
-
-"""
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-"""
-
-"""
-  INSTRUCTIONS
-  Put this in /usr/lib64/glusterfs/$version/filter to have it run automatically,
-  or else you'll have to run it by hand every time you change the volume
-  configuration.  Give it a list of volume names on which to enable the
-  protection functionality; it will deliberately ignore client volfiles for
-  other volumes, and all server volfiles.  It *will* include internal client
-  volfiles such as those used for NFS or rebalance/self-heal; this is a
-  deliberate choice so that it will catch deletions from those sources as well.
-"""
-
-volume_list = [ "jdtest" ]
-
-import copy
-import string
-import sys
-import types
-
-class Translator:
-    def __init__ (self, name):
-        self.name = name
-        self.xl_type = ""
-        self.opts = {}
-        self.subvols = []
-        self.dumped = False
-    def __repr__ (self):
-        return "<Translator %s>" % self.name
-
-def load (path):
-    # If it's a string, open it; otherwise, assume it's already a
-    # file-like object (most notably from urllib*).
-    if type(path) in types.StringTypes:
-        fp = file(path,"r")
-    else:
-        fp = path
-    all_xlators = {}
-    xlator = None
-    last_xlator = None
-    while True:
-        text = fp.readline()
-        if text == "":
-            break
-        text = text.split()
-        if not len(text):
-            continue
-        if text[0] == "volume":
-            if xlator:
-                raise RuntimeError, "nested volume definition"
-            xlator = Translator(text[1])
-            continue
-        if not xlator:
-            raise RuntimeError, "text outside volume definition"
-        if text[0] == "type":
-            xlator.xl_type = text[1]
-            continue
-        if text[0] == "option":
-            xlator.opts[text[1]] = string.join(text[2:])
-            continue
-        if text[0] == "subvolumes":
-            for sv in text[1:]:
-                xlator.subvols.append(all_xlators[sv])
-            continue
-        if text[0] == "end-volume":
-            all_xlators[xlator.name] = xlator
-            last_xlator = xlator
-            xlator = None
-            continue
-        raise RuntimeError, "unrecognized keyword %s" % text[0]
-    if xlator:
-        raise RuntimeError, "unclosed volume definition"
-    return all_xlators, last_xlator
-
-def generate (graph, last, stream=sys.stdout):
-    for sv in last.subvols:
-        if not sv.dumped:
-            generate(graph,sv,stream)
-            print >> stream, ""
-            sv.dumped = True
-    print >> stream, "volume %s" % last.name
-    print >> stream, "    type %s" % last.xl_type
-    for k, v in last.opts.iteritems():
-        print >> stream, "    option %s %s" % (k, v)
-    if last.subvols:
-        print >> stream, "    subvolumes %s" % string.join(
-            [ sv.name for sv in last.subvols ])
-    print >> stream, "end-volume"
-
-def push_filter (graph, old_xl, filt_type, opts={}):
-    new_type = "-" + filt_type.split("/")[1]
-    old_type = "-" + old_xl.xl_type.split("/")[1]
-    pos = old_xl.name.find(old_type)
-    if pos >= 0:
-        new_name = old_xl.name
-        old_name = new_name[:pos] + new_type + new_name[len(old_type)+pos:]
-    else:
-        new_name = old_xl.name + old_type
-        old_name = old_xl.name + new_type
-    new_xl = Translator(new_name)
-    new_xl.xl_type = old_xl.xl_type
-    new_xl.opts = old_xl.opts
-    new_xl.subvols = old_xl.subvols
-    graph[new_xl.name] = new_xl
-    old_xl.name = old_name
-    old_xl.xl_type = filt_type
-    old_xl.opts = opts
-    old_xl.subvols = [new_xl]
-    graph[old_xl.name] = old_xl
-
-if __name__ == "__main__":
-    path = sys.argv[1]
-    # Alow an override for debugging.
-    for extra in sys.argv[2:]:
-        volume_list.append(extra)
-    graph, last = load(path)
-    for v in volume_list:
-        if graph.has_key(v):
-            break
-    else:
-        print "No configured volumes found - aborting."
-        sys.exit(0)
-    for v in graph.values():
-        if v.xl_type == "cluster/distribute":
-            push_filter(graph,v,"features/prot_dht")
-        elif v.xl_type == "protocol/client":
-            push_filter(graph,v,"features/prot_client")
-    # We push debug/trace so that every fop gets a real frame, because DHT
-    # gets confused if STACK_WIND_TAIL causes certain fops to be invoked
-    # from anything other than a direct child.
-    for v in graph.values():
-        if v.xl_type == "features/prot_client":
-            push_filter(graph,v,"debug/trace")
-    generate(graph,last,stream=open(path,"w"))
diff --git a/extras/python/Makefile.am b/extras/python/Makefile.am
new file mode 100644
index 00000000000..7d81fa0319b
--- /dev/null
+++ b/extras/python/Makefile.am
@@ -0,0 +1,7 @@
+if HAVE_PYTHON
+# Install __init__.py into the Python site-packages area
+pypkgdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster
+pypkg_PYTHON = __init__.py
+endif
+
+EXTRA_DIST = __init__.py
diff --git a/xlators/features/glupy/src/__init__.py.in b/extras/python/__init__.py
index 3ad9513f40e..3ad9513f40e 100644
--- a/xlators/features/glupy/src/__init__.py.in
+++ b/extras/python/__init__.py
diff --git a/extras/contri-add.sh b/extras/quota/contri-add.sh
index 7db5edd5d20..7db5edd5d20 100755
--- a/extras/contri-add.sh
+++ b/extras/quota/contri-add.sh
diff --git a/extras/quota/log_accounting.sh b/extras/quota/log_accounting.sh
new file mode 100755
index 00000000000..e2dd87b84d7
--- /dev/null
+++ b/extras/quota/log_accounting.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# The script does an accounting of all directories using command 'du' and
+# using gluster. We can then compare the two to identify accounting mismatch
+# THere can be minor mismatch because gluster only accounts for the size of
+# files. Direcotries can take up upto 4kB space on FS per directory. THis
+# size is accounted by du and not by gluster. However the difference would
+# not be significant.
+
+mountpoint=$1
+volname=$2
+
+usage ()
+{
+    echo >&2 "usage: $0 <mountpoint> <volume name>"
+    exit
+}
+
+[ $# -lt 2 ] && usage
+
+cd $mountpoint
+du -h | head -n -1 | tr -d '.' |awk  '{ for (i = 2; i <= NF; i++) { printf("%s ", $i);}  print "" }' > /tmp/gluster_quota_1
+cat /tmp/gluster_quota_1 | sed 's/ $//' | sed 's/ /\\ /g' | sed 's/(/\\(/g' | sed 's/)/\\)/g' |xargs gluster v quota $volname list > /tmp/gluster_quota_2
+du -h | head -n -1 |awk  '{ for (i = 2; i <= NF; i++) { printf("%s %s", $i, $1);}  print "" }' | tr -d '.' >  /tmp/gluster_quota_3
+cat /tmp/gluster_quota_2 /tmp/gluster_quota_3 | sort > /tmp/gluster_quota_4
+find . -type d > /tmp/gluster_quota_5
+tar -cvf /tmp/gluster_quota_files.tar /tmp/gluster_quota_*
diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py
new file mode 100755
index 00000000000..e62f7fc52a3
--- /dev/null
+++ b/extras/quota/quota_fsck.py
@@ -0,0 +1,377 @@
+#!/usr/bin/python3
+# The following script enables, Detecting, Reporting and Fixing
+# anomalies in quota accounting. Run this script with -h option
+# for further details.
+
+'''
+   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+'''
+from __future__ import print_function
+import os, sys, re
+from stat import *
+import subprocess
+import argparse
+import xattr
+
+aggr_size = {}
+verbose_mode = False
+mnt_path = None
+brick_path = None
+obj_fix_count = 0
+file_count = 0
+dir_count = 0
+
+#CONSTANTS
+KB = 1024
+MB = 1048576
+GB = 1048576 * 1024
+TB = 1048576 * 1048576
+
+QUOTA_VERBOSE = 0
+QUOTA_META_ABSENT = 1
+QUOTA_SIZE_MISMATCH = 2
+
+IS_DIRTY ='0x3100'
+IS_CLEAN ='0x3000'
+
+
+epilog_msg='''
+            The script attempts to find any gluster accounting issues in the
+            filesystem at the given subtree. The script crawls the given
+            subdirectory tree doing a stat for all files and compares the
+            size reported by gluster quota with the size reported by stat
+            calls. Any mismatch is reported. In addition integrity of marker
+            xattrs are verified.
+            '''
+
+def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None):
+    if log_type == QUOTA_VERBOSE:
+        print('%-24s %-60s\nxattr_values: %s\n%s\n' % ("Verbose", path, xattr_dict, stbuf))
+    elif log_type == QUOTA_META_ABSENT:
+        print('%-24s %-60s\n%s\n' % ("Quota-Meta Absent", path, xattr_dict))
+    elif log_type == QUOTA_SIZE_MISMATCH:
+        print("mismatch")
+        if dir_size is not None:
+            print('%24s %60s %12s %12s' % ("Size Mismatch", path,
+                xattr_dict, dir_size))
+        else:
+            print('%-24s %-60s %-12s %-12s' % ("Size Mismatch", path, xattr_dict,
+                   stbuf.st_size))
+
+def size_differs_lot(s1, s2):
+    '''
+    There could be minor accounting differences between the stat based
+    accounting and gluster accounting. To avoid these from throwing lot
+    of false positives in our logs. using a threshold of 1M for now.
+    TODO: For a deeply nested directory, at higher levels in hierarchy
+    differences may not be significant, hence this check needs to be improved.
+    '''
+    if abs(s1-s2) > 0:
+        return True
+    else:
+        return False
+
+def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size):
+    '''
+            Hard links are messy.. we have to account them for their parent
+            directory. But, stop accounting at the most common ancestor.
+            Eg:
+                say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3
+
+            suppose we encounter the hard links h1 first , then h2 and then h3.
+            while accounting for h1, we account the size until root(d2->d1->/)
+            while accounting for h2, we need to account only till d3. (as d1
+            and / are accounted for this inode).
+            while accounting for h3 we should not account at all.. as all
+            its ancestors are already accounted for same inode.
+
+            curr_dict                : dict of hardlinks that were seen and
+                                       accounted by the current iteration.
+            accounted_dict           : dict of hardlinks that has already been
+                                       accounted for.
+
+            size                     : size of the object as accounted by the
+                                       curr_iteration.
+
+            Return vale:
+            curr_size                : size reduced by hardlink sizes for those
+                                       hardlinks that has already been accounted
+                                       in current subtree.
+            Also delete the duplicate link from curr_dict.
+    '''
+
+    dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys())
+    for link in dual_accounted_links:
+        curr_size = curr_size - curr_dict[link]
+        del curr_dict[link]
+    return curr_size
+
+
+def fix_xattr(file_name, mark_dirty):
+    global obj_fix_count
+    global mnt_path
+
+    if mnt_path is None:
+        return
+    if mark_dirty:
+        print("MARKING DIRTY: " + file_name)
+        out = subprocess.check_output (["/usr/bin/setfattr", "-n",
+                                       "trusted.glusterfs.quota.dirty",
+                                       "-v", IS_DIRTY, file_name])
+    rel_path = os.path.relpath(file_name, brick_path)
+    print("stat on "  + mnt_path + "/" + rel_path)
+    stbuf = os.lstat(mnt_path + "/" + rel_path)
+
+    obj_fix_count += 1
+
+def get_quota_xattr_brick(dpath):
+    out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference",
+                                    "-d", "-m.", "-e", "hex", dpath])
+    pairs = out.splitlines()
+
+    '''
+    Sample output to be parsed:
+    [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/
+    # file: export/b1/B0/d14/d13/
+    security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000
+    trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8
+    trusted.glusterfs.dht=0x000000010000000000000000ffffffff
+    trusted.glusterfs.dht.mds=0x00000000
+    trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e
+    trusted.glusterfs.quota.dirty=0x3000
+    trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e
+    '''
+
+    '''
+    xattr_dict dictionary holds quota related xattrs
+    eg:
+    '''
+
+    xattr_dict = {}
+    xattr_dict['parents'] = {}
+
+    for xattr in pairs[1:]:
+        xattr = xattr.decode("utf-8")
+        xattr_key = xattr.split("=")[0]
+        if xattr_key == "":
+            # skip any empty lines
+            continue
+        elif not re.search("quota", xattr_key):
+            # skip all non quota xattr.
+            continue
+
+        xattr_value = xattr.split("=")[1]
+        if re.search("contri", xattr_key):
+
+            xattr_version = xattr_key.split(".")[5]
+            if 'version' not in xattr_dict:
+                xattr_dict['version'] = xattr_version
+            else:
+                if xattr_version != xattr_dict['version']:
+                   print("Multiple xattr version found")
+
+
+            cur_parent = xattr_key.split(".")[3]
+            if cur_parent not in xattr_dict['parents']:
+                xattr_dict['parents'][cur_parent] = {}
+
+            contri_dict = xattr_dict['parents'][cur_parent]
+            if len(xattr_value) == 34:
+                # 34 bytes implies file contri xattr
+                # contri format =0x< 16bytes file size><16bytes file count>
+                # size is obtained in iatt, file count = 1, dir count=0
+                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
+                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
+                contri_dict['contri_dir_count'] = 0
+            else:
+                # This is a directory contri.
+                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
+                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
+                contri_dict['contri_dir_count'] = int(xattr_value[34:], 16)
+
+        elif re.search("size", xattr_key):
+            xattr_dict['size'] = int(xattr_value[2:18], 16)
+            xattr_dict['file_count'] = int(xattr_value[18:34], 16)
+            xattr_dict['dir_count'] = int(xattr_value[34:], 16)
+        elif re.search("dirty", xattr_key):
+            if xattr_value == IS_CLEAN:
+                xattr_dict['dirty'] = False
+            elif xattr_value == IS_DIRTY:
+                xattr_dict['dirty'] = True
+        elif re.search("limit_objects", xattr_key):
+            xattr_dict['limit_objects'] = int(xattr_value[2:18], 16)
+        elif re.search("limit_set", xattr_key):
+            xattr_dict['limit_set'] = int(xattr_value[2:18], 16)
+
+    return xattr_dict
+
+def verify_file_xattr(path, stbuf = None):
+
+    global file_count
+    file_count += 1
+
+    if stbuf is None:
+        stbuf = os.lstat(path)
+
+    xattr_dict = get_quota_xattr_brick(path)
+
+    for parent in xattr_dict['parents']:
+        contri_dict = xattr_dict['parents'][parent]
+
+        if 'contri_size' not in contri_dict or \
+           'contri_file_count' not in contri_dict or \
+           'contri_dir_count' not in contri_dict:
+            print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf)
+            fix_xattr(path, False)
+            return
+        elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size):
+            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf)
+            fix_xattr(path, False)
+            return
+
+    if verbose_mode is True:
+        print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf)
+
+
+def verify_dir_xattr(path, dir_size):
+
+    global dir_count
+    dir_count += 1
+    xattr_dict = get_quota_xattr_brick(path)
+
+    stbuf = os.lstat(path)
+
+    for parent in xattr_dict['parents']:
+        contri_dict = xattr_dict['parents'][parent]
+
+        if 'size' not in xattr_dict or 'contri_size' not in contri_dict:
+            print_msg(QUOTA_META_ABSENT, path)
+            fix_xattr(path, True)
+            return
+        elif size_differs_lot(dir_size, xattr_dict['size']) or \
+             size_differs_lot(contri_dict['contri_size'], xattr_dict['size']):
+            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size)
+            fix_xattr(path, True)
+            return
+
+    if verbose_mode is True:
+        print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size)
+
+
+def walktree(t_dir, hard_link_dict):
+    '''recursively descend the directory tree rooted at dir,
+       aggregating the size
+       t_dir            : directory to walk over.
+       hard_link_dict   : dict of inodes with multiple hard_links under t_dir
+    '''
+    global aggr_size
+    aggr_size[t_dir] = 0
+
+    for entry in os.listdir(t_dir):
+        pathname = os.path.join(t_dir, entry)
+        stbuf = os.lstat(pathname)
+        if S_ISDIR(stbuf.st_mode):
+            # It's a directory, recurse into it
+            if entry == '.glusterfs':
+                print("skipping " + pathname)
+                continue
+            descendent_hardlinks = {}
+            subtree_size = walktree(pathname, descendent_hardlinks)
+
+            subtree_size = fix_hardlink_accounting(descendent_hardlinks,
+                                                   hard_link_dict,
+                                                   subtree_size)
+
+            aggr_size[t_dir] = aggr_size[t_dir] + subtree_size
+
+        elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode):
+            # Even a symbolic link file may have multiple hardlinks.
+
+            file_size = stbuf.st_size
+            if stbuf.st_nlink > 2:
+                # send a single element dict to check if file is accounted.
+                file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size},
+                                                    hard_link_dict,
+                                                    stbuf.st_size)
+
+                if file_size == 0:
+                    print_msg("HARD_LINK (skipped)", pathname, "",
+                                stbuf)
+                else:
+                    print_msg("HARD_LINK (accounted)", pathname, "",
+                               stbuf)
+                    hard_link_dict[stbuf.st_ino] = stbuf.st_size
+
+            if t_dir in aggr_size:
+                aggr_size[t_dir] = aggr_size[t_dir] + file_size
+            else:
+                aggr_size[t_dir] = file_size
+            verify_file_xattr(pathname, stbuf)
+
+        else:
+            # Unknown file type, print a message
+            print('Skipping %s, due to file mode' % (pathname))
+
+    if t_dir not in aggr_size:
+        aggr_size[t_dir] = 0
+
+    verify_dir_xattr(t_dir, aggr_size[t_dir])
+    # du also accounts for t_directory sizes
+    # aggr_size[t_dir] += 4096
+
+    #cleanup
+    ret = aggr_size[t_dir]
+    del aggr_size[t_dir]
+    return ret
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg)
+    parser.add_argument('brick_path', nargs=1,
+                        help='The brick path (or any descendent sub-directory of brick path)',
+                        )
+    parser.add_argument('--full-logs', dest='verbose', action='store_true',
+                   help='''
+                         log all the xattr values and stat values reported
+                         for analysis. [CAUTION: This can give lot of output
+                         depending on FS depth. So one has to make sure enough
+                         disk space exists if redirecting to file]
+                        '''
+                        )
+    parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store',
+                   help='''
+                         fix accounting issues where the xattr values disagree
+                         with stat sizes reported by gluster. A mount is also
+                         required for this option to be used.
+                         [CAUTION: This will directly modify backend xattr]
+                        '''
+                        )
+    parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store',
+                   help='''
+                         limit the crawling and accounting verification/correction
+                         to a specific subdirectory.
+                        '''
+                        )
+
+    args = parser.parse_args()
+    verbose_mode = args.verbose
+    brick_path = args.brick_path[0]
+    sub_dir = args.sub_dir
+    mnt_path = args.mnt
+    hard_link_dict = {}
+    if sub_dir is not None:
+        walktree(os.path.join(brick_path, sub_dir), hard_link_dict)
+    else:
+        walktree(brick_path, hard_link_dict)
+
+    print("Files verified : " + str(file_count))
+    print("Directories verified : " + str(dir_count))
+    if mnt_path is not None:
+        print("Objects Fixed : " + str(obj_fix_count))
diff --git a/extras/quota/xattr_analysis.py b/extras/quota/xattr_analysis.py
new file mode 100755
index 00000000000..7bd7d96374c
--- /dev/null
+++ b/extras/quota/xattr_analysis.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# Below script has two purposes
+#  1. Display xattr of entire FS tree in a human readable form
+#  2. Display all the directory where contri and size mismatch.
+#      (If there are any directory with contri and size mismatch that are not dirty
+#       then that highlights a propagation issue)
+#  The script takes only one input LOG _FILE generated from the command,
+#  find <brick_path> | xargs  getfattr -d -m. -e hex  > log_gluster_xattr
+
+from __future__ import print_function
+import re
+import subprocess
+import sys
+from hurry.filesize import size
+
+if len(sys.argv) < 2:
+    sys.exit('Usage: %s log_gluster_xattr \n'
+              'to generate log_gluster_xattr use: \n'
+              'find <brick_path> | xargs  getfattr -d -m. -e hex  > log_gluster_xattr'
+              % sys.argv[0])
+LOG_FILE=sys.argv[1]
+
+def get_quota_xattr_brick():
+    out = subprocess.check_output (["/usr/bin/cat", LOG_FILE])
+    pairs = out.splitlines()
+
+    xdict = {}
+    mismatch_size = [('====contri_size===', '====size====')]
+    for xattr in pairs:
+        k = xattr.split("=")[0]
+        if re.search("# file:", k):
+            print(xdict)
+            filename=k
+            print("=====" + filename + "=======")
+            xdict = {}
+        elif k is "":
+            pass
+        else:
+            print(xattr)
+            v = xattr.split("=")[1]
+            if re.search("contri", k):
+                if len(v) == 34:
+                    # for files size is obtained in iatt, file count should be 1, dir count=0
+                    xdict['contri_file_count'] = int(v[18:34], 16)
+                    xdict['contri_dir_count'] = 0
+                else:
+                    xdict['contri_size'] = size(int(v[2:18], 16))
+                    xdict['contri_file_count'] = int(v[18:34], 16)
+                    xdict['contri_dir_count'] = int(v[34:], 16)
+            elif re.search("size", k):
+                xdict['size'] = size(int(v[2:18], 16))
+                xdict['file_count'] = int(v[18:34], 16)
+                xdict['dir_count'] = int(v[34:], 16)
+            elif re.search("dirty", k):
+                if v == '0x3000':
+                    xdict['dirty'] = False
+                elif v == '0x3100':
+                    xdict['dirty'] = True
+            elif re.search("limit_objects", k):
+                xdict['limit_objects'] = int(v[2:18], 16)
+            elif re.search("limit_set", k):
+                xdict['limit_set'] = size(int(v[2:18], 16))
+
+            if 'size' in xdict and 'contri_size' in xdict and xdict['size'] != xdict['contri_size']:
+                mismatch_size.append((xdict['contri_size'], xdict['size'], filename))
+
+    for values in mismatch_size:
+        print(values)
+
+
+if __name__ == '__main__':
+    get_quota_xattr_brick()
+
diff --git a/extras/rebalance.py b/extras/rebalance.py
index 80c614c5dfe..37c68ebbb42 100755
--- a/extras/rebalance.py
+++ b/extras/rebalance.py
@@ -1,4 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+from __future__ import print_function
 
 import atexit
 import copy
@@ -11,6 +13,7 @@ import subprocess
 import sys
 import tempfile
 import volfilter
+import platform
 
 # It's just more convenient to have named fields.
 class Brick:
@@ -37,20 +40,20 @@ class Brick:
 
 def get_bricks (host, vol):
         t = pipes.Template()
-        t.prepend("gluster --remote-host=%s system getspec %s"%(host,vol),".-")
-        return t.open(None,"r")
+        t.prepend("gluster --remote-host=%s system getspec %s"%(host, vol), ".-")
+        return t.open(None, "r")
 
 def generate_stanza (vf, all_xlators, cur_subvol):
         sv_list = []
         for sv in cur_subvol.subvols:
-                generate_stanza(vf,all_xlators,sv)
+                generate_stanza(vf, all_xlators, sv)
                 sv_list.append(sv.name)
-        vf.write("volume %s\n"%cur_subvol.name)
-        vf.write("  type %s\n"%cur_subvol.type)
-        for kvpair in cur_subvol.opts.iteritems():
-                vf.write("  option %s %s\n"%kvpair)
+        vf.write("volume %s\n" % cur_subvol.name)
+        vf.write("  type %s\n" % cur_subvol.type)
+        for kvpair in cur_subvol.opts.items():
+                vf.write("  option %s %s\n" % kvpair)
         if sv_list:
-                vf.write("  subvolumes %s\n"%string.join(sv_list))
+                vf.write("  subvolumes %s\n" % ''.join(sv_list))
         vf.write("end-volume\n\n")
 
 
@@ -58,14 +61,14 @@ def mount_brick (localpath, all_xlators, dht_subvol):
 
         # Generate a volfile.
         vf_name = localpath + ".vol"
-        vf = open(vf_name,"w")
-        generate_stanza(vf,all_xlators,dht_subvol)
+        vf = open(vf_name, "w")
+        generate_stanza(vf, all_xlators, dht_subvol)
         vf.flush()
         vf.close()
 
         # Create a brick directory and mount the brick there.
         os.mkdir(localpath)
-        subprocess.call(["glusterfs","-f",vf_name,localpath])
+        subprocess.call(["glusterfs", "-f", vf_name, localpath])
 
 # We use the command-line tools because there's no getxattr support in the
 # Python standard library (which is ridiculous IMO).  Adding the xattr package
@@ -79,16 +82,16 @@ def mount_brick (localpath, all_xlators, dht_subvol):
 def get_range (brick):
         t = pipes.Template()
         cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null"
-        t.prepend(cmd%brick,".-")
-        t.append("grep ^trusted.glusterfs.dht=","--")
-        f = t.open(None,"r")
+        t.prepend(cmd%brick, ".-")
+        t.append("grep ^trusted.glusterfs.dht=", "--")
+        f = t.open(None, "r")
         try:
                 value = f.readline().rstrip().split('=')[1][2:]
         except:
-                print "could not get layout for %s (might be OK)" % brick
+                print("could not get layout for %s (might be OK)" % brick)
                 return None
-        v_start = int("0x"+value[16:24],16)
-        v_end = int("0x"+value[24:32],16)
+        v_start = int("0x"+value[16:24], 16)
+        v_end = int("0x"+value[24:32], 16)
         return (v_start, v_end)
 
 def calc_sizes (bricks, total):
@@ -125,7 +128,7 @@ def normalize (in_bricks):
                                 curr_hash = b.r_end + 1
                                 break
                 else:
-                        print "gap found at 0x%08x" % curr_hash
+                        print("gap found at 0x%08x" % curr_hash)
                         sys.exit(1)
         return out_bricks + in_bricks, used
 
@@ -153,8 +156,8 @@ def get_score (bricks):
 
 if __name__ == "__main__":
 
-	my_usage = "%prog [options] server volume [directory]"
-	parser = optparse.OptionParser(usage=my_usage)
+        my_usage = "%prog [options] server volume [directory]"
+        parser = optparse.OptionParser(usage=my_usage)
         parser.add_option("-f", "--free-space", dest="free_space",
                           default=False, action="store_true",
                           help="use free space instead of total space")
@@ -164,7 +167,7 @@ if __name__ == "__main__":
         parser.add_option("-v", "--verbose", dest="verbose",
                           default=False, action="store_true",
                           help="verbose output")
-	options, args = parser.parse_args()
+        options, args = parser.parse_args()
 
         if len(args) == 3:
                 fix_dir = args[2]
@@ -182,9 +185,9 @@ if __name__ == "__main__":
         def cleanup_workdir ():
                 os.chdir(orig_dir)
                 if options.verbose:
-                        print "Cleaning up %s" % work_dir
+                        print("Cleaning up %s" % work_dir)
                 for b in bricks:
-                        subprocess.call(["umount",b.path])
+                        subprocess.call(["umount", b.path])
                 shutil.rmtree(work_dir)
         if not options.leave_mounted:
                 atexit.register(cleanup_workdir)
@@ -192,44 +195,51 @@ if __name__ == "__main__":
 
         # Mount each brick individually, so we can issue brick-specific calls.
         if options.verbose:
-                print "Mounting subvolumes..."
+                print("Mounting subvolumes...")
         index = 0
-        volfile_pipe = get_bricks(hostname,volname)
+        volfile_pipe = get_bricks(hostname, volname)
         all_xlators, last_xlator = volfilter.load(volfile_pipe)
         for dht_vol in all_xlators.itervalues():
                 if dht_vol.type == "cluster/distribute":
                         break
         else:
-                print "no DHT volume found"
+                print("no DHT volume found")
                 sys.exit(1)
         for sv in dht_vol.subvols:
                 #print "found subvol %s" % sv.name
                 lpath = "%s/brick%s" % (work_dir, index)
                 index += 1
-                mount_brick(lpath,all_xlators,sv)
-                bricks.append(Brick(lpath,sv.name))
+                mount_brick(lpath, all_xlators, sv)
+                bricks.append(Brick(lpath, sv.name))
         if index == 0:
-                print "no bricks"
+                print("no bricks")
                 sys.exit(1)
 
         # Collect all of the sizes.
         if options.verbose:
-                print "Collecting information..."
+                print("Collecting information...")
         total = 0
         for b in bricks:
                 info = os.statvfs(b.path)
+                # On FreeBSD f_bsize (info[0]) contains the optimal I/O size,
+                # not the block size as it's found on Linux. In this case we
+                # use f_frsize (info[1]).
+                if platform.system() == 'FreeBSD':
+                        bsize = info[1]
+                else:
+                        bsize = info[0]
                 # We want a standard unit even if different bricks use
                 # different block sizes.  The size is chosen to avoid overflows
                 # for very large bricks with very small block sizes, but also
                 # accommodate filesystems which use very large block sizes to
                 # cheat on benchmarks.
-                blocksper100mb = 104857600 / info[0]
+                blocksper100mb = 104857600 / bsize
                 if options.free_space:
                         size = info[3] / blocksper100mb
                 else:
                         size = info[2] / blocksper100mb
                 if size <= 0:
-                        print "brick %s has invalid size %d" % (b.path, size)
+                        print("brick %s has invalid size %d" % (b.path, size))
                         sys.exit(1)
                 b.set_size(size)
                 total += size
@@ -240,13 +250,13 @@ if __name__ == "__main__":
                 if hash_range is not None:
                         rs, re = hash_range
                         if rs > re:
-                                print "%s has backwards hash range" % b.path
+                                print("%s has backwards hash range" % b.path)
                                 sys.exit(1)
-                        b.set_range(hash_range[0],hash_range[1])
+                        b.set_range(hash_range[0], hash_range[1])
 
         if options.verbose:
-                print "Calculating new layouts..."
-        calc_sizes(bricks,total)
+                print("Calculating new layouts...")
+        calc_sizes(bricks, total)
         bricks, used = normalize(bricks)
 
         # We can't afford O(n!) here, but O(n^2) should be OK and the result
@@ -254,10 +264,10 @@ if __name__ == "__main__":
         while used < len(bricks):
                 best_place = used
                 best_score = get_score(bricks)
-                for i in xrange(used):
+                for i in range(used):
                         new_bricks = bricks[:]
                         del new_bricks[used]
-                        new_bricks.insert(i,bricks[used])
+                        new_bricks.insert(i, bricks[used])
                         new_score = get_score(new_bricks)
                         if new_score > best_score:
                                 best_place = i
@@ -265,7 +275,7 @@ if __name__ == "__main__":
                 if best_place != used:
                         nb = bricks[used]
                         del bricks[used]
-                        bricks.insert(best_place,nb)
+                        bricks.insert(best_place, nb)
                 used += 1
 
         # Finalize whatever we decided on.
@@ -275,25 +285,25 @@ if __name__ == "__main__":
                 curr_hash += b.good_size
                 b.r_end = curr_hash - 1
 
-        print "Here are the xattr values for your size-weighted layout:"
+        print("Here are the xattr values for your size-weighted layout:")
         for b in bricks:
-                print "  %s: 0x0000000200000000%08x%08x" % (
-                        b.sv_name, b.r_start, b.r_end)
+                print("  %s: 0x0000000200000000%08x%08x" % (
+                        b.sv_name, b.r_start, b.r_end))
 
         if fix_dir:
                 if options.verbose:
-                        print "Fixing layout for %s" % fix_dir
+                        print("Fixing layout for %s" % fix_dir)
                 for b in bricks:
                         value = "0x0000000200000000%08x%08x" % (
                                 b.r_start, b.r_end)
                         path = "%s/%s" % (b.path, fix_dir)
                         cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % (
                                 value, path)
-                        print cmd
+                        print(cmd)
 
         if options.leave_mounted:
-                print "The following subvolumes are still mounted:"
+                print("The following subvolumes are still mounted:")
                 for b in bricks:
-                        print "%s on %s" % (b.sv_name, b.path)
-                print "Don't forget to clean up when you're done."
+                        print("%s on %s" % (b.sv_name, b.path))
+                print("Don't forget to clean up when you're done.")
 
diff --git a/extras/run-gluster.tmpfiles.in b/extras/run-gluster.tmpfiles.in
index 49a2662c4c8..329f2dde6db 100644
--- a/extras/run-gluster.tmpfiles.in
+++ b/extras/run-gluster.tmpfiles.in
@@ -1,2 +1,2 @@
 # hardcoding /run for now, should be detected while building from source?
-d /run/gluster 0755 root root -
+d /run/gluster 0775 gluster gluster -
diff --git a/extras/snap_scheduler/Makefile.am b/extras/snap_scheduler/Makefile.am
index ffc157935a3..782f139016f 100644
--- a/extras/snap_scheduler/Makefile.am
+++ b/extras/snap_scheduler/Makefile.am
@@ -1,6 +1,8 @@
 snap_schedulerdir = $(sbindir)/
 
+if WITH_SERVER
 snap_scheduler_SCRIPTS = gcron.py snap_scheduler.py conf.py
+endif
 
 EXTRA_DIST = gcron.py snap_scheduler.py conf.py
 
diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py
index d72057861ff..0e4df77d481 100755
--- a/extras/snap_scheduler/gcron.py
+++ b/extras/snap_scheduler/gcron.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 #
 # Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
@@ -19,10 +19,10 @@ import logging.handlers
 import fcntl
 
 
-GCRON_TASKS = "/var/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks"
+GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks"
 GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks"
 GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag"
-LOCK_FILE_DIR = "/var/run/gluster/shared_storage/snaps/lock_files/"
+LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/"
 log = logging.getLogger("gcron-logger")
 start_time = 0.0
 
@@ -38,7 +38,8 @@ def initLogger(script_name):
     sh.setFormatter(formatter)
 
     process = subprocess.Popen(["gluster", "--print-logdir"],
-                               stdout=subprocess.PIPE)
+                               stdout=subprocess.PIPE,
+                               universal_newlines=True)
     out, err = process.communicate()
     if process.returncode == 0:
         logfile = os.path.join(out.strip(), script_name[:-3]+".log")
@@ -88,7 +89,7 @@ def takeSnap(volname="", snapname=""):
 def doJob(name, lockFile, jobFunc, volname):
     success = True
     try:
-        f = os.open(lockFile, os.O_RDWR | os.O_NONBLOCK)
+        f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK)
         try:
             fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             mtime = os.path.getmtime(lockFile)
@@ -105,11 +106,11 @@ def doJob(name, lockFile, jobFunc, volname):
             else:
                 log.info("Job %s has been processed already", name)
             fcntl.flock(f, fcntl.LOCK_UN)
-        except IOError as (errno, strerror):
+        except (OSError, IOError):
             log.info("Job %s is being processed by another agent", name)
         os.close(f)
-    except IOError as (errno, strerror):
-        log.debug("Failed to open lock file %s : %s", lockFile, strerror)
+    except (OSError, IOError) as e:
+        log.debug("Failed to open lock file %s : %s", lockFile, e)
         log.error("Failed to process job %s", name)
         success = False
 
@@ -122,19 +123,20 @@ def main():
     global start_time
     if sys.argv[1] == "--update":
         if not os.path.exists(GCRON_TASKS):
-            # Create a flag in /var/run/gluster which indicates that this nodes
-            # doesn't have access to GCRON_TASKS right now, so that
+            # Create a flag in /var/run/gluster which indicates that this
+            # node doesn't have access to GCRON_TASKS right now, so that
             # when the mount is available and GCRON_TASKS is available
             # the flag will tell this routine to reload GCRON_CROND_TASK
             try:
-                f = os.open(GCRON_RELOAD_FLAG, os.O_CREAT | os.O_NONBLOCK, 0644)
+                f = os.open(GCRON_RELOAD_FLAG,
+                            os.O_CREAT | os.O_NONBLOCK, 0o644)
                 os.close(f)
-            except OSError as (errno, strerror):
+            except OSError as e:
                 if errno != EEXIST:
                     log.error("Failed to create %s : %s",
-                              GCRON_RELOAD_FLAG, strerror)
+                              GCRON_RELOAD_FLAG, e)
                     output("Failed to create %s. Error: %s"
-                           % (GCRON_RELOAD_FLAG, strerror))
+                           % (GCRON_RELOAD_FLAG, e))
             return
 
         if not os.path.exists(GCRON_CROND_TASK):
@@ -153,9 +155,9 @@ def main():
                 if process.returncode != 0:
                     log.error("Failed to touch %s. Error: %s.",
                               GCRON_CROND_TASK, err)
-            except (IOError, OSError) as (errno, strerror):
+            except (IOError, OSError) as e:
                 log.error("Failed to touch %s. Error: %s.",
-                          GCRON_CROND_TASK, strerror)
+                          GCRON_CROND_TASK, e)
             return
         if os.lstat(GCRON_TASKS).st_mtime > \
            os.lstat(GCRON_CROND_TASK).st_mtime:
@@ -167,9 +169,9 @@ def main():
                 if process.returncode != 0:
                     log.error("Failed to touch %s. Error: %s.",
                               GCRON_CROND_TASK, err)
-            except IOError as (errno, strerror):
+            except IOError as e:
                 log.error("Failed to touch %s. Error: %s.",
-                          GCRON_CROND_TASK, strerror)
+                          GCRON_CROND_TASK, e)
         return
 
     volname = sys.argv[1]
diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py
index 23d5aa3b525..e8fcc449a9b 100755
--- a/extras/snap_scheduler/snap_scheduler.py
+++ b/extras/snap_scheduler/snap_scheduler.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 #
 # Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
@@ -21,13 +21,53 @@ import shutil
 from errno import EEXIST
 from conf import GLUSTERFS_LIBEXECDIR
 sys.path.insert(1, GLUSTERFS_LIBEXECDIR)
-from events.gf_event import gf_event
-from events import eventtypes
+
+EVENTS_ENABLED = True
+try:
+    from events.eventtypes import SNAPSHOT_SCHEDULER_INITIALISED \
+                         as EVENT_SNAPSHOT_SCHEDULER_INITIALISED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_INIT_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLED \
+                         as EVENT_SNAPSHOT_SCHEDULER_DISABLED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLED \
+                         as EVENT_SNAPSHOT_SCHEDULER_ENABLED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADDED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDITED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED
+except ImportError:
+    # Events APIs not installed, dummy eventtypes with None
+    EVENTS_ENABLED = False
+    EVENT_SNAPSHOT_SCHEDULER_INITIALISED = None
+    EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_DISABLED = None
+    EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_ENABLED = None
+    EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED = None
 
 SCRIPT_NAME = "snap_scheduler"
 scheduler_enabled = False
 log = logging.getLogger(SCRIPT_NAME)
-SHARED_STORAGE_DIR="/var/run/gluster/shared_storage"
+SHARED_STORAGE_DIR="/run/gluster/shared_storage"
 GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled"
 GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled"
 GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks"
@@ -109,7 +149,7 @@ def initLogger():
     sh.setFormatter(formatter)
 
     process = subprocess.Popen(["gluster", "--print-logdir"],
-                               stdout=subprocess.PIPE)
+                               stdout=subprocess.PIPE, universal_newlines=True)
     logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log")
 
     fh = logging.FileHandler(logfile)
@@ -167,11 +207,11 @@ def enable_scheduler():
                     os.remove(GCRON_TASKS)
                 try:
                     f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK,
-                                0644)
+                                0o644)
                     os.close(f)
-                except OSError as (errno, strerror):
+                except OSError as e:
                     log.error("Failed to open %s. Error: %s.",
-                              GCRON_ENABLED, strerror)
+                              GCRON_ENABLED, e)
                     ret = INTERNAL_ERROR
                     return ret
                 os.symlink(GCRON_ENABLED, GCRON_TASKS)
@@ -179,8 +219,9 @@ def enable_scheduler():
                 log.info("Snapshot scheduling is enabled")
                 output("Snapshot scheduling is enabled")
                 ret = 0
-            except OSError as (errno, strerror):
-                print_str = "Failed to enable snapshot scheduling. Error: "+strerror
+            except OSError as e:
+                print_str = ("Failed to enable snapshot scheduling."
+                             "Error: {{}}" + e)
                 log.error(print_str)
                 output(print_str)
                 ret = INTERNAL_ERROR
@@ -222,14 +263,15 @@ def disable_scheduler():
                     os.remove(GCRON_DISABLED)
                 if os.path.lexists(GCRON_TASKS):
                     os.remove(GCRON_TASKS)
-                f = os.open(GCRON_DISABLED, os.O_CREAT, 0644)
+                f = os.open(GCRON_DISABLED, os.O_CREAT, 0o644)
                 os.close(f)
                 os.symlink(GCRON_DISABLED, GCRON_TASKS)
                 log.info("Snapshot scheduling is disabled")
                 output("Snapshot scheduling is disabled")
                 ret = 0
-            except OSError as (errno, strerror):
-                print_str = "Failed to disable snapshot scheduling. Error: "+strerror
+            except OSError as e:
+                print_str = ("Failed to disable snapshot scheduling. Error: "
+                             + e)
                 log.error(print_str)
                 output(print_str)
                 ret = INTERNAL_ERROR
@@ -268,8 +310,8 @@ def load_tasks_from_file():
                 tasks[jobname] = schedule+":"+volname
             f.close()
         ret = 0
-    except IOError as (errno, strerror):
-        log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, strerror)
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, e)
         ret = INTERNAL_ERROR
 
     return ret
@@ -282,8 +324,8 @@ def get_current_scheduler():
             current_scheduler = f.readline().rstrip('\n')
             f.close()
         ret = 0
-    except IOError as (errno, strerror):
-        log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, strerror)
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, e)
         ret = INTERNAL_ERROR
 
     return ret
@@ -323,7 +365,7 @@ def list_schedules():
 
 def write_tasks_to_file():
     try:
-        with open(TMP_FILE, "w", 0644) as f:
+        with open(TMP_FILE, "w", 0o644) as f:
             # If tasks is empty, just create an empty tmp file
             if len(tasks) != 0:
                 for key in sorted(tasks):
@@ -336,8 +378,8 @@ def write_tasks_to_file():
                 f.flush()
                 os.fsync(f.fileno())
             f.close()
-    except IOError as (errno, strerror):
-        log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror)
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
         ret = INTERNAL_ERROR
         return ret
 
@@ -348,13 +390,13 @@ def write_tasks_to_file():
 
 def update_current_scheduler(data):
     try:
-        with open(TMP_FILE, "w", 0644) as f:
+        with open(TMP_FILE, "w", 0o644) as f:
             f.write("%s" % data)
             f.flush()
             os.fsync(f.fileno())
             f.close()
-    except IOError as (errno, strerror):
-        log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror)
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
         ret = INTERNAL_ERROR
         return ret
 
@@ -417,11 +459,11 @@ def add_schedules(jobname, schedule, volname):
                     job_lockfile = LOCK_FILE_DIR + jobname
                     try:
                         f = os.open(job_lockfile, os.O_CREAT | os.O_NONBLOCK,
-                                    0644)
+                                    0o644)
                         os.close(f)
-                    except OSError as (errno, strerror):
+                    except OSError as e:
                         log.error("Failed to open %s. Error: %s.",
-                                  job_lockfile, strerror)
+                                  job_lockfile, e)
                         ret = INTERNAL_ERROR
                         return ret
                     log.info("Successfully added snapshot schedule %s" %
@@ -449,9 +491,9 @@ def delete_schedules(jobname):
                 job_lockfile = LOCK_FILE_DIR+jobname
                 try:
                     os.remove(job_lockfile)
-                except OSError as (errno, strerror):
+                except OSError as e:
                     log.error("Failed to open %s. Error: %s.",
-                              job_lockfile, strerror)
+                              job_lockfile, e)
                     ret = INTERNAL_ERROR
                     return ret
                 log.info("Successfully deleted snapshot schedule %s"
@@ -505,18 +547,113 @@ def edit_schedules(jobname, schedule, volname):
 
     return ret
 
+def get_bool_val():
+    getsebool_cli = ["getsebool",
+                     "-a"]
+    p1 = subprocess.Popen(getsebool_cli, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    grep_cmd = ["grep",
+                "cron_system_cronjob_use_shares"]
+    p2 = subprocess.Popen(grep_cmd, stdin=p1.stdout,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    p1.stdout.close()
+    output, err = p2.communicate()
+    rv = p2.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return -1
+
+    bool_val = output.split()[2]
+    log.debug("Bool value = '%s'", bool_val)
+
+    return bool_val
+
+def get_selinux_status():
+    getenforce_cli = ["getenforce"]
+    log.debug("Running command '%s'", " ".join(getenforce_cli))
+
+    try:
+        p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+    except OSError as oserr:
+        log.error("Failed to run the command \"getenforce\". Error: %s" %\
+                  oserr)
+        return -1
+
+    output, err = p1.communicate()
+    rv = p1.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return -1
+    else:
+        selinux_status=output.rstrip()
+        log.debug("selinux status: %s", selinux_status)
+
+    return selinux_status
+
+def set_cronjob_user_share():
+    selinux_status = get_selinux_status()
+    if (selinux_status == -1):
+        log.error("Failed to get selinux status")
+        return -1
+    elif (selinux_status == "Disabled"):
+        return 0
+
+    bool_val = get_bool_val()
+    # In case of a failure (where the boolean value is not)
+    # present in the system, we should not proceed further
+    # We should only proceed when the value is "off"
+    if (bool_val == -1 or bool_val != "off"):
+        return 0
+
+    setsebool_cli = ["setsebool", "-P",
+                     "cron_system_cronjob_use_shares",
+                     "on"]
+    log.debug("Running command '%s'", " ".join(setsebool_cli))
+
+    p1 = subprocess.Popen(setsebool_cli, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    output, err = p1.communicate()
+    rv = p1.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return rv
+
+    bool_val = get_bool_val()
+    if (bool_val == "on"):
+        return 0
+    else:
+        # In case of an error or if boolean is not on
+        # we return a failure here
+        return -1
 
 def initialise_scheduler():
+    ret = set_cronjob_user_share()
+    if ret:
+        log.error("Failed to set selinux boolean "
+                  "cron_system_cronjob_use_shares to 'on'")
+        return ret
+
     try:
-        with open(TMP_FILE, "w+", 0644) as f:
+        with open(TMP_FILE, "w+", 0o644) as f:
             updater = ("* * * * * root PATH=$PATH:/usr/local/sbin:"
                        "/usr/sbin gcron.py --update\n")
             f.write("%s\n" % updater)
             f.flush()
             os.fsync(f.fileno())
             f.close()
-    except IOError as (errno, strerror):
-        log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror)
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
         ret = INIT_FAILED
         return ret
 
@@ -524,10 +661,10 @@ def initialise_scheduler():
 
     if not os.path.lexists(GCRON_TASKS):
         try:
-            f = open(GCRON_TASKS, "w", 0644)
+            f = open(GCRON_TASKS, "w", 0o644)
             f.close()
-        except IOError as (errno, strerror):
-            log.error("Failed to open %s. Error: %s.", GCRON_TASKS, strerror)
+        except IOError as e:
+            log.error("Failed to open %s. Error: %s.", GCRON_TASKS, e)
             ret = INIT_FAILED
             return ret
 
@@ -538,7 +675,7 @@ def initialise_scheduler():
 
     log.info("Successfully initialised snapshot scheduler for this node")
     output("Successfully initialised snapshot scheduler for this node")
-    gf_event (eventtypes.SNAPSHOT_SCHEDULER_INITIALISED, status="Success")
+    gf_event (EVENT_SNAPSHOT_SCHEDULER_INITIALISED, status="Success")
 
     ret = 0
     return ret
@@ -585,7 +722,7 @@ def perform_operation(args):
         ret = initialise_scheduler()
         if ret != 0:
             output("Failed to initialise snapshot scheduling")
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_INIT_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED,
                       error=print_error(ret))
         return ret
 
@@ -594,10 +731,10 @@ def perform_operation(args):
         ret = disable_scheduler()
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_DISABLED,
-                      status="Successfuly Disabled")
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED,
+                      status="Successfully Disabled")
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_DISABLE_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED,
                       error=print_error(ret))
         return ret
 
@@ -629,10 +766,10 @@ def perform_operation(args):
         ret = enable_scheduler()
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_ENABLED,
-                      status="Successfuly Enabled")
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLED,
+                      status="Successfully Enabled")
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_ENABLE_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED,
                       error=print_error(ret))
         return ret
 
@@ -641,10 +778,10 @@ def perform_operation(args):
         ret = disable_scheduler()
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_DISABLED,
-                      status="Successfuly Disabled")
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED,
+                      status="Successfully Disabled")
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_DISABLE_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED,
                       error=print_error(ret))
         return ret
 
@@ -661,10 +798,10 @@ def perform_operation(args):
         ret = add_schedules(args.jobname, args.schedule, args.volname)
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_ADDED,
-                      status="Successfuly added job "+args.jobname)
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED,
+                      status="Successfully added job "+args.jobname)
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED,
                       status="Failed to add job "+args.jobname,
                       error=print_error(ret))
         return ret
@@ -677,10 +814,10 @@ def perform_operation(args):
         ret = delete_schedules(args.jobname)
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_DELETED,
-                      status="Successfuly deleted job "+args.jobname)
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED,
+                      status="Successfully deleted job "+args.jobname)
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED,
                       status="Failed to delete job "+args.jobname,
                       error=print_error(ret))
         return ret
@@ -693,10 +830,10 @@ def perform_operation(args):
         ret = edit_schedules(args.jobname, args.schedule, args.volname)
         if ret == 0:
             subprocess.Popen(["touch", "-h", GCRON_TASKS])
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_EDITED,
-                      status="Successfuly edited job "+args.jobname)
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED,
+                      status="Successfully edited job "+args.jobname)
         else:
-            gf_event (eventtypes.SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED,
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED,
                       status="Failed to edit job "+args.jobname,
                       error=print_error(ret))
         return ret
@@ -704,6 +841,11 @@ def perform_operation(args):
     ret = INVALID_ARG
     return ret
 
+def gf_event(event_type, **kwargs):
+    if EVENTS_ENABLED:
+        from events.gf_event import gf_event as gfevent
+        gfevent(event_type, **kwargs)
+
 
 def main(argv):
     initLogger()
@@ -754,42 +896,42 @@ def main(argv):
     if not os.path.exists(SHARED_STORAGE_DIR+"/snaps/"):
         try:
             os.makedirs(SHARED_STORAGE_DIR+"/snaps/")
-        except OSError as (errno, strerror):
+        except OSError as e:
             if errno != EEXIST:
-                log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", strerror)
+                log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", e)
                 output("Failed to create %s. Error: %s"
-                       % (SHARED_STORAGE_DIR+"/snaps/", strerror))
+                       % (SHARED_STORAGE_DIR+"/snaps/", e))
                 return INTERNAL_ERROR
 
     if not os.path.exists(GCRON_ENABLED):
-        f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0644)
+        f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0o644)
         os.close(f)
 
     if not os.path.exists(LOCK_FILE_DIR):
         try:
             os.makedirs(LOCK_FILE_DIR)
-        except OSError as (errno, strerror):
+        except OSError as e:
             if errno != EEXIST:
-                log.error("Failed to create %s : %s", LOCK_FILE_DIR, strerror)
+                log.error("Failed to create %s : %s", LOCK_FILE_DIR, e)
                 output("Failed to create %s. Error: %s"
-                       % (LOCK_FILE_DIR, strerror))
+                       % (LOCK_FILE_DIR, e))
                 return INTERNAL_ERROR
 
     try:
-        f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0644)
+        f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0o644)
         try:
             fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             ret = perform_operation(args)
             fcntl.flock(f, fcntl.LOCK_UN)
-        except IOError as (errno, strerror):
+        except IOError:
             log.info("%s is being processed by another agent.", LOCK_FILE)
             output("Another snap_scheduler command is running. "
                    "Please try again after some time.")
             return ANOTHER_TRANSACTION_IN_PROGRESS
         os.close(f)
-    except OSError as (errno, strerror):
-        log.error("Failed to open %s : %s", LOCK_FILE, strerror)
-        output("Failed to open %s. Error: %s" % (LOCK_FILE, strerror))
+    except OSError as e:
+        log.error("Failed to open %s : %s", LOCK_FILE, e)
+        output("Failed to open %s. Error: %s" % (LOCK_FILE, e))
         return INTERNAL_ERROR
 
     return ret
diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb
new file mode 100755
index 00000000000..1aff43377db
--- /dev/null
+++ b/extras/statedumpparse.rb
@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+
+require 'time'
+require 'optparse'
+
+unless Array.instance_methods.include? :to_h
+  class Array
+    def to_h
+      h = {}
+      each { |k,v| h[k]=v }
+      h
+    end
+  end
+end
+
+# statedump.c:gf_proc_dump_mempool_info uses a five-dash record separator,
+# client.c:client_fd_lk_ctx_dump uses a six-dash record separator.
+ARRSEP = /^(-{5,6}=-{5,6})?$/
+HEAD = /^\[(.*)\]$/
+INPUT_FORMATS = %w[statedump json]
+
+format = 'json'
+input_format = 'statedump'
+tz = '+0000'
+memstat_select,memstat_reject = //,/\Z./
+OptionParser.new do |op|
+  op.banner << " [<] <STATEDUMP>"
+  op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s }
+  op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s }
+  op.on("--timezone=T",
+        "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s }
+  op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s|
+    memstat_select = Regexp.new s
+  }
+  op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s|
+    memstat_reject = Regexp.new s
+  }
+end.parse!
+
+
+if format =~ /\Amemstat(?:-(.*))?/
+  memstat_type = $1 || 'plain'
+  unless %w[plain human json].include? memstat_type
+    raise "unknown memstat type #{memstat_type.dump}"
+  end
+  format = 'memstat'
+end
+
+repr, logsep = case format
+when 'yaml'
+  require 'yaml'
+
+  [proc { |e| e.to_yaml }, "\n"]
+when 'json', 'memstat'
+  require 'json'
+
+  [proc { |e| e.to_json }, " "]
+else
+  raise "unkonwn format '#{format}'"
+end
+formatter = proc { |e| puts repr.call(e) }
+
+INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'"
+
+dumpinfo = {}
+
+# parse a statedump entry
+elem_cbk = proc { |s,&cbk|
+  arraylike = false
+  s.grep(/\S/).empty? and next
+  head = nil
+  while s.last =~ /^\s*$/
+    s.pop
+  end
+  body = catch { |misc2|
+    s[0] =~ HEAD ? (head = $1) : (throw misc2)
+    body = [[]]
+    s[1..-1].each { |l|
+      if l =~ ARRSEP
+        arraylike = true
+        body << []
+        next
+      end
+      body.last << l
+    }
+
+    body.reject(&:empty?).map { |e|
+      ea = e.map { |l|
+        k,v = l.split("=",2)
+        m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v
+        [k, m ? (m[2] ? Float(v) : Integer(v)) : v]
+      }
+      begin
+        ea.to_h
+      rescue
+        throw misc2
+      end
+    }
+  }
+
+  if body
+    cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])]
+  else
+    STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep)
+  end
+}
+
+# aggregator routine
+aggr = case format
+when 'memstat'
+  meminfo = {}
+  # commit memory-related entries to meminfo
+  proc { |k,r|
+    case k
+    when /memusage/
+      (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject
+    when "mempool"
+      r.each {|e|
+        kk = "mempool:#{e['pool-name']}"
+        (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject
+      }
+    end
+  }
+else
+  # just format data, don't actually aggregate anything
+  proc { |pair| formatter.call pair }
+end
+
+# processing the data
+case input_format
+when 'statedump'
+  acc = []
+  $<.each { |l|
+    l = l.strip
+    if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/
+      dumpinfo["_meta"]||={}
+      (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ")
+      next
+    end
+
+    if l =~ HEAD
+      elem_cbk.call(acc, &aggr)
+      acc = [l]
+      next
+    end
+
+    acc << l
+  }
+  elem_cbk.call(acc, &aggr)
+when 'json'
+  $<.each { |l|
+    r = JSON.load l
+    case r
+    when Array
+      aggr[r]
+    when Hash
+      dumpinfo.merge! r
+    end
+  }
+end
+
+# final actions: output aggregated data
+case format
+when 'memstat'
+  ma = meminfo.values.map(&:to_a).inject(:+)
+  totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h
+  tt = ma.transpose[1].inject(:+)
+
+  summary_sep,showm = case memstat_type
+  when 'json'
+    ["", proc { |k,v| puts({type: k, value: v}.to_json) }]
+  when 'plain', 'human'
+    # human-friendly number representation
+    hr = proc { |n|
+      qa = %w[B kB MB GB]
+      q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1
+      "%.2f%s" % [n.to_f / (1 << q*10), qa[q]]
+    }
+
+    templ = "%{val} %{key}"
+    tft = proc { |t| t }
+    nft = if memstat_type == 'human'
+      nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max
+      proc { |n|
+        hn = hr[n]
+        " " * (nw - hn.size) + hn
+      }
+    else
+      nw = tt.to_s.size
+      proc { |n| "%#{nw}d" % n }
+    end
+    ## Alternative template, key first:
+    # templ = "%{key} %{val}"
+    # tw = ma.transpose[0].map(&:size).max
+    # tft = proc { |t| t + " " * [tw - t.size, 0].max }
+    # nft = (memstat_type == 'human') ? hr : proc { |n| n }
+    ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }]
+  else
+    raise 'this should be impossible'
+  end
+
+  ma.sort_by { |k,v| v }.each(&showm)
+  print summary_sep
+  totals.each { |coll,t| showm.call "Total #{coll}", t }
+  showm.call "TOTAL", tt
+else
+  formatter.call dumpinfo
+end
diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh
index 69fd601273e..710aaf5fd3c 100755
--- a/extras/stop-all-gluster-processes.sh
+++ b/extras/stop-all-gluster-processes.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
 # Kill all the processes/services except glusterd
 #
@@ -72,7 +72,7 @@ kill_bricks_and_services()
     local pidfile
     local pid
 
-    for pidfile in $(find /var/lib/glusterd/ -name '*.pid');
+    for pidfile in $(find /var/run/gluster/ -name '*.pid');
     do
         local pid=$(cat ${pidfile});
         echo "sending SIG${signal} to pid: ${pid}";
diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c
index 74bd47e303e..e013a6e6e8a 100644
--- a/extras/stripe-merge.c
+++ b/extras/stripe-merge.c
@@ -28,7 +28,7 @@
 #include <stdint.h>
 #include <errno.h>
 #include <string.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <fnmatch.h>
 
 #define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index"
@@ -40,33 +40,33 @@
 #define INVALID_MODE UINT32_MAX
 
 struct file_stripe_info {
-	int stripe_count;
-	int stripe_size;
-	int coalesce;
-	mode_t mode;
-	int fd[0];
+    int stripe_count;
+    int stripe_size;
+    int coalesce;
+    mode_t mode;
+    int fd[0];
 };
 
-static int close_files(struct file_stripe_info *);
+static int
+close_files(struct file_stripe_info *);
 
-static struct
-file_stripe_info *alloc_file_stripe_info(int count)
+static struct file_stripe_info *
+alloc_file_stripe_info(int count)
 {
-	int i;
-	struct file_stripe_info *finfo;
+    int i;
+    struct file_stripe_info *finfo;
 
-	finfo = calloc(1, sizeof(struct file_stripe_info) +
-		(sizeof(int) * count));
-	if (!finfo)
-		return NULL;
+    finfo = calloc(1, sizeof(struct file_stripe_info) + (sizeof(int) * count));
+    if (!finfo)
+        return NULL;
 
-	for (i = 0; i < count; i++)
-		finfo->fd[i] = INVALID_FD;
+    for (i = 0; i < count; i++)
+        finfo->fd[i] = INVALID_FD;
 
-	finfo->mode = INVALID_MODE;
-	finfo->coalesce = INVALID_FD;
+    finfo->mode = INVALID_MODE;
+    finfo->coalesce = INVALID_FD;
 
-	return finfo;
+    return finfo;
 }
 
 /*
@@ -77,39 +77,39 @@ file_stripe_info *alloc_file_stripe_info(int count)
 static int
 get_stripe_attr_name(const char *path, const char *pattern, char **attrname)
 {
-	char attrbuf[4096];
-	char *ptr, *match = NULL;
-	int len, r, match_count = 0;
-
-	if (!path || !pattern || !attrname)
-		return -1;
-
-	len = listxattr(path, attrbuf, sizeof(attrbuf));
-	if (len < 0)
-		return len;
-
-	ptr = attrbuf;
-	while (ptr) {
-		r = fnmatch(pattern, ptr, 0);
-		if (!r) {
-			if (!match)
-				match = ptr;
-			match_count++;
-		} else if (r != FNM_NOMATCH) {
-			return -1;
-		}
-
-		len -= strlen(ptr) + 1;
-		if (len > 0)
-			ptr += strlen(ptr) + 1;
-		else
-			ptr = NULL;
-	}
-
-	if (match)
-		*attrname = strdup(match);
-
-	return match_count;
+    char attrbuf[4096];
+    char *ptr, *match = NULL;
+    int len, r, match_count = 0;
+
+    if (!path || !pattern || !attrname)
+        return -1;
+
+    len = listxattr(path, attrbuf, sizeof(attrbuf));
+    if (len < 0)
+        return len;
+
+    ptr = attrbuf;
+    while (ptr) {
+        r = fnmatch(pattern, ptr, 0);
+        if (!r) {
+            if (!match)
+                match = ptr;
+            match_count++;
+        } else if (r != FNM_NOMATCH) {
+            return -1;
+        }
+
+        len -= strlen(ptr) + 1;
+        if (len > 0)
+            ptr += strlen(ptr) + 1;
+        else
+            ptr = NULL;
+    }
+
+    if (match)
+        *attrname = strdup(match);
+
+    return match_count;
 }
 
 /*
@@ -118,19 +118,19 @@ get_stripe_attr_name(const char *path, const char *pattern, char **attrname)
 static int
 get_stripe_attr_val(const char *path, const char *attr, int *val)
 {
-	char attrbuf[4096];
-	int len;
+    char attrbuf[4096];
+    int len;
 
-	if (!path || !attr || !val)
-		return -1;
+    if (!path || !attr || !val)
+        return -1;
 
-	len = getxattr(path, attr, attrbuf, sizeof(attrbuf));
-	if (len < 0)
-		return len;
+    len = getxattr(path, attr, attrbuf, sizeof(attrbuf));
+    if (len < 0)
+        return len;
 
-	*val = atoi(attrbuf);
+    *val = atoi(attrbuf);
 
-	return 0;
+    return 0;
 }
 
 /*
@@ -145,29 +145,31 @@ get_stripe_attr_val(const char *path, const char *attr, int *val)
 static int
 get_attr(const char *path, const char *pattern, char **buf, int *val)
 {
-	int count = 1;
-
-	if (!buf)
-		return -1;
-
-	if (!*buf) {
-		count = get_stripe_attr_name(path, pattern, buf);
-		if (count > 1) {
-			/* pattern isn't good enough */
-			fprintf(stderr, "ERROR: duplicate attributes found "
-				"matching pattern: %s\n", pattern);
-			free(*buf);
-			*buf = NULL;
-			return count;
-		} else if (count < 1) {
-			return count;
-		}
-	}
-
-	if (get_stripe_attr_val(path, *buf, val) < 0)
-		return -1;
-
-	return count;
+    int count = 1;
+
+    if (!buf)
+        return -1;
+
+    if (!*buf) {
+        count = get_stripe_attr_name(path, pattern, buf);
+        if (count > 1) {
+            /* pattern isn't good enough */
+            fprintf(stderr,
+                    "ERROR: duplicate attributes found "
+                    "matching pattern: %s\n",
+                    pattern);
+            free(*buf);
+            *buf = NULL;
+            return count;
+        } else if (count < 1) {
+            return count;
+        }
+    }
+
+    if (get_stripe_attr_val(path, *buf, val) < 0)
+        return -1;
+
+    return count;
 }
 
 /*
@@ -178,164 +180,168 @@ get_attr(const char *path, const char *pattern, char **buf, int *val)
  * print a warning if any files are missing. We proceed without error in the
  * latter case to support partial recovery.
  */
-static struct
-file_stripe_info *validate_and_open_files(char *paths[], int count)
+static struct file_stripe_info *
+validate_and_open_files(char *paths[], int count)
 {
-	int i, val, tmp;
-	struct stat sbuf;
-	char *stripe_count_attr = NULL;
-	char *stripe_size_attr = NULL;
-	char *stripe_index_attr = NULL;
-	char *stripe_coalesce_attr = NULL;
-	struct file_stripe_info *finfo = NULL;
-
-	for (i = 0; i < count; i++) {
-		if (!paths[i])
-			goto err;
-
-		/*
-		 * Check the stripe count first so we can allocate the info
-		 * struct with the appropriate number of fds.
-		 */
-		if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT,
-				&stripe_count_attr, &val) != 1) {
-			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
-				paths[i], ATTRNAME_STRIPE_COUNT);
-			goto err;
-		}
-		if (!finfo) {
-			finfo = alloc_file_stripe_info(val);
-			if (!finfo)
-				goto err;
-
-			if (val != count)
-				fprintf(stderr, "WARNING: %s: stripe-count "
-					"(%d) != file count (%d). Result may "
-					"be incomplete.\n", paths[i], val,
-					count);
-
-			finfo->stripe_count = val;
-		} else if (val != finfo->stripe_count) {
-			fprintf(stderr, "ERROR %s: invalid stripe count: %d "
-				"(expected %d)\n", paths[i], val,
-				finfo->stripe_count);
-			goto err;
-		}
-
-		/*
-		 * Get and validate the chunk size.
-		 */
-		if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr,
-				&val) != 1) {
-			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
-				paths[i], ATTRNAME_STRIPE_SIZE);
-			goto err;
-		}
-
-		if (!finfo->stripe_size) {
-			finfo->stripe_size = val;
-		} else if (val != finfo->stripe_size) {
-			fprintf(stderr, "ERROR: %s: invalid stripe size: %d "
-				"(expected %d)\n", paths[i], val,
-				finfo->stripe_size);
-			goto err;
-		}
-
-		/*
-		 * stripe-coalesce is a backward compatible attribute. If the
-		 * attribute does not exist, assume a value of zero for the
-		 * traditional stripe format.
-		 */
-		tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE,
-				&stripe_coalesce_attr, &val);
-		if (!tmp) {
-			val = 0;
-		} else if (tmp != 1) {
-			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
-				paths[i], ATTRNAME_STRIPE_COALESCE);
-			goto err;
-		}
-
-		if (finfo->coalesce == INVALID_FD) {
-			finfo->coalesce = val;
-		} else if (val != finfo->coalesce) {
-			fprintf(stderr, "ERROR: %s: invalid coalesce flag\n",
-				paths[i]);
-			goto err;
-		}
-
-		/*
-		 * Get/validate the stripe index and open the file in the
-		 * appropriate fd slot.
-		 */
-		if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX,
-				&stripe_index_attr, &val) != 1) {
-			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
-				paths[i], ATTRNAME_STRIPE_INDEX);
-			goto err;
-		}
-		if (finfo->fd[val] != INVALID_FD) {
-			fprintf(stderr, "ERROR: %s: duplicate stripe index: "
-				"%d\n", paths[i], val);
-			goto err;
-		}
-
-		finfo->fd[val] = open(paths[i], O_RDONLY);
-		if (finfo->fd[val] < 0)
-			goto err;
-
-		/*
-		 * Get the creation mode for the file.
-		 */
-		if (fstat(finfo->fd[val], &sbuf) < 0)
-			goto err;
-		if (finfo->mode == INVALID_MODE) {
-			finfo->mode = sbuf.st_mode;
-		} else if (sbuf.st_mode != finfo->mode) {
-			fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]);
-			goto err;
-		}
-	}
-
-	free(stripe_count_attr);
-	free(stripe_size_attr);
-	free(stripe_index_attr);
-	free(stripe_coalesce_attr);
-
-	return finfo;
+    int i, val, tmp;
+    struct stat sbuf;
+    char *stripe_count_attr = NULL;
+    char *stripe_size_attr = NULL;
+    char *stripe_index_attr = NULL;
+    char *stripe_coalesce_attr = NULL;
+    struct file_stripe_info *finfo = NULL;
+
+    for (i = 0; i < count; i++) {
+        if (!paths[i])
+            goto err;
+
+        /*
+         * Check the stripe count first so we can allocate the info
+         * struct with the appropriate number of fds.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, &stripe_count_attr,
+                     &val) != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_COUNT);
+            goto err;
+        }
+        if (!finfo) {
+            finfo = alloc_file_stripe_info(val);
+            if (!finfo)
+                goto err;
+
+            if (val != count)
+                fprintf(stderr,
+                        "WARNING: %s: stripe-count "
+                        "(%d) != file count (%d). Result may "
+                        "be incomplete.\n",
+                        paths[i], val, count);
+
+            finfo->stripe_count = val;
+        } else if (val != finfo->stripe_count) {
+            fprintf(stderr,
+                    "ERROR %s: invalid stripe count: %d "
+                    "(expected %d)\n",
+                    paths[i], val, finfo->stripe_count);
+            goto err;
+        }
+
+        /*
+         * Get and validate the chunk size.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, &val) !=
+            1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_SIZE);
+            goto err;
+        }
+
+        if (!finfo->stripe_size) {
+            finfo->stripe_size = val;
+        } else if (val != finfo->stripe_size) {
+            fprintf(stderr,
+                    "ERROR: %s: invalid stripe size: %d "
+                    "(expected %d)\n",
+                    paths[i], val, finfo->stripe_size);
+            goto err;
+        }
+
+        /*
+         * stripe-coalesce is a backward compatible attribute. If the
+         * attribute does not exist, assume a value of zero for the
+         * traditional stripe format.
+         */
+        tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE,
+                       &stripe_coalesce_attr, &val);
+        if (!tmp) {
+            val = 0;
+        } else if (tmp != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_COALESCE);
+            goto err;
+        }
+
+        if (finfo->coalesce == INVALID_FD) {
+            finfo->coalesce = val;
+        } else if (val != finfo->coalesce) {
+            fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", paths[i]);
+            goto err;
+        }
+
+        /*
+         * Get/validate the stripe index and open the file in the
+         * appropriate fd slot.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, &stripe_index_attr,
+                     &val) != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_INDEX);
+            goto err;
+        }
+        if (finfo->fd[val] != INVALID_FD) {
+            fprintf(stderr,
+                    "ERROR: %s: duplicate stripe index: "
+                    "%d\n",
+                    paths[i], val);
+            goto err;
+        }
+
+        finfo->fd[val] = open(paths[i], O_RDONLY);
+        if (finfo->fd[val] < 0)
+            goto err;
+
+        /*
+         * Get the creation mode for the file.
+         */
+        if (fstat(finfo->fd[val], &sbuf) < 0)
+            goto err;
+        if (finfo->mode == INVALID_MODE) {
+            finfo->mode = sbuf.st_mode;
+        } else if (sbuf.st_mode != finfo->mode) {
+            fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]);
+            goto err;
+        }
+    }
+
+    free(stripe_count_attr);
+    free(stripe_size_attr);
+    free(stripe_index_attr);
+    free(stripe_coalesce_attr);
+
+    return finfo;
 err:
 
-	free(stripe_count_attr);
-	free(stripe_size_attr);
-	free(stripe_index_attr);
-	free(stripe_coalesce_attr);
+    free(stripe_count_attr);
+    free(stripe_size_attr);
+    free(stripe_index_attr);
+    free(stripe_coalesce_attr);
 
-	if (finfo) {
-		close_files(finfo);
-		free(finfo);
-	}
+    if (finfo) {
+        close_files(finfo);
+        free(finfo);
+    }
 
-	return NULL;
+    return NULL;
 }
 
 static int
 close_files(struct file_stripe_info *finfo)
 {
-	int i, ret;
+    int i, ret;
 
-	if (!finfo)
-		return -1;
+    if (!finfo)
+        return -1;
 
-	for (i = 0; i < finfo->stripe_count; i++) {
-		if (finfo->fd[i] == INVALID_FD)
-			continue;
+    for (i = 0; i < finfo->stripe_count; i++) {
+        if (finfo->fd[i] == INVALID_FD)
+            continue;
 
-		ret = close(finfo->fd[i]);
-		if (ret < 0)
-			return ret;
-	}
+        ret = close(finfo->fd[i]);
+        if (ret < 0)
+            return ret;
+    }
 
-	return ret;
+    return ret;
 }
 
 /*
@@ -351,43 +357,43 @@ close_files(struct file_stripe_info *finfo)
 static int
 generate_file_coalesce(int target, struct file_stripe_info *finfo)
 {
-	char *buf;
-	int ret = 0;
-	int r, w, i;
-
-	buf = malloc(finfo->stripe_size);
-	if (!buf)
-		return -1;
-
-	i = 0;
-	while (1) {
-		if (finfo->fd[i] == INVALID_FD) {
-			if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0)
-				break;
-
-			i = (i + 1) % finfo->stripe_count;
-			continue;
-		}
-
-		r = read(finfo->fd[i], buf, finfo->stripe_size);
-		if (r < 0) {
-			ret = r;
-			break;
-		}
-		if (!r)
-			break;
-
-		w = write(target, buf, r);
-		if (w < 0) {
-			ret = w;
-			break;
-		}
-
-		i = (i + 1) % finfo->stripe_count;
-	}
-
-	free(buf);
-	return ret;
+    char *buf;
+    int ret = 0;
+    int r, w, i;
+
+    buf = malloc(finfo->stripe_size);
+    if (!buf)
+        return -1;
+
+    i = 0;
+    while (1) {
+        if (finfo->fd[i] == INVALID_FD) {
+            if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0)
+                break;
+
+            i = (i + 1) % finfo->stripe_count;
+            continue;
+        }
+
+        r = read(finfo->fd[i], buf, finfo->stripe_size);
+        if (r < 0) {
+            ret = r;
+            break;
+        }
+        if (!r)
+            break;
+
+        w = write(target, buf, r);
+        if (w < 0) {
+            ret = w;
+            break;
+        }
+
+        i = (i + 1) % finfo->stripe_count;
+    }
+
+    free(buf);
+    return ret;
 }
 
 /*
@@ -398,97 +404,100 @@ generate_file_coalesce(int target, struct file_stripe_info *finfo)
 static int
 generate_file_traditional(int target, struct file_stripe_info *finfo)
 {
-	int i, j, max_ret, ret;
-	char buf[finfo->stripe_count][4096];
-
-	do {
-		char newbuf[4096] = {0, };
-
-		max_ret = 0;
-		for (i = 0; i < finfo->stripe_count; i++) {
-			memset(buf[i], 0, 4096);
-			ret = read(finfo->fd[i], buf[i], 4096);
-			if (ret > max_ret)
-				max_ret = ret;
-		}
-		for (i = 0; i < max_ret; i++)
-			for (j = 0; j < finfo->stripe_count; j++)
-				newbuf[i] |= buf[j][i];
-			write(target, newbuf, max_ret);
-	} while (max_ret);
-
-	return 0;
+    int i, j, max_ret, ret;
+    char buf[finfo->stripe_count][4096];
+
+    do {
+        char newbuf[4096] = {
+            0,
+        };
+
+        max_ret = 0;
+        for (i = 0; i < finfo->stripe_count; i++) {
+            memset(buf[i], 0, 4096);
+            ret = read(finfo->fd[i], buf[i], 4096);
+            if (ret > max_ret)
+                max_ret = ret;
+        }
+        for (i = 0; i < max_ret; i++)
+            for (j = 0; j < finfo->stripe_count; j++)
+                newbuf[i] |= buf[j][i];
+        write(target, newbuf, max_ret);
+    } while (max_ret);
+
+    return 0;
 }
 
 static int
 generate_file(int target, struct file_stripe_info *finfo)
 {
-	if (finfo->coalesce)
-		return generate_file_coalesce(target, finfo);
+    if (finfo->coalesce)
+        return generate_file_coalesce(target, finfo);
 
-	return generate_file_traditional(target, finfo);
+    return generate_file_traditional(target, finfo);
 }
 
 static void
 usage(char *name)
 {
-	fprintf(stderr, "Usage: %s [-o <outputfile>] <inputfile1> "
-		"<inputfile2> ...\n", name);
+    fprintf(stderr,
+            "Usage: %s [-o <outputfile>] <inputfile1> "
+            "<inputfile2> ...\n",
+            name);
 }
 
 int
 main(int argc, char *argv[])
 {
-	int file_count, opt;
-	char *opath = NULL;
-	int targetfd;
-	struct file_stripe_info *finfo;
-
-	while ((opt = getopt(argc, argv, "o:")) != -1) {
-		switch (opt) {
-		case 'o':
-			opath = optarg;
-			break;
-		default:
-			usage(argv[0]);
-			return -1;
-		}
-	}
-
-	file_count = argc - optind;
-
-	if (!opath || !file_count) {
-		usage(argv[0]);
-		return -1;
-	}
-
-	finfo = validate_and_open_files(&argv[optind], file_count);
-	if (!finfo)
-		goto err;
-
-	targetfd = open(opath, O_RDWR|O_CREAT, finfo->mode);
-	if (targetfd < 0)
-		goto err;
-
-	if (generate_file(targetfd, finfo) < 0)
-		goto err;
-
-	if (fsync(targetfd) < 0)
-		fprintf(stderr, "ERROR: %s\n", strerror(errno));
-	if (close(targetfd) < 0)
-		fprintf(stderr, "ERROR: %s\n", strerror(errno));
-
-	close_files(finfo);
-	free(finfo);
-
-	return 0;
+    int file_count, opt;
+    char *opath = NULL;
+    int targetfd;
+    struct file_stripe_info *finfo;
+
+    while ((opt = getopt(argc, argv, "o:")) != -1) {
+        switch (opt) {
+            case 'o':
+                opath = optarg;
+                break;
+            default:
+                usage(argv[0]);
+                return -1;
+        }
+    }
+
+    file_count = argc - optind;
+
+    if (!opath || !file_count) {
+        usage(argv[0]);
+        return -1;
+    }
+
+    finfo = validate_and_open_files(&argv[optind], file_count);
+    if (!finfo)
+        goto err;
+
+    targetfd = open(opath, O_RDWR | O_CREAT, finfo->mode);
+    if (targetfd < 0)
+        goto err;
+
+    if (generate_file(targetfd, finfo) < 0)
+        goto err;
+
+    if (fsync(targetfd) < 0)
+        fprintf(stderr, "ERROR: %s\n", strerror(errno));
+    if (close(targetfd) < 0)
+        fprintf(stderr, "ERROR: %s\n", strerror(errno));
+
+    close_files(finfo);
+    free(finfo);
+
+    return 0;
 
 err:
-	if (finfo) {
-		close_files(finfo);
-		free(finfo);
-	}
+    if (finfo) {
+        close_files(finfo);
+        free(finfo);
+    }
 
-	return -1;
+    return -1;
 }
-
diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am
index 5b9b117efaf..61446a9b84a 100644
--- a/extras/systemd/Makefile.am
+++ b/extras/systemd/Makefile.am
@@ -1,11 +1,17 @@
-CLEANFILES = glusterd.service glustereventsd.service
-EXTRA_DIST = glusterd.service.in glustereventsd.service.in
+CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service gluster-ta-volume.service
+EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in gluster-ta-volume.service.in
 
 if USE_SYSTEMD
+systemd_DATA = gluster-ta-volume.service
+endif
+
+if WITH_SERVER
+if USE_SYSTEMD
 # systemddir is already defined through configure.ac
-systemd_DATA = glusterd.service
+systemd_DATA += glusterd.service glusterfssharedstorage.service
 
 if BUILD_EVENTS
 systemd_DATA += glustereventsd.service
 endif
 endif
+endif
diff --git a/extras/systemd/gluster-ta-volume.service.in b/extras/systemd/gluster-ta-volume.service.in
new file mode 100644
index 00000000000..2802bca05bf
--- /dev/null
+++ b/extras/systemd/gluster-ta-volume.service.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=GlusterFS, Thin-arbiter process to maintain quorum for replica volume
+After=network.target
+
+[Service]
+Environment="LOG_LEVEL=WARNING"
+ExecStart=@prefix@/sbin/glusterfsd -N --volfile-id ta -f @GLUSTERD_WORKDIR@/thin-arbiter/thin-arbiter.vol --brick-port 24007 --xlator-option ta-server.transport.socket.listen-port=24007 -LWARNING
+Restart=always
+KillMode=process
+SuccessExitStatus=15
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in
index 26694cfc8ab..abb0d82911f 100644
--- a/extras/systemd/glusterd.service.in
+++ b/extras/systemd/glusterd.service.in
@@ -1,7 +1,10 @@
 [Unit]
 Description=GlusterFS, a clustered file-system server
-Requires=rpcbind.service
-After=network.target rpcbind.service
+Documentation=man:glusterd(8)
+StartLimitBurst=6
+StartLimitIntervalSec=3600
+Requires=@RPCBIND_SERVICE@
+After=network.target @RPCBIND_SERVICE@
 Before=network-online.target
 
 [Service]
@@ -9,9 +12,15 @@ Type=forking
 PIDFile=@localstatedir@/run/glusterd.pid
 LimitNOFILE=65536
 Environment="LOG_LEVEL=INFO"
-EnvironmentFile=-@sysconfdir@/sysconfig/glusterd
+EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd
 ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid  --log-level $LOG_LEVEL $GLUSTERD_OPTIONS
 KillMode=process
+TimeoutSec=300
+SuccessExitStatus=15
+Restart=on-abnormal
+RestartSec=60
+StartLimitBurst=6
+StartLimitInterval=3600
 
 [Install]
 WantedBy=multi-user.target
diff --git a/extras/systemd/glustereventsd.service.in b/extras/systemd/glustereventsd.service.in
index 4bfcf42f386..f80b78199f6 100644
--- a/extras/systemd/glustereventsd.service.in
+++ b/extras/systemd/glustereventsd.service.in
@@ -1,6 +1,8 @@
 [Unit]
 Description=Gluster Events Notifier
-After=syslog.target network.target
+After=network.target
+Documentation=man:glustereventsd(8)
+
 
 [Service]
 Environment=PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES_EXPANDED@:$PYTHONPATH
diff --git a/extras/systemd/glusterfssharedstorage.service.in b/extras/systemd/glusterfssharedstorage.service.in
new file mode 100644
index 00000000000..723ff49afb7
--- /dev/null
+++ b/extras/systemd/glusterfssharedstorage.service.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=Mount glusterfs sharedstorage
+Requires=glusterd.service remote-fs-pre.target local-fs.target
+
+[Service]
+Type=forking
+ExecStart=@GLUSTERFS_LIBEXECDIR@/mount-shared-storage.sh
+Restart=on-failure
+RestartSec=3
+RestartForceExitStatus=1
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c
index 8f74a25cf68..d120c053a69 100644
--- a/extras/test/ld-preload-test/ld-preload-lib.c
+++ b/extras/test/ld-preload-test/ld-preload-lib.c
@@ -34,594 +34,582 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <dirent.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <sys/sendfile.h>
 
 /* Err number that is assigned to errno so that test application can
  * verify that the function was intercepted correctly.
  */
-#define PRELOAD_ERRNO_VERF      6449
-#define set_errno()             (errno = PRELOAD_ERRNO_VERF)
+#define PRELOAD_ERRNO_VERF 6449
+#define set_errno() (errno = PRELOAD_ERRNO_VERF)
 
 void
-intercept (char *call, int tabs)
+intercept(char *call, int tabs)
 {
-        while (tabs > 0) {
-                fprintf (stdout, "\t");
-                --tabs;
-        }
+    while (tabs > 0) {
+        fprintf(stdout, "\t");
+        --tabs;
+    }
 
-        fprintf (stdout, "Intercepted by %s", call);
+    fprintf(stdout, "Intercepted by %s", call);
 }
 
 int
-creat64 (const char *pathname, mode_t mode)
+creat64(const char *pathname, mode_t mode)
 {
-        intercept ("creat64", 2);
-        set_errno ();
-        return -1;
+    intercept("creat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-creat (const char *pathname, mode_t mode)
+creat(const char *pathname, mode_t mode)
 {
-        intercept ("creat", 2);
-        set_errno ();
-        return -1;
+    intercept("creat", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-close (int fd)
+close(int fd)
 {
-        intercept ("close", 2);
-        set_errno ();
-        return -1;
+    intercept("close", 2);
+    set_errno();
+    return -1;
 }
 
 int
-open64 (const char *pathname, int flags, ...)
+open64(const char *pathname, int flags, ...)
 {
-        intercept ("open64", 2);
-        set_errno ();
-        return -1;
+    intercept("open64", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-open (const char *pathname, int flags, ...)
+open(const char *pathname, int flags, ...)
 {
-        intercept ("open", 2);
-        set_errno ();
-        return -1;
+    intercept("open", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-read (int fd, void *buf, size_t count)
+read(int fd, void *buf, size_t count)
 {
-        intercept ("read", 2);
-        set_errno ();
-        return -1;
+    intercept("read", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-readv (int fd, const struct iovec *vector, int count)
+readv(int fd, const struct iovec *vector, int count)
 {
-        intercept ("readv", 2);
-        set_errno ();
-        return -1;
+    intercept("readv", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pread (int fd, void *buf, size_t count, unsigned long offset)
+pread(int fd, void *buf, size_t count, unsigned long offset)
 {
-        intercept ("pread", 2);
-        set_errno ();
-        return -1;
+    intercept("pread", 2);
+    set_errno();
+    return -1;
 }
 
-
 ssize_t
-pread64 (int fd, void *buf, size_t count, uint64_t offset)
+pread64(int fd, void *buf, size_t count, uint64_t offset)
 {
-        intercept ("pread64", 2);
-        set_errno ();
-        return -1;
+    intercept("pread64", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-write (int fd, const void *buf, size_t count)
+write(int fd, const void *buf, size_t count)
 {
-        intercept ("write", 2);
-        set_errno ();
-        return -1;
+    intercept("write", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-writev (int fd, const struct iovec *vector, int count)
+writev(int fd, const struct iovec *vector, int count)
 {
-        intercept ("writev", 2);
-        set_errno ();
-        return -1;
+    intercept("writev", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pwrite (int fd, const void *buf, size_t count, unsigned long offset)
+pwrite(int fd, const void *buf, size_t count, unsigned long offset)
 {
-        intercept ("pwrite", 2);
-        set_errno ();
-        return -1;
+    intercept("pwrite", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pwrite64 (int fd, const void *buf, size_t count, uint64_t offset)
+pwrite64(int fd, const void *buf, size_t count, uint64_t offset)
 {
-        intercept ("pwrite64", 2);
-        set_errno ();
-        return -1;
+    intercept("pwrite64", 2);
+    set_errno();
+    return -1;
 }
 
-
 off_t
-lseek (int fildes, unsigned long offset, int whence)
+lseek(int fildes, unsigned long offset, int whence)
 {
-        intercept ("lseek", 2);
-        set_errno ();
-        return -1;
+    intercept("lseek", 2);
+    set_errno();
+    return -1;
 }
 
 off_t
-lseek64 (int fildes, uint64_t offset, int whence)
+lseek64(int fildes, uint64_t offset, int whence)
 {
-        intercept ("lseek64", 2);
-        set_errno ();
-        return -1;
+    intercept("lseek64", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-dup (int fd)
+dup(int fd)
 {
-        intercept ("dup", 2);
-        set_errno ();
-        return -1;
+    intercept("dup", 2);
+    set_errno();
+    return -1;
 }
 
 int
-dup2 (int oldfd, int newfd)
+dup2(int oldfd, int newfd)
 {
-        intercept ("dup2", 2);
-        set_errno ();
-        return -1;
+    intercept("dup2", 2);
+    set_errno();
+    return -1;
 }
 
 int
-mkdir (const char *pathname, mode_t mode)
+mkdir(const char *pathname, mode_t mode)
 {
-        intercept ("mkdir", 2);
-        set_errno ();
-        return -1;
+    intercept("mkdir", 2);
+    set_errno();
+    return -1;
 }
 
 int
-rmdir (const char *pathname)
+rmdir(const char *pathname)
 {
-        intercept ("rmdir", 2);
-        set_errno ();
-        return -1;
+    intercept("rmdir", 2);
+    set_errno();
+    return -1;
 }
 
 int
-chmod (const char *pathname, mode_t mode)
+chmod(const char *pathname, mode_t mode)
 {
-        intercept ("chmod", 2);
-        set_errno ();
-        return -1;
+    intercept("chmod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-chown (const char *pathname, uid_t owner, gid_t group)
+chown(const char *pathname, uid_t owner, gid_t group)
 {
-        intercept ("chown", 2);
-        set_errno ();
-        return -1;
+    intercept("chown", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fchmod (int fd, mode_t mode)
+fchmod(int fd, mode_t mode)
 {
-        intercept ("fchmod", 2);
-        set_errno ();
-        return -1;
+    intercept("fchmod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fchown (int fd, uid_t uid, gid_t gid)
+fchown(int fd, uid_t uid, gid_t gid)
 {
-        intercept ("fchown", 2);
-        set_errno ();
-        return -1;
+    intercept("fchown", 2);
+    set_errno();
+    return -1;
 }
 
-int fsync (int fd)
+int
+fsync(int fd)
 {
-        intercept ("fsync", 2);
-        set_errno ();
-        return -1;
+    intercept("fsync", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-ftruncate (int fd, off_t length)
+ftruncate(int fd, off_t length)
 {
-        intercept ("ftruncate", 1);
-        set_errno ();
-        return -1;
+    intercept("ftruncate", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-ftruncate64 (int fd, off_t length)
+ftruncate64(int fd, off_t length)
 {
-        intercept ("ftruncate64", 1);
-        set_errno ();
-        return -1;
+    intercept("ftruncate64", 1);
+    set_errno();
+    return -1;
 }
 
 int
-link (const char *oldpath, const char *newname)
+link(const char *oldpath, const char *newname)
 {
-        intercept ("link", 2);
-        set_errno ();
-        return -1;
+    intercept("link", 2);
+    set_errno();
+    return -1;
 }
 
 int
-rename (const char *oldpath, const char *newpath)
+rename(const char *oldpath, const char *newpath)
 {
-        intercept ("rename", 2);
-        set_errno ();
-        return -1;
+    intercept("rename", 2);
+    set_errno();
+    return -1;
 }
 
 int
-utimes (const char *path, const struct timeval times[2])
+utimes(const char *path, const struct timeval times[2])
 {
-        intercept ("utimes", 2);
-        set_errno ();
-        return -1;
+    intercept("utimes", 2);
+    set_errno();
+    return -1;
 }
 
 int
-utime (const char *path, const struct utimbuf *buf)
+futimes(int fd, const struct timeval times[2])
 {
-        intercept ("utime", 2);
-        set_errno ();
-        return -1;
+    intercept("futimes", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-mknod (const char *path, mode_t mode, dev_t dev)
+utime(const char *path, const struct utimbuf *buf)
 {
-        intercept ("mknod", 2);
-        set_errno ();
-        return -1;
+    intercept("utime", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__xmknod (int ver, const char *path, mode_t mode, dev_t *dev)
+mknod(const char *path, mode_t mode, dev_t dev)
 {
-        intercept ("__xmknod", 2);
-        set_errno ();
-        return -1;
+    intercept("mknod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-mkfifo (const char *path, mode_t mode)
+__xmknod(int ver, const char *path, mode_t mode, dev_t *dev)
 {
-        intercept ("mkfifo", 2);
-        set_errno ();
-        return -1;
+    intercept("__xmknod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-unlink (const char *path)
+mkfifo(const char *path, mode_t mode)
 {
-        intercept ("unlink", 2);
-        set_errno ();
-        return -1;
+    intercept("mkfifo", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-symlink (const char *oldpath, const char *newpath)
+unlink(const char *path)
 {
-        intercept ("symlink", 2);
-        set_errno ();
-        return -1;
+    intercept("unlink", 2);
+    set_errno();
+    return -1;
 }
 
 int
-readlink (const char *path, char *buf, size_t bufsize)
+symlink(const char *oldpath, const char *newpath)
 {
-        intercept ("readlink", 1);
-        set_errno ();
-        return -1;
+    intercept("symlink", 2);
+    set_errno();
+    return -1;
 }
 
+int
+readlink(const char *path, char *buf, size_t bufsize)
+{
+    intercept("readlink", 1);
+    set_errno();
+    return -1;
+}
 
 char *
-realpath (const char *path, char *resolved)
+realpath(const char *path, char *resolved)
 {
-        intercept ("realpath", 1);
-        set_errno ();
-        return NULL;
+    intercept("realpath", 1);
+    set_errno();
+    return NULL;
 }
 
-
 DIR *
-opendir (const char *path)
+opendir(const char *path)
 {
-        intercept ("opendir", 2);
-        set_errno ();
-        return NULL;
+    intercept("opendir", 2);
+    set_errno();
+    return NULL;
 }
 
-
 struct dirent *
-readdir (DIR *dir)
+readdir(DIR *dir)
 {
-        intercept ("readdir\t", 2);
-        set_errno ();
-        return NULL;
+    intercept("readdir\t", 2);
+    set_errno();
+    return NULL;
 }
 
 struct dirent *
-readdir64 (DIR *dir)
+readdir64(DIR *dir)
 {
-        intercept ("readdir64", 2);
-        set_errno ();
-        return NULL;
+    intercept("readdir64", 2);
+    set_errno();
+    return NULL;
 }
 
-
 int
-readdir_r (DIR *dir, struct dirent *entry, struct dirent **result)
+readdir_r(DIR *dir, struct dirent *entry, struct dirent **result)
 {
-        intercept ("readdir_r", 1);
-        set_errno ();
-        return -1;
+    intercept("readdir_r", 1);
+    set_errno();
+    return -1;
 }
 
 int
-readdir64_r (DIR *dir, struct dirent *entry, struct dirent **result)
+readdir64_r(DIR *dir, struct dirent *entry, struct dirent **result)
 {
-        intercept ("readdir64_r", 1);
-        set_errno ();
-        return -1;
+    intercept("readdir64_r", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-closedir (DIR *dh)
+closedir(DIR *dh)
 {
-        intercept ("closedir", 1);
-        set_errno ();
-        return -1;
+    intercept("closedir", 1);
+    set_errno();
+    return -1;
 }
 
 int
-__xstat (int ver, const char *path, struct stat *buf)
+__xstat(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__xstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__xstat\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-__xstat64 (int ver, const char *path, struct stat *buf)
+__xstat64(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__xstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__xstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-stat (const char *path, struct stat *buf)
+stat(const char *path, struct stat *buf)
 {
-        intercept ("stat", 2);
-        set_errno ();
-        return -1;
+    intercept("stat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-stat64 (const char *path, struct stat *buf)
+stat64(const char *path, struct stat *buf)
 {
-        intercept ("stat64", 2);
-        set_errno ();
-        return -1;
+    intercept("stat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__fxstat (int ver, int fd, struct stat *buf)
+__fxstat(int ver, int fd, struct stat *buf)
 {
-        intercept ("__fxstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__fxstat\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-__fxstat64 (int ver, int fd, struct stat *buf)
+__fxstat64(int ver, int fd, struct stat *buf)
 {
-        intercept ("__fxstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__fxstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fstat (int fd, struct stat *buf)
+fstat(int fd, struct stat *buf)
 {
-        intercept ("fstat", 2);
-        set_errno ();
-        return -1;
+    intercept("fstat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fstat64 (int fd , struct stat *buf)
+fstat64(int fd, struct stat *buf)
 {
-        intercept ("fstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("fstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__lxstat (int ver, const char *path, struct stat *buf)
+__lxstat(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__lxstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__lxstat\t", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__lxstat64 (int ver, const char *path, struct stat *buf)
+__lxstat64(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__lxstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__lxstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lstat (const char *path, struct stat *buf)
+lstat(const char *path, struct stat *buf)
 {
-        intercept ("lstat", 2);
-        set_errno ();
-        return -1;
+    intercept("lstat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lstat64 (const char *path, struct stat *buf)
+lstat64(const char *path, struct stat *buf)
 {
-        intercept ("lstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("lstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-statfs (const char *path, struct statfs *buf)
+statfs(const char *path, struct statfs *buf)
 {
-        intercept ("statfs", 2);
-        set_errno ();
-        return -1;
+    intercept("statfs", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-statfs64 (const char *path, struct statfs *buf)
+statfs64(const char *path, struct statfs *buf)
 {
-        intercept ("statfs64", 2);
-        set_errno ();
-        return -1;
+    intercept("statfs64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-statvfs (const char *path, struct statvfs *buf)
+statvfs(const char *path, struct statvfs *buf)
 {
-        intercept ("statvfs\t", 2);
-        set_errno ();
-        return -1;
+    intercept("statvfs\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-statvfs64 (const char *path, struct statvfs *buf)
+statvfs64(const char *path, struct statvfs *buf)
 {
-        intercept ("statvfs64", 2);
-        set_errno ();
-        return -1;
+    intercept("statvfs64", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-getxattr (const char *path, const char *name, void *value, size_t size)
+getxattr(const char *path, const char *name, void *value, size_t size)
 {
-        intercept ("getxattr", 1);
-        set_errno ();
-        return -1;
+    intercept("getxattr", 1);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-lgetxattr (const char *path, const char *name, void *value, size_t size)
+lgetxattr(const char *path, const char *name, void *value, size_t size)
 {
-        intercept ("lgetxattr", 1);
-        set_errno ();
-        return -1;
+    intercept("lgetxattr", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-remove (const char* path)
+remove(const char *path)
 {
-        intercept ("remove", 2);
-        set_errno ();
-        return -1;
+    intercept("remove", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lchown (const char *path, uid_t owner, gid_t group)
+lchown(const char *path, uid_t owner, gid_t group)
 {
-        intercept ("lchown", 2);
-        set_errno ();
-        return -1;
+    intercept("lchown", 2);
+    set_errno();
+    return -1;
 }
 
 void
-rewinddir (DIR *dirp)
+rewinddir(DIR *dirp)
 {
-        intercept ("rewinddir", 1);
-        set_errno ();
-        return;
+    intercept("rewinddir", 1);
+    set_errno();
+    return;
 }
 
 void
-seekdir (DIR *dirp, off_t offset)
+seekdir(DIR *dirp, off_t offset)
 {
-        intercept ("seekdir", 2);
-        set_errno ();
-        return;
+    intercept("seekdir", 2);
+    set_errno();
+    return;
 }
 
 off_t
-telldir (DIR *dirp)
+telldir(DIR *dirp)
 {
-        intercept ("telldir", 2);
-        set_errno ();
-        return -1;
+    intercept("telldir", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-sendfile (int out_fd, int in_fd, off_t *offset, size_t count)
+sendfile(int out_fd, int in_fd, off_t *offset, size_t count)
 {
-        intercept ("sendfile\t", 1);
-        set_errno ();
-        return -1;
+    intercept("sendfile\t", 1);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count)
+sendfile64(int out_fd, int in_fd, off_t *offset, size_t count)
 {
-        intercept ("sendfile64", 1);
-        set_errno ();
-        return -1;
+    intercept("sendfile64", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-fcntl (int fd, int cmd, ...)
+fcntl(int fd, int cmd, ...)
 {
-        intercept ("fcntl", 2);
-        set_errno ();
-        return -1;
+    intercept("fcntl", 2);
+    set_errno();
+    return -1;
 }
-
diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c
index cf8dd52c3e1..54dde8c7d54 100644
--- a/extras/test/ld-preload-test/ld-preload-test.c
+++ b/extras/test/ld-preload-test/ld-preload-test.c
@@ -46,322 +46,313 @@
 #include <sys/uio.h>
 #include <utime.h>
 #include <sys/time.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <sys/sendfile.h>
 
-
-#define PRELOAD_ERRNO_VERF      6449
+#define PRELOAD_ERRNO_VERF 6449
 void
 check_err(int ret, char *call, int tabs)
 {
-        while (tabs > 0) {
-                fprintf (stdout, "\t");
-                --tabs;
-        }
-        if (ret != -1) {
-                fprintf (stdout, "Not intercepted: %s\n", call);
-                return;
-        }
-
-        if (errno != PRELOAD_ERRNO_VERF) {
-                fprintf (stdout, "Not intercepted: %s: err: %s\n", call,
-                         strerror (errno));
-                return;
-        }
+    while (tabs > 0) {
+        fprintf(stdout, "\t");
+        --tabs;
+    }
+    if (ret != -1) {
+        fprintf(stdout, "Not intercepted: %s\n", call);
+        return;
+    }
 
-        fprintf (stdout, "Intercept verified: %s\n", call);
+    if (errno != PRELOAD_ERRNO_VERF) {
+        fprintf(stdout, "Not intercepted: %s: err: %s\n", call,
+                strerror(errno));
         return;
+    }
+
+    fprintf(stdout, "Intercept verified: %s\n", call);
+    return;
 }
 
 void
-usage (FILE *fp)
+usage(FILE *fp)
 {
-        fprintf (fp, "Usage: ld-preload-test <Options>\n");
-        fprintf (fp, "Options\n");
-        fprintf (fp, "\t--path\t\tPathname is used as the file/directory"
-                     " created for the test.\n");
-
+    fprintf(fp, "Usage: ld-preload-test <Options>\n");
+    fprintf(fp, "Options\n");
+    fprintf(fp,
+            "\t--path\t\tPathname is used as the file/directory"
+            " created for the test.\n");
 }
 
-
 int
-run_file_tests (char *testfile)
+run_file_tests(char *testfile)
 {
-        int             ret = -1;
-        struct stat     buf;
+    int ret = -1;
+    struct stat buf;
 
-        assert (testfile);
-        fprintf (stdout, "Testing creat");
-        ret = creat (testfile, S_IRWXU);
-        check_err (ret, "creat", 2);
+    assert(testfile);
+    fprintf(stdout, "Testing creat");
+    ret = creat(testfile, S_IRWXU);
+    check_err(ret, "creat", 2);
 
-        fprintf (stdout, "Testing close");
-        ret = close (ret);
-        check_err (ret, "close", 2);
+    fprintf(stdout, "Testing close");
+    ret = close(ret);
+    check_err(ret, "close", 2);
 
-        fprintf (stdout, "Testing open");
-        ret = open (testfile, O_RDONLY);
-        check_err (ret, "open", 2);
+    fprintf(stdout, "Testing open");
+    ret = open(testfile, O_RDONLY);
+    check_err(ret, "open", 2);
 
-        fprintf (stdout, "Testing read");
-        ret = read (0, NULL, 0);
-        check_err (ret, "read", 2);
+    fprintf(stdout, "Testing read");
+    ret = read(0, NULL, 0);
+    check_err(ret, "read", 2);
 
-        fprintf (stdout, "Testing readv");
-        ret = readv (0, NULL, 0);
-        check_err (ret, "readv", 2);
+    fprintf(stdout, "Testing readv");
+    ret = readv(0, NULL, 0);
+    check_err(ret, "readv", 2);
 
-        fprintf (stdout, "Testing pread");
-        ret = pread (0, NULL, 0, 0);
-        check_err (ret, "pread", 2);
+    fprintf(stdout, "Testing pread");
+    ret = pread(0, NULL, 0, 0);
+    check_err(ret, "pread", 2);
 
-        fprintf (stdout, "Testing write");
-        ret = write (0, NULL, 0);
-        check_err (ret, "write", 2);
+    fprintf(stdout, "Testing write");
+    ret = write(0, NULL, 0);
+    check_err(ret, "write", 2);
 
-        fprintf (stdout, "Testing writev");
-        ret = writev (0, NULL, 0);
-        check_err (ret, "writev", 2);
+    fprintf(stdout, "Testing writev");
+    ret = writev(0, NULL, 0);
+    check_err(ret, "writev", 2);
 
-        fprintf (stdout, "Testing pwrite");
-        ret = pwrite (0, NULL, 0, 0);
-        check_err (ret, "pwrite", 2);
+    fprintf(stdout, "Testing pwrite");
+    ret = pwrite(0, NULL, 0, 0);
+    check_err(ret, "pwrite", 2);
 
-        fprintf (stdout, "Testing lseek");
-        ret = lseek (0, 0, 0);
-        check_err (ret, "lseek", 2);
+    fprintf(stdout, "Testing lseek");
+    ret = lseek(0, 0, 0);
+    check_err(ret, "lseek", 2);
 
-        fprintf (stdout, "Testing dup");
-        ret = dup (0);
-        check_err (ret, "dup", 2);
+    fprintf(stdout, "Testing dup");
+    ret = dup(0);
+    check_err(ret, "dup", 2);
 
-        fprintf (stdout, "Testing dup2");
-        ret = dup2 (0, 0);
-        check_err (ret, "dup2", 2);
+    fprintf(stdout, "Testing dup2");
+    ret = dup2(0, 0);
+    check_err(ret, "dup2", 2);
 
-        fprintf (stdout, "Testing fchmod");
-        ret = fchmod (0, 0);
-        check_err (ret, "fchmod", 2);
+    fprintf(stdout, "Testing fchmod");
+    ret = fchmod(0, 0);
+    check_err(ret, "fchmod", 2);
 
-        fprintf (stdout, "Testing fchown");
-        ret = fchown (0, 0, 0);
-        check_err (ret, "fchown", 2);
+    fprintf(stdout, "Testing fchown");
+    ret = fchown(0, 0, 0);
+    check_err(ret, "fchown", 2);
 
-        fprintf (stdout, "Testing fsync");
-        ret = fsync (0);
-        check_err (ret, "fsync", 2);
+    fprintf(stdout, "Testing fsync");
+    ret = fsync(0);
+    check_err(ret, "fsync", 2);
 
-        fprintf (stdout, "Testing ftruncate");
-        ret = ftruncate (0, 0);
-        check_err (ret, "ftruncate", 1);
+    fprintf(stdout, "Testing ftruncate");
+    ret = ftruncate(0, 0);
+    check_err(ret, "ftruncate", 1);
 
-        fprintf (stdout, "Testing fstat");
-        ret = fstat (0, &buf);
-        check_err (ret, "fstat", 1);
+    fprintf(stdout, "Testing fstat");
+    ret = fstat(0, &buf);
+    check_err(ret, "fstat", 1);
 
-        fprintf (stdout, "Testing sendfile");
-        ret = sendfile (0, 0, NULL, 0);
-        check_err (ret, "sendfile", 1);
+    fprintf(stdout, "Testing sendfile");
+    ret = sendfile(0, 0, NULL, 0);
+    check_err(ret, "sendfile", 1);
 
-        fprintf (stdout, "Testing fcntl");
-        ret = fcntl (0, 0, NULL);
-        check_err (ret, "fcntl", 2);
+    fprintf(stdout, "Testing fcntl");
+    ret = fcntl(0, 0, NULL);
+    check_err(ret, "fcntl", 2);
 
-        fprintf (stdout, "Testing close");
-        ret = close (ret);
-        check_err (ret, "close", 2);
+    fprintf(stdout, "Testing close");
+    ret = close(ret);
+    check_err(ret, "close", 2);
 
-        fprintf (stdout, "Testing remove");
-        ret = remove (testfile);
-        check_err (ret, "remove", 2);
+    fprintf(stdout, "Testing remove");
+    ret = remove(testfile);
+    check_err(ret, "remove", 2);
 
-        return ret;
+    return ret;
 }
 
-
 int
-run_attr_tests (char *testfile)
+run_attr_tests(char *testfile)
 {
-        int             ret = -1;
-        char            *res = NULL;
-        struct stat     buf;
-        struct statfs   sbuf;
-        struct statvfs  svbuf;
-
-        assert (testfile);
-
-        fprintf (stdout, "Testing chmod");
-        ret = chmod (testfile, 0);
-        check_err (ret, "chmod", 2);
-
-        fprintf (stdout, "Testing chown");
-        ret = chown (testfile, 0, 0);
-        check_err (ret, "chown", 2);
-
-        fprintf (stdout, "Testing link");
-        ret = link (testfile, testfile);
-        check_err (ret, "link", 2);
-
-        fprintf (stdout, "Testing rename");
-        ret = rename (testfile, testfile);
-        check_err (ret, "rename", 2);
-
-        fprintf (stdout, "Testing utimes");
-        ret = utimes (testfile, NULL);
-        check_err (ret, "utimes", 2);
-
-        fprintf (stdout, "Testing utime");
-        ret = utime (testfile, NULL);
-        check_err (ret, "utime", 2);
-
-        fprintf (stdout, "Testing unlink");
-        ret = unlink (testfile);
-        check_err (ret, "unlink", 2);
-
-        fprintf (stdout, "Testing symlink");
-        ret = symlink (testfile, testfile);
-        check_err (ret, "symlink", 2);
-
-        fprintf (stdout, "Testing readlink");
-        ret = readlink (testfile, testfile, 0);
-        check_err (ret, "readlink", 2);
-
-        fprintf (stdout, "Testing realpath");
-        ret = 0;
-        res = realpath ((const char *)testfile, testfile);
-        if (!res)
-                ret = -1;
-        check_err (ret, "realpath", 2);
-
-        fprintf (stdout, "Testing stat");
-        ret = stat (testfile, &buf);
-        check_err (ret, "stat", 1);
-
-        fprintf (stdout, "Testing lstat");
-        ret = lstat (testfile, &buf);
-        check_err (ret, "lstat", 1);
-
-        fprintf (stdout, "Testing statfs");
-        ret = statfs (testfile, &sbuf);
-        check_err (ret, "statfs", 2);
-
-        fprintf (stdout, "Testing statvfs");
-        ret = statvfs (testfile, &svbuf);
-        check_err (ret, "statvfs", 1);
-
-        fprintf (stdout, "Testing getxattr");
-        ret = getxattr (testfile, NULL, NULL, 0);
-        check_err (ret, "getxattr", 2);
-
-        fprintf (stdout, "Testing lgetxattr");
-        ret = lgetxattr (testfile, NULL, NULL, 0);
-        check_err (ret, "lgetxattr", 1);
-
-        fprintf (stdout, "Testing lchown");
-        ret = lchown (testfile, 0, 0);
-        check_err (ret, "lchown", 2);
-        return 0;
+    int ret = -1;
+    char *res = NULL;
+    struct stat buf;
+    struct statfs sbuf;
+    struct statvfs svbuf;
+
+    assert(testfile);
+
+    fprintf(stdout, "Testing chmod");
+    ret = chmod(testfile, 0);
+    check_err(ret, "chmod", 2);
+
+    fprintf(stdout, "Testing chown");
+    ret = chown(testfile, 0, 0);
+    check_err(ret, "chown", 2);
+
+    fprintf(stdout, "Testing link");
+    ret = link(testfile, testfile);
+    check_err(ret, "link", 2);
+
+    fprintf(stdout, "Testing rename");
+    ret = rename(testfile, testfile);
+    check_err(ret, "rename", 2);
+
+    fprintf(stdout, "Testing utimes");
+    ret = utimes(testfile, NULL);
+    check_err(ret, "utimes", 2);
+
+    fprintf(stdout, "Testing utime");
+    ret = utime(testfile, NULL);
+    check_err(ret, "utime", 2);
+
+    fprintf(stdout, "Testing unlink");
+    ret = unlink(testfile);
+    check_err(ret, "unlink", 2);
+
+    fprintf(stdout, "Testing symlink");
+    ret = symlink(testfile, testfile);
+    check_err(ret, "symlink", 2);
+
+    fprintf(stdout, "Testing readlink");
+    ret = readlink(testfile, testfile, 0);
+    check_err(ret, "readlink", 2);
+
+    fprintf(stdout, "Testing realpath");
+    ret = 0;
+    res = realpath((const char *)testfile, testfile);
+    if (!res)
+        ret = -1;
+    check_err(ret, "realpath", 2);
+
+    fprintf(stdout, "Testing stat");
+    ret = stat(testfile, &buf);
+    check_err(ret, "stat", 1);
+
+    fprintf(stdout, "Testing lstat");
+    ret = lstat(testfile, &buf);
+    check_err(ret, "lstat", 1);
+
+    fprintf(stdout, "Testing statfs");
+    ret = statfs(testfile, &sbuf);
+    check_err(ret, "statfs", 2);
+
+    fprintf(stdout, "Testing statvfs");
+    ret = statvfs(testfile, &svbuf);
+    check_err(ret, "statvfs", 1);
+
+    fprintf(stdout, "Testing getxattr");
+    ret = getxattr(testfile, NULL, NULL, 0);
+    check_err(ret, "getxattr", 2);
+
+    fprintf(stdout, "Testing lgetxattr");
+    ret = lgetxattr(testfile, NULL, NULL, 0);
+    check_err(ret, "lgetxattr", 1);
+
+    fprintf(stdout, "Testing lchown");
+    ret = lchown(testfile, 0, 0);
+    check_err(ret, "lchown", 2);
+    return 0;
 }
 
-
 int
-run_dev_tests (char *testfile)
+run_dev_tests(char *testfile)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        assert (testfile);
+    assert(testfile);
 
-        fprintf (stdout, "Testing mknod");
-        ret = mknod (testfile, 0, 0);
-        check_err (ret, "mknod", 2);
+    fprintf(stdout, "Testing mknod");
+    ret = mknod(testfile, 0, 0);
+    check_err(ret, "mknod", 2);
 
-        fprintf (stdout, "Testing mkfifo");
-        ret = mkfifo (testfile, 0);
-        check_err (ret, "mkfifo", 2);
-        return 0;
+    fprintf(stdout, "Testing mkfifo");
+    ret = mkfifo(testfile, 0);
+    check_err(ret, "mkfifo", 2);
+    return 0;
 }
 
 int
-run_dir_tests (char *testpath)
+run_dir_tests(char *testpath)
 {
-        int             ret = -1;
-        DIR             *dh = NULL;
-        struct dirent   *dire = NULL;
-
-        assert (testpath);
-
-        fprintf (stdout, "Testing mkdir");
-        ret = mkdir (testpath, 0);
-        check_err (ret, "mkdir", 2);
-
-        fprintf (stdout, "Testing rmdir");
-        ret = rmdir (testpath);
-        check_err (ret, "rmdir", 2);
-
-        fprintf (stdout, "Testing opendir");
-        ret = 0;
-        dh = opendir (testpath);
-        if (!dh)
-                ret = -1;
-        check_err (ret, "opendir", 2);
-
-        fprintf (stdout, "Testing readdir");
-        ret = 0;
-        dire = readdir (dh);
-        if (!dire)
-                ret = -1;
-        check_err (ret, "readdir", 1);
-
-        fprintf (stdout, "Testing readdir_r");
-        ret = readdir_r (dh, dire, &dire);
-        check_err (ret, "readdir_r", 1);
-
-        fprintf (stdout, "Testing rewinddir");
-        rewinddir (dh);
-        check_err (-1, "rewinddir", 1);
-
-        fprintf (stdout, "Testing seekdir");
-        seekdir (dh, 0);
-        check_err (-1, "seekdir", 2);
-
-        fprintf (stdout, "Testing telldir");
-        ret = telldir (dh);
-        check_err (ret, "telldir", 2);
-
-        fprintf (stdout, "Testing closedir");
-        ret = closedir (dh);
-        check_err (ret, "closedir", 2);
-        return 0;
+    int ret = -1;
+    DIR *dh = NULL;
+    struct dirent *dire = NULL;
+
+    assert(testpath);
+
+    fprintf(stdout, "Testing mkdir");
+    ret = mkdir(testpath, 0);
+    check_err(ret, "mkdir", 2);
+
+    fprintf(stdout, "Testing rmdir");
+    ret = rmdir(testpath);
+    check_err(ret, "rmdir", 2);
+
+    fprintf(stdout, "Testing opendir");
+    ret = 0;
+    dh = opendir(testpath);
+    if (!dh)
+        ret = -1;
+    check_err(ret, "opendir", 2);
+
+    fprintf(stdout, "Testing readdir");
+    ret = 0;
+    dire = readdir(dh);
+    if (!dire)
+        ret = -1;
+    check_err(ret, "readdir", 1);
+
+    fprintf(stdout, "Testing readdir_r");
+    ret = readdir_r(dh, dire, &dire);
+    check_err(ret, "readdir_r", 1);
+
+    fprintf(stdout, "Testing rewinddir");
+    rewinddir(dh);
+    check_err(-1, "rewinddir", 1);
+
+    fprintf(stdout, "Testing seekdir");
+    seekdir(dh, 0);
+    check_err(-1, "seekdir", 2);
+
+    fprintf(stdout, "Testing telldir");
+    ret = telldir(dh);
+    check_err(ret, "telldir", 2);
+
+    fprintf(stdout, "Testing closedir");
+    ret = closedir(dh);
+    check_err(ret, "closedir", 2);
+    return 0;
 }
 
-
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        char            *testpath = NULL;
-        int             x = 0;
-
-        for (;x < argc; ++x) {
-                if (strcmp (argv[x], "--path") == 0) {
-                        testpath = argv[x+1];
-                        continue;
-                }
+    char *testpath = NULL;
+    int x = 0;
 
+    for (; x < argc; ++x) {
+        if (strcmp(argv[x], "--path") == 0) {
+            testpath = argv[x + 1];
+            continue;
         }
+    }
 
-        if (!testpath) {
-                fprintf (stderr, "--path not specified\n");
-                usage (stderr);
-                return -1;
-        }
+    if (!testpath) {
+        fprintf(stderr, "--path not specified\n");
+        usage(stderr);
+        return -1;
+    }
 
-        run_file_tests (testpath);
-        run_dir_tests (testpath);
-        run_attr_tests (testpath);
-        run_dev_tests (testpath);
+    run_file_tests(testpath);
+    run_dir_tests(testpath);
+    run_attr_tests(testpath);
+    run_dev_tests(testpath);
 
-        return 0;
+    return 0;
 }
-
-
diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c
index 4184079d043..509952b4180 100644
--- a/extras/test/open-fd-tests.c
+++ b/extras/test/open-fd-tests.c
@@ -4,61 +4,64 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <errno.h>
 #include <string.h>
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        int          ret      = -1;
-        int          fd       = 0;
-        char        *filename = NULL;
-        int          loop     = 0;
-        struct stat  stbuf    = {0,};
-        char         string[1024] = {0,};
+    int ret = -1;
+    int fd = 0;
+    char *filename = NULL;
+    int loop = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char string[1024] = {
+        0,
+    };
 
-        if (argc > 1)
-                filename = argv[1];
+    if (argc > 1)
+        filename = argv[1];
 
-        if (!filename)
-                filename = "temp-fd-test-file";
+    if (!filename)
+        filename = "temp-fd-test-file";
 
-        fd = open (filename, O_RDWR|O_CREAT|O_TRUNC);
-        if (fd < 0) {
-                fd = 0;
-                fprintf (stderr, "open failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        while (loop < 1000) {
-                /* Use it as a mechanism to test time delays */
-                memset (string, 0, 1024);
-                scanf ("%s", string);
+    fd = open(filename, O_RDWR | O_CREAT | O_TRUNC);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
 
-                ret = write (fd, string, strlen (string));
-                if (ret != strlen (string)) {
-                        fprintf (stderr, "write failed : %s (%s %d)\n",
-                                 strerror (errno), string, loop);
-                        goto out;
-                }
+    while (loop < 1000) {
+        /* Use it as a mechanism to test time delays */
+        memset(string, 0, 1024);
+        scanf("%s", string);
 
-                ret = write (fd, "\n", 1);
-                if (ret != 1) {
-                        fprintf (stderr, "write failed : %s (%d)\n",
-                                 strerror (errno), loop);
-                        goto out;
-                }
+        ret = write(fd, string, strlen(string));
+        if (ret != strlen(string)) {
+            fprintf(stderr, "write failed : %s (%s %d)\n", strerror(errno),
+                    string, loop);
+            goto out;
+        }
 
-                loop++;
+        ret = write(fd, "\n", 1);
+        if (ret != 1) {
+            fprintf(stderr, "write failed : %s (%d)\n", strerror(errno), loop);
+            goto out;
         }
 
-        fprintf (stdout, "finishing the test after %d loops\n", loop);
+        loop++;
+    }
+
+    fprintf(stdout, "finishing the test after %d loops\n", loop);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (fd)
-                close (fd);
+    if (fd)
+        close(fd);
 
-        return ret;
+    return ret;
 }
diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c
index 219dd6a2da2..1d9c125db67 100644
--- a/extras/test/test-ffop.c
+++ b/extras/test/test-ffop.c
@@ -3,777 +3,825 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <errno.h>
 #include <string.h>
 #include <dirent.h>
 
-int fd_based_fops_1 (char *filename); //for fd based fops after unlink
-int fd_based_fops_2 (char *filename); //for fd based fops before unlink
-int dup_fd_based_fops (char *filename); // fops based on fd after dup
-int path_based_fops (char *filename); //for fops based on path
-int dir_based_fops (char *filename); // for fops which operate on directory
-int link_based_fops (char *filename); //for fops which operate in link files (symlinks)
-int test_open_modes (char *filename); // to test open syscall with open modes available.
-int generic_open_read_write (char *filename, int flag); // generic function which does open write and read.
+int
+fd_based_fops_1(char *filename);  // for fd based fops after unlink
+int
+fd_based_fops_2(char *filename);  // for fd based fops before unlink
+int
+dup_fd_based_fops(char *filename);  // fops based on fd after dup
+int
+path_based_fops(char *filename);  // for fops based on path
+int
+dir_based_fops(char *filename);  // for fops which operate on directory
+int
+link_based_fops(
+    char *filename);  // for fops which operate in link files (symlinks)
+int
+test_open_modes(
+    char *filename);  // to test open syscall with open modes available.
+int
+generic_open_read_write(
+    char *filename,
+    int flag);  // generic function which does open write and read.
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        int   ret           = -1;
-        char  filename[255] = {0,};
-
-        if (argc > 1)
-                strcpy(filename, argv[1]);
-        else
-                strcpy(filename, "temp-xattr-test-file");
-
-        ret = fd_based_fops_1 (strcat(filename, "_1"));
-        if (ret < 0)
-                fprintf (stderr, "fd based file operation 1 failed\n");
-        else
-                fprintf (stdout, "fd based file operation 1 passed\n");
-
-        ret = fd_based_fops_2 (strcat(filename, "_2"));
-        if (ret < 0)
-                fprintf (stderr, "fd based file operation 2 failed\n");
-        else
-                fprintf (stdout, "fd based file operation 2 passed\n");
-
-        ret = dup_fd_based_fops (strcat (filename, "_3"));
-        if (ret < 0)
-                fprintf (stderr, "dup fd based file operation failed\n");
-        else
-                fprintf (stdout, "dup fd based file operation passed\n");
-
-        ret = path_based_fops (strcat (filename, "_4"));
-        if (ret < 0)
-                fprintf (stderr, "path based file operation failed\n");
-        else
-                fprintf (stdout, "path based file operation passed\n");
-
-        ret = dir_based_fops (strcat (filename, "_5"));
-        if (ret < 0)
-                fprintf (stderr, "directory based file operation failed\n");
-        else
-                fprintf (stdout, "directory based file operation passed\n");
-
-        ret = link_based_fops (strcat (filename, "_5"));
-        if (ret < 0)
-                fprintf (stderr, "link based file operation failed\n");
-        else
-                fprintf (stdout, "link based file operation passed\n");
-
-        ret = test_open_modes (strcat (filename, "_5"));
-        if (ret < 0)
-                fprintf (stderr, "testing modes of 'open' call failed\n");
-        else
-                fprintf (stdout, "testing modes of 'open' call passed\n");
+    int ret = -1;
+    char filename[255] = {
+        0,
+    };
+
+    if (argc > 1)
+        strcpy(filename, argv[1]);
+    else
+        strcpy(filename, "temp-xattr-test-file");
+
+    ret = fd_based_fops_1(strcat(filename, "_1"));
+    if (ret < 0)
+        fprintf(stderr, "fd based file operation 1 failed\n");
+    else
+        fprintf(stdout, "fd based file operation 1 passed\n");
+
+    ret = fd_based_fops_2(strcat(filename, "_2"));
+    if (ret < 0)
+        fprintf(stderr, "fd based file operation 2 failed\n");
+    else
+        fprintf(stdout, "fd based file operation 2 passed\n");
+
+    ret = dup_fd_based_fops(strcat(filename, "_3"));
+    if (ret < 0)
+        fprintf(stderr, "dup fd based file operation failed\n");
+    else
+        fprintf(stdout, "dup fd based file operation passed\n");
+
+    ret = path_based_fops(strcat(filename, "_4"));
+    if (ret < 0)
+        fprintf(stderr, "path based file operation failed\n");
+    else
+        fprintf(stdout, "path based file operation passed\n");
+
+    ret = dir_based_fops(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "directory based file operation failed\n");
+    else
+        fprintf(stdout, "directory based file operation passed\n");
+
+    ret = link_based_fops(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "link based file operation failed\n");
+    else
+        fprintf(stdout, "link based file operation passed\n");
+
+    ret = test_open_modes(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "testing modes of 'open' call failed\n");
+    else
+        fprintf(stdout, "testing modes of 'open' call passed\n");
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-fd_based_fops_1 (char *filename)
+fd_based_fops_1(char *filename)
 {
-        int         fd        = 0;
-        int         ret       = -1;
-        struct stat stbuf     = {0,};
-        char        wstr[50]  = {0,};
-        char        rstr[50]  = {0,};
-
-        fd = open (filename, O_RDWR|O_CREAT);
-        if (fd < 0) {
-                fd = 0;
-                fprintf (stderr, "open failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = unlink (filename);
-        if (ret < 0) {
-                fprintf (stderr, "unlink failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (wstr, "This is my string\n");
-        ret = write (fd, wstr, strlen(wstr));
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "write failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lseek (fd, 0, SEEK_SET);
-        if (ret < 0) {
-                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = read (fd, rstr, strlen(wstr));
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "read failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = memcmp (rstr, wstr, strlen (wstr));
-        if (ret != 0) {
-                ret = -1;
-                fprintf (stderr, "read returning junk\n");
-                goto out;
-        }
-
-        ret = ftruncate (fd, 0);
-        if (ret < 0) {
-                fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fstat (fd, &stbuf);
-        if (ret < 0) {
-                fprintf (stderr, "fstat failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchmod (fd, 0640);
-        if (ret < 0) {
-                fprintf (stderr, "fchmod failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchown (fd, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "fchown failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsync (fd);
-        if (ret < 0) {
-                fprintf (stderr, "fsync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fdatasync (fd);
-        if (ret < 0) {
-                fprintf (stderr, "fdatasync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = flistxattr (fd, NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "flistxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fremovexattr (fd, "trusted.xattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
+    int fd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = unlink(filename);
+    if (ret < 0) {
+        fprintf(stderr, "unlink failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(wstr, "This is my string\n");
+    ret = write(fd, wstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(fd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = ftruncate(fd, 0);
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fstat(fd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(fd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(fd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(fd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(fd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (fd)
-                close (fd);
+    if (fd)
+        close(fd);
 
-        return ret;
+    return ret;
 }
 
-
 int
-fd_based_fops_2 (char *filename)
+fd_based_fops_2(char *filename)
 {
-        int         fd    = 0;
-        int         ret   = -1;
-        struct stat stbuf = {0,};
-        char        wstr[50]  = {0,};
-        char        rstr[50]  = {0,};
-
-        fd = open (filename, O_RDWR|O_CREAT);
-        if (fd < 0) {
-                fd = 0;
-                fprintf (stderr, "open failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = ftruncate (fd, 0);
-
-        if (ret < 0) {
-                fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (wstr, "This is my second string\n");
-        ret = write (fd, wstr, strlen (wstr));
-        if (ret < 0) {
-                ret = -1;
-                fprintf (stderr, "write failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        lseek (fd, 0, SEEK_SET);
-        if (ret < 0) {
-                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = read (fd, rstr, strlen (wstr));
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "read failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = memcmp (rstr, wstr, strlen (wstr));
-        if (ret != 0) {
-                ret = -1;
-                fprintf (stderr, "read returning junk\n");
-                goto out;
-        }
-
-        ret = fstat (fd, &stbuf);
-        if (ret < 0) {
-                fprintf (stderr, "fstat failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchmod (fd, 0640);
-        if (ret < 0) {
-                fprintf (stderr, "fchmod failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchown (fd, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "fchown failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsync (fd);
-        if (ret < 0) {
-                fprintf (stderr, "fsync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fdatasync (fd);
-        if (ret < 0) {
-                fprintf (stderr, "fdatasync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = flistxattr (fd, NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "flistxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fremovexattr (fd, "trusted.xattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno));
-                goto out;
-        }
+    int fd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = ftruncate(fd, 0);
+
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(wstr, "This is my second string\n");
+    ret = write(fd, wstr, strlen(wstr));
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(fd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = fstat(fd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(fd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(fd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(fd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(fd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
 
 out:
-        if (fd)
-                close (fd);
-        unlink (filename);
+    if (fd)
+        close(fd);
+    unlink(filename);
 
-        return ret;
+    return ret;
 }
 
 int
-path_based_fops (char *filename)
+path_based_fops(char *filename)
 {
-        int         ret              = -1;
-        int         fd               = 0;
-        struct stat stbuf            = {0,};
-        char        newfilename[255] = {0,};
-
-        fd = creat (filename, 0644);
-        if (fd < 0) {
-                fprintf (stderr, "creat failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = truncate (filename, 0);
-        if (ret < 0) {
-                fprintf (stderr, "truncate failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = stat (filename, &stbuf);
-        if (ret < 0) {
-                fprintf (stderr, "stat failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = chmod (filename, 0640);
-        if (ret < 0) {
-                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = chown (filename, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "chown failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = setxattr (filename, "trusted.xattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "setxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = listxattr (filename, NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "listxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = getxattr (filename, "trusted.xattr-test", NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "getxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = removexattr (filename, "trusted.xattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "removexattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = access (filename, R_OK|W_OK);
-        if (ret < 0) {
-                fprintf (stderr, "access failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (newfilename, filename);
-        strcat(newfilename, "_new");
-        ret = rename (filename, newfilename);
-        if (ret < 0) {
-                fprintf (stderr, "rename failed: %s\n", strerror (errno));
-                goto out;
-        }
-        unlink (newfilename);
+    int ret = -1;
+    int fd = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char newfilename[255] = {
+        0,
+    };
+
+    fd = creat(filename, 0644);
+    if (fd < 0) {
+        fprintf(stderr, "creat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = truncate(filename, 0);
+    if (ret < 0) {
+        fprintf(stderr, "truncate failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = stat(filename, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "stat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chmod(filename, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chown(filename, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "chown failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = setxattr(filename, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "setxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = listxattr(filename, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "listxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = getxattr(filename, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "getxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = removexattr(filename, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "removexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = access(filename, R_OK | W_OK);
+    if (ret < 0) {
+        fprintf(stderr, "access failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newfilename, filename);
+    strcat(newfilename, "_new");
+    ret = rename(filename, newfilename);
+    if (ret < 0) {
+        fprintf(stderr, "rename failed: %s\n", strerror(errno));
+        goto out;
+    }
+    unlink(newfilename);
 
 out:
-        if (fd)
-                close (fd);
+    if (fd)
+        close(fd);
 
-        unlink (filename);
-        return ret;
+    unlink(filename);
+    return ret;
 }
 
 int
-dup_fd_based_fops (char *filename)
+dup_fd_based_fops(char *filename)
 {
-        int         fd        = 0;
-        int         newfd     = 0;
-        int         ret       = -1;
-        struct stat stbuf     = {0,};
-        char        wstr[50]  = {0,};
-        char        rstr[50]  = {0,};
-
-        fd = open (filename, O_RDWR|O_CREAT);
-        if (fd < 0) {
-                fd = 0;
-                fprintf (stderr, "open failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        newfd = dup (fd);
-        if (newfd < 0) {
-                ret = -1;
-                fprintf (stderr, "dup failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        close (fd);
-
-        strcpy (wstr, "This is my string\n");
-        ret = write (newfd, wstr, strlen(wstr));
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "write failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lseek (newfd, 0, SEEK_SET);
-        if (ret < 0) {
-                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = read (newfd, rstr, strlen(wstr));
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "read failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = memcmp (rstr, wstr, strlen (wstr));
-        if (ret != 0) {
-                ret = -1;
-                fprintf (stderr, "read returning junk\n");
-                goto out;
-        }
-
-        ret = ftruncate (newfd, 0);
-        if (ret < 0) {
-                fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fstat (newfd, &stbuf);
-        if (ret < 0) {
-                fprintf (stderr, "fstat failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchmod (newfd, 0640);
-        if (ret < 0) {
-                fprintf (stderr, "fchmod failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fchown (newfd, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "fchown failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsync (newfd);
-        if (ret < 0) {
-                fprintf (stderr, "fsync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fsetxattr (newfd, "trusted.xattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fdatasync (newfd);
-        if (ret < 0) {
-                fprintf (stderr, "fdatasync failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = flistxattr (newfd, NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "flistxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fgetxattr (newfd, "trusted.xattr-test", NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = fremovexattr (newfd, "trusted.xattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
+    int fd = 0;
+    int newfd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    newfd = dup(fd);
+    if (newfd < 0) {
+        ret = -1;
+        fprintf(stderr, "dup failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    close(fd);
+
+    strcpy(wstr, "This is my string\n");
+    ret = write(newfd, wstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lseek(newfd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(newfd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = ftruncate(newfd, 0);
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fstat(newfd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(newfd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(newfd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(newfd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(newfd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(newfd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(newfd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(newfd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(newfd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (newfd)
-                close (newfd);
-        ret = unlink (filename);
-        if (ret < 0)
-                fprintf (stderr, "unlink failed : %s\n", strerror (errno));
+    if (newfd)
+        close(newfd);
+    ret = unlink(filename);
+    if (ret < 0)
+        fprintf(stderr, "unlink failed : %s\n", strerror(errno));
 
-        return ret;
+    return ret;
 }
 
 int
-dir_based_fops (char *dirname)
+dir_based_fops(char *dirname)
 {
-        int            ret           = -1;
-        DIR           *dp            = NULL;
-        char           buff[255]     = {0,};
-        struct dirent *dbuff         = {0,};
-        struct stat    stbuff        = {0,};
-        char           newdname[255] = {0,};
-        char          *cwd           = NULL;
-
-        ret = mkdir (dirname, 0755);
-        if (ret < 0) {
-                fprintf (stderr, "mkdir failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        dp = opendir (dirname);
-        if (dp == NULL) {
-                fprintf (stderr, "opendir failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        dbuff = readdir (dp);
-        if (NULL == dbuff) {
-                fprintf (stderr, "readdir failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = closedir (dp);
-        if (ret < 0) {
-                fprintf (stderr, "closedir failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = stat (dirname, &stbuff);
-        if (ret < 0) {
-                fprintf (stderr, "stat failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = chmod (dirname, 0744);
-        if (ret < 0) {
-                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = chown (dirname, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = setxattr (dirname, "trusted.xattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "setxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = listxattr (dirname, NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "listxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = getxattr (dirname, "trusted.xattr-test", NULL, 0);
-        if (ret <= 0) {
-                ret = -1;
-                fprintf (stderr, "getxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = removexattr (dirname, "trusted.xattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "removexattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (newdname, dirname);
-        strcat (newdname, "/../");
-        ret = chdir (newdname);
-        if (ret < 0) {
-                fprintf (stderr, "chdir failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        cwd = getcwd (buff, 255);
-        if (NULL == cwd) {
-                fprintf (stderr, "getcwd failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (newdname, dirname);
-        strcat (newdname, "new");
-        ret = rename (dirname, newdname);
-        if (ret < 0) {
-                fprintf (stderr, "rename failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = rmdir (newdname);
-        if (ret < 0) {
-                fprintf (stderr, "rmdir failed: %s\n", strerror (errno));
-                return ret;
-        }
+    int ret = -1;
+    DIR *dp = NULL;
+    char buff[255] = {
+        0,
+    };
+    struct dirent *dbuff = {
+        0,
+    };
+    struct stat stbuff = {
+        0,
+    };
+    char newdname[255] = {
+        0,
+    };
+    char *cwd = NULL;
+
+    ret = mkdir(dirname, 0755);
+    if (ret < 0) {
+        fprintf(stderr, "mkdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    dp = opendir(dirname);
+    if (dp == NULL) {
+        fprintf(stderr, "opendir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    dbuff = readdir(dp);
+    if (NULL == dbuff) {
+        fprintf(stderr, "readdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = closedir(dp);
+    if (ret < 0) {
+        fprintf(stderr, "closedir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = stat(dirname, &stbuff);
+    if (ret < 0) {
+        fprintf(stderr, "stat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chmod(dirname, 0744);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chown(dirname, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = setxattr(dirname, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "setxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = listxattr(dirname, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "listxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = getxattr(dirname, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "getxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = removexattr(dirname, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "removexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newdname, dirname);
+    strcat(newdname, "/../");
+    ret = chdir(newdname);
+    if (ret < 0) {
+        fprintf(stderr, "chdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    cwd = getcwd(buff, 255);
+    if (NULL == cwd) {
+        fprintf(stderr, "getcwd failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newdname, dirname);
+    strcat(newdname, "new");
+    ret = rename(dirname, newdname);
+    if (ret < 0) {
+        fprintf(stderr, "rename failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = rmdir(newdname);
+    if (ret < 0) {
+        fprintf(stderr, "rmdir failed: %s\n", strerror(errno));
+        return ret;
+    }
 
 out:
-        rmdir (dirname);
-        return ret;
+    rmdir(dirname);
+    return ret;
 }
 
 int
-link_based_fops (char *filename)
+link_based_fops(char *filename)
 {
-        int         ret           = -1;
-        int         fd            = 0;
-        char        newname[255]  = {0,};
-        char        linkname[255] = {0,};
-        struct stat lstbuf        = {0,};
-
-        fd = creat (filename, 0644);
-        if (fd < 0) {
-                fd = 0;
-                fprintf (stderr, "creat failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (newname, filename);
-        strcat (newname, "_hlink");
-        ret = link (filename, newname);
-        if (ret < 0) {
-                fprintf (stderr, "link failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = unlink (filename);
-        if (ret < 0) {
-                fprintf (stderr, "unlink failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        strcpy (linkname, filename);
-        strcat (linkname, "_slink");
-        ret = symlink (newname, linkname);
-        if (ret < 0) {
-                fprintf (stderr, "symlink failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lstat (linkname, &lstbuf);
-        if (ret < 0) {
-                fprintf (stderr, "lstbuf failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lchown (linkname, 10001, 10001);
-        if (ret < 0) {
-                fprintf (stderr, "lchown failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lsetxattr (linkname, "trusted.lxattr-test", "working", 8, 0);
-        if (ret < 0) {
-                fprintf (stderr, "lsetxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = llistxattr (linkname, NULL, 0);
-        if (ret < 0) {
-                ret = -1;
-                fprintf (stderr, "llistxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lgetxattr (linkname, "trusted.lxattr-test", NULL, 0);
-        if (ret < 0) {
-                ret = -1;
-                fprintf (stderr, "lgetxattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
-        ret = lremovexattr (linkname, "trusted.lxattr-test");
-        if (ret < 0) {
-                fprintf (stderr, "lremovexattr failed: %s\n", strerror (errno));
-                goto out;
-        }
-
+    int ret = -1;
+    int fd = 0;
+    char newname[255] = {
+        0,
+    };
+    char linkname[255] = {
+        0,
+    };
+    struct stat lstbuf = {
+        0,
+    };
+
+    fd = creat(filename, 0644);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "creat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newname, filename);
+    strcat(newname, "_hlink");
+    ret = link(filename, newname);
+    if (ret < 0) {
+        fprintf(stderr, "link failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = unlink(filename);
+    if (ret < 0) {
+        fprintf(stderr, "unlink failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(linkname, filename);
+    strcat(linkname, "_slink");
+    ret = symlink(newname, linkname);
+    if (ret < 0) {
+        fprintf(stderr, "symlink failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lstat(linkname, &lstbuf);
+    if (ret < 0) {
+        fprintf(stderr, "lstbuf failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lchown(linkname, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "lchown failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lsetxattr(linkname, "trusted.lxattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "lsetxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = llistxattr(linkname, NULL, 0);
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "llistxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lgetxattr(linkname, "trusted.lxattr-test", NULL, 0);
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "lgetxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lremovexattr(linkname, "trusted.lxattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "lremovexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
 
 out:
-        if (fd)
-                close(fd);
-        unlink (linkname);
-        unlink (newname);
+    if (fd)
+        close(fd);
+    unlink(linkname);
+    unlink(newname);
 }
 
 int
-test_open_modes (char *filename)
+test_open_modes(char *filename)
 {
-        int ret = -1;
-
-        ret = generic_open_read_write (filename, O_CREAT|O_WRONLY);
-        if (3 != ret) {
-               fprintf (stderr, "flag O_CREAT|O_WRONLY failed: \n");
-               goto out;
-        }
-
-        ret = generic_open_read_write (filename, O_CREAT|O_RDWR);
-        if (ret != 0) {
-               fprintf (stderr, "flag O_CREAT|O_RDWR failed\n");
-               goto out;
-        }
-
-        ret = generic_open_read_write (filename, O_CREAT|O_RDONLY);
-        if (ret != 0) {
-                fprintf (stderr, "flag O_CREAT|O_RDONLY failed\n");
-                goto out;
-        }
-
-        ret = creat (filename, 0644);
-        close (ret);
-        ret = generic_open_read_write (filename, O_WRONLY);
-        if (3 != ret) {
-               fprintf (stderr, "flag O_WRONLY failed\n");
-               goto out;
-        }
-
-        ret = creat (filename, 0644);
-        close (ret);
-        ret = generic_open_read_write (filename, O_RDWR);
-        if (0 != ret) {
-               fprintf (stderr, "flag O_RDWR failed\n");
-               goto out;
-        }
-
-        ret = creat (filename, 0644);
-        close (ret);
-        ret = generic_open_read_write (filename, O_RDONLY);
-        if (0 != ret) {
-               fprintf (stderr, "flag O_RDONLY failed\n");
-               goto out;
-        }
-
-        ret = creat (filename, 0644);
-        close (ret);
-        ret = generic_open_read_write (filename, O_TRUNC|O_WRONLY);
-        if (3 != ret) {
-               fprintf (stderr, "flag O_TRUNC|O_WRONLY failed\n");
-               goto out;
-        }
+    int ret = -1;
+
+    ret = generic_open_read_write(filename, O_CREAT | O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_WRONLY failed: \n");
+        goto out;
+    }
+
+    ret = generic_open_read_write(filename, O_CREAT | O_RDWR);
+    if (ret != 0) {
+        fprintf(stderr, "flag O_CREAT|O_RDWR failed\n");
+        goto out;
+    }
+
+    ret = generic_open_read_write(filename, O_CREAT | O_RDONLY);
+    if (ret != 0) {
+        fprintf(stderr, "flag O_CREAT|O_RDONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_WRONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_RDWR);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_RDWR failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_RDONLY);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_RDONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_TRUNC | O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_TRUNC|O_WRONLY failed\n");
+        goto out;
+    }
 
 #if 0 /* undefined behaviour, unable to reliably test */
         ret = creat (filename, 0644);
@@ -785,84 +833,88 @@ test_open_modes (char *filename)
         }
 #endif
 
-        ret = generic_open_read_write (filename, O_CREAT|O_RDWR|O_SYNC);
-        if (0 != ret) {
-               fprintf (stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n");
-               goto out;
-        }
+    ret = generic_open_read_write(filename, O_CREAT | O_RDWR | O_SYNC);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n");
+        goto out;
+    }
 
-        ret = creat (filename, 0644);
-        close (ret);
-        ret = generic_open_read_write (filename, O_CREAT|O_EXCL);
-        if (0 != ret) {
-                fprintf (stderr, "flag O_CREAT|O_EXCL failed\n");
-                goto out;
-        }
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_CREAT | O_EXCL);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_EXCL failed\n");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int generic_open_read_write (char *filename, int flag)
+int
+generic_open_read_write(char *filename, int flag)
 {
-        int  fd          = 0;
-        int  ret         = -1;
-        char wstring[50] = {0,};
-        char rstring[50] = {0,};
-
-        fd = open (filename, flag);
-        if (fd < 0) {
-                if (flag == O_CREAT|O_EXCL && errno == EEXIST) {
-                        unlink (filename);
-                        return 0;
-                }
-                else {
-                        fd = 0;
-                        fprintf (stderr, "open failed: %s\n", strerror (errno));
-                        return 1;
-                }
-        }
-
-        strcpy (wstring, "My string to write\n");
-        ret = write (fd, wstring, strlen(wstring));
-        if (ret <= 0) {
-                if (errno != EBADF) {
-                        fprintf (stderr, "write failed: %s\n", strerror (errno));
-                        close (fd);
-                        unlink(filename);
-                        return 2;
-                }
-        }
-
-        ret = lseek (fd, 0, SEEK_SET);
-        if (ret < 0) {
-                close (fd);
-                unlink(filename);
-                return 4;
-        }
-
-        ret = read (fd, rstring, strlen(wstring));
-        if (ret < 0) {
-                close (fd);
-                unlink (filename);
-                return 3;
-        }
-
-        /* Compare the rstring with wstring. But we do not want to return
-         * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or
-         * O_TRUNC|O_RDONLY. Because in that case we are not writing
-         * anything to the file.*/
-
-        ret = memcmp (wstring, rstring, strlen (wstring));
-        if (0 != ret  && !(flag == O_CREAT|O_RDONLY || flag == O_RDONLY ||\
-            flag == O_TRUNC|O_RDONLY)) {
-                fprintf (stderr, "read is returning junk\n");
-                close (fd);
-                unlink (filename);
-                return 4;
-        }
-
-        close (fd);
-        unlink (filename);
-        return 0;
+    int fd = 0;
+    int ret = -1;
+    char wstring[50] = {
+        0,
+    };
+    char rstring[50] = {
+        0,
+    };
+
+    fd = open(filename, flag);
+    if (fd < 0) {
+        if (flag == O_CREAT | O_EXCL && errno == EEXIST) {
+            unlink(filename);
+            return 0;
+        } else {
+            fd = 0;
+            fprintf(stderr, "open failed: %s\n", strerror(errno));
+            return 1;
+        }
+    }
+
+    strcpy(wstring, "My string to write\n");
+    ret = write(fd, wstring, strlen(wstring));
+    if (ret <= 0) {
+        if (errno != EBADF) {
+            fprintf(stderr, "write failed: %s\n", strerror(errno));
+            close(fd);
+            unlink(filename);
+            return 2;
+        }
+    }
+
+    ret = lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        close(fd);
+        unlink(filename);
+        return 4;
+    }
+
+    ret = read(fd, rstring, strlen(wstring));
+    if (ret < 0) {
+        close(fd);
+        unlink(filename);
+        return 3;
+    }
+
+    /* Compare the rstring with wstring. But we do not want to return
+     * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or
+     * O_TRUNC|O_RDONLY. Because in that case we are not writing
+     * anything to the file.*/
+
+    ret = memcmp(wstring, rstring, strlen(wstring));
+    if (0 != ret && !(flag == O_CREAT | O_RDONLY || flag == O_RDONLY ||
+                      flag == O_TRUNC | O_RDONLY)) {
+        fprintf(stderr, "read is returning junk\n");
+        close(fd);
+        unlink(filename);
+        return 4;
+    }
+
+    close(fd);
+    unlink(filename);
+    return 0;
 }
diff --git a/extras/thin-arbiter/setup-thin-arbiter.sh b/extras/thin-arbiter/setup-thin-arbiter.sh
new file mode 100755
index 00000000000..0681b30ef3f
--- /dev/null
+++ b/extras/thin-arbiter/setup-thin-arbiter.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+#  Copyright (c) 2018-2019 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+
+#  This tool has been developed to setup thin-arbiter process on a node.
+#  Seting up a thin arbiter process involves following files -
+#  1 - thin-arbiter.vol
+#  Thin-arbiter (TA) process will use the graph in this file to load the
+#  required translators.
+#  2 - gluster-ta-volume.service (generated by gluster-ta-volume.service.in)
+#  TA process would be running as systemd service.
+#
+#  TA process uses a location to save TA id files for every subvolume.
+#  This location can be taken as input from user. Once provided and the
+#  TA process is started on a node, it can not be changed using this
+#  script or by any other mean. The same location should be used in
+#  the gluster CLI when creating thin-arbiter volumes.
+
+MYPATH=`dirname $0`
+
+volloc="/var/lib/glusterd/thin-arbiter"
+mkdir -p $volloc
+
+if [ -f /etc/glusterfs/thin-arbiter.vol ]; then
+    volfile=/etc/glusterfs/thin-arbiter.vol
+else
+    volfile=$MYPATH/thin-arbiter.vol
+fi
+
+tafile="$volloc/thin-arbiter.vol"
+
+
+help () {
+    echo " "
+    echo '  This tool helps to setup thin-arbiter (TA) process on a node.
+  TA process uses a location to save TA id files for every subvolume.
+  This location can be taken as input from user. Once provided and the
+  TA process is started on a node, it can not be changed using this script
+  or by any other mean. The same location should be used in gluster CLI
+  when creating thin-arbiter volumes.
+
+  usage: setup-thin-arbiter.sh [-s] [-h]
+    options:
+    -s - Setup thin-arbiter file path and start process
+    -h - Show this help message and exit
+'
+}
+
+volfile_set_brick_path () {
+    while read -r line
+    do
+        dir=`echo "$line" | cut -d' ' -f 2`
+        if [ "$dir" = "directory" ]; then
+            bpath=`echo "$line" | cut -d' ' -f 3`
+            sed -i -- 's?'$bpath'?'$1'?g' $tafile
+            return
+        fi
+    done < $tafile
+}
+
+check_ta_proc () {
+    pro=`ps aux | grep thin-arbiter.vol | grep "volfile-id"`
+    if [ "${pro}" = '' ]; then
+            echo ""
+    else
+        curr_loc=`cat $volloc/thin-arbiter.vol | grep option | grep directory`
+        loc=`echo "${curr_loc##* }"`
+        echo "******************************************************"
+        echo "Error:"
+        echo "Thin-arbiter process is running with thin-arbiter path = $loc"
+        echo "Can not change TA path on this host now."
+        echo "$pro"
+        echo "******************************************************"
+        exit 1
+    fi
+}
+
+getpath () {
+    check_ta_proc
+    echo "******************************************************"
+    echo "User will be required to enter a path/folder for arbiter volume."
+    echo "Please note that this path will be used for ALL VOLUMES using this"
+    echo "node to host thin-arbiter. After setting, if a volume"
+    echo "has been created using this host and path then path for"
+    echo "thin-arbiter can not be changed "
+    echo "******************************************************"
+    echo " "
+    while true;
+    do
+        echo -n "Enter brick path for thin arbiter volumes: "
+        echo " "
+        read tapath
+        if [ "${tapath}" = '' ]; then
+            echo "Please enter valid path"
+            continue
+        else
+            echo "Entered brick path : $tapath "
+            echo "Please note that this brick path will be used for ALL"
+            echo "VOLUMES using this node to host thin-arbiter brick"
+            echo -n "Want to continue? (y/N): "
+            echo " "
+            read cont
+
+            if [ "${cont}" = 'N' ] || [ "${cont}" = 'n' ]; then
+                exit 0
+            else
+                break
+            fi
+        fi
+    done
+}
+
+setup () {
+    getpath
+    mkdir -p $tapath/.glusterfs/indices
+    if [ -d $tapath/.glusterfs/indices ]; then
+        echo " "
+    else
+        echo "Could not create $tapath/.glusterfs/indices directory, check provided ta path."
+        exit 1
+    fi
+
+    cp -f --backup --suffix=_old $volfile $volloc/thin-arbiter.vol
+    volfile_set_brick_path "$tapath"
+
+    echo "Directory path to be used for thin-arbiter volume is: $tapath"
+    echo " "
+    echo "========================================================"
+
+    if [ -f /usr/lib/systemd/system/gluster-ta-volume.service ]; then
+        echo "Starting thin-arbiter process"
+    else
+        cp $MYPATH/../systemd/gluster-ta-volume.service /etc/systemd/system/
+        echo "Starting thin-arbiter process"
+        chmod 0644 /etc/systemd/system/gluster-ta-volume.service
+    fi
+
+    systemctl daemon-reload
+    systemctl enable gluster-ta-volume
+    systemctl stop gluster-ta-volume
+    systemctl start gluster-ta-volume
+
+    if [ $? == 0 ]; then
+        echo "thin-arbiter process has been setup and running"
+    else
+        echo "Failed to setup thin arbiter"
+        exit 1
+    fi
+
+}
+
+main()
+{
+
+    if [ "$#" -ne 1 ]; then
+        help
+        exit 0
+    fi
+
+    while getopts "sh" opt; do
+        case $opt in
+            h)
+                help
+                exit 0
+                ;;
+            s)
+                setup
+                exit 0
+                ;;
+            *)
+                help
+                exit 0
+                ;;
+        esac
+    done
+}
+
+main "$@"
diff --git a/extras/thin-arbiter/thin-arbiter.vol b/extras/thin-arbiter/thin-arbiter.vol
new file mode 100644
index 00000000000..c76babc7b3c
--- /dev/null
+++ b/extras/thin-arbiter/thin-arbiter.vol
@@ -0,0 +1,57 @@
+volume ta-posix
+    type storage/posix
+    option directory /mnt/thin-arbiter
+end-volume
+
+volume ta-thin-arbiter
+    type features/thin-arbiter
+    subvolumes ta-posix
+end-volume
+
+volume ta-locks
+    type features/locks
+    option notify-contention yes
+    subvolumes ta-thin-arbiter
+end-volume
+
+volume ta-upcall
+    type features/upcall
+    option cache-invalidation off
+    subvolumes ta-locks
+end-volume
+
+volume ta-io-threads
+    type performance/io-threads
+    subvolumes ta-upcall
+end-volume
+
+volume ta-index
+    type features/index
+    option xattrop-pending-watchlist trusted.afr.ta-
+    option xattrop-dirty-watchlist trusted.afr.dirty
+    option index-base /mnt/thin-arbiter/.glusterfs/indices
+    subvolumes ta-io-threads
+end-volume
+
+volume /mnt/thin-arbiter
+    type debug/io-stats
+    option count-fop-hits off
+    option latency-measurement off
+    option unique-id /mnt/thin-arbiter
+    subvolumes ta-index
+end-volume
+
+volume ta-server
+    type protocol/server
+    option transport.listen-backlog 10
+    option transport.socket.keepalive-count 9
+    option transport.socket.keepalive-interval 2
+    option transport.socket.keepalive-time 20
+    option transport.tcp-user-timeout 0
+    option transport.socket.keepalive 1
+    option auth.addr./mnt/thin-arbiter.allow *
+    option auth-path /mnt/thin-arbiter
+    option transport.address-family inet
+    option transport-type tcp
+    subvolumes /mnt/thin-arbiter
+end-volume
diff --git a/extras/volfilter.py b/extras/volfilter.py
index 0ca456a7882..5558a1beff4 100644
--- a/extras/volfilter.py
+++ b/extras/volfilter.py
@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU General Public License * along
 # with HekaFS.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import print_function
 import copy
 import string
 import sys
@@ -35,7 +36,7 @@ good_xlators = [
 	"storage/posix",
 ]
 
-def copy_stack (old_xl,suffix,recursive=False):
+def copy_stack (old_xl, suffix, recursive=False):
 	if recursive:
 		new_name = old_xl.name + "-" + suffix
 	else:
@@ -45,7 +46,7 @@ def copy_stack (old_xl,suffix,recursive=False):
 	# The results with normal assignment here are . . . amusing.
 	new_xl.opts = copy.deepcopy(old_xl.opts)
 	for sv in old_xl.subvols:
-		new_xl.subvols.append(copy_stack(sv,suffix,True))
+		new_xl.subvols.append(copy_stack(sv, suffix, True))
 	# Patch up the path at the bottom.
 	if new_xl.type == "storage/posix":
 		new_xl.opts["directory"] += ("/" + suffix)
@@ -63,10 +64,10 @@ def cleanup (parent, graph):
 			parent.opts["transport-type"] = "ssl"
 		sv = []
 		for child in parent.subvols:
-			sv.append(cleanup(child,graph))
+			sv.append(cleanup(child, graph))
 		parent.subvols = sv
 	else:
-		parent = cleanup(parent.subvols[0],graph)
+		parent = cleanup(parent.subvols[0], graph)
 	return parent
 
 class Translator:
@@ -82,8 +83,8 @@ class Translator:
 def load (path):
 	# If it's a string, open it; otherwise, assume it's already a
 	# file-like object (most notably from urllib*).
-	if type(path) in types.StringTypes:
-		fp = file(path,"r")
+	if type(path) in (str,):
+		fp = file(path, "r")
 	else:
 		fp = path
 	all_xlators = {}
@@ -98,16 +99,16 @@ def load (path):
 			continue
 		if text[0] == "volume":
 			if xlator:
-				raise RuntimeError, "nested volume definition"
+				raise RuntimeError("nested volume definition")
 			xlator = Translator(text[1])
 			continue
 		if not xlator:
-			raise RuntimeError, "text outside volume definition"
+			raise RuntimeError("text outside volume definition")
 		if text[0] == "type":
 			xlator.type = text[1]
 			continue
 		if text[0] == "option":
-			xlator.opts[text[1]] = string.join(text[2:])
+			xlator.opts[text[1]] = ''.join(text[2:])
 			continue
 		if text[0] == "subvolumes":
 			for sv in text[1:]:
@@ -118,25 +119,25 @@ def load (path):
 			last_xlator = xlator
 			xlator = None
 			continue
-		raise RuntimeError, "unrecognized keyword %s" % text[0]
+		raise RuntimeError("unrecognized keyword %s" % text[0])
 	if xlator:
-		raise RuntimeError, "unclosed volume definition"
+		raise RuntimeError("unclosed volume definition")
 	return all_xlators, last_xlator
 
 def generate (graph, last, stream=sys.stdout):
 	for sv in last.subvols:
 		if not sv.dumped:
-			generate(graph,sv,stream)
-			print >> stream, ""
+			generate(graph, sv, stream)
+			print("", file=stream)
 			sv.dumped = True
-	print >> stream, "volume %s" % last.name
-	print >> stream, "    type %s" % last.type
-	for k, v in last.opts.iteritems():
-		print >> stream, "    option %s %s" % (k, v)
+	print("volume %s" % last.name, file=stream)
+	print("    type %s" % last.type, file=stream)
+	for k, v in last.opts.items():
+		print("    option %s %s" % (k, v), file=stream)
 	if last.subvols:
-		print >> stream, "    subvolumes %s" % string.join(
-			[ sv.name for sv in last.subvols ])
-	print >> stream, "end-volume"
+		print("    subvolumes %s" % ''.join(
+			[ sv.name for sv in last.subvols ]), file=stream)
+	print("end-volume", file=stream)
 
 def push_filter (graph, old_xl, filt_type, opts={}):
 	suffix = "-" + old_xl.type.split("/")[1]
@@ -156,7 +157,7 @@ def push_filter (graph, old_xl, filt_type, opts={}):
 
 def delete (graph, victim):
 	if len(victim.subvols) != 1:
-		raise RuntimeError, "attempt to delete non-unary translator"
+		raise RuntimeError("attempt to delete non-unary translator")
 	for xl in graph.itervalues():
 		while xl.subvols.count(victim):
 			i = xl.subvols.index(victim)
@@ -164,4 +165,4 @@ def delete (graph, victim):
 
 if __name__ == "__main__":
 	graph, last = load(sys.argv[1])
-	generate(graph,last)
+	generate(graph, last)
diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases
index e19b99c79c8..901c12418e3 100644
--- a/extras/who-wrote-glusterfs/gitdm.aliases
+++ b/extras/who-wrote-glusterfs/gitdm.aliases
@@ -16,11 +16,13 @@ anush@gluster.com	ashetty@redhat.com
 csaba@gluster.com	csaba@redhat.com
 csaba@lowlife.hu	csaba@redhat.com
 csaba@zresearch.com	csaba@redhat.com
+gd@samba.org            gd@redhat.com
 harsha@gluster.com	fharshav@redhat.com
 harsha@zresearch.com	fharshav@redhat.com
 harsha@dev.gluster.com	fharshav@redhat.com
 harsha@harshavardhana.net	fharshav@redhat.com
 jclift@redhat.com       jclift@gluster.org
+kkeithle@linux.keithley.org	kkeithle@redhat.com
 kkeithle@f16node1.kkeithle.usersys.redhat.com	kkeithle@redhat.com
 kaushal@gluster.com	kaushal@redhat.com
 kaushikbv@gluster.com	kbudiger@redhat.com
@@ -32,6 +34,9 @@ me@louiszuckerman.com	louiszuckerman@gmail.com
 msvbhat@gmail.com	vbhat@redhat.com
 nullpai@gmail.com       ppai@redhat.com
 vishwanath@gluster.com	vbhat@redhat.com
+obnox@samba.org         madam@redhat.com
+oleksandr@natalenko.name        o.natalenko@lanet.ua
+patrick@puiterwijk.org  puiterwijk@fedoraproject.org
 pavan@dev.gluster.com	pavan@gluster.com
 zaitcev@yahoo.com	zaitcev@kotori.zaitcev.us
 pranithk@gluster.com	pkarampu@redhat.com
@@ -41,6 +46,8 @@ raghavendra@zresearch.com	rgowdapp@redhat.com
 rahulcssjce@gmail.com	rahulcs@redhat.com
 rajesh@gluster.com	rajesh@redhat.com
 rajesh.amaravathi@gmail.com	rajesh@redhat.com
+root@ravi2.(none)       ravishankar@redhat.com
+sabansal@localhost.localdomain  sabansal@redhat.com
 shehjart@zresearch.com	shehjart@gluster.com
 venky@gluster.com	vshankar@redhat.com
 vijay@gluster.com	vbellur@redhat.com
@@ -48,3 +55,4 @@ vijay@dev.gluster.com	vbellur@redhat.com
 vijaykumar.koppad@gmail.com	vkoppad@redhat.com
 vikas@zresearch.com	vikas@gluster.com
 shishirng@gluster.com	sgowda@redhat.com
+potatogim@potatogim.net	potatogim@gluesys.com
diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map
index 39526f0f99c..7cd2bbd605b 100644
--- a/extras/who-wrote-glusterfs/gitdm.domain-map
+++ b/extras/who-wrote-glusterfs/gitdm.domain-map
@@ -2,15 +2,28 @@
 # Here is a set of mappings of domain names onto employer names.
 #
 active.by       ActiveCloud
+appeartv.com    Appear TV
 cern.ch         CERN
+cmss.chinamobile.com    China Mobile(Suzhou) Software Technology
+datalab.es      DataLab S.L.
+fb.com          Facebook
+fedoraproject.org       Fedora Project
 gluster.com     Red Hat
-gmail.com       (unknown)
+gmail.com       (personal contributions)
 gooddata.com	GoodData
 hastexo.com     hastexo
+horde.com       (personal contributions)
 ibm.com		IBM
+io.com          IO
+lanet.ua        Lanet Network
 linbit.com	LINBIT
+nectec.or.th    NECTEC
 netbsd.org	NetBSD
 netdirect.ca	Net Direct
+nokia.com       Nokia
 redhat.com	Red Hat
 stepping-stone.ch       stepping stone GmbH
+xtaotech.com    XTAO Co.
+yahoo.in        (personal contributions)
 zresearch.com   Red Hat
+gluesys.com     Gluesys
diff --git a/geo-replication/Makefile.am b/geo-replication/Makefile.am
index 556951d9fb7..591b23d0eaf 100644
--- a/geo-replication/Makefile.am
+++ b/geo-replication/Makefile.am
@@ -1,3 +1,8 @@
 SUBDIRS = syncdaemon src
 
 CLEANFILES =
+
+EXTRA_DIST = gsyncd.conf.in
+
+gsyncdconfdir = $(sysconfdir)/glusterfs/
+gsyncdconf_DATA = gsyncd.conf
diff --git a/geo-replication/gsyncd.conf.in b/geo-replication/gsyncd.conf.in
new file mode 100644
index 00000000000..9688c79fab7
--- /dev/null
+++ b/geo-replication/gsyncd.conf.in
@@ -0,0 +1,349 @@
+[__meta__]
+version = 4.0
+
+[master-bricks]
+configurable=false
+
+[slave-bricks]
+configurable=false
+
+[master-volume-id]
+configurable=false
+
+[slave-volume-id]
+configurable=false
+
+[master-replica-count]
+configurable=false
+type=int
+value=1
+
+[master-disperse-count]
+configurable=false
+type=int
+value=1
+
+[master-distribution-count]
+configurable=false
+type=int
+value=1
+
+[glusterd-workdir]
+value = @GLUSTERD_WORKDIR@
+
+[gluster-logdir]
+value = /var/log/glusterfs
+
+[gluster-rundir]
+value = /var/run/gluster
+
+[gsyncd-miscdir]
+value = /var/lib/misc/gluster/gsyncd
+
+[stime-xattr-prefix]
+value=
+
+[checkpoint]
+value=0
+help=Set Checkpoint
+validation=unixtime
+type=int
+
+[gluster-cli-options]
+value=
+help=Gluster CLI Options
+
+[pid-file]
+value=${gluster_rundir}/gsyncd-${master}-${primary_slave_host}-${slavevol}.pid
+configurable=false
+template = true
+help=PID file path
+
+[state-file]
+value=${glusterd_workdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/monitor.status
+configurable=false
+template=true
+help=Status File path
+
+[georep-session-working-dir]
+value=${glusterd_workdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/
+template=true
+help=Session Working directory
+configurable=false
+
+[access-mount]
+value=false
+type=bool
+validation=bool
+help=Do not lazy unmount the master volume. This allows admin to access the mount for debugging.
+
+[slave-access-mount]
+value=false
+type=bool
+validation=bool
+help=Do not lazy unmount the slave volume. This allows admin to access the mount for debugging.
+
+[isolated-slaves]
+value=
+help=List of Slave nodes which are isolated
+
+[changelog-batch-size]
+# Max size of Changelogs to process per batch, Changelogs Processing is
+# not limited by the number of changelogs but instead based on
+# size of the changelog file, One sample changelog file size was 145408
+# with ~1000 CREATE and ~1000 DATA. 5 such files in one batch is 727040
+# If geo-rep worker crashes while processing a batch, it has to retry only
+# that batch since stime will get updated after each batch.
+value=727040
+help=Max size of Changelogs to process per batch.
+type=int
+
+[slave-timeout]
+value=120
+type=int
+help=Timeout in seconds for Slave Gsyncd. If no activity from master for this timeout, Slave gsyncd will be disconnected. Set Timeout to zero to skip this check.
+
+[connection-timeout]
+value=60
+type=int
+help=Timeout for mounts
+
+[replica-failover-interval]
+value=1
+type=int
+help=Minimum time interval in seconds for passive worker to become Active
+
+[changelog-archive-format]
+value=%Y%m
+help=Processed changelogs will be archived in working directory. Pattern for archive file
+
+[use-meta-volume]
+value=false
+type=bool
+help=Use this to set Active Passive mode to meta-volume.
+
+[meta-volume-mnt]
+value=/run/gluster/shared_storage
+help=Meta Volume or Shared Volume mount path
+
+[allow-network]
+value=
+
+[change-interval]
+value=5
+type=int
+
+[sync-method]
+value=rsync
+help=Sync method for data sync. Available methods are tar over ssh and rsync. Default is rsync.
+validation=choice
+allowed_values=tarssh,rsync
+
+[remote-gsyncd]
+value =
+help=If SSH keys are not secured with gsyncd prefix then use this configuration to set the actual path of gsyncd(Usually /usr/libexec/glusterfs/gsyncd)
+
+[gluster-command-dir]
+value=@SBIN_DIR@
+help=Directory where Gluster binaries exist on master
+
+[slave-gluster-command-dir]
+value=@SBIN_DIR@
+help=Directory where Gluster binaries exist on slave
+
+[gluster-params]
+value = aux-gfid-mount acl
+help=Parameters for Gluster Geo-rep mount in Master
+
+[slave-gluster-params]
+value = aux-gfid-mount acl
+help=Parameters for Gluster Geo-rep mount in Slave
+
+[ignore-deletes]
+value = false
+type=bool
+help=Do not sync deletes in Slave
+
+[special-sync-mode]
+# tunables for failover/failback mechanism:
+# None   - gsyncd behaves as normal
+# blind  - gsyncd works with xtime pairs to identify
+#          candidates for synchronization
+# wrapup - same as normal mode but does not assign
+#          xtimes to orphaned files
+# see crawl() for usage of the above tunables
+value =
+help=
+
+[gfid-conflict-resolution]
+value = true
+validation=bool
+type=bool
+help=Disables automatic gfid conflict resolution while syncing
+
+[working-dir]
+value = ${gsyncd_miscdir}/${master}_${primary_slave_host}_${slavevol}/
+template=true
+configurable=false
+help=Working directory for storing Changelogs
+
+[change-detector]
+value=changelog
+help=Change detector
+validation=choice
+allowed_values=changelog,xsync
+
+[cli-log-file]
+value=${gluster_logdir}/geo-replication/cli.log
+template=true
+configurable=false
+
+[cli-log-level]
+value=INFO
+help=Set CLI Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/gsyncd.log
+configurable=false
+template=true
+
+[changelog-log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/changes-${local_id}.log
+configurable=false
+template=true
+
+[gluster-log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/mnt-${local_id}.log
+template=true
+configurable=false
+
+[slave-log-file]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/gsyncd.log
+template=true
+configurable=false
+
+[slave-gluster-log-file]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/mnt-${master_node}-${master_brick_id}.log
+template=true
+configurable=false
+
+[slave-gluster-log-file-mbr]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/mnt-mbr-${master_node}-${master_brick_id}.log
+template=true
+configurable=false
+
+[log-level]
+value=INFO
+help=Set Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[gluster-log-level]
+value=INFO
+help=Set Gluster mount Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[changelog-log-level]
+value=INFO
+help=Set Changelog Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[slave-log-level]
+value=INFO
+help=Set Slave Gsyncd Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[slave-gluster-log-level]
+value=INFO
+help=Set Slave Gluster mount Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[ssh-port]
+value=22
+validation=minmax
+min=1
+max=65535
+help=Set SSH port
+type=int
+
+[ssh-command]
+value=ssh
+help=Set ssh binary path
+validation=execpath
+
+[tar-command]
+value=tar
+help=Set tar command path
+validation=execpath
+
+[ssh-options]
+value = -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${glusterd_workdir}/geo-replication/secret.pem
+template=true
+
+[ssh-options-tar]
+value = -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${glusterd_workdir}/geo-replication/tar_ssh.pem
+template=true
+
+[gluster-command]
+value=gluster
+help=Set gluster binary path
+validation=execpath
+
+[sync-jobs]
+value=3
+help=Number of Syncer jobs
+validation=minmax
+min=1
+max=100
+type=int
+
+[rsync-command]
+value=rsync
+help=Set rsync command path
+validation=execpath
+
+[rsync-options]
+value=
+
+[rsync-ssh-options]
+value=
+
+[rsync-opt-ignore-missing-args]
+value=true
+type=bool
+
+[rsync-opt-existing]
+value=true
+type=bool
+
+[log-rsync-performance]
+value=false
+help=Log Rsync performance
+validation=bool
+type=bool
+
+[use-rsync-xattrs]
+value=false
+type=bool
+
+[sync-xattrs]
+value=true
+type=bool
+
+[sync-acls]
+value=true
+type=bool
+
+[max-rsync-retries]
+value=10
+type=int
+
+[state_socket_unencoded]
+# Unused, For backward compatibility
+value=
diff --git a/geo-replication/setup.py b/geo-replication/setup.py
index 6d678baa2f7..0eae469d2d6 100644
--- a/geo-replication/setup.py
+++ b/geo-replication/setup.py
@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
-
+#
 # This file is licensed to you under your choice of the GNU Lesser
 # General Public License, version 3 or any later version (LGPLv3 or
 # later), or the GNU General Public License, version 2 (GPLv2), in all
@@ -20,11 +20,11 @@ setup(
     name=name,
     version="",
     description='GlusterFS Geo Replication',
-    license='',
+    license='GPLV2 and LGPLV3+',
     author='Red Hat, Inc.',
     author_email='gluster-devel@gluster.org',
     url='http://www.gluster.org',
-    packages=['syncdaemon', ],
+    packages=[name, ],
     test_suite='nose.collector',
     install_requires=[],
     scripts=[],
diff --git a/geo-replication/src/Makefile.am b/geo-replication/src/Makefile.am
index 87435d5718c..9937a0bd026 100644
--- a/geo-replication/src/Makefile.am
+++ b/geo-replication/src/Makefile.am
@@ -1,4 +1,4 @@
-gsyncddir = $(libexecdir)/glusterfs
+gsyncddir = $(GLUSTERFS_LIBEXECDIR)
 
 gsyncd_SCRIPTS = gverify.sh peer_gsec_create \
 	set_geo_rep_pem_keys.sh peer_mountbroker peer_mountbroker.py \
@@ -21,7 +21,7 @@ noinst_HEADERS = procdiggy.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" -DUSE_LIBGLUSTERFS \
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\" -DUSE_LIBGLUSTERFS \
 	-DSBIN_DIR=\"$(sbindir)\" -DPYTHON=\"$(PYTHON)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
@@ -35,11 +35,11 @@ $(top_builddir)/libglusterfs/src/libglusterfs.la:
 install-exec-hook:
 	$(mkdir_p) $(DESTDIR)$(sbindir)
 	rm -f $(DESTDIR)$(sbindir)/gluster-mountbroker
-	ln -s $(libexecdir)/glusterfs/peer_mountbroker.py \
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_mountbroker.py \
 		$(DESTDIR)$(sbindir)/gluster-mountbroker
 
 	rm -f $(DESTDIR)$(sbindir)/gluster-georep-sshkey
-	ln -s $(libexecdir)/glusterfs/peer_georep-sshkey.py \
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_georep-sshkey.py \
 		$(DESTDIR)$(sbindir)/gluster-georep-sshkey
 
 
diff --git a/geo-replication/src/gsyncd.c b/geo-replication/src/gsyncd.c
index 2c48ca5d540..b5aeec5bf33 100644
--- a/geo-replication/src/gsyncd.c
+++ b/geo-replication/src/gsyncd.c
@@ -7,8 +7,8 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "compat.h"
-#include "syscall.h"
+#include <glusterfs/compat.h>
+#include <glusterfs/syscall.h>
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -24,13 +24,13 @@
  * We unconditionally pass then while building gsyncd binary.
  */
 #ifdef USE_LIBGLUSTERFS
-#include "glusterfs.h"
-#include "globals.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/defaults.h>
 #endif
 
-#include "common-utils.h"
-#include "run.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/run.h>
 #include "procdiggy.h"
 
 #define _GLUSTERD_CALLED_ "_GLUSTERD_CALLED_"
@@ -42,395 +42,361 @@
 int restricted = 0;
 
 static int
-duplexpand (void **buf, size_t tsiz, size_t *len)
+duplexpand(void **buf, size_t tsiz, size_t *len)
 {
-        size_t osiz = tsiz * *len;
-        char *p = realloc (*buf, osiz << 1);
-        if (!p) {
-                free(*buf);
-                return -1;
-        }
+    size_t osiz = tsiz * *len;
+    char *p = realloc(*buf, osiz << 1);
+    if (!p) {
+        return -1;
+    }
 
-        memset (p + osiz, 0, osiz);
-        *buf = p;
-        *len <<= 1;
+    memset(p + osiz, 0, osiz);
+    *buf = p;
+    *len <<= 1;
 
-        return 0;
+    return 0;
 }
 
 static int
-str2argv (char *str, char ***argv)
+str2argv(char *str, char ***argv)
 {
-        char *p         = NULL;
-        char *savetok   = NULL;
-        char *temp      = NULL;
-        char *temp1     = NULL;
-        int argc        = 0;
-        size_t argv_len = 32;
-        int ret         = 0;
-        int i           = 0;
-
-        assert (str);
-        temp = str = strdup (str);
-        if (!str)
+    char *p = NULL;
+    char *savetok = NULL;
+    char *temp = NULL;
+    char *temp1 = NULL;
+    int argc = 0;
+    size_t argv_len = 32;
+    int ret = 0;
+    int i = 0;
+
+    assert(str);
+    temp = str = strdup(str);
+    if (!str)
+        goto error;
+
+    *argv = calloc(argv_len, sizeof(**argv));
+    if (!*argv)
+        goto error;
+
+    while ((p = strtok_r(str, " ", &savetok))) {
+        str = NULL;
+
+        argc++;
+        if (argc == argv_len) {
+            ret = duplexpand((void *)argv, sizeof(**argv), &argv_len);
+            if (ret == -1)
                 goto error;
-
-        *argv = calloc (argv_len, sizeof (**argv));
-        if (!*argv)
-                goto error;
-
-        while ((p = strtok_r (str, " ", &savetok))) {
-                str = NULL;
-
-                argc++;
-                if (argc == argv_len) {
-                        ret = duplexpand ((void *)argv,
-                                          sizeof (**argv),
-                                          &argv_len);
-                        if (ret == -1)
-                                goto error;
-                }
-                temp1 = strdup (p);
-                if (!temp1)
-                        goto error;
-                (*argv)[argc - 1] = temp1;
         }
-
-        free(temp);
-        return argc;
-
- error:
-        fprintf (stderr, "out of memory\n");
-        free(temp);
-        for (i = 0; i < argc - 1; i++)
-                free((*argv)[i]);
-        free(*argv);
-        return -1;
+        temp1 = strdup(p);
+        if (!temp1)
+            goto error;
+        (*argv)[argc - 1] = temp1;
+    }
+
+    free(temp);
+    return argc;
+
+error:
+    fprintf(stderr, "out of memory\n");
+    free(temp);
+    for (i = 0; i < argc - 1; i++)
+        free((*argv)[i]);
+    free(*argv);
+    return -1;
 }
 
 static int
-invoke_gsyncd (int argc, char **argv)
+invoke_gsyncd(int argc, char **argv)
 {
-        char config_file[PATH_MAX] = {0,};
-        size_t gluster_workdir_len = 0;
-        runner_t runner            = {0,};
-        int i                      = 0;
-        int j                      = 0;
-        char *nargv[argc + 4];
-        char *python = NULL;
-
-        if (restricted) {
-                size_t len;
-                /* in restricted mode we forcibly use the system-wide config */
-                runinit (&runner);
-                runner_add_args (&runner, SBIN_DIR"/gluster",
-                                 "--remote-host=localhost",
-                                 "--log-file=-", "system::", "getwd",
-                                 NULL);
-                runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-                if (runner_start (&runner) == 0 &&
-                    fgets (config_file, PATH_MAX,
-                           runner_chio (&runner, STDOUT_FILENO)) != NULL &&
-                    (len = strlen (config_file)) &&
-                    config_file[len - 1] == '\n' &&
-                    runner_end (&runner) == 0)
-                        gluster_workdir_len = len - 1;
-
-                if (gluster_workdir_len) {
-                        if (gluster_workdir_len + 1 + strlen (GSYNCD_CONF_TEMPLATE) + 1 >
-                            PATH_MAX)
-                                goto error;
-                        config_file[gluster_workdir_len] = '/';
-                        strcat (config_file, GSYNCD_CONF_TEMPLATE);
-                } else
-                        goto error;
-
-                if (setenv ("_GSYNCD_RESTRICTED_", "1", 1) == -1)
-                        goto error;
-        }
+    int i = 0;
+    int j = 0;
+    char *nargv[argc + 4];
+    char *python = NULL;
 
-        if (chdir ("/") == -1)
-                goto error;
+    if (chdir("/") == -1)
+        goto error;
 
-        j = 0;
-        python = getenv("PYTHON");
-        if(!python)
-                python = PYTHON;
-        nargv[j++] = python;
-        nargv[j++] = GSYNCD_PREFIX"/python/syncdaemon/"GSYNCD_PY;
-        for (i = 1; i < argc; i++)
-                nargv[j++] = argv[i];
-        if (config_file[0]) {
-                nargv[j++] = "-c";
-                nargv[j++] = config_file;
-        }
-        nargv[j++] = NULL;
+    j = 0;
+    python = getenv("PYTHON");
+    if (!python)
+        python = PYTHON;
+    nargv[j++] = python;
+    nargv[j++] = GSYNCD_PREFIX "/python/syncdaemon/" GSYNCD_PY;
+    for (i = 1; i < argc; i++)
+        nargv[j++] = argv[i];
 
-        execvp (python, nargv);
+    nargv[j++] = NULL;
 
-        fprintf (stderr, "exec of '%s' failed\n", python);
-        return 127;
+    execvp(python, nargv);
 
- error:
-        fprintf (stderr, "gsyncd initializaion failed\n");
-        return 1;
-}
+    fprintf(stderr, "exec of '%s' failed\n", python);
+    return 127;
 
+error:
+    fprintf(stderr, "gsyncd initializaion failed\n");
+    return 1;
+}
 
 static int
-find_gsyncd (pid_t pid, pid_t ppid, char *name, void *data)
+find_gsyncd(pid_t pid, pid_t ppid, char *name, void *data)
 {
-        char buf[NAME_MAX * 2] = {0,};
-        char path[PATH_MAX]    = {0,};
-        char *p                = NULL;
-        int zeros              = 0;
-        int ret                = 0;
-        int fd                 = -1;
-        pid_t *pida            = (pid_t *)data;
-
-        if (ppid != pida[0])
-                return 0;
-
-        snprintf (path, sizeof path, PROC"/%d/cmdline", pid);
-        fd = open (path, O_RDONLY);
-        if (fd == -1)
-                return 0;
-        ret = sys_read (fd, buf, sizeof (buf));
-        sys_close (fd);
-        if (ret == -1)
-                return 0;
-        for (zeros = 0, p = buf; zeros < 2 && p < buf + ret; p++)
-                zeros += !*p;
-
-        ret = 0;
-        switch (zeros) {
+    char buf[NAME_MAX * 2] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *p = NULL;
+    int zeros = 0;
+    int ret = 0;
+    int fd = -1;
+    pid_t *pida = (pid_t *)data;
+
+    if (ppid != pida[0])
+        return 0;
+
+    snprintf(path, sizeof path, PROC "/%d/cmdline", pid);
+    fd = open(path, O_RDONLY);
+    if (fd == -1)
+        return 0;
+    ret = sys_read(fd, buf, sizeof(buf));
+    sys_close(fd);
+    if (ret == -1)
+        return 0;
+    for (zeros = 0, p = buf; zeros < 2 && p < buf + ret; p++)
+        zeros += !*p;
+
+    ret = 0;
+    switch (zeros) {
         case 2:
-                if ((strcmp (basename (buf), basename (PYTHON)) ||
-                     strcmp (basename (buf + strlen (buf) + 1), GSYNCD_PY)) == 0) {
-                        ret = 1;
-                        break;
-                }
-                /* fallthrough */
+            if ((strcmp(basename(buf), basename(PYTHON)) ||
+                 strcmp(basename(buf + strlen(buf) + 1), GSYNCD_PY)) == 0) {
+                ret = 1;
+                break;
+            }
+            /* fallthrough */
         case 1:
-                if (strcmp (basename (buf), GSYNCD_PY) == 0)
-                        ret = 1;
+            if (strcmp(basename(buf), GSYNCD_PY) == 0)
+                ret = 1;
+    }
+
+    if (ret == 1) {
+        if (pida[1] != -1) {
+            fprintf(stderr, GSYNCD_PY " sibling is not unique");
+            return -1;
         }
+        pida[1] = pid;
+    }
 
-        if (ret == 1) {
-                if (pida[1] != -1) {
-                        fprintf (stderr, GSYNCD_PY" sibling is not unique");
-                        return -1;
-                }
-                pida[1] = pid;
-        }
-
-        return 0;
+    return 0;
 }
 
 static int
-invoke_rsync (int argc, char **argv)
+invoke_rsync(int argc, char **argv)
 {
-        int i                  = 0;
-        char path[PATH_MAX]    = {0,};
-        pid_t pid              = -1;
-        pid_t ppid             = -1;
-        pid_t pida[]           = {-1, -1};
-        char *name             = NULL;
-        char buf[PATH_MAX + 1] = {0,};
-        int ret                = 0;
-
-        assert (argv[argc] == NULL);
-
-        if (argc < 2 || strcmp (argv[1], "--server") != 0)
-                goto error;
-
-        for (i = 2; i < argc && argv[i][0] == '-'; i++);
-
-        if (!(i == argc - 2 && strcmp (argv[i], ".") == 0 && argv[i + 1][0] == '/')) {
-                fprintf (stderr, "need an rsync invocation without protected args\n");
-                goto error;
-        }
-
-        /* look up sshd we are spawned from */
-        for (pid = getpid () ;; pid = ppid) {
-                ppid = pidinfo (pid, &name);
-                if (ppid < 0) {
-                        fprintf (stderr, "sshd ancestor not found\n");
-                        goto error;
-                }
-                if (strcmp (name, "sshd") == 0) {
-                        GF_FREE (name);
-                        break;
-                }
-                GF_FREE (name);
+    int i = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    pid_t pid = -1;
+    pid_t ppid = -1;
+    pid_t pida[] = {-1, -1};
+    char *name = NULL;
+    char buf[PATH_MAX + 1] = {
+        0,
+    };
+    int ret = 0;
+
+    assert(argv[argc] == NULL);
+
+    if (argc < 2 || strcmp(argv[1], "--server") != 0)
+        goto error;
+
+    for (i = 2; i < argc && argv[i][0] == '-'; i++)
+        ;
+
+    if (!(i == argc - 2 && strcmp(argv[i], ".") == 0 &&
+          argv[i + 1][0] == '/')) {
+        fprintf(stderr, "need an rsync invocation without protected args\n");
+        goto error;
+    }
+
+    /* look up sshd we are spawned from */
+    for (pid = getpid();; pid = ppid) {
+        ppid = pidinfo(pid, &name);
+        if (ppid < 0) {
+            fprintf(stderr, "sshd ancestor not found\n");
+            goto error;
         }
-        /* look up "ssh-sibling" gsyncd */
-        pida[0] = pid;
-        ret = prociter (find_gsyncd, pida);
-        if (ret == -1 || pida[1] == -1) {
-                fprintf (stderr, "gsyncd sibling not found\n");
-                goto error;
-        }
-        /* check if rsync target matches gsyncd target */
-        snprintf (path, sizeof path, PROC"/%d/cwd", pida[1]);
-        ret = sys_readlink (path, buf, sizeof (buf));
-        if (ret == -1 || ret == sizeof (buf))
-                goto error;
-        if (strcmp (argv[argc - 1], "/") == 0 /* root dir cannot be a target */ ||
-            (strcmp (argv[argc - 1], path) /* match against gluster target */ &&
-             strcmp (argv[argc - 1], buf) /* match against file target */) != 0) {
-                fprintf (stderr, "rsync target does not match "GEOREP" session\n");
-                goto error;
+        if (strcmp(name, "sshd") == 0) {
+            GF_FREE(name);
+            break;
         }
-
-        argv[0] = RSYNC;
-
-        execvp (RSYNC, argv);
-
-        fprintf (stderr, "exec of "RSYNC" failed\n");
-        return 127;
-
- error:
-        fprintf (stderr, "disallowed "RSYNC" invocation\n");
-        return 1;
+        GF_FREE(name);
+    }
+    /* look up "ssh-sibling" gsyncd */
+    pida[0] = pid;
+    ret = prociter(find_gsyncd, pida);
+    if (ret == -1 || pida[1] == -1) {
+        fprintf(stderr, "gsyncd sibling not found\n");
+        goto error;
+    }
+    /* check if rsync target matches gsyncd target */
+    snprintf(path, sizeof path, PROC "/%d/cwd", pida[1]);
+    ret = sys_readlink(path, buf, sizeof(buf));
+    if (ret == -1 || ret == sizeof(buf))
+        goto error;
+    if (strcmp(argv[argc - 1], "/") == 0 /* root dir cannot be a target */ ||
+        (strcmp(argv[argc - 1], path) /* match against gluster target */ &&
+         strcmp(argv[argc - 1], buf) /* match against file target */) != 0) {
+        fprintf(stderr, "rsync target does not match " GEOREP " session\n");
+        goto error;
+    }
+
+    argv[0] = RSYNC;
+
+    execvp(RSYNC, argv);
+
+    fprintf(stderr, "exec of " RSYNC " failed\n");
+    return 127;
+
+error:
+    fprintf(stderr, "disallowed " RSYNC " invocation\n");
+    return 1;
 }
 
 static int
-invoke_gluster (int argc, char **argv)
+invoke_gluster(int argc, char **argv)
 {
-        int i = 0;
-        int j = 0;
-        int optsover = 0;
-        char *ov = NULL;
-
-        for (i = 1; i < argc; i++) {
-                ov = strtail (argv[i], "--");
-                if (ov && !optsover) {
-                        if (*ov == '\0')
-                                optsover = 1;
-                        continue;
-                }
-                switch (++j) {
-                case 1:
-                        if (strcmp (argv[i], "volume") != 0)
-                                goto error;
-                        break;
-                case 2:
-                        if (strcmp (argv[i], "info") != 0)
-                                goto error;
-                        break;
-                case 3:
-                        break;
-                default:
-                        goto error;
-                }
+    int i = 0;
+    int j = 0;
+    int optsover = 0;
+    char *ov = NULL;
+
+    for (i = 1; i < argc; i++) {
+        ov = strtail(argv[i], "--");
+        if (ov && !optsover) {
+            if (*ov == '\0')
+                optsover = 1;
+            continue;
+        }
+        switch (++j) {
+            case 1:
+                if (strcmp(argv[i], "volume") != 0)
+                    goto error;
+                break;
+            case 2:
+                if (strcmp(argv[i], "info") != 0)
+                    goto error;
+                break;
+            case 3:
+                break;
+            default:
+                goto error;
         }
+    }
 
-        argv[0] = "gluster";
-        execvp (SBIN_DIR"/gluster", argv);
-        fprintf (stderr, "exec of gluster failed\n");
-        return 127;
+    argv[0] = "gluster";
+    execvp(SBIN_DIR "/gluster", argv);
+    fprintf(stderr, "exec of gluster failed\n");
+    return 127;
 
- error:
-        fprintf (stderr, "disallowed gluster invocation\n");
-        return 1;
+error:
+    fprintf(stderr, "disallowed gluster invocation\n");
+    return 1;
 }
 
 struct invocable {
-        char *name;
-        int (*invoker) (int argc, char **argv);
+    char *name;
+    int (*invoker)(int argc, char **argv);
 };
 
-struct invocable invocables[] = {
-        { "rsync",   invoke_rsync  },
-        { "gsyncd",  invoke_gsyncd },
-        { "gluster", invoke_gluster },
-        { NULL, NULL}
-};
+struct invocable invocables[] = {{"rsync", invoke_rsync},
+                                 {"gsyncd", invoke_gsyncd},
+                                 {"gluster", invoke_gluster},
+                                 {NULL, NULL}};
 
 int
-main (int argc, char **argv)
+main(int argc, char **argv)
 {
-        int               ret   = -1;
-        char             *evas  = NULL;
-        struct invocable *i     = NULL;
-        char             *b     = NULL;
-        char             *sargv = NULL;
-        int               j     = 0;
+    int ret = -1;
+    char *evas = NULL;
+    struct invocable *i = NULL;
+    char *b = NULL;
+    char *sargv = NULL;
+    int j = 0;
 
 #ifdef USE_LIBGLUSTERFS
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                return ENOMEM;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        return ENOMEM;
 
-        if (glusterfs_globals_init (ctx))
-                return 1;
+    if (glusterfs_globals_init(ctx))
+        return 1;
 
-        THIS->ctx = ctx;
-        ret = default_mem_acct_init (THIS);
-        if (ret) {
-                fprintf (stderr, "internal error: mem accounting failed\n");
-                return 1;
-        }
+    THIS->ctx = ctx;
+    ret = default_mem_acct_init(THIS);
+    if (ret) {
+        fprintf(stderr, "internal error: mem accounting failed\n");
+        return 1;
+    }
 #endif
 
-        evas = getenv (_GLUSTERD_CALLED_);
-        if (evas && strcmp (evas, "1") == 0)
-                /* OK, we know glusterd called us, no need to look for further config
-                 *...although this conclusion should not inherit to our children
-                 */
-                unsetenv (_GLUSTERD_CALLED_);
-        else {
-                /* we regard all gsyncd invocations unsafe
-                 * that do not come from glusterd and
-                 * therefore restrict it
-                 */
-                restricted = 1;
-
-                if (!getenv (_GSYNCD_DISPATCHED_)) {
-                        evas = getenv ("SSH_ORIGINAL_COMMAND");
-                        if (evas)
-                                sargv = evas;
-                        else {
-                                evas = getenv ("SHELL");
-                                if (evas && strcmp (basename (evas), "gsyncd") == 0 &&
-                                    argc == 3 && strcmp (argv[1], "-c") == 0)
-                                        sargv = argv[2];
-                        }
-                }
-
+    evas = getenv(_GLUSTERD_CALLED_);
+    if (evas && strcmp(evas, "1") == 0)
+        /* OK, we know glusterd called us, no need to look for further config
+         *...although this conclusion should not inherit to our children
+         */
+        unsetenv(_GLUSTERD_CALLED_);
+    else {
+        /* we regard all gsyncd invocations unsafe
+         * that do not come from glusterd and
+         * therefore restrict it
+         */
+        restricted = 1;
+
+        if (!getenv(_GSYNCD_DISPATCHED_)) {
+            evas = getenv("SSH_ORIGINAL_COMMAND");
+            if (evas)
+                sargv = evas;
+            else {
+                evas = getenv("SHELL");
+                if (evas && strcmp(basename(evas), "gsyncd") == 0 &&
+                    argc == 3 && strcmp(argv[1], "-c") == 0)
+                    sargv = argv[2];
+            }
         }
+    }
 
-        if (!(sargv && restricted))
-                return invoke_gsyncd (argc, argv);
+    if (!(sargv && restricted))
+        return invoke_gsyncd(argc, argv);
 
-        argc = str2argv (sargv, &argv);
+    argc = str2argv(sargv, &argv);
 
-        if (argc == -1) {
-                fprintf (stderr, "internal error\n");
-                return 1;
-        }
-
-        if (setenv (_GSYNCD_DISPATCHED_, "1", 1) == -1) {
-                fprintf (stderr, "internal error\n");
-                goto out;
-        }
+    if (argc == -1) {
+        fprintf(stderr, "internal error\n");
+        return 1;
+    }
 
+    if (setenv(_GSYNCD_DISPATCHED_, "1", 1) == -1) {
+        fprintf(stderr, "internal error\n");
+        goto out;
+    }
 
-        b = basename (argv[0]);
-        for (i = invocables; i->name; i++) {
-                if (strcmp (b, i->name) == 0)
-                        return i->invoker (argc, argv);
-        }
+    b = basename(argv[0]);
+    for (i = invocables; i->name; i++) {
+        if (strcmp(b, i->name) == 0)
+            return i->invoker(argc, argv);
+    }
 
-        fprintf (stderr, "invoking %s in restricted SSH session is not allowed\n",
-                 b);
+    fprintf(stderr, "invoking %s in restricted SSH session is not allowed\n",
+            b);
 
 out:
-        for (j = 1; j < argc; j++)
-                free(argv[j]);
-        free(argv);
-        return 1;
+    for (j = 1; j < argc; j++)
+        free(argv[j]);
+    free(argv);
+    return 1;
 }
diff --git a/geo-replication/src/gverify.sh b/geo-replication/src/gverify.sh
index 9b1328aa631..f5f70d245e0 100755
--- a/geo-replication/src/gverify.sh
+++ b/geo-replication/src/gverify.sh
@@ -7,15 +7,24 @@
 # Considering buffer_size 100MB
 BUFFER_SIZE=104857600;
 SSH_PORT=$5;
-slave_log_file=`gluster --print-logdir`/geo-replication-slaves/slave.log
+master_log_file=`gluster --print-logdir`/geo-replication/gverify-mastermnt.log
+slave_log_file=`gluster --print-logdir`/geo-replication/gverify-slavemnt.log
 
 function SSHM()
 {
-    ssh -p ${SSH_PORT} -q \
-	-oPasswordAuthentication=no \
-	-oStrictHostKeyChecking=no \
-	-oControlMaster=yes \
-	"$@";
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    fi
 }
 
 function get_inode_num()
@@ -85,6 +94,7 @@ echo $cmd_line;
 function master_stats()
 {
     MASTERVOL=$1;
+    local inet6=$2;
     local d;
     local i;
     local disk_size;
@@ -93,7 +103,12 @@ function master_stats()
     local m_status;
 
     d=$(mktemp -d -t ${0##*/}.XXXXXX 2>/dev/null);
-    glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --volfile-id $MASTERVOL -l $slave_log_file $d;
+    if [ "$inet6" = "inet6" ]; then
+        glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --xlator-option="transport.address-family=inet6" --volfile-id $MASTERVOL -l $master_log_file $d;
+    else
+        glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --volfile-id $MASTERVOL -l $master_log_file $d;
+    fi
+
     i=$(get_inode_num $d);
     if [[ "$i" -ne "1" ]]; then
         echo 0:0;
@@ -115,12 +130,18 @@ function slave_stats()
     SLAVEUSER=$1;
     SLAVEHOST=$2;
     SLAVEVOL=$3;
+    local inet6=$4;
     local cmd_line;
     local ver;
     local status;
 
     d=$(mktemp -d -t ${0##*/}.XXXXXX 2>/dev/null);
-    glusterfs --xlator-option="*dht.lookup-unhashed=off" --volfile-server $SLAVEHOST --volfile-id $SLAVEVOL -l $slave_log_file $d;
+    if [ "$inet6" = "inet6" ]; then
+        glusterfs --xlator-option="*dht.lookup-unhashed=off" --xlator-option="transport.address-family=inet6" --volfile-server $SLAVEHOST --volfile-id $SLAVEVOL -l $slave_log_file $d;
+    else
+        glusterfs --xlator-option="*dht.lookup-unhashed=off" --volfile-server $SLAVEHOST --volfile-id $SLAVEVOL -l $slave_log_file $d;
+    fi
+
     i=$(get_inode_num $d);
     if [[ "$i" -ne "1" ]]; then
         echo 0:0;
@@ -158,6 +179,10 @@ function main()
     log_file=$6
     > $log_file
 
+    inet6=$7
+    local cmd_line
+    local ver
+
     # Use FORCE_BLOCKER flag in the error message to differentiate
     # between the errors which the force command should bypass
 
@@ -172,15 +197,32 @@ function main()
         exit 1;
     fi;
 
-    ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    fi
+
     if [ $? -ne 0 ]; then
         echo "FORCE_BLOCKER|Passwordless ssh login has not been setup with $3 for user $2." > $log_file
         exit 1;
     fi;
 
+    cmd_line=$(cmd_slave);
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ver=$(ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 bash -c "'$cmd_line'")
+    else
+        ver=$(ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 bash -c "'$cmd_line'")
+    fi
+
+    if [ -z "$ver" ]; then
+        echo "FORCE_BLOCKER|gluster command not found on $3 for user $2." > $log_file
+        exit 1;
+    fi;
+
     ERRORS=0;
-    master_data=$(master_stats $1);
-    slave_data=$(slave_stats $2 $3 $4);
+    master_data=$(master_stats $1 ${inet6});
+    slave_data=$(slave_stats $2 $3 $4 ${inet6});
     master_disk_size=$(echo $master_data | cut -f1 -d':');
     slave_disk_size=$(echo $slave_data | cut -f1 -d':');
     master_used_size=$(echo $master_data | cut -f2 -d':');
@@ -190,12 +232,12 @@ function main()
     slave_no_of_files=$(echo $slave_data | cut -f4 -d':');
 
     if [[ "x$master_disk_size" = "x" || "x$master_version" = "x" || "$master_disk_size" -eq "0" ]]; then
-        echo "FORCE_BLOCKER|Unable to fetch master volume details. Please check the master cluster and master volume." > $log_file;
+        echo "FORCE_BLOCKER|Unable to mount and fetch master volume details. Please check the log: $master_log_file" > $log_file;
 	exit 1;
     fi;
 
     if [[ "x$slave_disk_size" = "x" || "x$slave_version" = "x" || "$slave_disk_size" -eq "0" ]]; then
-	echo "FORCE_BLOCKER|Unable to fetch slave volume details. Please check the slave cluster and slave volume." > $log_file;
+	echo "FORCE_BLOCKER|Unable to mount and fetch slave volume details. Please check the log: $slave_log_file" > $log_file;
 	exit 1;
     fi;
 
@@ -223,7 +265,7 @@ function main()
     fi;
 
     if [[ $master_version != $slave_version ]]; then
-        echo "Gluster version mismatch between master and slave." >> $log_file;
+        echo "Gluster version mismatch between master and slave. Master version: $master_version Slave version: $slave_version" >> $log_file;
         ERRORS=$(($ERRORS + 1));
     fi;
 
diff --git a/geo-replication/src/peer_georep-sshkey.py.in b/geo-replication/src/peer_georep-sshkey.py.in
index 400f29d64f3..58696e9a616 100644
--- a/geo-replication/src/peer_georep-sshkey.py.in
+++ b/geo-replication/src/peer_georep-sshkey.py.in
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 #  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
@@ -30,8 +30,8 @@ from prettytable import PrettyTable
 
 SECRET_PEM = "@GLUSTERD_WORKDIR@/geo-replication/secret.pem"
 TAR_SSH_PEM = "@GLUSTERD_WORKDIR@/geo-replication/tar_ssh.pem"
-GSYNCD_CMD = 'command="@GLUSTERFS_LIBEXECDIR@/gsyncd"  '
-TAR_CMD = 'command="tar ${SSH_ORIGINAL_COMMAND#* }"  '
+GSYNCD_CMD = 'command="@GLUSTERFS_LIBEXECDIR@/gsyncd" '
+TAR_CMD = 'command="tar ${SSH_ORIGINAL_COMMAND#* }" '
 COMMON_SECRET_FILE = "@GLUSTERD_WORKDIR@/geo-replication/common_secret.pem.pub"
 
 
diff --git a/geo-replication/src/peer_gsec_create.in b/geo-replication/src/peer_gsec_create.in
index 05c1638bdcd..6d4a4847013 100755
--- a/geo-replication/src/peer_gsec_create.in
+++ b/geo-replication/src/peer_gsec_create.in
@@ -18,7 +18,7 @@ if [ "Xcontainer" = "X$1" ]; then
     output1=`cat "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub`
     output2=`cat "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub`
 else
-    output1=`echo command=\"${libexecdir}/glusterfs/gsyncd\" " "``cat "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub`
-    output2=`echo command=\"tar \$\{SSH_ORIGINAL_COMMAND#* \}\" " "``cat "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub`
+    output1=`echo command=\"${libexecdir}/glusterfs/gsyncd\" ""``cat "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub`
+    output2=`echo command=\"tar \$\{SSH_ORIGINAL_COMMAND#* \}\" ""``cat "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub`
 fi
 echo -e "$output1\n$output2"
diff --git a/geo-replication/src/peer_mountbroker.in b/geo-replication/src/peer_mountbroker.in
index ec514f038e1..8ecf38ded41 100644
--- a/geo-replication/src/peer_mountbroker.in
+++ b/geo-replication/src/peer_mountbroker.in
@@ -1,10 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
+
+from __future__ import print_function
+
 import os
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 import json
 import sys
 
-
 PROG_DESCRIPTION = """
 GlusterFS Mountbroker user management
 """
@@ -14,19 +16,19 @@ args = None
 
 def ok(message=""):
     if (not args and "-j" in sys.argv) or (args and args.json):
-        print json.dumps({"ok": True, "message": message})
+        print(json.dumps({"ok": True, "message": message}))
     else:
         if message:
-            print message
+            print(message)
 
     sys.exit(0)
 
 
 def notok(message=""):
     if (not args and "-j" in sys.argv) or (args and args.json):
-        print json.dumps({"ok": False, "message": message})
+        print(json.dumps({"ok": False, "message": message}))
     else:
-        print "error: %s" % message
+        print("error: %s" % message)
 
     # Always return zero due to limitation while executing
     # as `gluster system:: execute`
@@ -64,7 +66,7 @@ class MountbrokerUserMgmt(object):
     def _get_write_data(self):
         op = "volume management\n"
         op += "    type mgmt/glusterd\n"
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             op += "    option %s %s\n" % (k, v)
         for line in self.commented_lines:
             op += "    %s\n" % line
@@ -87,7 +89,7 @@ class MountbrokerUserMgmt(object):
 
     def add_user(self, user, volumes):
         vols = set()
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             if k.startswith("mountbroker-geo-replication.") \
                and user == k.split(".")[-1]:
                 vols.update(v.split(","))
@@ -98,7 +100,7 @@ class MountbrokerUserMgmt(object):
 
     def remove_volume(self, user, volumes):
         vols = set()
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             if k.startswith("mountbroker-geo-replication.") \
                and user == k.split(".")[-1]:
                 vols.update(v.split(","))
@@ -118,7 +120,7 @@ class MountbrokerUserMgmt(object):
     def info(self):
         data = {"users": []}
 
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             if k.startswith("mountbroker-geo-replication."):
                 data["users"].append(
                     {"name": k.split(".")[-1], "volumes": v.split(",")}
@@ -132,7 +134,7 @@ class MountbrokerUserMgmt(object):
 def format_info(data):
     op = "%s %s\n" % ("Option".ljust(50), "Value".ljust(50))
     op += ("-" * 101) + "\n"
-    for key, value in data.iteritems():
+    for key, value in data.items():
         if key != "users":
             op += "%s %s\n" % (key.ljust(50), value)
 
diff --git a/geo-replication/src/peer_mountbroker.py.in b/geo-replication/src/peer_mountbroker.py.in
index be182c5a7de..40b90ffc560 100644
--- a/geo-replication/src/peer_mountbroker.py.in
+++ b/geo-replication/src/peer_mountbroker.py.in
@@ -1,4 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
+
+from __future__ import print_function
+
 import os
 from errno import EEXIST, ENOENT
 
@@ -8,6 +11,7 @@ from gluster.cliutils import (execute, Cmd, node_output_ok,
 from prettytable import PrettyTable
 
 LOG_DIR = "@localstatedir@/log/glusterfs/geo-replication-slaves"
+CLI_LOG = "@localstatedir@/log/glusterfs/cli.log"
 GEOREP_DIR = "@GLUSTERD_WORKDIR@/geo-replication"
 GLUSTERD_VOLFILE = "@GLUSTERD_VOLFILE@"
 
@@ -43,12 +47,12 @@ class MountbrokerUserMgmt(object):
             for line in f:
                 line = line.strip()
                 if line.startswith("option "):
-                    key, value = line.split(" ")[1:]
+                    key, value = line.split()[1:]
                     self._options[key] = value
                 if line.startswith("#"):
                     self.commented_lines.append(line)
 
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             if k.startswith("mountbroker-geo-replication."):
                 user = k.split(".")[-1]
                 self.user_volumes[user] = set(v.split(","))
@@ -59,7 +63,7 @@ class MountbrokerUserMgmt(object):
     def _get_write_data(self):
         op = "volume management\n"
         op += "    type mgmt/glusterd\n"
-        for k, v in self._options.iteritems():
+        for k, v in self._options.items():
             if k.startswith("mountbroker-geo-replication."):
                 # Users will be added seperately
                 continue
@@ -142,7 +146,7 @@ class NodeSetup(Cmd):
     # chgrp -R <grp> /var/log/glusterfs/geo-replication-slaves
     # chgrp -R <grp> /var/lib/glusterd/geo-replication
     # chmod -R 770 /var/log/glusterfs/geo-replication-slaves
-    # chmod -R 770 /var/lib/glusterd/geo-replication
+    # chmod 770 /var/lib/glusterd/geo-replication
     # mkdir -p <mnt_root>
     # chmod 0711 <mnt_root>
     # If selinux,
@@ -192,8 +196,13 @@ class NodeSetup(Cmd):
 
         execute(["chgrp", "-R", args.group, GEOREP_DIR])
         execute(["chgrp", "-R", args.group, LOG_DIR])
-        execute(["chmod", "-R", "770", GEOREP_DIR])
-        execute(["chmod", "-R", "770", args.group, LOG_DIR])
+        execute(["chgrp", args.group, CLI_LOG])
+        execute(["chmod", "770", GEOREP_DIR])
+        execute(["find", LOG_DIR, "-type", "d", "-exec", "chmod", "770", "{}",
+                 "+"])
+        execute(["find", LOG_DIR, "-type", "f", "-exec", "chmod", "660", "{}",
+                 "+"])
+        execute(["chmod", "660", CLI_LOG])
 
         m.set_mount_root_and_group(args.mount_root, args.group)
         m.save()
@@ -213,8 +222,10 @@ class CliSetup(Cmd):
     name = "setup"
 
     def args(self, parser):
-        parser.add_argument("mount_root")
-        parser.add_argument("group")
+        parser.add_argument("mount_root",
+                            help="Path to the mountbroker-root directory.")
+        parser.add_argument("group",
+                            help="Group to be used for setup.")
 
     def run(self, args):
         out = execute_in_peers("node-setup", [args.mount_root,
@@ -270,7 +281,7 @@ class CliStatus(Cmd):
 
         for p in out:
             node_data = p.output
-            if node_data == "":
+            if node_data == "" or node_data == "N/A":
                 node_data = {}
 
             users_row_data = ""
@@ -324,8 +335,10 @@ class CliAdd(Cmd):
     name = "add"
 
     def args(self, parser):
-        parser.add_argument("volume")
-        parser.add_argument("user")
+        parser.add_argument("volume",
+                            help="Volume to be added.")
+        parser.add_argument("user",
+                            help="User for which volume is to be added.")
 
     def run(self, args):
         out = execute_in_peers("node-add", [args.volume,
@@ -365,8 +378,9 @@ class CliRemove(Cmd):
     name = "remove"
 
     def args(self, parser):
-        parser.add_argument("--volume", default=".")
-        parser.add_argument("--user", default=".")
+        parser.add_argument("--volume", default=".", help="Volume to be removed.")
+        parser.add_argument("--user", default=".",
+                            help="User for which volume has to be removed.")
 
     def run(self, args):
         out = execute_in_peers("node-remove", [args.volume,
diff --git a/geo-replication/src/procdiggy.c b/geo-replication/src/procdiggy.c
index 35125d20ba8..8068ef79a42 100644
--- a/geo-replication/src/procdiggy.c
+++ b/geo-replication/src/procdiggy.c
@@ -15,110 +15,122 @@
 #include <ctype.h>
 #include <sys/param.h> /* for PATH_MAX */
 
-#include "common-utils.h"
-#include "syscall.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syscall.h>
 #include "procdiggy.h"
 
 pid_t
-pidinfo (pid_t pid, char **name)
+pidinfo(pid_t pid, char **name)
 {
-        char buf[NAME_MAX * 2] = {0,};
-        FILE *f                = NULL;
-        char path[PATH_MAX]    = {0,};
-        char *p                = NULL;
-        int ret                = 0;
-
-        snprintf (path, sizeof path, PROC"/%d/status", pid);
-
-        f = fopen (path, "r");
-        if (!f)
-                return -1;
-
-        if (name)
-                *name = NULL;
-        for (;;) {
-                size_t len;
-                memset (buf, 0, sizeof (buf));
-                if (fgets (buf, sizeof (buf), f) == NULL ||
-                    (len = strlen (buf)) == 0 ||
-                    buf[len - 1] != '\n') {
-                        pid = -1;
-                        goto out;
-                }
-                buf[len - 1] = '\0';
-
-                if (name && !*name) {
-                        p = strtail (buf, "Name:");
-                        if (p) {
-                                while (isspace (*++p));
-                                *name = gf_strdup (p);
-                                if (!*name) {
-                                        pid = -2;
-                                        goto out;
-                                }
-                                continue;
-                        }
+    char buf[NAME_MAX * 2] = {
+        0,
+    };
+    FILE *f = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *p = NULL;
+    int ret = 0;
+    pid_t lpid = -1;
+
+    if (name)
+        *name = NULL;
+
+    snprintf(path, sizeof path, PROC "/%d/status", pid);
+
+    f = fopen(path, "r");
+    if (!f)
+        return -1;
+
+    for (;;) {
+        size_t len;
+        memset(buf, 0, sizeof(buf));
+        if (fgets(buf, sizeof(buf), f) == NULL || (len = strlen(buf)) == 0 ||
+            buf[len - 1] != '\n') {
+            lpid = -1;
+            goto out;
+        }
+        buf[len - 1] = '\0';
+
+        if (name && !*name) {
+            p = strtail(buf, "Name:");
+            if (p) {
+                while (isspace(*++p))
+                    ;
+                *name = gf_strdup(p);
+                if (!*name) {
+                    lpid = -2;
+                    goto out;
                 }
-
-                p = strtail (buf, "PPid:");
-                if (p)
-                        break;
+                continue;
+            }
         }
 
-        while (isspace (*++p));
-        ret = gf_string2int (p, &pid);
-        if (ret == -1)
-                pid = -1;
-
- out:
-        fclose (f);
-        if (pid == -1 && name && *name)
-                GF_FREE (name);
-        if (pid == -2)
-                fprintf (stderr, "out of memory\n");
-        return pid;
+        p = strtail(buf, "PPid:");
+        if (p)
+            break;
+    }
+
+    while (isspace(*++p))
+        ;
+    ret = gf_string2int(p, &lpid);
+    if (ret == -1)
+        lpid = -1;
+
+out:
+    fclose(f);
+    if (lpid == -1 && name && *name)
+        GF_FREE(*name);
+    if (lpid == -2)
+        fprintf(stderr, "out of memory\n");
+    return lpid;
 }
 
 int
-prociter (int (*proch) (pid_t pid, pid_t ppid, char *tmpname, void *data),
-          void *data)
+prociter(int (*proch)(pid_t pid, pid_t ppid, char *tmpname, void *data),
+         void *data)
 {
-        char *name        = NULL;
-        DIR *d            = NULL;
-        struct dirent *de = NULL;
-        struct dirent scratch[2] = {{0,},};
-        pid_t pid         = -1;
-        pid_t ppid        = -1;
-        int ret           = 0;
-
-        d = sys_opendir (PROC);
-        if (!d)
-                return -1;
-
-        for (;;) {
-                errno = 0;
-                de = sys_readdir (d, scratch);
-                if (!de || errno != 0)
-                        break;
-
-                if (gf_string2int (de->d_name, &pid) != -1 && pid >= 0) {
-                        ppid = pidinfo (pid, &name);
-                        switch (ppid) {
-                        case -1: continue;
-                        case -2: ret = -1; break;
-                        }
-                        ret = proch (pid, ppid, name, data);
-                        GF_FREE (name);
-                        if (ret)
-                                break;
-                }
-        }
-        sys_closedir (d);
-        if (!de && errno) {
-                fprintf (stderr, "failed to traverse "PROC" (%s)\n",
-                         strerror (errno));
-                ret = -1;
+    char *name = NULL;
+    DIR *d = NULL;
+    struct dirent *de = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    pid_t pid = -1;
+    pid_t ppid = -1;
+    int ret = 0;
+
+    d = sys_opendir(PROC);
+    if (!d)
+        return -1;
+
+    for (;;) {
+        errno = 0;
+        de = sys_readdir(d, scratch);
+        if (!de || errno != 0)
+            break;
+
+        if (gf_string2int(de->d_name, &pid) != -1 && pid >= 0) {
+            ppid = pidinfo(pid, &name);
+            switch (ppid) {
+                case -1:
+                    continue;
+                case -2:
+                    break;
+            }
+            ret = proch(pid, ppid, name, data);
+            GF_FREE(name);
+            if (ret)
+                break;
         }
-
-        return ret;
+    }
+    sys_closedir(d);
+    if (!de && errno) {
+        fprintf(stderr, "failed to traverse " PROC " (%s)\n", strerror(errno));
+        ret = -1;
+    }
+
+    return ret;
 }
diff --git a/geo-replication/src/procdiggy.h b/geo-replication/src/procdiggy.h
index 56dfc4eb213..e17ccd31c89 100644
--- a/geo-replication/src/procdiggy.h
+++ b/geo-replication/src/procdiggy.h
@@ -13,8 +13,9 @@
 
 #define PROC "/proc"
 
-pid_t pidinfo (pid_t pid, char **name);
-
-int prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
-              void *data);
+pid_t
+pidinfo(pid_t pid, char **name);
 
+int
+prociter(int (*proch)(pid_t pid, pid_t ppid, char *name, void *data),
+         void *data);
diff --git a/geo-replication/src/set_geo_rep_pem_keys.sh b/geo-replication/src/set_geo_rep_pem_keys.sh
index ae23f4ff0c6..8a43fa39d1f 100755
--- a/geo-replication/src/set_geo_rep_pem_keys.sh
+++ b/geo-replication/src/set_geo_rep_pem_keys.sh
@@ -47,6 +47,7 @@ function main()
         cp $home_dir/${COMMON_SECRET_PEM_PUB} ${GLUSTERD_WORKDIR}/geo-replication/
         gluster system:: copy file /geo-replication/${COMMON_SECRET_PEM_PUB}
         gluster system:: execute add_secret_pub $user geo-replication/${master_vol}_${slave_vol}_common_secret.pem.pub
+        gluster vol set ${slave_vol} features.read-only on
     else
         echo "$home_dir/common_secret.pem.pub not present. Please run geo-replication command on master with push-pem option to generate the file"
         exit 1;
diff --git a/geo-replication/syncdaemon/Makefile.am b/geo-replication/syncdaemon/Makefile.am
index 7cdaf45ddec..d70e3368faf 100644
--- a/geo-replication/syncdaemon/Makefile.am
+++ b/geo-replication/syncdaemon/Makefile.am
@@ -1,8 +1,8 @@
-syncdaemondir = $(libexecdir)/glusterfs/python/syncdaemon
+syncdaemondir = $(GLUSTERFS_LIBEXECDIR)/python/syncdaemon
 
-syncdaemon_PYTHON = gconf.py gsyncd.py __init__.py master.py README.md repce.py \
-	resource.py configinterface.py syncdutils.py monitor.py libcxattr.py \
-	$(top_builddir)/contrib/ipaddr-py/ipaddr.py libgfchangelog.py changelogagent.py \
-	gsyncdstatus.py conf.py
+syncdaemon_PYTHON = rconf.py gsyncd.py __init__.py master.py README.md repce.py \
+	resource.py syncdutils.py monitor.py libcxattr.py gsyncdconfig.py \
+	libgfchangelog.py gsyncdstatus.py conf.py logutils.py \
+	subcmds.py argsupgrade.py py2py3.py
 
 CLEANFILES =
diff --git a/geo-replication/syncdaemon/README.md b/geo-replication/syncdaemon/README.md
index 2a202e3f99e..5ab785ae669 100644
--- a/geo-replication/syncdaemon/README.md
+++ b/geo-replication/syncdaemon/README.md
@@ -19,7 +19,6 @@ INSTALLATION
 
 As of now, the supported way of operation is running from the source directory or using the RPMs given.
 
-If you use Python 2.4.x, you need to install the [Ctypes module](http://python.net/crew/theller/ctypes/).
 
 CONFIGURATION
 -------------
diff --git a/geo-replication/syncdaemon/__codecheck.py b/geo-replication/syncdaemon/__codecheck.py
index 45dbd26bb64..9437147f7d9 100644
--- a/geo-replication/syncdaemon/__codecheck.py
+++ b/geo-replication/syncdaemon/__codecheck.py
@@ -8,6 +8,7 @@
 # cases as published by the Free Software Foundation.
 #
 
+from __future__ import print_function
 import os
 import os.path
 import sys
@@ -45,7 +46,7 @@ class IPNetwork(list):
     gsyncd = sys.modules['gsyncd']
     for a in [['--help'], ['--version'],
               ['--canonicalize-escape-url', '/foo']]:
-        print('>>> invoking program with args: %s' % ' '.join(a))
+        print(('>>> invoking program with args: %s' % ' '.join(a)))
         pid = os.fork()
         if not pid:
             sys_argv_set(a)
diff --git a/geo-replication/syncdaemon/argsupgrade.py b/geo-replication/syncdaemon/argsupgrade.py
new file mode 100644
index 00000000000..7af40633ef8
--- /dev/null
+++ b/geo-replication/syncdaemon/argsupgrade.py
@@ -0,0 +1,359 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+# Converts old style args into new style args
+
+from __future__ import print_function
+import sys
+from argparse import ArgumentParser
+import socket
+import os
+
+from syncdutils import GsyncdError
+from conf import GLUSTERD_WORKDIR
+
+
+def gethostbyname(hnam):
+    """gethostbyname wrapper"""
+    try:
+        return socket.gethostbyname(hnam)
+    except socket.gaierror:
+        ex = sys.exc_info()[1]
+        raise GsyncdError("failed to resolve %s: %s" %
+                          (hnam, ex.strerror))
+
+
+def slave_url(urldata):
+    urldata = urldata.replace("ssh://", "")
+    host, vol = urldata.split("::")
+    vol = vol.split(":")[0]
+    return "%s::%s" % (host, vol)
+
+
+def init_gsyncd_template_conf():
+    path = GLUSTERD_WORKDIR + "/geo-replication/gsyncd_template.conf"
+    dname = os.path.dirname(path)
+    if not os.path.exists(dname):
+        try:
+            os.mkdir(dname)
+        except OSError:
+            pass
+
+    if not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def init_gsyncd_session_conf(master, slave):
+    slave = slave_url(slave)
+    master = master.strip(":")
+    slavehost, slavevol = slave.split("::")
+    slavehost = slavehost.split("@")[-1]
+
+    # Session Config File
+    path = "%s/geo-replication/%s_%s_%s/gsyncd.conf" % (
+        GLUSTERD_WORKDIR, master, slavehost, slavevol)
+
+    if os.path.exists(os.path.dirname(path)) and not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def init_gsyncd_conf(path):
+    dname = os.path.dirname(path)
+    if not os.path.exists(dname):
+        try:
+            os.mkdir(dname)
+        except OSError:
+            pass
+
+    if os.path.exists(dname) and not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def upgrade():
+    # Create dummy template conf(empty), hack to avoid glusterd
+    # fail when it does stat to check the existence.
+    init_gsyncd_template_conf()
+
+    inet6 = False
+    if "--inet6" in sys.argv:
+        inet6 = True
+
+    if "--monitor" in sys.argv:
+        # python gsyncd.py --path=/bricks/b1
+        # --monitor -c gsyncd.conf
+        # --iprefix=/var :gv1
+        # --glusterd-uuid=f26ac7a8-eb1b-4ea7-959c-80b27d3e43d0
+        # f241::gv2
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--glusterd-uuid")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        p.add_argument("--path", action="append")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # Overwrite the sys.argv after rearrange
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+        sys.argv = [
+            sys.argv[0],
+            "monitor",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--local-node-id",
+            pargs.glusterd_uuid
+        ]
+    elif "--status-get" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2
+        # --status-get --path /bricks/b1
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--path")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        sys.argv = [
+            sys.argv[0],
+            "status",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--local-path",
+            pargs.path
+        ]
+    elif "--canonicalize-url" in sys.argv:
+        # This can accept multiple URLs and converts each URL to the
+        # format ssh://USER@IP:gluster://127.0.0.1:VOLUME
+        # This format not used in gsyncd, but added for glusterd compatibility
+        p = ArgumentParser()
+        p.add_argument("--canonicalize-url", nargs="+")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        for url in pargs.canonicalize_url:
+            host, vol = url.split("::")
+            host = host.replace("ssh://", "")
+            remote_addr = host
+            if "@" not in remote_addr:
+                remote_addr = "root@" + remote_addr
+
+            user, hname = remote_addr.split("@")
+
+            if not inet6:
+                hname = gethostbyname(hname)
+
+            print(("ssh://%s@%s:gluster://127.0.0.1:%s" % (
+                user, hname, vol)))
+
+        sys.exit(0)
+    elif "--normalize-url" in sys.argv:
+        # Adds schema prefix as ssh://
+        # This format not used in gsyncd, but added for glusterd compatibility
+        p = ArgumentParser()
+        p.add_argument("--normalize-url")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+        print(("ssh://%s" % slave_url(pargs.normalize_url)))
+        sys.exit(0)
+    elif "--config-get-all" in sys.argv:
+        #  -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-get-all
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        sys.argv = [
+            sys.argv[0],
+            "config-get",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--show-defaults",
+            "--use-underscore"
+        ]
+    elif "--verify" in sys.argv and "spawning" in sys.argv:
+        # Just checks that able to spawn gsyncd or not
+        sys.exit(0)
+    elif "--slavevoluuid-get" in sys.argv:
+        # --slavevoluuid-get f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--slavevoluuid-get")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+        host, vol = pargs.slavevoluuid_get.split("::")
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "voluuidget",
+            host,
+            vol
+        ]
+    elif "--config-set-rx" in sys.argv:
+        # Not required since default conf is not generated
+        # and custom conf generated only when required
+        # -c gsyncd.conf --config-set-rx remote-gsyncd
+        # /usr/local/libexec/glusterfs/gsyncd . .
+        # Touch the gsyncd.conf file and create session
+        # directory if required
+        p = ArgumentParser()
+        p.add_argument("-c", dest="config_file")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # If not template conf then it is trying to create
+        # session config, create a empty file instead
+        if pargs.config_file.endswith("gsyncd.conf"):
+            init_gsyncd_conf(pargs.config_file)
+        sys.exit(0)
+    elif "--create" in sys.argv:
+        # To update monitor status file
+        # --create Created -c gsyncd.conf
+        # --iprefix=/var :gv1 f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--create")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "monitor-status",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            pargs.create
+        ]
+    elif "--config-get" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-get pid-file
+        p = ArgumentParser()
+        p.add_argument("--config-get")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-get",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--only-value",
+            "--show-defaults",
+            "--name",
+            pargs.config_get.replace("_", "-")
+        ]
+    elif "--config-set" in sys.argv:
+        # ignore session-owner
+        if "session-owner" in sys.argv:
+            sys.exit(0)
+
+        # --path=/bricks/b1  -c gsyncd.conf :gv1 f241::gv2
+        # --config-set log_level DEBUG
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--config-set", action='store_true')
+        p.add_argument("name")
+        p.add_argument("--value")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-set",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--name=%s" % pargs.name,
+            "--value=%s" % pargs.value
+        ]
+    elif "--config-check" in sys.argv:
+        # --config-check georep_session_working_dir
+        p = ArgumentParser()
+        p.add_argument("--config-check")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-check",
+            pargs.config_check.replace("_", "-")
+        ]
+    elif "--config-del" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-del log_level
+        p = ArgumentParser()
+        p.add_argument("--config-del")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-reset",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            pargs.config_del.replace("_", "-")
+        ]
+    elif "--delete" in sys.argv:
+        # --delete -c gsyncd.conf --iprefix=/var
+        # --path-list=--path=/bricks/b1  :gv1 f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--reset-sync-time", action="store_true")
+        p.add_argument("--path-list")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--iprefix")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        paths = pargs.path_list.split("--path=")
+        paths = ["--path=%s" % x.strip() for x in paths if x.strip() != ""]
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "delete",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave)
+        ]
+        sys.argv += paths
+
+        if pargs.reset_sync_time:
+            sys.argv.append("--reset-sync-time")
+
+    if inet6:
+        # Add `--inet6` as first argument
+        sys.argv = [sys.argv[0], "--inet6"] + sys.argv[1:]
diff --git a/geo-replication/syncdaemon/changelogagent.py b/geo-replication/syncdaemon/changelogagent.py
deleted file mode 100644
index 731dbd06f57..00000000000
--- a/geo-replication/syncdaemon/changelogagent.py
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
-# This file is part of GlusterFS.
-
-# This file is licensed to you under your choice of the GNU Lesser
-# General Public License, version 3 or any later version (LGPLv3 or
-# later), or the GNU General Public License, version 2 (GPLv2), in all
-# cases as published by the Free Software Foundation.
-#
-
-import os
-import logging
-import syncdutils
-from syncdutils import select, CHANGELOG_AGENT_SERVER_VERSION
-from repce import RepceServer
-
-
-class _MetaChangelog(object):
-
-    def __getattr__(self, meth):
-        from libgfchangelog import Changes as LChanges
-        xmeth = [m for m in dir(LChanges) if m[0] != '_']
-        if meth not in xmeth:
-            return
-        for m in xmeth:
-            setattr(self, m, getattr(LChanges, m))
-        return getattr(self, meth)
-
-Changes = _MetaChangelog()
-
-
-class Changelog(object):
-    def version(self):
-        return CHANGELOG_AGENT_SERVER_VERSION
-
-    def init(self):
-        return Changes.cl_init()
-
-    def register(self, cl_brick, cl_dir, cl_log, cl_level, retries=0):
-        return Changes.cl_register(cl_brick, cl_dir, cl_log, cl_level, retries)
-
-    def scan(self):
-        return Changes.cl_scan()
-
-    def getchanges(self):
-        return Changes.cl_getchanges()
-
-    def done(self, clfile):
-        return Changes.cl_done(clfile)
-
-    def history(self, changelog_path, start, end, num_parallel):
-        return Changes.cl_history_changelog(changelog_path, start, end,
-                                            num_parallel)
-
-    def history_scan(self):
-        return Changes.cl_history_scan()
-
-    def history_getchanges(self):
-        return Changes.cl_history_getchanges()
-
-    def history_done(self, clfile):
-        return Changes.cl_history_done(clfile)
-
-
-class ChangelogAgent(object):
-    def __init__(self, obj, fd_tup):
-        (inf, ouf, rw, ww) = fd_tup.split(',')
-        repce = RepceServer(obj, int(inf), int(ouf), 1)
-        t = syncdutils.Thread(target=lambda: (repce.service_loop(),
-                                              syncdutils.finalize()))
-        t.start()
-        logging.info('Agent listining...')
-
-        select((), (), ())
-
-
-def agent(obj, fd_tup):
-    return ChangelogAgent(obj, fd_tup)
diff --git a/geo-replication/syncdaemon/conf.py.in b/geo-replication/syncdaemon/conf.py.in
index 88072789496..2042fa9cdfb 100644
--- a/geo-replication/syncdaemon/conf.py.in
+++ b/geo-replication/syncdaemon/conf.py.in
@@ -12,3 +12,6 @@ GLUSTERFS_LIBEXECDIR = '@GLUSTERFS_LIBEXECDIR@'
 GLUSTERD_WORKDIR = "@GLUSTERD_WORKDIR@"
 
 LOCALSTATEDIR = "@localstatedir@"
+UUID_FILE = "@GLUSTERD_WORKDIR@/glusterd.info"
+GLUSTERFS_CONFDIR = "@SYSCONF_DIR@/glusterfs"
+GCONF_VERSION = 4.0
diff --git a/geo-replication/syncdaemon/configinterface.py b/geo-replication/syncdaemon/configinterface.py
deleted file mode 100644
index adcefb8ab52..00000000000
--- a/geo-replication/syncdaemon/configinterface.py
+++ /dev/null
@@ -1,415 +0,0 @@
-#
-# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
-# This file is part of GlusterFS.
-
-# This file is licensed to you under your choice of the GNU Lesser
-# General Public License, version 3 or any later version (LGPLv3 or
-# later), or the GNU General Public License, version 2 (GPLv2), in all
-# cases as published by the Free Software Foundation.
-#
-
-try:
-    import ConfigParser
-except ImportError:
-    # py 3
-    import configparser as ConfigParser
-import re
-from string import Template
-import os
-import errno
-import sys
-from stat import ST_DEV, ST_INO, ST_MTIME
-import tempfile
-import shutil
-
-from syncdutils import escape, unescape, norm, update_file, GsyncdError
-from conf import GLUSTERD_WORKDIR, LOCALSTATEDIR
-
-SECT_ORD = '__section_order__'
-SECT_META = '__meta__'
-config_version = 2.0
-
-re_type = type(re.compile(''))
-
-TMPL_CONFIG_FILE = GLUSTERD_WORKDIR + "/geo-replication/gsyncd_template.conf"
-
-# (SECTION, OPTION, OLD VALUE, NEW VALUE)
-CONFIGS = (
-    ("peersrx . .",
-     "georep_session_working_dir",
-     "",
-     GLUSTERD_WORKDIR + "/geo-replication/${mastervol}_${remotehost}_"
-     "${slavevol}/"),
-    ("peersrx .",
-     "gluster_params",
-     "aux-gfid-mount xlator-option=\*-dht.assert-no-child-down=true",
-     "aux-gfid-mount"),
-    ("peersrx .",
-     "gluster_params",
-     "aux-gfid-mount",
-     "aux-gfid-mount acl"),
-    ("peersrx . .",
-     "ssh_command_tar",
-     "",
-     "ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no "
-     "-i " + GLUSTERD_WORKDIR + "/geo-replication/tar_ssh.pem"),
-    ("peersrx . .",
-     "changelog_log_file",
-     "",
-     "${iprefix}/log/glusterfs/geo-replication/${mastervol}"
-     "/${eSlave}${local_id}-changes.log"),
-    ("peersrx . .",
-     "working_dir",
-     LOCALSTATEDIR + "/run/gluster/${mastervol}/${eSlave}",
-     "${iprefix}/lib/misc/glusterfsd/${mastervol}/${eSlave}"),
-    ("peersrx . .",
-     "ignore_deletes",
-     "true",
-     "false"),
-    ("peersrx . .",
-     "pid-file",
-     GLUSTERD_WORKDIR + "/geo-replication/${mastervol}_${remotehost}_"
-     "${slavevol}/${eSlave}.pid",
-     GLUSTERD_WORKDIR + "/geo-replication/${mastervol}_${remotehost}_"
-     "${slavevol}/monitor.pid"),
-    ("peersrx . .",
-     "state-file",
-     GLUSTERD_WORKDIR + "/geo-replication/${mastervol}_${remotehost}_"
-     "${slavevol}/${eSlave}.status",
-     GLUSTERD_WORKDIR + "/geo-replication/${mastervol}_${remotehost}_"
-     "${slavevol}/monitor.status"),
-)
-
-
-def upgrade_config_file(path, confdata):
-    config_change = False
-    config = ConfigParser.RawConfigParser()
-    # If confdata.rx present then glusterd is adding config values,
-    # it will create config file if not exists. config.read is fine in
-    # this case since any other error will be raised during write.
-    if getattr(confdata, "rx", False):
-        config.read(path)
-    else:
-        with open(path) as fp:
-            config.readfp(fp)
-
-    for sec, opt, oldval, newval in CONFIGS:
-        try:
-            val = config.get(sec, opt)
-        except ConfigParser.NoOptionError:
-            # if new config opt not exists
-            config_change = True
-            config.set(sec, opt, newval)
-            continue
-        except ConfigParser.Error:
-            """
-            When gsyncd invoked at the time of create, config file
-            will not be their. Ignore any ConfigParser errors
-            """
-            continue
-
-        if val == newval:
-            # value is same as new val
-            continue
-
-        if val == oldval:
-            # config value needs update
-            config_change = True
-            config.set(sec, opt, newval)
-
-    # To convert from old peers section format to new peers section format.
-    # Old format: peers gluster://<master ip>:<master vol> \
-    #              ssh://root@<slave ip>:gluster://<master ip>:<slave vol>
-    # New format: peers <master vol name> <slave vol name>
-    for old_sect in config.sections():
-        if old_sect.startswith("peers "):
-            peers_data = old_sect.split(" ")
-            mvol = peers_data[1].split("%3A")[-1]
-            svol = peers_data[2].split("%3A")[-1]
-            new_sect = "peers {0} {1}".format(mvol, svol)
-
-            if old_sect == new_sect:
-                # Already in new format "peers mastervol slavevol"
-                continue
-
-            # Create new section if not exists
-            try:
-                config.add_section(new_sect)
-            except ConfigParser.DuplicateSectionError:
-                pass
-
-            config_change = True
-            # Add all the items of old_sect to new_sect
-            for key, val in config.items(old_sect):
-                config.set(new_sect, key, val)
-
-            # Delete old section
-            config.remove_section(old_sect)
-
-    if config_change:
-        tempConfigFile = tempfile.NamedTemporaryFile(mode="wb", delete=False)
-        with open(tempConfigFile.name, 'wb') as configFile:
-            config.write(configFile)
-
-        # If src and dst are two different file system, then os.rename
-        # fails, In this case if temp file created in /tmp and if /tmp is
-        # separate fs then os.rename gives following error, so use shutil
-        # OSError: [Errno 18] Invalid cross-device link
-        # mail.python.org/pipermail/python-list/2005-February/342893.html
-        shutil.move(tempConfigFile.name, path)
-
-
-class MultiDict(object):
-
-    """a virtual dict-like class which functions as the union
-    of underlying dicts"""
-
-    def __init__(self, *dd):
-        self.dicts = dd
-
-    def __getitem__(self, key):
-        val = None
-        for d in self.dicts:
-            if d.get(key) is not None:
-                val = d[key]
-        if val is None:
-            raise KeyError(key)
-        return val
-
-
-class GConffile(object):
-
-    """A high-level interface to ConfigParser which flattens the two-tiered
-       config layout by implenting automatic section dispatch based on initial
-       parameters.
-
-    Also ensure section ordering in terms of their time of addition -- a compat
-    hack for Python < 2.7.
-    """
-
-    def _normconfig(self):
-        """normalize config keys by s/-/_/g"""
-        for n, s in self.config._sections.items():
-            if n.find('__') == 0:
-                continue
-            s2 = type(s)()
-            for k, v in s.items():
-                if k.find('__') != 0:
-                    k = norm(k)
-                s2[k] = v
-            self.config._sections[n] = s2
-
-    def __init__(self, path, peers, confdata, *dd):
-        """
-        - .path: location of config file
-        - .config: underlying ConfigParser instance
-        - .peers: on behalf of whom we flatten .config
-          (master, or master-slave url pair)
-        - .auxdicts: template subtituents
-        """
-        self.peers = peers
-        self.path = path
-        self.auxdicts = dd
-        self.config = ConfigParser.RawConfigParser()
-        if getattr(confdata, "rx", False):
-            self.config.read(path)
-        else:
-            with open(path) as fp:
-                self.config.readfp(fp)
-
-        self.dev, self.ino, self.mtime = -1, -1, -1
-        self._normconfig()
-
-    def _load(self):
-        try:
-            sres = os.stat(self.path)
-            self.dev = sres[ST_DEV]
-            self.ino = sres[ST_INO]
-            self.mtime = sres[ST_MTIME]
-        except (OSError, IOError):
-            if sys.exc_info()[1].errno == errno.ENOENT:
-                sres = None
-
-        self.config = ConfigParser.RawConfigParser()
-        with open(self.path) as fp:
-            self.config.readfp(fp)
-        self._normconfig()
-
-    def get_realtime(self, opt):
-        try:
-            sres = os.stat(self.path)
-        except (OSError, IOError):
-            if sys.exc_info()[1].errno == errno.ENOENT:
-                sres = None
-            else:
-                raise
-
-        # compare file system stat with that of our stream file handle
-        if not sres or sres[ST_DEV] != self.dev or \
-           sres[ST_INO] != self.ino or self.mtime != sres[ST_MTIME]:
-            self._load()
-
-        return self.get(opt, printValue=False)
-
-    def section(self, rx=False):
-        """get the section name of the section representing .peers
-        in .config"""
-        peers = self.peers
-        if not peers:
-            peers = ['.', '.']
-            rx = True
-        if rx:
-            return ' '.join(['peersrx'] + [escape(u) for u in peers])
-        else:
-            return ' '.join(['peers'] + [u.split(':')[-1] for u in peers])
-
-    @staticmethod
-    def parse_section(section):
-        """retrieve peers sequence encoded by section name
-           (as urls or regexen, depending on section type)
-        """
-        sl = section.split()
-        st = sl.pop(0)
-        sl = [unescape(u) for u in sl]
-        if st == 'peersrx':
-            sl = [re.compile(u) for u in sl]
-        return sl
-
-    def ord_sections(self):
-        """Return an ordered list of sections.
-
-        Ordering happens based on the auxiliary
-        SECT_ORD section storing indices for each
-        section added through the config API.
-
-        To not to go corrupt in case of manually
-        written config files, we take care to append
-        also those sections which are not registered
-        in SECT_ORD.
-
-        Needed for python 2.{4,5,6} where ConfigParser
-        cannot yet order sections/options internally.
-        """
-        so = {}
-        if self.config.has_section(SECT_ORD):
-            so = self.config._sections[SECT_ORD]
-        so2 = {}
-        for k, v in so.items():
-            if k != '__name__':
-                so2[k] = int(v)
-        tv = 0
-        if so2:
-            tv = max(so2.values()) + 1
-        ss = [s for s in self.config.sections() if s.find('__') != 0]
-        for s in ss:
-            if s in so.keys():
-                continue
-            so2[s] = tv
-            tv += 1
-
-        def scmp(x, y):
-            return cmp(*(so2[s] for s in (x, y)))
-        ss.sort(scmp)
-        return ss
-
-    def update_to(self, dct, allow_unresolved=False):
-        """update @dct from key/values of ours.
-
-        key/values are collected from .config by filtering the regexp sections
-        according to match, and from .section. The values are treated as
-        templates, which are substituted from .auxdicts and (in case of regexp
-        sections) match groups.
-        """
-        if not self.peers:
-            raise GsyncdError('no peers given, cannot select matching options')
-
-        def update_from_sect(sect, mud):
-            for k, v in self.config._sections[sect].items():
-                if k == '__name__':
-                    continue
-                if allow_unresolved:
-                    dct[k] = Template(v).safe_substitute(mud)
-                else:
-                    dct[k] = Template(v).substitute(mud)
-        for sect in self.ord_sections():
-            sp = self.parse_section(sect)
-            if isinstance(sp[0], re_type) and len(sp) == len(self.peers):
-                match = True
-                mad = {}
-                for i in range(len(sp)):
-                    m = sp[i].search(self.peers[i])
-                    if not m:
-                        match = False
-                        break
-                    for j in range(len(m.groups())):
-                        mad['match%d_%d' % (i + 1, j + 1)] = m.groups()[j]
-                if match:
-                    update_from_sect(sect, MultiDict(dct, mad, *self.auxdicts))
-        if self.config.has_section(self.section()):
-            update_from_sect(self.section(), MultiDict(dct, *self.auxdicts))
-
-    def get(self, opt=None, printValue=True):
-        """print the matching key/value pairs from .config,
-           or if @opt given, the value for @opt (according to the
-           logic described in .update_to)
-        """
-        d = {}
-        self.update_to(d, allow_unresolved=True)
-        if opt:
-            opt = norm(opt)
-            v = d.get(opt)
-            if v:
-                if printValue:
-                    print(v)
-                else:
-                    return v
-        else:
-            for k, v in d.iteritems():
-                if k == '__name__':
-                    continue
-                print("%s: %s" % (k, v))
-
-    def write(self, trfn, opt, *a, **kw):
-        """update on-disk config transactionally
-
-        @trfn is the transaction function
-        """
-        def mergeconf(f):
-            self.config = ConfigParser.RawConfigParser()
-            self.config.readfp(f)
-            self._normconfig()
-            if not self.config.has_section(SECT_META):
-                self.config.add_section(SECT_META)
-            self.config.set(SECT_META, 'version', config_version)
-            return trfn(norm(opt), *a, **kw)
-
-        def updateconf(f):
-            self.config.write(f)
-        update_file(self.path, updateconf, mergeconf)
-
-    def _set(self, opt, val, rx=False):
-        """set @opt to @val in .section"""
-        sect = self.section(rx)
-        if not self.config.has_section(sect):
-            self.config.add_section(sect)
-            # regarding SECT_ORD, cf. ord_sections
-            if not self.config.has_section(SECT_ORD):
-                self.config.add_section(SECT_ORD)
-            self.config.set(
-                SECT_ORD, sect, len(self.config._sections[SECT_ORD]))
-        self.config.set(sect, opt, val)
-        return True
-
-    def set(self, opt, *a, **kw):
-        """perform ._set transactionally"""
-        self.write(self._set, opt, *a, **kw)
-
-    def _delete(self, opt, rx=False):
-        """delete @opt from .section"""
-        sect = self.section(rx)
-        if self.config.has_section(sect):
-            return self.config.remove_option(sect, opt)
-
-    def delete(self, opt, *a, **kw):
-        """perform ._delete transactionally"""
-        self.write(self._delete, opt, *a, **kw)
diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py
index ee2a9b334d3..257ed72c6ae 100644
--- a/geo-replication/syncdaemon/gsyncd.py
+++ b/geo-replication/syncdaemon/gsyncd.py
@@ -1,748 +1,325 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
 #
-# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
-# This file is part of GlusterFS.
-
-# This file is licensed to you under your choice of the GNU Lesser
-# General Public License, version 3 or any later version (LGPLv3 or
-# later), or the GNU General Public License, version 2 (GPLv2), in all
-# cases as published by the Free Software Foundation.
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 #
 
+from argparse import ArgumentParser
+import time
 import os
-import os.path
-import glob
+from errno import EEXIST
 import sys
-import time
 import logging
-import shutil
-import optparse
-import fcntl
-import fnmatch
-from optparse import OptionParser, SUPPRESS_HELP
-from logging import Logger, handlers
-from errno import ENOENT
-
-from ipaddr import IPAddress, IPNetwork
-
-from gconf import gconf
-from syncdutils import FreeObject, norm, grabpidfile, finalize
-from syncdutils import log_raise_exception, privileged, boolify
-from syncdutils import GsyncdError, select, set_term_handler
-from configinterface import GConffile, upgrade_config_file, TMPL_CONFIG_FILE
-import resource
-from monitor import monitor
-import xml.etree.ElementTree as XET
-from subprocess import PIPE
-import subprocess
-from changelogagent import agent, Changelog
-from gsyncdstatus import set_monitor_status, GeorepStatus
-from libcxattr import Xattr
-import struct
-
-ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
-
-
-class GLogger(Logger):
-
-    """Logger customizations for gsyncd.
-
-    It implements a log format similar to that of glusterfs.
-    """
-
-    def makeRecord(self, name, level, *a):
-        rv = Logger.makeRecord(self, name, level, *a)
-        rv.nsecs = (rv.created - int(rv.created)) * 1000000
-        fr = sys._getframe(4)
-        callee = fr.f_locals.get('self')
-        if callee:
-            ctx = str(type(callee)).split("'")[1].split('.')[-1]
-        else:
-            ctx = '<top>'
-        if not hasattr(rv, 'funcName'):
-            rv.funcName = fr.f_code.co_name
-        rv.lvlnam = logging.getLevelName(level)[0]
-        rv.ctx = ctx
-        return rv
-
-    @classmethod
-    def setup(cls, **kw):
-        lbl = kw.get('label', "")
-        if lbl:
-            lbl = '(' + lbl + ')'
-        lprm = {'datefmt': "%Y-%m-%d %H:%M:%S",
-                'format': "[%(asctime)s.%(nsecs)d] %(lvlnam)s [%(module)s" +
-                lbl + ":%(lineno)s:%(funcName)s] %(ctx)s: %(message)s"}
-        lprm.update(kw)
-        lvl = kw.get('level', logging.INFO)
-        lprm['level'] = lvl
-        logging.root = cls("root", lvl)
-        logging.setLoggerClass(cls)
-        logging.getLogger().handlers = []
-        logging.getLogger().setLevel(lprm['level'])
-        logging.Formatter.converter = time.gmtime  # Log in GMT/UTC time
-
-        if 'filename' in lprm:
-            try:
-                logging_handler = handlers.WatchedFileHandler(lprm['filename'])
-                formatter = logging.Formatter(fmt=lprm['format'],
-                                              datefmt=lprm['datefmt'])
-                logging_handler.setFormatter(formatter)
-                logging.getLogger().addHandler(logging_handler)
-            except AttributeError:
-                # Python version < 2.6 will not have WatchedFileHandler
-                # so fallback to logging without any handler.
-                # Note: logrotate will not work if Python version is < 2.6
-                logging.basicConfig(**lprm)
-        else:
-            # If filename not passed(not available in lprm) then it may be
-            # streaming.(Ex: {"stream": "/dev/stdout"})
-            logging.basicConfig(**lprm)
-
-    @classmethod
-    def _gsyncd_loginit(cls, **kw):
-        lkw = {}
-        if gconf.log_level:
-            lkw['level'] = gconf.log_level
-        if kw.get('log_file'):
-            if kw['log_file'] in ('-', '/dev/stderr'):
-                lkw['stream'] = sys.stderr
-            elif kw['log_file'] == '/dev/stdout':
-                lkw['stream'] = sys.stdout
-            else:
-                lkw['filename'] = kw['log_file']
-
-        cls.setup(label=kw.get('label'), **lkw)
-
-        lkw.update({'saved_label': kw.get('label')})
-        gconf.log_metadata = lkw
-        gconf.log_exit = True
-
-
-# Given slave host and its volume name, get corresponding volume uuid
-def slave_vol_uuid_get(host, vol):
-    po = subprocess.Popen(['gluster', '--xml', '--remote-host=' + host,
-                           'volume', 'info', vol], bufsize=0,
-                          stdin=None, stdout=PIPE, stderr=PIPE)
-    vix, err = po.communicate()
-    if po.returncode != 0:
-        logging.info("Volume info failed, unable to get "
-                     "volume uuid of %s present in %s,"
-                     "returning empty string: %s" %
-                     (vol, host, po.returncode))
-        return ""
-    vi = XET.fromstring(vix)
-    if vi.find('opRet').text != '0':
-        logging.info("Unable to get volume uuid of %s, "
-                     "present in %s returning empty string: %s" %
-                     (vol, host, vi.find('opErrstr').text))
-        return ""
 
-    try:
-        voluuid = vi.find("volInfo/volumes/volume/id").text
-    except (ParseError, AttributeError, ValueError) as e:
-        logging.info("Parsing failed to volume uuid of %s, "
-                     "present in %s returning empty string: %s" %
-                     (vol, host, e))
-        voluuid = ""
-
-    return voluuid
-
-
-def startup(**kw):
-    """set up logging, pidfile grabbing, daemonization"""
-    if getattr(gconf, 'pid_file', None) and kw.get('go_daemon') != 'postconn':
-        if not grabpidfile():
-            sys.stderr.write("pidfile is taken, exiting.\n")
-            sys.exit(2)
-        gconf.pid_file_owned = True
-
-    if kw.get('go_daemon') == 'should':
-        x, y = os.pipe()
-        gconf.cpid = os.fork()
-        if gconf.cpid:
-            os.close(x)
-            sys.exit()
-        os.close(y)
-        os.setsid()
-        dn = os.open(os.devnull, os.O_RDWR)
-        for f in (sys.stdin, sys.stdout, sys.stderr):
-            os.dup2(dn, f.fileno())
-        if getattr(gconf, 'pid_file', None):
-            if not grabpidfile(gconf.pid_file + '.tmp'):
-                raise GsyncdError("cannot grab temporary pidfile")
-            os.rename(gconf.pid_file + '.tmp', gconf.pid_file)
-        # wait for parent to terminate
-        # so we can start up with
-        # no messing from the dirty
-        # ol' bustard
-        select((x,), (), ())
-        os.close(x)
-
-    GLogger._gsyncd_loginit(**kw)
-
-
-def _unlink(path):
-    try:
-        os.unlink(path)
-    except (OSError, IOError):
-        if sys.exc_info()[1].errno == ENOENT:
-            pass
-        else:
-            raise GsyncdError('Unlink error: %s' % path)
+from logutils import setup_logging
+import gsyncdconfig as gconf
+from rconf import rconf
+import subcmds
+from conf import GLUSTERD_WORKDIR, GLUSTERFS_CONFDIR, GCONF_VERSION
+from syncdutils import (set_term_handler, finalize, lf,
+                        log_raise_exception, FreeObject, escape)
+import argsupgrade
+
+
+GSYNCD_VERSION = "gsyncd.py %s.0" % GCONF_VERSION
 
 
 def main():
-    """main routine, signal/exception handling boilerplates"""
-    gconf.starttime = time.time()
+    rconf.starttime = time.time()
+
+    # If old Glusterd sends commands in old format, below function
+    # converts the sys.argv to new format. This conversion is added
+    # temporarily for backward compatibility. This can be removed
+    # once integrated with Glusterd2
+    # This modifies sys.argv globally, so rest of the code works as usual
+    argsupgrade.upgrade()
+
+    # Default argparse version handler prints to stderr, which is fixed in
+    # 3.x series but not in 2.x, using custom parser to fix this issue
+    if "--version" in sys.argv:
+        print(GSYNCD_VERSION)
+        sys.exit(0)
+
+    parser = ArgumentParser()
+    parser.add_argument("--inet6", action="store_true")
+    sp = parser.add_subparsers(dest="subcmd")
+
+    # Monitor Status File update
+    p = sp.add_parser("monitor-status")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("status", help="Update Monitor Status")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Monitor
+    p = sp.add_parser("monitor")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--pause-on-start",
+                   action="store_true",
+                   help="Start with Paused state")
+    p.add_argument("--local-node-id", help="Local Node ID")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--use-gconf-volinfo", action="store_true")
+
+    # Worker
+    p = sp.add_parser("worker")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("--local-path", help="Local Brick Path")
+    p.add_argument("--feedback-fd", type=int,
+                   help="feedback fd between monitor and worker")
+    p.add_argument("--local-node", help="Local master node")
+    p.add_argument("--local-node-id", help="Local Node ID")
+    p.add_argument("--subvol-num", type=int, help="Subvolume number")
+    p.add_argument("--is-hottier", action="store_true",
+                   help="Is this brick part of hot tier")
+    p.add_argument("--resource-remote",
+                   help="Remote node to connect to Slave Volume")
+    p.add_argument("--resource-remote-id",
+                   help="Remote node ID to connect to Slave Volume")
+    p.add_argument("--slave-id", help="Slave Volume ID")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Slave
+    p = sp.add_parser("slave")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("--session-owner")
+    p.add_argument("--master-brick",
+                   help="Master brick which is connected to the Slave")
+    p.add_argument("--master-node",
+                   help="Master node which is connected to the Slave")
+    p.add_argument("--master-node-id",
+                   help="Master node ID which is connected to the Slave")
+    p.add_argument("--local-node", help="Local Slave node")
+    p.add_argument("--local-node-id", help="Local Slave ID")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # All configurations which are configured via "slave-" options
+    # DO NOT add default values for these configurations, default values
+    # will be picked from template config file
+    p.add_argument("--slave-timeout", type=int,
+                   help="Timeout to end gsyncd at Slave side")
+    p.add_argument("--use-rsync-xattrs", action="store_true")
+    p.add_argument("--slave-log-level", help="Slave Gsyncd Log level")
+    p.add_argument("--slave-gluster-log-level",
+                   help="Slave Gluster mount Log level")
+    p.add_argument("--slave-gluster-command-dir",
+                   help="Directory where Gluster binaries exist on slave")
+    p.add_argument("--slave-access-mount", action="store_true",
+                   help="Do not lazy umount the slave volume")
+    p.add_argument("--master-dist-count", type=int,
+                   help="Master Distribution count")
+
+    # Status
+    p = sp.add_parser("status")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--local-path", help="Local Brick Path")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--json", action="store_true")
+
+    # Config-check
+    p = sp.add_parser("config-check")
+    p.add_argument("name", help="Config Name")
+    p.add_argument("--value", help="Config Value")
+    p.add_argument("--debug", action="store_true")
+
+    # Config-get
+    p = sp.add_parser("config-get")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("--name", help="Config Name")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--show-defaults", action="store_true")
+    p.add_argument("--only-value", action="store_true")
+    p.add_argument("--use-underscore", action="store_true")
+    p.add_argument("--json", action="store_true")
+
+    # Config-set
+    p = sp.add_parser("config-set")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-n", "--name", help="Config Name")
+    p.add_argument("-v", "--value", help="Config Value")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Config-reset
+    p = sp.add_parser("config-reset")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("name", help="Config Name")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # voluuidget
+    p = sp.add_parser("voluuidget")
+    p.add_argument("host", help="Hostname")
+    p.add_argument("volname", help="Volume Name")
+    p.add_argument("--debug", action="store_true")
+
+    # Delete
+    p = sp.add_parser("delete")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument('--path', dest='paths', action="append")
+    p.add_argument("--reset-sync-time", action="store_true",
+                   help="Reset Sync Time")
+    p.add_argument("--debug", action="store_true")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Extra template values, All arguments are already part of template
+    # variables, use this for adding extra variables
+    extra_tmpl_args = {}
+
+    # Add First/Primary Slave host, user and volume
+    if getattr(args, "slave", None) is not None:
+        hostdata, slavevol = args.slave.split("::")
+        hostdata = hostdata.split("@")
+        slavehost = hostdata[-1]
+        slaveuser = "root"
+        if len(hostdata) == 2:
+            slaveuser = hostdata[0]
+        extra_tmpl_args["primary_slave_host"] = slavehost
+        extra_tmpl_args["slaveuser"] = slaveuser
+        extra_tmpl_args["slavevol"] = slavevol
+
+    # Add Bricks encoded path
+    if getattr(args, "local_path", None) is not None:
+        extra_tmpl_args["local_id"] = escape(args.local_path)
+
+    # Add Master Bricks encoded path(For Slave)
+    if getattr(args, "master_brick", None) is not None:
+        extra_tmpl_args["master_brick_id"] = escape(args.master_brick)
+
+    # Load configurations
+    config_file = getattr(args, "config_file", None)
+
+    # Subcmd accepts config file argument but not passed
+    # Set default path for config file in that case
+    # If an subcmd accepts config file then it also accepts
+    # master and Slave arguments.
+    if config_file is None and hasattr(args, "config_file") \
+        and args.subcmd != "slave":
+        config_file = "%s/geo-replication/%s_%s_%s/gsyncd.conf" % (
+            GLUSTERD_WORKDIR,
+            args.master,
+            extra_tmpl_args["primary_slave_host"],
+            extra_tmpl_args["slavevol"])
+
+    # If Config file path not exists, log error and continue using default conf
+    config_file_error_msg = None
+    if config_file is not None and not os.path.exists(config_file):
+        # Logging not yet initialized, create the error message to
+        # log later and reset the config_file to None
+        config_file_error_msg = lf(
+            "Session config file not exists, using the default config",
+            path=config_file)
+        config_file = None
+
+    rconf.config_file = config_file
+
+    # Override gconf values from argument values only if it is slave gsyncd
+    override_from_args = False
+    if args.subcmd == "slave":
+        override_from_args = True
+
+    if config_file is not None and \
+       args.subcmd in ["monitor", "config-get", "config-set", "config-reset"]:
+        ret = gconf.is_config_file_old(config_file, args.master, extra_tmpl_args["slavevol"])
+        if ret is not None:
+           gconf.config_upgrade(config_file, ret)
+
+    # Load Config file
+    gconf.load(GLUSTERFS_CONFDIR + "/gsyncd.conf",
+               config_file,
+               vars(args),
+               extra_tmpl_args,
+               override_from_args)
+
+    # Default label to print in log file
+    label = args.subcmd
+    if args.subcmd in ("worker"):
+        # If Worker, then add brick path also to label
+        label = "%s %s" % (args.subcmd, args.local_path)
+    elif args.subcmd == "slave":
+        # If Slave add Master node and Brick details
+        label = "%s %s%s" % (args.subcmd, args.master_node, args.master_brick)
+
+    # Setup Logger
+    # Default log file
+    log_file = gconf.get("cli-log-file")
+    log_level = gconf.get("cli-log-level")
+    if getattr(args, "master", None) is not None and \
+       getattr(args, "slave", None) is not None:
+        log_file = gconf.get("log-file")
+        log_level = gconf.get("log-level")
+
+    # Use different log file location for Slave log file
+    if args.subcmd == "slave":
+        log_file = gconf.get("slave-log-file")
+        log_level = gconf.get("slave-log-level")
+
+    if args.debug:
+        log_file = "-"
+        log_level = "DEBUG"
+
+    # Create Logdir if not exists
+    try:
+        if log_file != "-":
+            os.mkdir(os.path.dirname(log_file))
+    except OSError as e:
+        if e.errno != EEXIST:
+            raise
+
+    setup_logging(
+        log_file=log_file,
+        level=log_level,
+        label=label
+    )
+
+    if config_file_error_msg is not None:
+        logging.warn(config_file_error_msg)
+
+    # Log message for loaded config file
+    if config_file is not None:
+        logging.debug(lf("Using session config file", path=config_file))
+
     set_term_handler()
-    GLogger.setup()
     excont = FreeObject(exval=0)
+
+    # Gets the function name based on the input argument. For example
+    # if subcommand passed as argument is monitor then it looks for
+    # function with name "subcmd_monitor" in subcmds file
+    func = getattr(subcmds, "subcmd_" + args.subcmd.replace("-", "_"), None)
+
     try:
         try:
-            main_i()
+            if func is not None:
+                rconf.args = args
+                func(args)
         except:
             log_raise_exception(excont)
     finally:
         finalize(exval=excont.exval)
 
 
-def main_i():
-    """internal main routine
-
-    parse command line, decide what action will be taken;
-    we can either:
-    - query/manipulate configuration
-    - format gsyncd urls using gsyncd's url parsing engine
-    - start service in following modes, in given stages:
-      - agent: startup(), ChangelogAgent()
-      - monitor: startup(), monitor()
-      - master: startup(), connect_remote(), connect(), service_loop()
-      - slave: startup(), connect(), service_loop()
-    """
-    rconf = {'go_daemon': 'should'}
-
-    def store_abs(opt, optstr, val, parser):
-        if val and val != '-':
-            val = os.path.abspath(val)
-        setattr(parser.values, opt.dest, val)
-
-    def store_local(opt, optstr, val, parser):
-        rconf[opt.dest] = val
-
-    def store_local_curry(val):
-        return lambda o, oo, vx, p: store_local(o, oo, val, p)
-
-    def store_local_obj(op, dmake):
-        return lambda o, oo, vx, p: store_local(
-            o, oo, FreeObject(op=op, **dmake(vx)), p)
-
-    op = OptionParser(
-        usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
-    op.add_option('--gluster-command-dir', metavar='DIR', default='')
-    op.add_option('--gluster-log-file', metavar='LOGF',
-                  default=os.devnull, type=str, action='callback',
-                  callback=store_abs)
-    op.add_option('--gluster-log-level', metavar='LVL')
-    op.add_option('--changelog-log-level', metavar='LVL', default="INFO")
-    op.add_option('--gluster-params', metavar='PRMS', default='')
-    op.add_option(
-        '--glusterd-uuid', metavar='UUID', type=str, default='',
-        help=SUPPRESS_HELP)
-    op.add_option(
-        '--gluster-cli-options', metavar='OPTS', default='--log-file=-')
-    op.add_option('--mountbroker', metavar='LABEL')
-    op.add_option('-p', '--pid-file', metavar='PIDF', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('-l', '--log-file', metavar='LOGF', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--iprefix', metavar='LOGD', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--changelog-log-file', metavar='LOGF', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--log-file-mbr', metavar='LOGF', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--state-file', metavar='STATF', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--state-detail-file', metavar='STATF',
-                  type=str, action='callback', callback=store_abs)
-    op.add_option('--georep-session-working-dir', metavar='STATF',
-                  type=str, action='callback', callback=store_abs)
-    op.add_option('--ignore-deletes', default=False, action='store_true')
-    op.add_option('--isolated-slave', default=False, action='store_true')
-    op.add_option('--use-rsync-xattrs', default=False, action='store_true')
-    op.add_option('--sync-xattrs', default=True, action='store_true')
-    op.add_option('--sync-acls', default=True, action='store_true')
-    op.add_option('--log-rsync-performance', default=False,
-                  action='store_true')
-    op.add_option('--max-rsync-retries', type=int, default=10)
-    op.add_option('--pause-on-start', default=False, action='store_true')
-    op.add_option('-L', '--log-level', metavar='LVL')
-    op.add_option('-r', '--remote-gsyncd', metavar='CMD',
-                  default=os.path.abspath(sys.argv[0]))
-    op.add_option('--volume-id', metavar='UUID')
-    op.add_option('--slave-id', metavar='ID')
-    op.add_option('--session-owner', metavar='ID')
-    op.add_option('--local-id', metavar='ID', help=SUPPRESS_HELP, default='')
-    op.add_option(
-        '--local-path', metavar='PATH', help=SUPPRESS_HELP, default='')
-    op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
-    op.add_option('--ssh-port', metavar='PORT', type=int, default=22)
-    op.add_option('--ssh-command-tar', metavar='CMD', default='ssh')
-    op.add_option('--rsync-command', metavar='CMD', default='rsync')
-    op.add_option('--rsync-options', metavar='OPTS', default='')
-    op.add_option('--rsync-ssh-options', metavar='OPTS', default='--compress')
-    op.add_option('--timeout', metavar='SEC', type=int, default=120)
-    op.add_option('--connection-timeout', metavar='SEC',
-                  type=int, default=60, help=SUPPRESS_HELP)
-    op.add_option('--sync-jobs', metavar='N', type=int, default=3)
-    op.add_option('--replica-failover-interval', metavar='N',
-                  type=int, default=1)
-    op.add_option('--changelog-archive-format', metavar='N',
-                  type=str, default="%Y%m")
-    op.add_option('--use-meta-volume', default=False, action='store_true')
-    op.add_option('--meta-volume-mnt', metavar='N',
-                  type=str, default="/var/run/gluster/shared_storage")
-    op.add_option(
-        '--turns', metavar='N', type=int, default=0, help=SUPPRESS_HELP)
-    op.add_option('--allow-network', metavar='IPS', default='')
-    op.add_option('--socketdir', metavar='DIR')
-    op.add_option('--state-socket-unencoded', metavar='SOCKF',
-                  type=str, action='callback', callback=store_abs)
-    op.add_option('--checkpoint', metavar='LABEL', default='0')
-
-    # tunables for failover/failback mechanism:
-    # None   - gsyncd behaves as normal
-    # blind  - gsyncd works with xtime pairs to identify
-    #          candidates for synchronization
-    # wrapup - same as normal mode but does not assign
-    #          xtimes to orphaned files
-    # see crawl() for usage of the above tunables
-    op.add_option('--special-sync-mode', type=str, help=SUPPRESS_HELP)
-
-    # changelog or xtime? (TODO: Change the default)
-    op.add_option(
-        '--change-detector', metavar='MODE', type=str, default='xtime')
-    # sleep interval for change detection (xtime crawl uses a hardcoded 1
-    # second sleep time)
-    op.add_option('--change-interval', metavar='SEC', type=int, default=3)
-    # working directory for changelog based mechanism
-    op.add_option('--working-dir', metavar='DIR', type=str,
-                  action='callback', callback=store_abs)
-    op.add_option('--use-tarssh', default=False, action='store_true')
-
-    op.add_option('-c', '--config-file', metavar='CONF',
-                  type=str, action='callback', callback=store_local)
-    # duh. need to specify dest or value will be mapped to None :S
-    op.add_option('--monitor', dest='monitor', action='callback',
-                  callback=store_local_curry(True))
-    op.add_option('--agent', dest='agent', action='callback',
-                  callback=store_local_curry(True))
-    op.add_option('--resource-local', dest='resource_local',
-                  type=str, action='callback', callback=store_local)
-    op.add_option('--resource-remote', dest='resource_remote',
-                  type=str, action='callback', callback=store_local)
-    op.add_option('--feedback-fd', dest='feedback_fd', type=int,
-                  help=SUPPRESS_HELP, action='callback', callback=store_local)
-    op.add_option('--rpc-fd', dest='rpc_fd', type=str, help=SUPPRESS_HELP)
-    op.add_option('--subvol-num', dest='subvol_num', type=str,
-                  help=SUPPRESS_HELP)
-    op.add_option('--listen', dest='listen', help=SUPPRESS_HELP,
-                  action='callback', callback=store_local_curry(True))
-    op.add_option('-N', '--no-daemon', dest="go_daemon",
-                  action='callback', callback=store_local_curry('dont'))
-    op.add_option('--verify', type=str, dest="verify",
-                  action='callback', callback=store_local)
-    op.add_option('--slavevoluuid-get', type=str, dest="slavevoluuid_get",
-                  action='callback', callback=store_local)
-    op.add_option('--create', type=str, dest="create",
-                  action='callback', callback=store_local)
-    op.add_option('--delete', dest='delete', action='callback',
-                  callback=store_local_curry(True))
-    op.add_option('--path-list', dest='path_list', action='callback',
-                  type=str, callback=store_local)
-    op.add_option('--reset-sync-time', default=False, action='store_true')
-    op.add_option('--status-get', dest='status_get', action='callback',
-                  callback=store_local_curry(True))
-    op.add_option('--debug', dest="go_daemon", action='callback',
-                  callback=lambda *a: (store_local_curry('dont')(*a),
-                                       setattr(
-                                           a[-1].values, 'log_file', '-'),
-                                       setattr(a[-1].values, 'log_level',
-                                               'DEBUG'),
-                                       setattr(a[-1].values,
-                                               'changelog_log_file', '-')))
-    op.add_option('--path', type=str, action='append')
-
-    for a in ('check', 'get'):
-        op.add_option('--config-' + a, metavar='OPT', type=str, dest='config',
-                      action='callback',
-                      callback=store_local_obj(a, lambda vx: {'opt': vx}))
-    op.add_option('--config-get-all', dest='config', action='callback',
-                  callback=store_local_obj('get', lambda vx: {'opt': None}))
-    for m in ('', '-rx', '-glob'):
-        # call this code 'Pythonic' eh?
-        # have to define a one-shot local function to be able
-        # to inject (a value depending on the)
-        # iteration variable into the inner lambda
-        def conf_mod_opt_regex_variant(rx):
-            op.add_option('--config-set' + m, metavar='OPT VAL', type=str,
-                          nargs=2, dest='config', action='callback',
-                          callback=store_local_obj('set', lambda vx: {
-                              'opt': vx[0], 'val': vx[1], 'rx': rx}))
-            op.add_option('--config-del' + m, metavar='OPT', type=str,
-                          dest='config', action='callback',
-                          callback=store_local_obj('del', lambda vx: {
-                              'opt': vx, 'rx': rx}))
-        conf_mod_opt_regex_variant(m and m[1:] or False)
-
-    op.add_option('--normalize-url', dest='url_print',
-                  action='callback', callback=store_local_curry('normal'))
-    op.add_option('--canonicalize-url', dest='url_print',
-                  action='callback', callback=store_local_curry('canon'))
-    op.add_option('--canonicalize-escape-url', dest='url_print',
-                  action='callback', callback=store_local_curry('canon_esc'))
-    op.add_option('--is-hottier', default=False, action='store_true')
-
-    tunables = [norm(o.get_opt_string()[2:])
-                for o in op.option_list
-                if (o.callback in (store_abs, 'store_true', None) and
-                    o.get_opt_string() not in ('--version', '--help'))]
-    remote_tunables = ['listen', 'go_daemon', 'timeout',
-                       'session_owner', 'config_file', 'use_rsync_xattrs']
-    rq_remote_tunables = {'listen': True}
-
-    # precedence for sources of values: 1) commandline, 2) cfg file, 3)
-    # defaults for this to work out we need to tell apart defaults from
-    # explicitly set options... so churn out the defaults here and call
-    # the parser with virgin values container.
-    defaults = op.get_default_values()
-    opts, args = op.parse_args(values=optparse.Values())
-    # slave url cleanup, if input comes with vol uuid as follows
-    # 'ssh://fvm1::gv2:07dfddca-94bb-4841-a051-a7e582811467'
-    temp_args = []
-    for arg in args:
-        # Split based on ::
-        data = arg.split("::")
-        if len(data)>1:
-            slavevol_name = data[1].split(":")[0]
-            temp_args.append("%s::%s" % (data[0], slavevol_name))
-        else:
-            temp_args.append(data[0])
-    args = temp_args
-    args_orig = args[:]
-
-    voluuid_get = rconf.get('slavevoluuid_get')
-    if voluuid_get:
-        slave_host, slave_vol = voluuid_get.split("::")
-        svol_uuid = slave_vol_uuid_get(slave_host, slave_vol)
-        print svol_uuid
-        return
-
-    r = rconf.get('resource_local')
-    if r:
-        if len(args) == 0:
-            args.append(None)
-        args[0] = r
-    r = rconf.get('resource_remote')
-    if r:
-        if len(args) == 0:
-            raise GsyncdError('local resource unspecfied')
-        elif len(args) == 1:
-            args.append(None)
-        args[1] = r
-    confdata = rconf.get('config')
-    if not (len(args) == 2 or
-            (len(args) == 1 and rconf.get('listen')) or
-            (len(args) <= 2 and confdata) or
-            rconf.get('url_print')):
-        sys.stderr.write("error: incorrect number of arguments\n\n")
-        sys.stderr.write(op.get_usage() + "\n")
-        sys.exit(1)
-
-    verify = rconf.get('verify')
-    if verify:
-        logging.info(verify)
-        logging.info("Able to spawn gsyncd.py")
-        return
-
-    restricted = os.getenv('_GSYNCD_RESTRICTED_')
-
-    if restricted:
-        allopts = {}
-        allopts.update(opts.__dict__)
-        allopts.update(rconf)
-        bannedtuns = set(allopts.keys()) - set(remote_tunables)
-        if bannedtuns:
-            raise GsyncdError('following tunables cannot be set with '
-                              'restricted SSH invocaton: ' +
-                              ', '.join(bannedtuns))
-        for k, v in rq_remote_tunables.items():
-            if not k in allopts or allopts[k] != v:
-                raise GsyncdError('tunable %s is not set to value %s required '
-                                  'for restricted SSH invocaton' %
-                                  (k, v))
-
-    confrx = getattr(confdata, 'rx', None)
-
-    def makersc(aa, check=True):
-        if not aa:
-            return ([], None, None)
-        ra = [resource.parse_url(u) for u in aa]
-        local = ra[0]
-        remote = None
-        if len(ra) > 1:
-            remote = ra[1]
-        if check and not local.can_connect_to(remote):
-            raise GsyncdError("%s cannot work with %s" %
-                              (local.path, remote and remote.path))
-        return (ra, local, remote)
-    if confrx:
-        # peers are regexen, don't try to parse them
-        if confrx == 'glob':
-            args = ['\A' + fnmatch.translate(a) for a in args]
-        canon_peers = args
-        namedict = {}
-    else:
-        dc = rconf.get('url_print')
-        rscs, local, remote = makersc(args_orig, not dc)
-        if dc:
-            for r in rscs:
-                print(r.get_url(**{'normal': {},
-                                   'canon': {'canonical': True},
-                                   'canon_esc': {'canonical': True,
-                                                 'escaped': True}}[dc]))
-            return
-        pa = ([], [], [])
-        urlprms = (
-            {}, {'canonical': True}, {'canonical': True, 'escaped': True})
-        for x in rscs:
-            for i in range(len(pa)):
-                pa[i].append(x.get_url(**urlprms[i]))
-        _, canon_peers, canon_esc_peers = pa
-        # creating the namedict, a dict representing various ways of referring
-        # to / repreenting peers to be fillable in config templates
-        mods = (lambda x: x, lambda x: x[
-                0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
-        if remote:
-            rmap = {local: ('local', 'master'), remote: ('remote', 'slave')}
-        else:
-            rmap = {local: ('local', 'slave')}
-        namedict = {}
-        for i in range(len(rscs)):
-            x = rscs[i]
-            for name in rmap[x]:
-                for j in range(3):
-                    namedict[mods[j](name)] = pa[j][i]
-                namedict[name + 'vol'] = x.volume
-                if name == 'remote':
-                    namedict['remotehost'] = x.remotehost
-    if not 'config_file' in rconf:
-        rconf['config_file'] = TMPL_CONFIG_FILE
-
-    upgrade_config_file(rconf['config_file'], confdata)
-    gcnf = GConffile(
-        rconf['config_file'], canon_peers, confdata,
-        defaults.__dict__, opts.__dict__, namedict)
-
-    checkpoint_change = False
-    if confdata:
-        opt_ok = norm(confdata.opt) in tunables + [None]
-        if confdata.op == 'check':
-            if opt_ok:
-                sys.exit(0)
-            else:
-                sys.exit(1)
-        elif not opt_ok:
-            raise GsyncdError("not a valid option: " + confdata.opt)
-        if confdata.op == 'get':
-            gcnf.get(confdata.opt)
-        elif confdata.op == 'set':
-            gcnf.set(confdata.opt, confdata.val, confdata.rx)
-        elif confdata.op == 'del':
-            gcnf.delete(confdata.opt, confdata.rx)
-        # when modifying checkpoint, it's important to make a log
-        # of that, so in that case we go on to set up logging even
-        # if its just config invocation
-        if confdata.opt == 'checkpoint' and confdata.op in ('set', 'del') and \
-           not confdata.rx:
-            checkpoint_change = True
-        if not checkpoint_change:
-            return
-
-    gconf.__dict__.update(defaults.__dict__)
-    gcnf.update_to(gconf.__dict__)
-    gconf.__dict__.update(opts.__dict__)
-    gconf.configinterface = gcnf
-
-    delete = rconf.get('delete')
-    if delete:
-        logging.info('geo-replication delete')
-        # remove the stime xattr from all the brick paths so that
-        # a re-create of a session will start sync all over again
-        stime_xattr_name = getattr(gconf, 'master.stime_xattr_name', None)
-
-        # Delete pid file, status file, socket file
-        cleanup_paths = []
-        if getattr(gconf, 'pid_file', None):
-            cleanup_paths.append(gconf.pid_file)
-
-        if getattr(gconf, 'state_file', None):
-            cleanup_paths.append(gconf.state_file)
-
-        if getattr(gconf, 'state_detail_file', None):
-            cleanup_paths.append(gconf.state_detail_file)
-
-        if getattr(gconf, 'state_socket_unencoded', None):
-            cleanup_paths.append(gconf.state_socket_unencoded)
-
-        cleanup_paths.append(rconf['config_file'][:-11] + "*")
-
-        # Cleanup changelog working dirs
-        if getattr(gconf, 'working_dir', None):
-            try:
-                shutil.rmtree(gconf.working_dir)
-            except (IOError, OSError):
-                if sys.exc_info()[1].errno == ENOENT:
-                    pass
-                else:
-                    raise GsyncdError(
-                        'Error while removing working dir: %s' %
-                        gconf.working_dir)
-
-        for path in cleanup_paths:
-            # To delete temp files
-            for f in glob.glob(path + "*"):
-                _unlink(f)
-
-        reset_sync_time = boolify(gconf.reset_sync_time)
-        if reset_sync_time and stime_xattr_name:
-            path_list = rconf.get('path_list')
-            paths = []
-            for p in path_list.split('--path='):
-                stripped_path = p.strip()
-                if stripped_path != "":
-                    # set stime to (0,0) to trigger full volume content resync
-                    # to slave on session recreation
-                    # look at master.py::Xcrawl   hint: zero_zero
-                    Xattr.lsetxattr(stripped_path, stime_xattr_name,
-                                    struct.pack("!II", 0, 0))
-
-        return
-
-    if restricted and gconf.allow_network:
-        ssh_conn = os.getenv('SSH_CONNECTION')
-        if not ssh_conn:
-            # legacy env var
-            ssh_conn = os.getenv('SSH_CLIENT')
-        if ssh_conn:
-            allowed_networks = [IPNetwork(a)
-                                for a in gconf.allow_network.split(',')]
-            client_ip = IPAddress(ssh_conn.split()[0])
-            allowed = False
-            for nw in allowed_networks:
-                if client_ip in nw:
-                    allowed = True
-                    break
-            if not allowed:
-                raise GsyncdError("client IP address is not allowed")
-
-    ffd = rconf.get('feedback_fd')
-    if ffd:
-        fcntl.fcntl(ffd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
-
-    # normalize loglevel
-    lvl0 = gconf.log_level
-    if isinstance(lvl0, str):
-        lvl1 = lvl0.upper()
-        lvl2 = logging.getLevelName(lvl1)
-        # I have _never_ _ever_ seen such an utterly braindead
-        # error condition
-        if lvl2 == "Level " + lvl1:
-            raise GsyncdError('cannot recognize log level "%s"' % lvl0)
-        gconf.log_level = lvl2
-
-    if not privileged() and gconf.log_file_mbr:
-        gconf.log_file = gconf.log_file_mbr
-
-    if checkpoint_change:
-        try:
-            GLogger._gsyncd_loginit(log_file=gconf.log_file, label='conf')
-            if confdata.op == 'set':
-                logging.info('checkpoint %s set' % confdata.val)
-            elif confdata.op == 'del':
-                logging.info('checkpoint info was reset')
-        except IOError:
-            if sys.exc_info()[1].errno == ENOENT:
-                # directory of log path is not present,
-                # which happens if we get here from
-                # a peer-multiplexed "config-set checkpoint"
-                # (as that directory is created only on the
-                # original node)
-                pass
-            else:
-                raise
-        return
-
-    create = rconf.get('create')
-    if create:
-        if getattr(gconf, 'state_file', None):
-            set_monitor_status(gconf.state_file, create)
-        return
-
-    go_daemon = rconf['go_daemon']
-    be_monitor = rconf.get('monitor')
-    be_agent = rconf.get('agent')
-
-    rscs, local, remote = makersc(args)
-
-    status_get = rconf.get('status_get')
-    if status_get:
-        for brick in gconf.path:
-            brick_status = GeorepStatus(gconf.state_file, brick,
-                                        getattr(gconf, "pid_file", None))
-            checkpoint_time = int(getattr(gconf, "checkpoint", "0"))
-            brick_status.print_status(checkpoint_time=checkpoint_time)
-        return
-
-    if not be_monitor and isinstance(remote, resource.SSH) and \
-       go_daemon == 'should':
-        go_daemon = 'postconn'
-        log_file = None
-    else:
-        log_file = gconf.log_file
-    if be_monitor:
-        label = 'monitor'
-    elif be_agent:
-        label = 'agent'
-    elif remote:
-        # master
-        label = gconf.local_path
-    else:
-        label = 'slave'
-    startup(go_daemon=go_daemon, log_file=log_file, label=label)
-    resource.Popen.init_errhandler()
-
-    if be_agent:
-        os.setsid()
-        logging.debug('rpc_fd: %s' % repr(gconf.rpc_fd))
-        return agent(Changelog(), gconf.rpc_fd)
-
-    if be_monitor:
-        return monitor(*rscs)
-
-    logging.info("syncing: %s" % " -> ".join(r.url for r in rscs))
-    if remote:
-        go_daemon = remote.connect_remote(go_daemon=go_daemon)
-        if go_daemon:
-            startup(go_daemon=go_daemon, log_file=gconf.log_file)
-            # complete remote connection in child
-            remote.connect_remote(go_daemon='done')
-    local.connect()
-    if ffd:
-        os.close(ffd)
-    local.service_loop(*[r for r in [remote] if r])
-
-
 if __name__ == "__main__":
     main()
diff --git a/geo-replication/syncdaemon/gsyncdconfig.py b/geo-replication/syncdaemon/gsyncdconfig.py
new file mode 100644
index 00000000000..8848071997a
--- /dev/null
+++ b/geo-replication/syncdaemon/gsyncdconfig.py
@@ -0,0 +1,485 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+try:
+    from ConfigParser import RawConfigParser, NoSectionError
+except ImportError:
+    from configparser import RawConfigParser, NoSectionError
+import os
+import shutil
+from string import Template
+from datetime import datetime
+from threading import Lock
+
+
+# Global object which can be used in other modules
+# once load_config is called
+_gconf = {}
+
+
+class GconfNotConfigurable(Exception):
+    pass
+
+
+class GconfInvalidValue(Exception):
+    pass
+
+
+class Gconf(object):
+    def __init__(self, default_conf_file, custom_conf_file=None,
+                 args={}, extra_tmpl_args={}, override_from_args=False):
+        self.lock = Lock()
+        self.default_conf_file = default_conf_file
+        self.custom_conf_file = custom_conf_file
+        self.tmp_conf_file = None
+        self.gconf = {}
+        self.gconfdata = {}
+        self.gconf_typecast = {}
+        self.template_conf = []
+        self.non_configurable_configs = []
+        self.prev_mtime = 0
+        if custom_conf_file is not None:
+            self.tmp_conf_file = custom_conf_file + ".tmp"
+
+        self.session_conf_items = []
+        self.args = args
+        self.extra_tmpl_args = extra_tmpl_args
+        self.override_from_args = override_from_args
+        # Store default values only if overwritten, Only for JSON/CLI output
+        self.default_values = {}
+        self._load()
+
+    def _tmpl_substitute(self):
+        tmpl_values = {}
+        for k, v in self.gconf.items():
+            tmpl_values[k.replace("-", "_")] = v
+
+        # override the config file values with the one user passed
+        for k, v in self.args.items():
+            # override the existing value only if set by user
+            if v is not None:
+                tmpl_values[k] = v
+
+        for k, v in self.extra_tmpl_args.items():
+            tmpl_values[k] = v
+
+        for k, v in self.gconf.items():
+            if k in self.template_conf and \
+               (isinstance(v, str) or isinstance(v, unicode)):
+                self.gconf[k] = Template(v).safe_substitute(tmpl_values)
+
+    def _do_typecast(self):
+        for k, v in self.gconf.items():
+            cast_func = globals().get(
+                "to_" + self.gconf_typecast.get(k, "string"), None)
+            if cast_func is not None:
+                self.gconf[k] = cast_func(v)
+                if self.default_values.get(k, None) is not None:
+                    self.default_values[k] = cast_func(v)
+
+    def reset(self, name):
+        # If custom conf file is not set then it is only read only configs
+        if self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        # If a config can not be modified
+        if name != "all" and not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        cnf = RawConfigParser()
+        with open(self.custom_conf_file) as f:
+            cnf.readfp(f)
+
+            # Nothing to Reset, Not configured
+            if name != "all":
+                if not cnf.has_option("vars", name):
+                    return True
+
+                # Remove option from custom conf file
+                cnf.remove_option("vars", name)
+            else:
+                # Remove and add empty section, do not disturb if config file
+                # already has any other section
+                try:
+                    cnf.remove_section("vars")
+                except NoSectionError:
+                    pass
+
+                cnf.add_section("vars")
+
+        with open(self.tmp_conf_file, "w") as fw:
+            cnf.write(fw)
+
+        os.rename(self.tmp_conf_file, self.custom_conf_file)
+
+        self.reload()
+
+        return True
+
+    def set(self, name, value):
+        if self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        if not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        if not self._is_valid_value(name, value):
+            raise GconfInvalidValue()
+
+        curr_val = self.gconf.get(name, None)
+        if curr_val == value:
+            return True
+
+        cnf = RawConfigParser()
+        with open(self.custom_conf_file) as f:
+            cnf.readfp(f)
+
+        if not cnf.has_section("vars"):
+            cnf.add_section("vars")
+
+        cnf.set("vars", name, value)
+        with open(self.tmp_conf_file, "w") as fw:
+            cnf.write(fw)
+
+        os.rename(self.tmp_conf_file, self.custom_conf_file)
+
+        self.reload()
+
+        return True
+
+    def check(self, name, value=None, with_conffile=True):
+        if with_conffile and self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        if not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        if value is not None and not self._is_valid_value(name, value):
+            raise GconfInvalidValue()
+
+
+    def _load_with_lock(self):
+        with self.lock:
+            self._load()
+
+    def _load(self):
+        self.gconf = {}
+        self.template_conf = []
+        self.gconf_typecast = {}
+        self.non_configurable_configs = []
+        self.session_conf_items = []
+        self.default_values = {}
+
+        conf = RawConfigParser()
+        # Default Template config file
+        with open(self.default_conf_file) as f:
+            conf.readfp(f)
+
+        # Custom Config file
+        if self.custom_conf_file is not None:
+            with open(self.custom_conf_file) as f:
+                conf.readfp(f)
+
+        # Get version from default conf file
+        self.version = conf.get("__meta__", "version")
+
+        # Populate default values
+        for sect in conf.sections():
+            if sect in ["__meta__", "vars"]:
+                continue
+
+            # Collect list of available options with help details
+            self.gconfdata[sect] = {}
+            for k, v in conf.items(sect):
+                self.gconfdata[sect][k] = v.strip()
+
+            # Collect the Type cast information
+            if conf.has_option(sect, "type"):
+                self.gconf_typecast[sect] = conf.get(sect, "type")
+
+            # Prepare list of configurable conf
+            if conf.has_option(sect, "configurable"):
+                if conf.get(sect, "configurable").lower() == "false":
+                    self.non_configurable_configs.append(sect)
+
+            # if it is a template conf value which needs to be substituted
+            if conf.has_option(sect, "template"):
+                if conf.get(sect, "template").lower().strip() == "true":
+                    self.template_conf.append(sect)
+
+            # Set default values
+            if conf.has_option(sect, "value"):
+                self.gconf[sect] = conf.get(sect, "value").strip()
+
+        # Load the custom conf elements and overwrite
+        if conf.has_section("vars"):
+            for k, v in conf.items("vars"):
+                self.session_conf_items.append(k)
+                self.default_values[k] = self.gconf.get(k, "")
+                self.gconf[k] = v.strip()
+
+        # Overwrite the Slave configurations which are sent as
+        # arguments to gsyncd slave
+        if self.override_from_args:
+            for k, v in self.args.items():
+                k = k.replace("_", "-")
+                if k.startswith("slave-") and k in self.gconf:
+                    self.gconf[k] = v
+
+        self._tmpl_substitute()
+        self._do_typecast()
+
+    def reload(self, with_lock=True):
+        if self._is_config_changed():
+            if with_lock:
+                self._load_with_lock()
+            else:
+                self._load()
+
+    def get(self, name, default_value=None, with_lock=True):
+        if with_lock:
+            with self.lock:
+                return self.gconf.get(name, default_value)
+        else:
+            return self.gconf.get(name, default_value)
+
+    def getall(self, show_defaults=False, show_non_configurable=False):
+        cnf = {}
+        if not show_defaults:
+            for k in self.session_conf_items:
+                if k not in self.non_configurable_configs:
+                    dv = self.default_values.get(k, "")
+                    cnf[k] = {
+                        "value": self.get(k),
+                        "default": dv,
+                        "configurable": True,
+                        "modified": False if dv == "" else True
+                    }
+            return cnf
+
+        # Show all configs including defaults
+        for k, v in self.gconf.items():
+            configurable = False if k in self.non_configurable_configs \
+                           else True
+            dv = self.default_values.get(k, "")
+            modified = False if dv == "" else True
+            if show_non_configurable:
+                cnf[k] = {
+                    "value": v,
+                    "default": dv,
+                    "configurable": configurable,
+                    "modified": modified
+                }
+            else:
+                if k not in self.non_configurable_configs:
+                    cnf[k] = {
+                        "value": v,
+                        "default": dv,
+                        "configurable": configurable,
+                        "modified": modified
+                    }
+
+        return cnf
+
+    def getr(self, name, default_value=None):
+        with self.lock:
+            self.reload(with_lock=False)
+            return self.get(name, default_value, with_lock=False)
+
+    def get_help(self, name=None):
+        pass
+
+    def _is_configurable(self, name):
+        item = self.gconfdata.get(name, None)
+        if item is None:
+            return False
+
+        return item.get("configurable", True)
+
+    def _is_valid_value(self, name, value):
+        item = self.gconfdata.get(name, None)
+        if item is None:
+            return False
+
+        # If validation func not defined
+        if item.get("validation", None) is None:
+            return True
+
+        # minmax validation
+        if item["validation"] == "minmax":
+            return validate_minmax(value, item["min"], item["max"])
+
+        if item["validation"] == "choice":
+            return validate_choice(value, item["allowed_values"])
+
+        if item["validation"] == "bool":
+            return validate_bool(value)
+
+        if item["validation"] == "execpath":
+            return validate_execpath(value)
+
+        if item["validation"] == "unixtime":
+            return validate_unixtime(value)
+
+        if item["validation"] == "int":
+            return validate_int(value)
+
+        return False
+
+    def _is_config_changed(self):
+        if self.custom_conf_file is not None and \
+           os.path.exists(self.custom_conf_file):
+                st = os.lstat(self.custom_conf_file)
+                if st.st_mtime > self.prev_mtime:
+                    self.prev_mtime = st.st_mtime
+                    return True
+
+        return False
+
+def is_config_file_old(config_file, mastervol, slavevol):
+    cnf = RawConfigParser()
+    cnf.read(config_file)
+    session_section = "peers %s %s" % (mastervol, slavevol)
+    try:
+        return dict(cnf.items(session_section))
+    except NoSectionError:
+        return None
+
+def config_upgrade(config_file, ret):
+    config_file_backup = os.path.join(os.path.dirname(config_file), "gsyncd.conf.bkp")
+
+    #copy old config file in a backup file
+    shutil.copyfile(config_file, config_file_backup)
+
+    #write a new config file
+    config = RawConfigParser()
+    config.add_section('vars')
+
+    for key, value in ret.items():
+        #handle option name changes
+        if key == "use_tarssh":
+            new_key = "sync-method"
+            if value == "true":
+                new_value = "tarssh"
+            else:
+                new_value = "rsync"
+            config.set('vars', new_key, new_value)
+        elif key == "timeout":
+            new_key = "slave-timeout"
+            config.set('vars', new_key, value)
+        #for changes like: ignore_deletes to ignore-deletes
+        else:
+            new_key = key.replace("_", "-")
+            config.set('vars', new_key, value)
+
+    with open(config_file, 'w') as configfile:
+        config.write(configfile)
+
+
+def validate_int(value):
+    try:
+        _ = int(value)
+        return True
+    except ValueError:
+        return False
+
+
+def validate_unixtime(value):
+    try:
+        y = datetime.fromtimestamp(int(value)).strftime("%Y")
+        if y == "1970":
+            return False
+
+        return True
+    except ValueError:
+        return False
+
+
+def validate_minmax(value, minval, maxval):
+    try:
+        value = int(value)
+        minval = int(minval)
+        maxval = int(maxval)
+        return value >= minval and value <= maxval
+    except ValueError:
+        return False
+
+
+def validate_choice(value, allowed_values):
+    allowed_values = allowed_values.split(",")
+    allowed_values = [v.strip() for v in allowed_values]
+
+    return value in allowed_values
+
+
+def validate_bool(value):
+    return value in ["true", "false"]
+
+
+def validate_execpath(value):
+    return os.path.isfile(value) and os.access(value, os.X_OK)
+
+
+def validate_filepath(value):
+    return os.path.isfile(value)
+
+
+def validate_path(value):
+    return os.path.exists(value)
+
+
+def to_int(value):
+    return int(value)
+
+
+def to_float(value):
+    return float(value)
+
+
+def to_bool(value):
+    if isinstance(value, bool):
+        return value
+    return True if value in ["true", "True"] else False
+
+
+def get(name, default_value=None):
+    return _gconf.get(name, default_value)
+
+
+def getall(show_defaults=False, show_non_configurable=False):
+    return _gconf.getall(show_defaults=show_defaults,
+                         show_non_configurable=show_non_configurable)
+
+
+def getr(name, default_value=None):
+    return _gconf.getr(name, default_value)
+
+
+def load(default_conf, custom_conf=None, args={}, extra_tmpl_args={},
+         override_from_args=False):
+    global _gconf
+    _gconf = Gconf(default_conf, custom_conf, args, extra_tmpl_args,
+                   override_from_args)
+
+
+def setconfig(name, value):
+    global _gconf
+    _gconf.set(name, value)
+
+
+def resetconfig(name):
+    global _gconf
+    _gconf.reset(name)
+
+
+def check(name, value=None, with_conffile=True):
+    global _gconf
+    _gconf.check(name, value=value, with_conffile=with_conffile)
diff --git a/geo-replication/syncdaemon/gsyncdstatus.py b/geo-replication/syncdaemon/gsyncdstatus.py
index d36ddce865e..1a655ff8887 100644
--- a/geo-replication/syncdaemon/gsyncdstatus.py
+++ b/geo-replication/syncdaemon/gsyncdstatus.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
@@ -9,14 +9,22 @@
 # cases as published by the Free Software Foundation.
 #
 
+from __future__ import print_function
 import fcntl
 import os
 import tempfile
-import urllib
+try:
+    import urllib.parse as urllib
+except ImportError:
+    import urllib
 import json
 import time
 from datetime import datetime
 from errno import EACCES, EAGAIN, ENOENT
+import logging
+
+from syncdutils import (EVENT_GEOREP_ACTIVE, EVENT_GEOREP_PASSIVE, gf_event,
+                        EVENT_GEOREP_CHECKPOINT_COMPLETED, lf)
 
 DEFAULT_STATUS = "N/A"
 MONITOR_STATUS = ("Created", "Started", "Paused", "Stopped")
@@ -95,6 +103,7 @@ class LockedOpen(object):
         return f
 
     def __exit__(self, _exc_type, _exc_value, _traceback):
+        fcntl.flock(self.fileobj, fcntl.LOCK_UN)
         self.fileobj.close()
 
 
@@ -115,7 +124,12 @@ def set_monitor_status(status_file, status):
 
 
 class GeorepStatus(object):
-    def __init__(self, monitor_status_file, brick, monitor_pid_file=None):
+    def __init__(self, monitor_status_file, master_node, brick, master_node_id,
+                 master, slave, monitor_pid_file=None):
+        self.master = master
+        slv_data = slave.split("::")
+        self.slave_host = slv_data[0]
+        self.slave_volume = slv_data[1].split(":")[0]  # Remove Slave UUID
         self.work_dir = os.path.dirname(monitor_status_file)
         self.monitor_status_file = monitor_status_file
         self.filename = os.path.join(self.work_dir,
@@ -126,18 +140,35 @@ class GeorepStatus(object):
         os.close(fd)
         fd = os.open(self.monitor_status_file, os.O_CREAT | os.O_RDWR)
         os.close(fd)
+        self.master_node = master_node
+        self.master_node_id = master_node_id
         self.brick = brick
         self.default_values = get_default_values()
         self.monitor_pid_file = monitor_pid_file
 
+    def send_event(self, event_type, **kwargs):
+        gf_event(event_type,
+                 master_volume=self.master,
+                 master_node=self.master_node,
+                 master_node_id=self.master_node_id,
+                 slave_host=self.slave_host,
+                 slave_volume=self.slave_volume,
+                 brick_path=self.brick,
+                 **kwargs)
+
     def _update(self, mergerfunc):
+        data = self.default_values
         with LockedOpen(self.filename, 'r+') as f:
             try:
-                data = json.load(f)
+                data.update(json.load(f))
             except ValueError:
-                data = self.default_values
+                pass
 
             data = mergerfunc(data)
+            # If Data is not changed by merger func
+            if not data:
+                return False
+
             with tempfile.NamedTemporaryFile(
                     'w',
                     dir=os.path.dirname(self.filename),
@@ -150,6 +181,7 @@ class GeorepStatus(object):
                             os.O_DIRECTORY)
             os.fsync(dirfd)
             os.close(dirfd)
+            return True
 
     def reset_on_worker_start(self):
         def merger(data):
@@ -164,10 +196,20 @@ class GeorepStatus(object):
 
     def set_field(self, key, value):
         def merger(data):
+            # Current data and prev data is same
+            if data[key] == value:
+                return {}
+
             data[key] = value
             return json.dumps(data)
 
-        self._update(merger)
+        return self._update(merger)
+
+    def trigger_gf_event_checkpoint_completion(self, checkpoint_time,
+                                               checkpoint_completion_time):
+        self.send_event(EVENT_GEOREP_CHECKPOINT_COMPLETED,
+                        checkpoint_time=checkpoint_time,
+                        checkpoint_completion_time=checkpoint_completion_time)
 
     def set_last_synced(self, value, checkpoint_time):
         def merger(data):
@@ -185,18 +227,30 @@ class GeorepStatus(object):
             # previously then update the checkpoint completed time
             if checkpoint_time > 0 and checkpoint_time <= value[0]:
                 if data["checkpoint_completed"] == "No":
+                    curr_time = int(time.time())
                     data["checkpoint_time"] = checkpoint_time
-                    data["checkpoint_completion_time"] = int(time.time())
+                    data["checkpoint_completion_time"] = curr_time
                     data["checkpoint_completed"] = "Yes"
+                    logging.info(lf("Checkpoint completed",
+                                    checkpoint_time=human_time_utc(
+                                        checkpoint_time),
+                                    completion_time=human_time_utc(curr_time)))
+                    self.trigger_gf_event_checkpoint_completion(
+                        checkpoint_time, curr_time)
+
             return json.dumps(data)
 
         self._update(merger)
 
     def set_worker_status(self, status):
-        self.set_field("worker_status", status)
+        if self.set_field("worker_status", status):
+            logging.info(lf("Worker Status Change",
+                            status=status))
 
     def set_worker_crawl_status(self, status):
-        self.set_field("crawl_status", status)
+        if self.set_field("crawl_status", status):
+            logging.info(lf("Crawl Status Change",
+                            status=status))
 
     def set_slave_node(self, slave_node):
         def merger(data):
@@ -222,10 +276,16 @@ class GeorepStatus(object):
         self._update(merger)
 
     def set_active(self):
-        self.set_field("worker_status", "Active")
+        if self.set_field("worker_status", "Active"):
+            logging.info(lf("Worker Status Change",
+                            status="Active"))
+            self.send_event(EVENT_GEOREP_ACTIVE)
 
     def set_passive(self):
-        self.set_field("worker_status", "Passive")
+        if self.set_field("worker_status", "Passive"):
+            logging.info(lf("Worker Status Change",
+                            status="Passive"))
+            self.send_event(EVENT_GEOREP_PASSIVE)
 
     def get_monitor_status(self):
         data = ""
@@ -345,6 +405,15 @@ class GeorepStatus(object):
 
         return data
 
-    def print_status(self, checkpoint_time=0):
-        for key, value in self.get_status(checkpoint_time).items():
-            print ("%s: %s" % (key, value))
+    def print_status(self, checkpoint_time=0, json_output=False):
+        status_out = self.get_status(checkpoint_time)
+        if json_output:
+            out = {}
+            # Convert all values as string
+            for k, v in status_out.items():
+                out[k] = str(v)
+            print(json.dumps(out))
+            return
+
+        for key, value in status_out.items():
+            print(("%s: %s" % (key, value)))
diff --git a/geo-replication/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py
index 3671e102c7f..e6406c36bd7 100644
--- a/geo-replication/syncdaemon/libcxattr.py
+++ b/geo-replication/syncdaemon/libcxattr.py
@@ -9,13 +9,14 @@
 #
 
 import os
-from ctypes import CDLL, create_string_buffer, get_errno
-from ctypes.util import find_library
+from ctypes import CDLL, get_errno
+from py2py3 import (bytearray_to_str, gr_create_string_buffer,
+                    gr_query_xattr, gr_lsetxattr, gr_lremovexattr)
 
 
 class Xattr(object):
 
-    """singleton that wraps the extended attribues system
+    """singleton that wraps the extended attributes system
        interface for python using ctypes
 
        Just implement it to the degree we need it, in particular
@@ -25,7 +26,7 @@ class Xattr(object):
          sizes we expect
     """
 
-    libc = CDLL(find_library("c"), use_errno=True)
+    libc = CDLL("libc.so.6", use_errno=True)
 
     @classmethod
     def geterrno(cls):
@@ -39,20 +40,23 @@ class Xattr(object):
     @classmethod
     def _query_xattr(cls, path, siz, syscall, *a):
         if siz:
-            buf = create_string_buffer('\0' * siz)
+            buf = gr_create_string_buffer(siz)
         else:
             buf = None
         ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
         if ret == -1:
             cls.raise_oserr()
         if siz:
-            return buf.raw[:ret]
+            # py2 and py3 compatibility. Convert bytes array
+            # to string
+            result = bytearray_to_str(buf.raw)
+            return result[:ret]
         else:
             return ret
 
     @classmethod
     def lgetxattr(cls, path, attr, siz=0):
-        return cls._query_xattr(path, siz, 'lgetxattr', attr)
+        return gr_query_xattr(cls, path, siz, 'lgetxattr', attr)
 
     @classmethod
     def lgetxattr_buf(cls, path, attr):
@@ -66,7 +70,7 @@ class Xattr(object):
 
     @classmethod
     def llistxattr(cls, path, siz=0):
-        ret = cls._query_xattr(path, siz, 'llistxattr')
+        ret = gr_query_xattr(cls, path, siz, 'llistxattr')
         if isinstance(ret, str):
             ret = ret.strip('\0')
             ret = ret.split('\0') if ret else []
@@ -74,13 +78,13 @@ class Xattr(object):
 
     @classmethod
     def lsetxattr(cls, path, attr, val):
-        ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
+        ret = gr_lsetxattr(cls, path, attr, val)
         if ret == -1:
             cls.raise_oserr()
 
     @classmethod
     def lremovexattr(cls, path, attr):
-        ret = cls.libc.lremovexattr(path, attr)
+        ret = gr_lremovexattr(cls, path, attr)
         if ret == -1:
             cls.raise_oserr()
 
diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py
index d87b56cd941..a3bda7282c0 100644
--- a/geo-replication/syncdaemon/libgfchangelog.py
+++ b/geo-replication/syncdaemon/libgfchangelog.py
@@ -9,129 +9,135 @@
 #
 
 import os
-from ctypes import CDLL, RTLD_GLOBAL, create_string_buffer, get_errno, byref, c_ulong
+from ctypes import CDLL, RTLD_GLOBAL, get_errno, byref, c_ulong
 from ctypes.util import find_library
 from syncdutils import ChangelogException, ChangelogHistoryNotAvailable
+from py2py3 import (gr_cl_history_changelog, gr_cl_done,
+                    gr_create_string_buffer, gr_cl_register,
+                    gr_cl_history_done, bytearray_to_str)
 
 
-class Changes(object):
-    libgfc = CDLL(find_library("gfchangelog"), mode=RTLD_GLOBAL, use_errno=True)
-
-    @classmethod
-    def geterrno(cls):
-        return get_errno()
-
-    @classmethod
-    def raise_changelog_err(cls):
-        errn = cls.geterrno()
-        raise ChangelogException(errn, os.strerror(errn))
-
-    @classmethod
-    def _get_api(cls, call):
-        return getattr(cls.libgfc, call)
-
-    @classmethod
-    def cl_init(cls):
-        ret = cls._get_api('gf_changelog_init')(None)
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_register(cls, brick, path, log_file, log_level, retries=0):
-        ret = cls._get_api('gf_changelog_register')(brick, path,
-                                                    log_file,
-                                                    log_level, retries)
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_scan(cls):
-        ret = cls._get_api('gf_changelog_scan')()
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_startfresh(cls):
-        ret = cls._get_api('gf_changelog_start_fresh')()
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_getchanges(cls):
-        """ remove hardcoding for path name length """
-        def clsort(f):
-            return f.split('.')[-1]
-        changes = []
-        buf = create_string_buffer('\0', 4096)
-        call = cls._get_api('gf_changelog_next_change')
-
-        while True:
-            ret = call(buf, 4096)
-            if ret in (0, -1):
-                break
-            changes.append(buf.raw[:ret - 1])
-        if ret == -1:
-            cls.raise_changelog_err()
-        # cleanup tracker
-        cls.cl_startfresh()
-        return sorted(changes, key=clsort)
-
-    @classmethod
-    def cl_done(cls, clfile):
-        ret = cls._get_api('gf_changelog_done')(clfile)
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_history_scan(cls):
-        ret = cls._get_api('gf_history_changelog_scan')()
-        if ret == -1:
-            cls.raise_changelog_err()
-
-        return ret
-
-    @classmethod
-    def cl_history_changelog(cls, changelog_path, start, end, num_parallel):
-        actual_end = c_ulong()
-        ret = cls._get_api('gf_history_changelog')(changelog_path, start, end,
-                                                   num_parallel,
-                                                   byref(actual_end))
-        if ret == -1:
-            cls.raise_changelog_err()
-
-        if ret == -2:
-            raise ChangelogHistoryNotAvailable()
-
-        return (ret, actual_end.value)
-
-    @classmethod
-    def cl_history_startfresh(cls):
-        ret = cls._get_api('gf_history_changelog_start_fresh')()
-        if ret == -1:
-            cls.raise_changelog_err()
-
-    @classmethod
-    def cl_history_getchanges(cls):
-        """ remove hardcoding for path name length """
-        def clsort(f):
-            return f.split('.')[-1]
-
-        changes = []
-        buf = create_string_buffer('\0', 4096)
-        call = cls._get_api('gf_history_changelog_next_change')
-
-        while True:
-            ret = call(buf, 4096)
-            if ret in (0, -1):
-                break
-            changes.append(buf.raw[:ret - 1])
-        if ret == -1:
-            cls.raise_changelog_err()
-
-        return sorted(changes, key=clsort)
-
-    @classmethod
-    def cl_history_done(cls, clfile):
-        ret = cls._get_api('gf_history_changelog_done')(clfile)
-        if ret == -1:
-            cls.raise_changelog_err()
+libgfc = CDLL(
+    find_library("gfchangelog"),
+    mode=RTLD_GLOBAL,
+    use_errno=True
+)
+
+
+def _raise_changelog_err():
+    errn = get_errno()
+    raise ChangelogException(errn, os.strerror(errn))
+
+
+def _init():
+    if libgfc.gf_changelog_init(None) == -1:
+        _raise_changelog_err()
+
+
+def register(brick, path, log_file, log_level, retries=0):
+    _init()
+
+    ret = gr_cl_register(libgfc, brick, path, log_file, log_level, retries)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def scan():
+    ret = libgfc.gf_changelog_scan()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def startfresh():
+    ret = libgfc.gf_changelog_start_fresh()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def getchanges():
+    def clsort(cfile):
+        return cfile.split('.')[-1]
+
+    changes = []
+    buf = gr_create_string_buffer(4096)
+    call = libgfc.gf_changelog_next_change
+
+    while True:
+        ret = call(buf, 4096)
+        if ret in (0, -1):
+            break
+
+        # py2 and py3 compatibility
+        result = bytearray_to_str(buf.raw[:ret - 1])
+        changes.append(result)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+    # cleanup tracker
+    startfresh()
+
+    return sorted(changes, key=clsort)
+
+
+def done(clfile):
+    ret = gr_cl_done(libgfc, clfile)
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def history_scan():
+    ret = libgfc.gf_history_changelog_scan()
+    if ret == -1:
+        _raise_changelog_err()
+
+    return ret
+
+
+def history_changelog(changelog_path, start, end, num_parallel):
+    actual_end = c_ulong()
+    ret = gr_cl_history_changelog(libgfc, changelog_path, start, end,
+                                  num_parallel, byref(actual_end))
+    if ret == -1:
+        _raise_changelog_err()
+
+    if ret == -2:
+        raise ChangelogHistoryNotAvailable()
+
+    return (ret, actual_end.value)
+
+
+def history_startfresh():
+    ret = libgfc.gf_history_changelog_start_fresh()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def history_getchanges():
+    def clsort(cfile):
+        return cfile.split('.')[-1]
+
+    changes = []
+    buf = gr_create_string_buffer(4096)
+    call = libgfc.gf_history_changelog_next_change
+
+    while True:
+        ret = call(buf, 4096)
+        if ret in (0, -1):
+            break
+
+        # py2 and py3 compatibility
+        result = bytearray_to_str(buf.raw[:ret - 1])
+        changes.append(result)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+    return sorted(changes, key=clsort)
+
+
+def history_done(clfile):
+    ret = gr_cl_history_done(libgfc, clfile)
+    if ret == -1:
+        _raise_changelog_err()
diff --git a/geo-replication/syncdaemon/logutils.py b/geo-replication/syncdaemon/logutils.py
new file mode 100644
index 00000000000..01ae7852f23
--- /dev/null
+++ b/geo-replication/syncdaemon/logutils.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import logging
+from logging import Logger, handlers
+import sys
+import time
+
+
+class GLogger(Logger):
+
+    """Logger customizations for gsyncd.
+
+    It implements a log format similar to that of glusterfs.
+    """
+
+    def makeRecord(self, name, level, *a):
+        rv = Logger.makeRecord(self, name, level, *a)
+        rv.nsecs = (rv.created - int(rv.created)) * 1000000
+        fr = sys._getframe(4)
+        callee = fr.f_locals.get('self')
+        if callee:
+            ctx = str(type(callee)).split("'")[1].split('.')[-1]
+        else:
+            ctx = '<top>'
+        if not hasattr(rv, 'funcName'):
+            rv.funcName = fr.f_code.co_name
+        rv.lvlnam = logging.getLevelName(level)[0]
+        rv.ctx = ctx
+        return rv
+
+
+LOGFMT = ("[%(asctime)s.%(nsecs)d] %(lvlnam)s [%(module)s{0}"
+          ":%(lineno)s:%(funcName)s] %(ctx)s: %(message)s")
+
+
+def setup_logging(level="INFO", label="", log_file=""):
+    if label:
+        label = "(" + label + ")"
+
+    filename = None
+    stream = None
+    if log_file:
+        if log_file in ('-', '/dev/stderr'):
+            stream = sys.stderr
+        elif log_file == '/dev/stdout':
+            stream = sys.stdout
+        else:
+            filename = log_file
+
+    datefmt = "%Y-%m-%d %H:%M:%S"
+    fmt = LOGFMT.format(label)
+    logging.root = GLogger("root", level)
+    logging.setLoggerClass(GLogger)
+    logging.Formatter.converter = time.gmtime  # Log in GMT/UTC time
+    logging.getLogger().handlers = []
+    logging.getLogger().setLevel(level)
+
+    if filename is not None:
+        logging_handler = handlers.WatchedFileHandler(filename)
+        formatter = logging.Formatter(fmt=fmt,
+                                      datefmt=datefmt)
+        logging_handler.setFormatter(formatter)
+        logging.getLogger().addHandler(logging_handler)
+    else:
+        logging.basicConfig(stream=stream,
+                            format=fmt,
+                            datefmt=datefmt,
+                            level=level)
diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py
index b5ddac05c29..9501aeae6b5 100644
--- a/geo-replication/syncdaemon/master.py
+++ b/geo-replication/syncdaemon/master.py
@@ -12,20 +12,23 @@ import os
 import sys
 import time
 import stat
-import json
 import logging
 import fcntl
 import string
 import errno
 import tarfile
-from errno import ENOENT, ENODATA, EEXIST, EACCES, EAGAIN, ESTALE
+from errno import ENOENT, ENODATA, EEXIST, EACCES, EAGAIN, ESTALE, EINTR
 from threading import Condition, Lock
 from datetime import datetime
-from gconf import gconf
-from syncdutils import Thread, GsyncdError, boolify, escape
-from syncdutils import unescape, gauxpfx, md5hex, selfkill
-from syncdutils import lstat, errno_wrap, FreeObject
-from syncdutils import NoStimeAvailable, PartialHistoryAvailable
+
+import gsyncdconfig as gconf
+import libgfchangelog
+from rconf import rconf
+from syncdutils import (Thread, GsyncdError, escape_space_newline,
+                        unescape_space_newline, gauxpfx, escape,
+                        lstat, errno_wrap, FreeObject, lf, matching_disk_gfid,
+                        NoStimeAvailable, PartialHistoryAvailable,
+                        host_brick_split)
 
 URXTIME = (-1, 0)
 
@@ -37,14 +40,6 @@ URXTIME = (-1, 0)
 # crawl before starting live changelog crawl.
 CHANGELOG_ROLLOVER_TIME = 15
 
-# Max size of Changelogs to process per batch, Changelogs Processing is
-# not limited by the number of changelogs but instead based on
-# size of the changelog file, One sample changelog file size was 145408
-# with ~1000 CREATE and ~1000 DATA. 5 such files in one batch is 727040
-# If geo-rep worker crashes while processing a batch, it has to retry only
-# that batch since stime will get updated after each batch.
-MAX_CHANGELOG_BATCH_SIZE = 727040
-
 # Utility functions to help us to get to closer proximity
 # of the DRY principle (no, don't look for elevated or
 # perspectivistic things here)
@@ -62,8 +57,8 @@ def _volinfo_hook_relax_foreign(self):
     fgn_vi = volinfo_sys[self.KFGN]
     if fgn_vi:
         expiry = fgn_vi['timeout'] - int(time.time()) + 1
-        logging.info('foreign volume info found, waiting %d sec for expiry' %
-                     expiry)
+        logging.info(lf('foreign volume info found, waiting for expiry',
+                        expiry=expiry))
         time.sleep(expiry)
         volinfo_sys = self.get_sys_volinfo()
     return volinfo_sys
@@ -72,6 +67,9 @@ def _volinfo_hook_relax_foreign(self):
 def edct(op, **ed):
     dct = {}
     dct['op'] = op
+    # This is used in automatic gfid conflict resolution.
+    # When marked True, it's skipped during re-processing.
+    dct['skip_entry'] = False
     for k in ed:
         if k == 'stat':
             st = ed[k]
@@ -93,23 +91,41 @@ def gmaster_builder(excrawl=None):
     """produce the GMaster class variant corresponding
        to sync mode"""
     this = sys.modules[__name__]
-    modemixin = gconf.special_sync_mode
+    modemixin = gconf.get("special-sync-mode")
     if not modemixin:
         modemixin = 'normal'
-    changemixin = 'xsync' if gconf.change_detector == 'xsync' \
-                  else excrawl or gconf.change_detector
-    logging.info('setting up %s change detection mode' % changemixin)
+
+    if gconf.get("change-detector") == 'xsync':
+        changemixin = 'xsync'
+    elif excrawl:
+        changemixin = excrawl
+    else:
+        changemixin = gconf.get("change-detector")
+
+    logging.debug(lf('setting up change detection mode',
+                     mode=changemixin))
     modemixin = getattr(this, modemixin.capitalize() + 'Mixin')
     crawlmixin = getattr(this, 'GMaster' + changemixin.capitalize() + 'Mixin')
-    sendmarkmixin = boolify(
-        gconf.use_rsync_xattrs) and SendmarkRsyncMixin or SendmarkNormalMixin
-    purgemixin = boolify(
-        gconf.ignore_deletes) and PurgeNoopMixin or PurgeNormalMixin
-    syncengine = boolify(gconf.use_tarssh) and TarSSHEngine or RsyncEngine
+
+    if gconf.get("use-rsync-xattrs"):
+        sendmarkmixin = SendmarkRsyncMixin
+    else:
+        sendmarkmixin = SendmarkNormalMixin
+
+    if gconf.get("ignore-deletes"):
+        purgemixin = PurgeNoopMixin
+    else:
+        purgemixin = PurgeNormalMixin
+
+    if gconf.get("sync-method") == "tarssh":
+        syncengine = TarSSHEngine
+    else:
+        syncengine = RsyncEngine
 
     class _GMaster(crawlmixin, modemixin, sendmarkmixin,
                    purgemixin, syncengine):
         pass
+
     return _GMaster
 
 
@@ -146,9 +162,9 @@ class NormalMixin(object):
         return xt0 >= xt1
 
     def make_xtime_opts(self, is_master, opts):
-        if not 'create' in opts:
+        if 'create' not in opts:
             opts['create'] = is_master
-        if not 'default_xtime' in opts:
+        if 'default_xtime' not in opts:
             opts['default_xtime'] = URXTIME
 
     def xtime_low(self, rsc, path, **opts):
@@ -167,7 +183,9 @@ class NormalMixin(object):
                 xt = _xtime_now()
                 rsc.server.aggregated.set_xtime(path, self.uuid, xt)
             else:
-                xt = opts['default_xtime']
+                zero_zero = (0, 0)
+                if xt != zero_zero:
+                    xt = opts['default_xtime']
         return xt
 
     def keepalive_payload_hook(self, timo, gap):
@@ -179,7 +197,7 @@ class NormalMixin(object):
             vi = vi.copy()
             vi['timeout'] = int(time.time()) + timo
         else:
-            # send keep-alives more frequently to
+            # send keep-alive more frequently to
             # avoid a delay in announcing our volume info
             # to slave if it becomes established in the
             # meantime
@@ -217,9 +235,9 @@ class RecoverMixin(NormalMixin):
 
     @staticmethod
     def make_xtime_opts(is_master, opts):
-        if not 'create' in opts:
+        if 'create' not in opts:
             opts['create'] = False
-        if not 'default_xtime' in opts:
+        if 'default_xtime' not in opts:
             opts['default_xtime'] = URXTIME
 
     def keepalive_payload_hook(self, timo, gap):
@@ -262,7 +280,7 @@ class TarSSHEngine(object):
     """
 
     def a_syncdata(self, files):
-        logging.debug('files: %s' % (files))
+        logging.debug(lf("Files", files=files))
 
         for f in files:
             pb = self.syncer.add(f)
@@ -270,7 +288,7 @@ class TarSSHEngine(object):
             def regjob(se, xte, pb):
                 rv = pb.wait()
                 if rv[0]:
-                    logging.debug('synced ' + se)
+                    logging.debug(lf('synced', file=se))
                     return True
                 else:
                     # stat check for file presence
@@ -296,16 +314,16 @@ class RsyncEngine(object):
     """Sync engine that uses rsync(1) for data transfers"""
 
     def a_syncdata(self, files):
-        logging.debug('files: %s' % (files))
+        logging.debug(lf("files", files=files))
 
         for f in files:
-            logging.debug('candidate for syncing %s' % f)
+            logging.debug(lf('candidate for syncing', file=f))
             pb = self.syncer.add(f)
 
             def regjob(se, xte, pb):
                 rv = pb.wait()
                 if rv[0]:
-                    logging.debug('synced ' + se)
+                    logging.debug(lf('synced', file=se))
                     return True
                 else:
                     # stat to check if the file exist
@@ -390,11 +408,9 @@ class GMasterCommon(object):
         self.master = master
         self.slave = slave
         self.jobtab = {}
-        if boolify(gconf.use_tarssh):
-            logging.info("using 'tar over ssh' as the sync engine")
+        if gconf.get("sync-method") == "tarssh":
             self.syncer = Syncer(slave, self.slave.tarssh, [2])
         else:
-            logging.info("using 'rsync' as the sync engine")
             # partial transfer (cf. rsync(1)), that's normal
             self.syncer = Syncer(slave, self.slave.rsync, [23, 24])
         # crawls vs. turns:
@@ -408,7 +424,7 @@ class GMasterCommon(object):
         # 0.
         self.crawls = 0
         self.turns = 0
-        self.total_turns = int(gconf.turns)
+        self.total_turns = rconf.turns
         self.crawl_start = datetime.now()
         self.lastreport = {'crawls': 0, 'turns': 0, 'time': 0}
         self.start = None
@@ -421,7 +437,7 @@ class GMasterCommon(object):
 
     def init_keep_alive(cls):
         """start the keep-alive thread """
-        timo = int(gconf.timeout or 0)
+        timo = gconf.get("slave-timeout", 0)
         if timo > 0:
             def keep_alive():
                 while True:
@@ -434,30 +450,22 @@ class GMasterCommon(object):
     def mgmt_lock(self):
 
         """Take management volume lock """
-        if gconf.mgmt_lock_fd:
+        if rconf.mgmt_lock_fd:
             try:
-                fcntl.lockf(gconf.mgmt_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                if not gconf.active_earlier:
-                    gconf.active_earlier = True
-                    logging.info("Got lock : %s : Becoming ACTIVE"
-                                 % gconf.local_path)
+                fcntl.lockf(rconf.mgmt_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
                 return True
             except:
                 ex = sys.exc_info()[1]
                 if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
-                    if not gconf.passive_earlier:
-                        gconf.passive_earlier = True
-                        logging.info("Didn't get lock : %s : Becoming PASSIVE"
-                                     % gconf.local_path)
                     return False
                 raise
 
         fd = None
-        bname = str(self.uuid) + "_" + str(gconf.slave_id) + "_subvol_" \
-            + str(gconf.subvol_num) + ".lock"
-        mgmt_lock_dir = os.path.join(gconf.meta_volume_mnt, "geo-rep")
+        bname = str(self.uuid) + "_" + rconf.args.slave_id + "_subvol_" \
+            + str(rconf.args.subvol_num) + ".lock"
+        mgmt_lock_dir = os.path.join(gconf.get("meta-volume-mnt"), "geo-rep")
         path = os.path.join(mgmt_lock_dir, bname)
-        logging.debug("lock_file_path: %s" % path)
+        logging.debug(lf("lock file path", path=path))
         try:
             fd = os.open(path, os.O_CREAT | os.O_RDWR)
         except OSError:
@@ -478,29 +486,22 @@ class GMasterCommon(object):
         try:
             fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
             # Save latest FD for future use
-            gconf.mgmt_lock_fd = fd
+            rconf.mgmt_lock_fd = fd
         except:
             ex = sys.exc_info()[1]
             if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
                 # cannot grab, it's taken
-                if not gconf.passive_earlier:
-                    gconf.passive_earlier = True
-                    logging.info("Didn't get lock : %s : Becoming PASSIVE"
-                                 % gconf.local_path)
-                gconf.mgmt_lock_fd = fd
+                rconf.mgmt_lock_fd = fd
                 return False
             raise
 
-        if not gconf.active_earlier:
-            gconf.active_earlier = True
-            logging.info("Got lock : %s : Becoming ACTIVE" % gconf.local_path)
         return True
 
     def should_crawl(self):
-        if not boolify(gconf.use_meta_volume):
-            return gconf.glusterd_uuid in self.master.server.node_uuid()
+        if not gconf.get("use-meta-volume"):
+            return rconf.args.local_node_id in self.master.server.node_uuid()
 
-        if not os.path.ismount(gconf.meta_volume_mnt):
+        if not os.path.ismount(gconf.get("meta-volume-mnt")):
             logging.error("Meta-volume is not mounted. Worker Exiting...")
             sys.exit(1)
         return self.mgmt_lock()
@@ -517,8 +518,8 @@ class GMasterCommon(object):
 
         # If crawlwrap is called when partial history available,
         # then it sets register_time which is the time when geo-rep
-        # worker registerd to changelog consumption. Since nsec is
-        # not considered in register time, their are chances of skipping
+        # worker registered to changelog consumption. Since nsec is
+        # not considered in register time, there are chances of skipping
         # changes detection in xsync crawl. This limit will be reset when
         # crawlwrap is called again.
         self.live_changelog_start_time = None
@@ -528,25 +529,24 @@ class GMasterCommon(object):
         # no need to maintain volinfo state machine.
         # in a cascading setup, each geo-replication session is
         # independent (ie. 'volume-mark' and 'xtime' are not
-        # propogated). This is because the slave's xtime is now
+        # propagated). This is because the slave's xtime is now
         # stored on the master itself. 'volume-mark' just identifies
         # that we are in a cascading setup and need to enable
         # 'geo-replication.ignore-pid-check' option.
         volinfo_sys = self.volinfo_hook()
         self.volinfo = volinfo_sys[self.KNAT]
         inter_master = volinfo_sys[self.KFGN]
-        logging.info("%s master with volume id %s ..." %
-                     (inter_master and "intermediate" or "primary",
-                      self.uuid))
-        gconf.configinterface.set('volume_id', self.uuid)
+        logging.debug("%s master with volume id %s ..." %
+                      (inter_master and "intermediate" or "primary",
+                       self.uuid))
+        rconf.volume_id = self.uuid
         if self.volinfo:
             if self.volinfo['retval']:
-                logging.warn("master cluster's info may not be valid %d" %
-                             self.volinfo['retval'])
+                logging.warn(lf("master cluster's info may not be valid",
+                                error=self.volinfo['retval']))
         else:
             raise GsyncdError("master volinfo unavailable")
         self.lastreport['time'] = time.time()
-        logging.info('crawl interval: %d seconds' % self.sleep_interval)
 
         t0 = time.time()
         crawl = self.should_crawl()
@@ -557,14 +557,14 @@ class GMasterCommon(object):
             self.start = time.time()
             should_display_info = self.start - self.lastreport['time'] >= 60
             if should_display_info:
-                logging.info("%d crawls, %d turns",
-                             self.crawls - self.lastreport['crawls'],
-                             self.turns - self.lastreport['turns'])
+                logging.debug("%d crawls, %d turns",
+                              self.crawls - self.lastreport['crawls'],
+                              self.turns - self.lastreport['turns'])
                 self.lastreport.update(crawls=self.crawls,
                                        turns=self.turns,
                                        time=self.start)
             t1 = time.time()
-            if int(t1 - t0) >= int(gconf.replica_failover_interval):
+            if int(t1 - t0) >= gconf.get("replica-failover-interval"):
                 crawl = self.should_crawl()
                 t0 = t1
             self.update_worker_remote_node()
@@ -574,13 +574,16 @@ class GMasterCommon(object):
                 # which is min of cluster (but max of the replicas)
                 brick_stime = self.xtime('.', self.slave)
                 cluster_stime = self.master.server.aggregated.stime_mnt(
-                    '.', '.'.join([str(self.uuid), str(gconf.slave_id)]))
-                logging.debug("Cluster stime: %s | Brick stime: %s" %
-                              (repr(cluster_stime), repr(brick_stime)))
+                    '.', '.'.join([str(self.uuid), rconf.args.slave_id]))
+                logging.debug(lf("Crawl info",
+                                 cluster_stime=cluster_stime,
+                                 brick_stime=brick_stime))
+
                 if not isinstance(cluster_stime, int):
                     if brick_stime < cluster_stime:
                         self.slave.server.set_stime(
                             self.FLAT_DIR_HIERARCHY, self.uuid, cluster_stime)
+                        self.upd_stime(cluster_stime)
                         # Purge all changelogs available in processing dir
                         # less than cluster_stime
                         proc_dir = os.path.join(self.tempdir,
@@ -697,17 +700,44 @@ class GMasterChangelogMixin(GMasterCommon):
     TYPE_GFID = "D "
     TYPE_ENTRY = "E "
 
+    MAX_EF_RETRIES = 10
+    MAX_OE_RETRIES = 10
+
     # flat directory hierarchy for gfid based access
     FLAT_DIR_HIERARCHY = '.'
 
     CHANGELOG_CONN_RETRIES = 5
 
+    def init_fop_batch_stats(self):
+        self.batch_stats = {
+            "CREATE": 0,
+            "MKNOD": 0,
+            "UNLINK": 0,
+            "MKDIR": 0,
+            "RMDIR": 0,
+            "LINK": 0,
+            "SYMLINK": 0,
+            "RENAME": 0,
+            "SETATTR": 0,
+            "SETXATTR": 0,
+            "XATTROP": 0,
+            "DATA": 0,
+            "ENTRY_SYNC_TIME": 0,
+            "META_SYNC_TIME": 0,
+            "DATA_START_TIME": 0
+        }
+
+    def update_fop_batch_stats(self, ty):
+        if ty in ['FSETXATTR']:
+            ty = 'SETXATTR'
+        self.batch_stats[ty] = self.batch_stats.get(ty, 0) + 1
+
     def archive_and_purge_changelogs(self, changelogs):
         # Creates tar file instead of tar.gz, since changelogs will
         # be appended to existing tar. archive name is
         # archive_<YEAR><MONTH>.tar
         archive_name = "archive_%s.tar" % datetime.today().strftime(
-            gconf.changelog_archive_format)
+            gconf.get("changelog-archive-format"))
 
         try:
             tar = tarfile.open(os.path.join(self.processed_changelogs_dir,
@@ -743,13 +773,9 @@ class GMasterChangelogMixin(GMasterCommon):
                 else:
                     raise
 
-    def fallback_xsync(self):
-        logging.info('falling back to xsync mode')
-        gconf.configinterface.set('change-detector', 'xsync')
-        selfkill()
-
     def setup_working_dir(self):
-        workdir = os.path.join(gconf.working_dir, md5hex(gconf.local_path))
+        workdir = os.path.join(gconf.get("working-dir"),
+                               escape(rconf.args.local_path))
         logging.debug('changelog working dir %s' % workdir)
         return workdir
 
@@ -759,11 +785,207 @@ class GMasterChangelogMixin(GMasterCommon):
             st = lstat(os.path.join(gfid_prefix, failure[0][entry_key]))
             if not isinstance(st, int):
                 num_failures += 1
-                logging.error('%s FAILED: %s' % (log_prefix,
-                                                 repr(failure)))
+                logging.error(lf('%s FAILED' % log_prefix,
+                                 data=failure))
+                if failure[0]['op'] == 'MKDIR':
+                    raise GsyncdError("The above directory failed to sync."
+                                      " Please fix it to proceed further.")
 
         self.status.inc_value("failures", num_failures)
 
+    def fix_possible_entry_failures(self, failures, retry_count, entries):
+        pfx = gauxpfx()
+        fix_entry_ops = []
+        failures1 = []
+        remove_gfids = set()
+        for failure in failures:
+            if failure[2]['name_mismatch']:
+                pbname = failure[2]['slave_entry']
+            elif failure[2]['dst']:
+                pbname = failure[0]['entry1']
+            else:
+                pbname = failure[0]['entry']
+
+            op = failure[0]['op']
+            # name exists but gfid is different
+            if failure[2]['gfid_mismatch'] or failure[2]['name_mismatch']:
+                slave_gfid = failure[2]['slave_gfid']
+                st = lstat(os.path.join(pfx, slave_gfid))
+                # Takes care of scenarios with no hardlinks
+                if isinstance(st, int) and st == ENOENT:
+                    logging.debug(lf('Entry not present on master. Fixing gfid '
+                                    'mismatch in slave. Deleting the entry',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    # Add deletion to fix_entry_ops list
+                    if failure[2]['slave_isdir']:
+                        fix_entry_ops.append(
+                            edct('RMDIR',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+                    else:
+                        fix_entry_ops.append(
+                            edct('UNLINK',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+                    remove_gfids.add(slave_gfid)
+                    if op in ['RENAME']:
+                        # If renamed gfid doesn't exists on master, remove
+                        # rename entry and unlink src on slave
+                        st = lstat(os.path.join(pfx, failure[0]['gfid']))
+                        if isinstance(st, int) and st == ENOENT:
+                            logging.debug("Unlink source %s" % repr(failure))
+                            remove_gfids.add(failure[0]['gfid'])
+                            fix_entry_ops.append(
+                                edct('UNLINK',
+                                     gfid=failure[0]['gfid'],
+                                     entry=failure[0]['entry']))
+                # Takes care of scenarios of hardlinks/renames on master
+                elif not isinstance(st, int):
+                    if matching_disk_gfid(slave_gfid, pbname):
+                        # Safe to ignore the failure as master contains same
+                        # file with same gfid. Remove entry from entries list
+                        logging.debug(lf('Fixing gfid mismatch in slave. '
+                                        ' Safe to ignore, take out entry',
+                                        retry_count=retry_count,
+                                        entry=repr(failure)))
+                        remove_gfids.add(failure[0]['gfid'])
+                        if op == 'RENAME':
+                            fix_entry_ops.append(
+                                edct('UNLINK',
+                                     gfid=failure[0]['gfid'],
+                                     entry=failure[0]['entry']))
+                    # The file exists on master but with different name.
+                    # Probably renamed and got missed during xsync crawl.
+                    elif failure[2]['slave_isdir']:
+                        realpath = os.readlink(os.path.join(
+                                               rconf.args.local_path,
+                                               ".glusterfs",
+                                               slave_gfid[0:2],
+                                               slave_gfid[2:4],
+                                               slave_gfid))
+                        dst_entry = os.path.join(pfx, realpath.split('/')[-2],
+                                                 realpath.split('/')[-1])
+                        src_entry = pbname
+                        logging.debug(lf('Fixing dir name/gfid mismatch in '
+                                        'slave', retry_count=retry_count,
+                                        entry=repr(failure)))
+                        if src_entry == dst_entry:
+                            # Safe to ignore the failure as master contains
+                            # same directory as in slave with same gfid.
+                            # Remove the failure entry from entries list
+                            logging.debug(lf('Fixing dir name/gfid mismatch'
+                                            ' in slave. Safe to ignore, '
+                                            'take out entry',
+                                            retry_count=retry_count,
+                                            entry=repr(failure)))
+                            try:
+                                entries.remove(failure[0])
+                            except ValueError:
+                                pass
+                        else:
+                            rename_dict = edct('RENAME', gfid=slave_gfid,
+                                               entry=src_entry,
+                                               entry1=dst_entry, stat=st,
+                                               link=None)
+                            logging.debug(lf('Fixing dir name/gfid mismatch'
+                                            ' in slave. Renaming',
+                                            retry_count=retry_count,
+                                            entry=repr(rename_dict)))
+                            fix_entry_ops.append(rename_dict)
+                    else:
+                        # A hardlink file exists with different name or
+                        # renamed file exists and we are sure from
+                        # matching_disk_gfid check that the entry doesn't
+                        # exist with same gfid so we can safely delete on slave
+                        logging.debug(lf('Fixing file gfid mismatch in slave. '
+                                        'Hardlink/Rename Case. Deleting entry',
+                                        retry_count=retry_count,
+                                        entry=repr(failure)))
+                        fix_entry_ops.append(
+                            edct('UNLINK',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+            elif failure[1] == ENOENT:
+                if op in ['RENAME']:
+                    pbname = failure[0]['entry1']
+                else:
+                    pbname = failure[0]['entry']
+
+                pargfid = pbname.split('/')[1]
+                st = lstat(os.path.join(pfx, pargfid))
+                # Safe to ignore the failure as master doesn't contain
+                # parent directory.
+                if isinstance(st, int):
+                    logging.debug(lf('Fixing ENOENT error in slave. Parent '
+                                    'does not exist on master. Safe to '
+                                    'ignore, take out entry',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    try:
+                        entries.remove(failure[0])
+                    except ValueError:
+                        pass
+                else:
+                    logging.debug(lf('Fixing ENOENT error in slave. Create '
+                                    'parent directory on slave.',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    realpath = os.readlink(os.path.join(rconf.args.local_path,
+                                                        ".glusterfs",
+                                                        pargfid[0:2],
+                                                        pargfid[2:4],
+                                                        pargfid))
+                    dir_entry = os.path.join(pfx, realpath.split('/')[-2],
+                                             realpath.split('/')[-1])
+                    fix_entry_ops.append(
+                        edct('MKDIR', gfid=pargfid, entry=dir_entry,
+                             mode=st.st_mode, uid=st.st_uid, gid=st.st_gid))
+
+        logging.debug("remove_gfids: %s" % repr(remove_gfids))
+        if remove_gfids:
+            for e in entries:
+                if e['op'] in ['MKDIR', 'MKNOD', 'CREATE', 'RENAME'] \
+                   and e['gfid'] in remove_gfids:
+                    logging.debug("Removed entry op from retrial list: entry: %s" % repr(e))
+                    e['skip_entry'] = True
+
+        if fix_entry_ops:
+            # Process deletions of entries whose gfids are mismatched
+            failures1 = self.slave.server.entry_ops(fix_entry_ops)
+
+        return (failures1, fix_entry_ops)
+
+    def handle_entry_failures(self, failures, entries):
+        retries = 0
+        pending_failures = False
+        failures1 = []
+        failures2 = []
+        entry_ops1 = []
+        entry_ops2 = []
+
+        if failures:
+            pending_failures = True
+            failures1 = failures
+            entry_ops1 = entries
+
+            while pending_failures and retries < self.MAX_EF_RETRIES:
+                retries += 1
+                (failures2, entry_ops2) = self.fix_possible_entry_failures(
+                    failures1, retries, entry_ops1)
+                if not failures2:
+                    pending_failures = False
+                    logging.info(lf('Successfully fixed entry ops with gfid '
+                                 'mismatch', retry_count=retries))
+                else:
+                    pending_failures = True
+                    failures1 = failures2
+                    entry_ops1 = entry_ops2
+
+            if pending_failures:
+                for failure in failures1:
+                    logging.error("Failed to fix entry ops %s", repr(failure))
+
     def process_change(self, change, done, retry):
         pfx = gauxpfx()
         clist = []
@@ -808,9 +1030,9 @@ class GMasterChangelogMixin(GMasterCommon):
             # skip ENTRY operation if hot tier brick
             if self.name == 'live_changelog' or \
                self.name == 'history_changelog':
-                if boolify(gconf.is_hottier) and et == self.TYPE_ENTRY:
-                    logging.debug('skip ENTRY op: %s if hot tier brick'
-                                  % (ec[self.POS_TYPE]))
+                if rconf.args.is_hottier and et == self.TYPE_ENTRY:
+                    logging.debug(lf('skip ENTRY op if hot tier brick',
+                                     op=ec[self.POS_TYPE]))
                     continue
 
             # Data and Meta operations are decided while parsing
@@ -828,8 +1050,11 @@ class GMasterChangelogMixin(GMasterCommon):
                 # itself, so no need to stat()...
                 ty = ec[self.POS_TYPE]
 
+                self.update_fop_batch_stats(ec[self.POS_TYPE])
+
                 # PARGFID/BNAME
-                en = unescape(os.path.join(pfx, ec[self.POS_ENTRY1]))
+                en = unescape_space_newline(
+                    os.path.join(pfx, ec[self.POS_ENTRY1]))
                 # GFID of the entry
                 gfid = ec[self.POS_GFID]
 
@@ -837,7 +1062,8 @@ class GMasterChangelogMixin(GMasterCommon):
                     # The index of PARGFID/BNAME for UNLINK, RMDIR
                     # is no more the last index. It varies based on
                     # changelog.capture-del-path is enabled or not.
-                    en = unescape(os.path.join(pfx, ec[self.UNLINK_ENTRY]))
+                    en = unescape_space_newline(
+                        os.path.join(pfx, ec[self.UNLINK_ENTRY]))
 
                     # Remove from DATA list, so that rsync will
                     # not fail
@@ -847,14 +1073,19 @@ class GMasterChangelogMixin(GMasterCommon):
                         # file got unlinked, May be historical Changelog
                         datas.remove(pt)
 
-                    if not boolify(gconf.ignore_deletes):
+                    if ty in ['RMDIR'] and not isinstance(st, int):
+                        logging.info(lf('Ignoring rmdir. Directory present in '
+                                        'master', gfid=gfid, pgfid_bname=en))
+                        continue
+
+                    if not gconf.get("ignore-deletes"):
                         if not ignore_entry_ops:
                             entries.append(edct(ty, gfid=gfid, entry=en))
                 elif ty in ['CREATE', 'MKDIR', 'MKNOD']:
                     # Special case: record mknod as link
                     if ty in ['MKNOD']:
                         mode = int(ec[2])
-                        if mode & 01000:
+                        if mode & 0o1000:
                                 # Avoid stat'ing the file as it
                                 # may be deleted in the interim
                                 st = FreeObject(st_mode=int(ec[2]),
@@ -886,32 +1117,61 @@ class GMasterChangelogMixin(GMasterCommon):
 
                     rl = None
                     if st and stat.S_ISLNK(st.st_mode):
-                        rl = errno_wrap(os.readlink, [en], [ENOENT], [ESTALE])
+                        rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                        [ESTALE, EINTR])
                         if isinstance(rl, int):
                             rl = None
 
-                    e1 = unescape(os.path.join(pfx, ec[self.POS_ENTRY1 - 1]))
+                    e1 = unescape_space_newline(
+                        os.path.join(pfx, ec[self.POS_ENTRY1 - 1]))
                     entries.append(edct(ty, gfid=gfid, entry=e1, entry1=en,
                                         stat=st, link=rl))
+                    # If src doesn't exist while doing rename, destination
+                    # is created. If data is not followed by rename, this
+                    # remains zero byte file on slave. Hence add data entry
+                    # for renames
+                    datas.add(os.path.join(pfx, gfid))
                 else:
                     # stat() to get mode and other information
+                    if not matching_disk_gfid(gfid, en):
+                        logging.debug(lf('Ignoring entry, purged in the '
+                                      'interim', file=en, gfid=gfid))
+                        continue
+
                     go = os.path.join(pfx, gfid)
                     st = lstat(go)
                     if isinstance(st, int):
-                        logging.debug('file %s got purged in the interim' % go)
+                        logging.debug(lf('Ignoring entry, purged in the '
+                                      'interim', file=en, gfid=gfid))
                         continue
 
                     if ty == 'LINK':
-                        entries.append(edct(ty, stat=st, entry=en, gfid=gfid))
+                        rl = None
+                        if st and stat.S_ISLNK(st.st_mode):
+                            rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                            [ESTALE, EINTR])
+                            if isinstance(rl, int):
+                                rl = None
+                        entries.append(edct(ty, stat=st, entry=en, gfid=gfid,
+                                       link=rl))
+                        # If src doesn't exist while doing link, destination
+                        # is created based on file type. If data is not
+                        # followed by link, this remains zero byte file on
+                        # slave. Hence add data entry for links
+                        if rl is None:
+                            datas.add(os.path.join(pfx, gfid))
                     elif ty == 'SYMLINK':
-                        rl = errno_wrap(os.readlink, [en], [ENOENT], [ESTALE])
+                        rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                        [ESTALE, EINTR])
                         if isinstance(rl, int):
                             continue
 
                         entries.append(
                             edct(ty, stat=st, entry=en, gfid=gfid, link=rl))
                     else:
-                        logging.warn('ignoring %s [op %s]' % (gfid, ty))
+                        logging.warn(lf('ignoring op',
+                                        gfid=gfid,
+                                        type=ty))
             elif et == self.TYPE_GFID:
                 # If self.unlinked_gfids is available, then that means it is
                 # retrying the changelog second time. Do not add the GFID's
@@ -922,6 +1182,7 @@ class GMasterChangelogMixin(GMasterCommon):
                 else:
                     datas.add(os.path.join(pfx, ec[0]))
             elif et == self.TYPE_META:
+                self.update_fop_batch_stats(ec[self.POS_TYPE])
                 if ec[1] == 'SETATTR':  # only setattr's for now...
                     if len(ec) == 5:
                         # In xsync crawl, we already have stat data
@@ -934,28 +1195,48 @@ class GMasterChangelogMixin(GMasterCommon):
                                                       st_mtime=ec[6])))
                     else:
                         meta_gfid.add((os.path.join(pfx, ec[0]), ))
-                elif ec[1] == 'SETXATTR' or ec[1] == 'XATTROP' or \
-                     ec[1] == 'FXATTROP':
+                elif ec[1] in ['SETXATTR', 'XATTROP', 'FXATTROP']:
                     # To sync xattr/acls use rsync/tar, --xattrs and --acls
                     # switch to rsync and tar
-                    if not boolify(gconf.use_tarssh) and \
-                       (boolify(gconf.sync_xattrs) or boolify(gconf.sync_acls)):
+                    if not gconf.get("sync-method") == "tarssh" and \
+                       (gconf.get("sync-xattrs") or gconf.get("sync-acls")):
                         datas.add(os.path.join(pfx, ec[0]))
             else:
-                logging.warn('got invalid changelog type: %s' % (et))
+                logging.warn(lf('got invalid fop type',
+                                type=et))
         logging.debug('entries: %s' % repr(entries))
 
         # Increment counters for Status
-        self.status.inc_value("entry", len(entries))
         self.files_in_batch += len(datas)
         self.status.inc_value("data", len(datas))
 
+        self.batch_stats["DATA"] += self.files_in_batch - \
+            self.batch_stats["SETXATTR"] - \
+            self.batch_stats["XATTROP"]
+
+        entry_start_time = time.time()
         # sync namespace
         if entries and not ignore_entry_ops:
             # Increment counters for Status
             self.status.inc_value("entry", len(entries))
 
             failures = self.slave.server.entry_ops(entries)
+
+            if gconf.get("gfid-conflict-resolution"):
+                count = 0
+                if failures:
+                    logging.info(lf('Entry ops failed with gfid mismatch',
+                                count=len(failures)))
+                while failures and count < self.MAX_OE_RETRIES:
+                    count += 1
+                    self.handle_entry_failures(failures, entries)
+                    logging.info(lf('Retry original entries', count=count))
+                    failures = self.slave.server.entry_ops(entries)
+                    if not failures:
+                        logging.info("Successfully fixed all entry ops with "
+                                     "gfid mismatch")
+                        break
+
             self.log_failures(failures, 'gfid', gauxpfx(), 'ENTRY')
             self.status.dec_value("entry", len(entries))
 
@@ -966,6 +1247,8 @@ class GMasterChangelogMixin(GMasterCommon):
                 self.status.set_field("last_synced_entry",
                                       entry_stime_to_update[0])
 
+        self.batch_stats["ENTRY_SYNC_TIME"] += time.time() - entry_start_time
+
         if ignore_entry_ops:
             # Book keeping, to show in logs the range of Changelogs skipped
             self.num_skipped_entry_changelogs += 1
@@ -974,6 +1257,7 @@ class GMasterChangelogMixin(GMasterCommon):
 
             self.skipped_entry_changelogs_last = change_ts
 
+        meta_start_time = time.time()
         # sync metadata
         if meta_gfid:
             meta_entries = []
@@ -983,14 +1267,20 @@ class GMasterChangelogMixin(GMasterCommon):
                 else:
                     st = lstat(go[0])
                 if isinstance(st, int):
-                    logging.debug('file %s got purged in the interim' % go[0])
+                    logging.debug(lf('file got purged in the interim',
+                                     file=go[0]))
                     continue
                 meta_entries.append(edct('META', go=go[0], stat=st))
             if meta_entries:
-                self.status.inc_value("meta", len(entries))
+                self.status.inc_value("meta", len(meta_entries))
                 failures = self.slave.server.meta_ops(meta_entries)
                 self.log_failures(failures, 'go', '', 'META')
-                self.status.dec_value("meta", len(entries))
+                self.status.dec_value("meta", len(meta_entries))
+
+        self.batch_stats["META_SYNC_TIME"] += time.time() - meta_start_time
+
+        if self.batch_stats["DATA_START_TIME"] == 0:
+            self.batch_stats["DATA_START_TIME"] = time.time()
 
         # sync data
         if datas:
@@ -1009,6 +1299,7 @@ class GMasterChangelogMixin(GMasterCommon):
         self.skipped_entry_changelogs_last = None
         self.num_skipped_entry_changelogs = 0
         self.batch_start_time = time.time()
+        self.init_fop_batch_stats()
 
         while True:
             # first, fire all changelog transfers in parallel. entry and
@@ -1019,7 +1310,7 @@ class GMasterChangelogMixin(GMasterCommon):
             # with data of other changelogs.
 
             if retry:
-                if tries == (int(gconf.max_rsync_retries) - 1):
+                if tries == (gconf.get("max-rsync-retries") - 1):
                     # Enable Error logging if it is last retry
                     self.syncer.enable_errorlog()
 
@@ -1033,7 +1324,8 @@ class GMasterChangelogMixin(GMasterCommon):
                     self.a_syncdata(self.datas_in_batch)
             else:
                 for change in changes:
-                    logging.debug('processing change %s' % change)
+                    logging.debug(lf('processing change',
+                                     changelog=change))
                     self.process_change(change, done, retry)
                     if not retry:
                         # number of changelogs processed in the batch
@@ -1064,7 +1356,7 @@ class GMasterChangelogMixin(GMasterCommon):
                 if done:
                     xtl = (int(change.split('.')[-1]) - 1, 0)
                     self.upd_stime(xtl)
-                    map(self.changelog_done_func, changes)
+                    list(map(self.changelog_done_func, changes))
                     self.archive_and_purge_changelogs(changes)
 
                 # Reset Data counter after sync
@@ -1076,10 +1368,10 @@ class GMasterChangelogMixin(GMasterCommon):
             # We do not know which changelog transfer failed, retry everything.
             retry = True
             tries += 1
-            if tries == int(gconf.max_rsync_retries):
-                logging.error('changelogs %s could not be processed '
-                              'completely - moving on...' %
-                              ' '.join(map(os.path.basename, changes)))
+            if tries == gconf.get("max-rsync-retries"):
+                logging.error(lf('changelogs could not be processed '
+                                 'completely - moving on...',
+                                 files=list(map(os.path.basename, changes))))
 
                 # Reset data counter on failure
                 self.status.dec_value("data", self.files_in_batch)
@@ -1089,7 +1381,7 @@ class GMasterChangelogMixin(GMasterCommon):
                 if done:
                     xtl = (int(change.split('.')[-1]) - 1, 0)
                     self.upd_stime(xtl)
-                    map(self.changelog_done_func, changes)
+                    list(map(self.changelog_done_func, changes))
                     self.archive_and_purge_changelogs(changes)
                 break
             # it's either entry_ops() or Rsync that failed to do it's
@@ -1099,36 +1391,56 @@ class GMasterChangelogMixin(GMasterCommon):
             # entry_ops() that failed... so we retry the _whole_ changelog
             # again.
             # TODO: remove entry retries when it's gets fixed.
-            logging.warn('incomplete sync, retrying changelogs: %s' %
-                         ' '.join(map(os.path.basename, changes)))
+            logging.warn(lf('incomplete sync, retrying changelogs',
+                            files=list(map(os.path.basename, changes))))
 
             # Reset the Data counter before Retry
             self.status.dec_value("data", self.files_in_batch)
             self.files_in_batch = 0
+            self.init_fop_batch_stats()
             time.sleep(0.5)
 
         # Log the Skipped Entry ops range if any
         if self.skipped_entry_changelogs_first is not None and \
            self.skipped_entry_changelogs_last is not None:
-            logging.info("Skipping already processed entry "
-                         "ops from CHANGELOG.{0} to CHANGELOG.{1} "
-                         "Num: {2}".format(
-                             self.skipped_entry_changelogs_first,
-                             self.skipped_entry_changelogs_last,
-                             self.num_skipped_entry_changelogs))
+            logging.info(lf("Skipping already processed entry ops",
+                            from_changelog=self.skipped_entry_changelogs_first,
+                            to_changelog=self.skipped_entry_changelogs_last,
+                            num_changelogs=self.num_skipped_entry_changelogs))
 
         # Log Current batch details
         if changes:
             logging.info(
-                "{0} mode completed in {1:.4f} seconds "
-                "({2} - {3} Num: {4}) stime: {5}, entry_stime: {6}".format(
-                    self.name,
-                    time.time() - self.batch_start_time,
-                    changes[0].split("/")[-1],
-                    changes[-1].split("/")[-1],
-                    len(changes),
-                    repr(self.get_data_stime()),
-                    repr(self.get_entry_stime())))
+                lf("Entry Time Taken",
+                   UNL=self.batch_stats["UNLINK"],
+                   RMD=self.batch_stats["RMDIR"],
+                   CRE=self.batch_stats["CREATE"],
+                   MKN=self.batch_stats["MKNOD"],
+                   MKD=self.batch_stats["MKDIR"],
+                   REN=self.batch_stats["RENAME"],
+                   LIN=self.batch_stats["LINK"],
+                   SYM=self.batch_stats["SYMLINK"],
+                   duration="%.4f" % self.batch_stats["ENTRY_SYNC_TIME"]))
+
+            logging.info(
+                lf("Data/Metadata Time Taken",
+                   SETA=self.batch_stats["SETATTR"],
+                   meta_duration="%.4f" % self.batch_stats["META_SYNC_TIME"],
+                   SETX=self.batch_stats["SETXATTR"],
+                   XATT=self.batch_stats["XATTROP"],
+                   DATA=self.batch_stats["DATA"],
+                   data_duration="%.4f" % (
+                       time.time() - self.batch_stats["DATA_START_TIME"])))
+
+            logging.info(
+                lf("Batch Completed",
+                   mode=self.name,
+                   duration="%.4f" % (time.time() - self.batch_start_time),
+                   changelog_start=changes[0].split(".")[-1],
+                   changelog_end=changes[-1].split(".")[-1],
+                   num_changelogs=len(changes),
+                   stime=self.get_data_stime(),
+                   entry_stime=self.get_entry_stime()))
 
     def upd_entry_stime(self, stime):
         self.slave.server.set_entry_stime(self.FLAT_DIR_HIERARCHY,
@@ -1142,19 +1454,20 @@ class GMasterChangelogMixin(GMasterCommon):
             self.sendmark(path, stime)
 
         # Update last_synced_time in status file based on stime
-        chkpt_time = gconf.configinterface.get_realtime(
-            "checkpoint")
-        checkpoint_time = 0
-        if chkpt_time is not None:
-            checkpoint_time = int(chkpt_time)
+        # only update stime if stime xattr set to Brick root
+        if path == self.FLAT_DIR_HIERARCHY:
+            chkpt_time = gconf.getr("checkpoint")
+            checkpoint_time = 0
+            if chkpt_time is not None:
+                checkpoint_time = int(chkpt_time)
 
-        self.status.set_last_synced(stime, checkpoint_time)
+            self.status.set_last_synced(stime, checkpoint_time)
 
     def update_worker_remote_node(self):
-        node = sys.argv[-1]
+        node = rconf.args.resource_remote
         node_data = node.split("@")
         node = node_data[-1]
-        remote_node_ip = node.split(":")[0]
+        remote_node_ip, _ = host_brick_split(node)
         self.status.set_slave_node(remote_node_ip)
 
     def changelogs_batch_process(self, changes):
@@ -1162,7 +1475,7 @@ class GMasterChangelogMixin(GMasterCommon):
         current_size = 0
         for c in changes:
             si = os.lstat(c).st_size
-            if (si + current_size) > MAX_CHANGELOG_BATCH_SIZE:
+            if (si + current_size) > gconf.get("changelog-batch-size"):
                 # Create new batch if single Changelog file greater than
                 # Max Size! or current batch size exceeds Max size
                 changelogs_batches.append([c])
@@ -1176,7 +1489,8 @@ class GMasterChangelogMixin(GMasterCommon):
                     changelogs_batches[-1].append(c)
 
         for batch in changelogs_batches:
-            logging.debug('processing changes %s' % repr(batch))
+            logging.debug(lf('processing changes',
+                             batch=batch))
             self.process(batch)
 
     def crawl(self):
@@ -1186,28 +1500,28 @@ class GMasterChangelogMixin(GMasterCommon):
         # that are _historical_ to that time.
         data_stime = self.get_data_stime()
 
-        self.changelog_agent.scan()
+        libgfchangelog.scan()
         self.crawls += 1
-        changes = self.changelog_agent.getchanges()
+        changes = libgfchangelog.getchanges()
         if changes:
             if data_stime:
-                logging.info("slave's time: %s" % repr(data_stime))
+                logging.info(lf("slave's time",
+                                stime=data_stime))
                 processed = [x for x in changes
                              if int(x.split('.')[-1]) < data_stime[0]]
                 for pr in processed:
-                    logging.info(
-                        'skipping already processed change: %s...' %
-                        os.path.basename(pr))
+                    logging.debug(
+                        lf('skipping already processed change',
+                           changelog=os.path.basename(pr)))
                     self.changelog_done_func(pr)
                     changes.remove(pr)
                 self.archive_and_purge_changelogs(processed)
 
         self.changelogs_batch_process(changes)
 
-    def register(self, register_time, changelog_agent, status):
-        self.changelog_agent = changelog_agent
-        self.sleep_interval = int(gconf.change_interval)
-        self.changelog_done_func = self.changelog_agent.done
+    def register(self, register_time, status):
+        self.sleep_interval = gconf.get("change-interval")
+        self.changelog_done_func = libgfchangelog.done
         self.tempdir = self.setup_working_dir()
         self.processed_changelogs_dir = os.path.join(self.tempdir,
                                                      ".processed")
@@ -1216,11 +1530,10 @@ class GMasterChangelogMixin(GMasterCommon):
 
 
 class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
-    def register(self, register_time, changelog_agent, status):
-        self.changelog_agent = changelog_agent
+    def register(self, register_time, status):
         self.changelog_register_time = register_time
         self.history_crawl_start_time = register_time
-        self.changelog_done_func = self.changelog_agent.history_done
+        self.changelog_done_func = libgfchangelog.history_done
         self.history_turns = 0
         self.tempdir = self.setup_working_dir()
         self.processed_changelogs_dir = os.path.join(self.tempdir,
@@ -1234,25 +1547,31 @@ class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
         data_stime = self.get_data_stime()
 
         end_time = int(time.time())
-        logging.info('starting history crawl... turns: %s, stime: %s, '
-                     'etime: %s, entry_stime: %s'
-                     % (self.history_turns, repr(data_stime),
-                        repr(end_time), self.get_entry_stime()))
+
+        #as start of historical crawl marks Geo-rep worker restart
+        if gconf.get("ignore-deletes"):
+            logging.info(lf('ignore-deletes config option is set',
+                         stime=data_stime))
+
+        logging.info(lf('starting history crawl',
+                        turns=self.history_turns,
+                        stime=data_stime,
+                        etime=end_time,
+                        entry_stime=self.get_entry_stime()))
 
         if not data_stime or data_stime == URXTIME:
-            logging.info("stime not available, abandoning history crawl")
             raise NoStimeAvailable()
 
         # Changelogs backend path is hardcoded as
         # <BRICK_PATH>/.glusterfs/changelogs, if user configured to different
         # location then consuming history will not work(Known issue as of now)
-        changelog_path = os.path.join(gconf.local_path,
+        changelog_path = os.path.join(rconf.args.local_path,
                                       ".glusterfs/changelogs")
-        ret, actual_end = self.changelog_agent.history(
+        ret, actual_end = libgfchangelog.history_changelog(
             changelog_path,
             data_stime[0],
             end_time,
-            int(gconf.sync_jobs))
+            gconf.get("sync-jobs"))
 
         # scan followed by getchanges till scan returns zero.
         # history_scan() is blocking call, till it gets the number
@@ -1260,18 +1579,19 @@ class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
         # to be processed. returns positive value as number of changelogs
         # to be processed, which will be fetched using
         # history_getchanges()
-        while self.changelog_agent.history_scan() > 0:
+        while libgfchangelog.history_scan() > 0:
             self.crawls += 1
 
-            changes = self.changelog_agent.history_getchanges()
+            changes = libgfchangelog.history_getchanges()
             if changes:
                 if data_stime:
-                    logging.info("slave's time: %s" % repr(data_stime))
+                    logging.info(lf("slave's time",
+                                    stime=data_stime))
                     processed = [x for x in changes
                                  if int(x.split('.')[-1]) < data_stime[0]]
                     for pr in processed:
-                        logging.info('skipping already processed change: '
-                                     '%s...' % os.path.basename(pr))
+                        logging.debug(lf('skipping already processed change',
+                                         changelog=os.path.basename(pr)))
                         self.changelog_done_func(pr)
                         changes.remove(pr)
 
@@ -1279,10 +1599,10 @@ class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
 
         history_turn_time = int(time.time()) - self.history_crawl_start_time
 
-        logging.info('finished history crawl syncing, endtime: %s, '
-                     'stime: %s, entry_stime: %s'
-                     % (actual_end, repr(self.get_data_stime()),
-                        self.get_entry_stime()))
+        logging.info(lf('finished history crawl',
+                        endtime=actual_end,
+                        stime=self.get_data_stime(),
+                        entry_stime=self.get_entry_stime()))
 
         # If TS returned from history_changelog is < register_time
         # then FS crawl may be required, since history is only available
@@ -1296,7 +1616,7 @@ class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
                 self.history_crawl_start_time = int(time.time())
                 self.crawl()
             else:
-                # This exeption will be catched in resource.py and
+                # This exception will be caught in resource.py and
                 # fallback to xsync for the small gap.
                 raise PartialHistoryAvailable(str(actual_end))
 
@@ -1315,17 +1635,18 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
 
     XSYNC_MAX_ENTRIES = 1 << 13
 
-    def register(self, register_time=None, changelog_agent=None, status=None):
+    def register(self, register_time=None, status=None):
         self.status = status
         self.counter = 0
         self.comlist = []
         self.stimes = []
         self.sleep_interval = 60
         self.tempdir = self.setup_working_dir()
+        logging.info(lf('Working dir',
+                        path=self.tempdir))
         self.tempdir = os.path.join(self.tempdir, 'xsync')
         self.processed_changelogs_dir = self.tempdir
         self.name = "xsync"
-        logging.info('xsync temp directory: %s' % self.tempdir)
         try:
             os.makedirs(self.tempdir)
         except OSError:
@@ -1334,6 +1655,11 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
                 pass
             else:
                 raise
+        # Purge stale unprocessed xsync changelogs
+        for f in os.listdir(self.tempdir):
+            if f.startswith("XSYNC-CHANGELOG"):
+                os.remove(os.path.join(self.tempdir, f))
+
 
     def crawl(self):
         """
@@ -1346,25 +1672,28 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
             self.Xcrawl()
         t = Thread(target=Xsyncer)
         t.start()
-        logging.info('starting hybrid crawl..., stime: %s'
-                     % repr(self.get_data_stime()))
+        logging.info(lf('starting hybrid crawl',
+                        stime=self.get_data_stime()))
         self.status.set_worker_crawl_status("Hybrid Crawl")
         while True:
             try:
                 item = self.comlist.pop(0)
                 if item[0] == 'finale':
-                    logging.info('finished hybrid crawl syncing, stime: %s'
-                                 % repr(self.get_data_stime()))
+                    logging.info(lf('finished hybrid crawl',
+                                    stime=self.get_data_stime()))
                     break
                 elif item[0] == 'xsync':
-                    logging.info('processing xsync changelog %s' % (item[1]))
+                    logging.info(lf('processing xsync changelog',
+                                    path=item[1]))
                     self.process([item[1]], 0)
                     self.archive_and_purge_changelogs([item[1]])
                 elif item[0] == 'stime':
-                    logging.debug('setting slave time: %s' % repr(item[1]))
+                    logging.debug(lf('setting slave time',
+                                     time=item[1]))
                     self.upd_stime(item[1][1], item[1][0])
                 else:
-                    logging.warn('unknown tuple in comlist (%s)' % repr(item))
+                    logging.warn(lf('unknown tuple in comlist',
+                                    entry=item))
             except IndexError:
                 time.sleep(1)
 
@@ -1424,7 +1753,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
     def is_sticky(self, path, mo):
         """check for DHTs linkto sticky bit file"""
         sticky = False
-        if mo & 01000:
+        if mo & 0o1000:
             sticky = self.master.server.linkto_check(path)
         return sticky
 
@@ -1442,8 +1771,9 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
             xtr_root = self.xtime('.', self.slave)
             if isinstance(xtr_root, int):
                 if xtr_root != ENOENT:
-                    logging.warn("slave cluster not returning the "
-                                 "correct xtime for root (%d)" % xtr_root)
+                    logging.warn(lf("slave cluster not returning the "
+                                    "xtime for root",
+                                    error=xtr_root))
                 xtr_root = self.minus_infinity
         xtl = self.xtime(path)
         if isinstance(xtl, int):
@@ -1451,8 +1781,10 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
         xtr = self.xtime(path, self.slave)
         if isinstance(xtr, int):
             if xtr != ENOENT:
-                logging.warn("slave cluster not returning the "
-                             "correct xtime for %s (%d)" % (path, xtr))
+                logging.warn(lf("slave cluster not returning the "
+                                "xtime for dir",
+                                path=path,
+                                error=xtr))
             xtr = self.minus_infinity
         xtr = max(xtr, xtr_root)
         zero_zero = (0, 0)
@@ -1467,27 +1799,32 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
         dem = self.master.server.entries(path)
         pargfid = self.master.server.gfid(path)
         if isinstance(pargfid, int):
-            logging.warn('skipping directory %s' % (path))
+            logging.warn(lf('skipping directory',
+                            path=path))
         for e in dem:
             bname = e
             e = os.path.join(path, e)
             xte = self.xtime(e)
             if isinstance(xte, int):
-                logging.warn("irregular xtime for %s: %s" %
-                             (e, errno.errorcode[xte]))
+                logging.warn(lf("irregular xtime",
+                                path=e,
+                                error=errno.errorcode[xte]))
                 continue
             if not self.need_sync(e, xte, xtr):
                 continue
             st = self.master.server.lstat(e)
             if isinstance(st, int):
-                logging.warn('%s got purged in the interim ...' % e)
+                logging.warn(lf('got purged in the interim',
+                                path=e))
                 continue
             if self.is_sticky(e, st.st_mode):
-                logging.debug('ignoring sticky bit file %s' % e)
+                logging.debug(lf('ignoring sticky bit file',
+                                 path=e))
                 continue
             gfid = self.master.server.gfid(e)
             if isinstance(gfid, int):
-                logging.warn('skipping entry %s..' % e)
+                logging.warn(lf('skipping entry',
+                                path=e))
                 continue
             mo = st.st_mode
             self.counter += 1 if ((stat.S_ISDIR(mo) or
@@ -1497,8 +1834,10 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
                 self.sync_done(self.stimes, False)
                 self.stimes = []
             if stat.S_ISDIR(mo):
-                self.write_entry_change("E", [gfid, 'MKDIR', str(mo),
-                    str(0), str(0), escape(os.path.join(pargfid, bname))])
+                self.write_entry_change("E",
+                                        [gfid, 'MKDIR', str(mo),
+                                         str(0), str(0), escape_space_newline(
+                                             os.path.join(pargfid, bname))])
                 self.write_entry_change("M", [gfid, "SETATTR", str(st.st_uid),
                                               str(st.st_gid), str(st.st_mode),
                                               str(st.st_atime),
@@ -1517,8 +1856,8 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
                 self.stimes.append((e, stime_to_update))
             elif stat.S_ISLNK(mo):
                 self.write_entry_change(
-                    "E", [gfid, 'SYMLINK', escape(os.path.join(pargfid,
-                                                               bname))])
+                    "E", [gfid, 'SYMLINK', escape_space_newline(
+                        os.path.join(pargfid, bname))])
             elif stat.S_ISREG(mo):
                 nlink = st.st_nlink
                 nlink -= 1  # fixup backend stat link count
@@ -1529,12 +1868,13 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
                     self.write_entry_change("E",
                                             [gfid, 'MKNOD', str(mo),
                                              str(0), str(0),
-                                             escape(os.path.join(
-                                                 pargfid, bname))])
+                                             escape_space_newline(
+                                                 os.path.join(
+                                                     pargfid, bname))])
                 else:
                     self.write_entry_change(
-                        "E", [gfid, 'LINK', escape(os.path.join(pargfid,
-                                                                bname))])
+                        "E", [gfid, 'LINK', escape_space_newline(
+                            os.path.join(pargfid, bname))])
                 self.write_entry_change("D", [gfid])
         if path == '.':
             stime_to_update = xtl
@@ -1631,11 +1971,11 @@ class Syncer(object):
         self.pb = PostBox()
         self.sync_engine = sync_engine
         self.errnos_ok = resilient_errnos
-        for i in range(int(gconf.sync_jobs)):
-            t = Thread(target=self.syncjob)
+        for i in range(gconf.get("sync-jobs")):
+            t = Thread(target=self.syncjob, args=(i + 1, ))
             t.start()
 
-    def syncjob(self):
+    def syncjob(self, job_id):
         """the life of a worker"""
         while True:
             pb = None
@@ -1648,7 +1988,14 @@ class Syncer(object):
                     break
                 time.sleep(0.5)
             pb.close()
+            start = time.time()
             po = self.sync_engine(pb, self.log_err)
+            logging.info(lf("Sync Time Taken",
+                            job=job_id,
+                            num_files=len(pb),
+                            return_code=po.returncode,
+                            duration="%.4f" % (time.time() - start)))
+
             if po.returncode == 0:
                 ret = (True, 0)
             elif po.returncode in self.errnos_ok:
diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py
index 22cd1cc3a86..6aa7b9dfc99 100644
--- a/geo-replication/syncdaemon/monitor.py
+++ b/geo-replication/syncdaemon/monitor.py
@@ -13,22 +13,23 @@ import sys
 import time
 import signal
 import logging
-import uuid
 import xml.etree.ElementTree as XET
-from subprocess import PIPE
-from resource import Popen, FILE, GLUSTER, SSH
 from threading import Lock
 from errno import ECHILD, ESRCH
-import re
 import random
-from gconf import gconf
-from syncdutils import select, waitpid, errno_wrap
-from syncdutils import set_term_handler, is_host_local, GsyncdError
-from syncdutils import escape, Thread, finalize, memoize
-from syncdutils import gf_event, EVENT_GEOREP_FAULTY
 
+from resource import SSH
+import gsyncdconfig as gconf
+import libgfchangelog
+from rconf import rconf
+from syncdutils import (select, waitpid, errno_wrap, lf, grabpidfile,
+                        set_term_handler, GsyncdError,
+                        Thread, finalize, Volinfo, VolinfoFromGconf,
+                        gf_event, EVENT_GEOREP_FAULTY, get_up_nodes,
+                        unshare_propagation_supported)
 from gsyncdstatus import GeorepStatus, set_monitor_status
-
+import py2py3
+from py2py3 import pipe
 
 ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
 
@@ -37,6 +38,8 @@ def get_subvol_num(brick_idx, vol, hot):
     tier = vol.is_tier()
     disperse_count = vol.disperse_count(tier, hot)
     replica_count = vol.replica_count(tier, hot)
+    distribute_count = vol.distribution_count(tier, hot)
+    gconf.setconfig("master-distribution-count", distribute_count)
 
     if (tier and not hot):
         brick_idx = brick_idx - vol.get_hot_bricks_count(tier)
@@ -55,118 +58,6 @@ def get_subvol_num(brick_idx, vol, hot):
         return str(cnt)
 
 
-def get_slave_bricks_status(host, vol):
-    po = Popen(['gluster', '--xml', '--remote-host=' + host,
-                'volume', 'status', vol, "detail"],
-               stdout=PIPE, stderr=PIPE)
-    vix = po.stdout.read()
-    po.wait()
-    po.terminate_geterr(fail_on_err=False)
-    if po.returncode != 0:
-        logging.info("Volume status command failed, unable to get "
-                     "list of up nodes of %s, returning empty list: %s" %
-                     (vol, po.returncode))
-        return []
-    vi = XET.fromstring(vix)
-    if vi.find('opRet').text != '0':
-        logging.info("Unable to get list of up nodes of %s, "
-                     "returning empty list: %s" %
-                     (vol, vi.find('opErrstr').text))
-        return []
-
-    up_hosts = set()
-
-    try:
-        for el in vi.findall('volStatus/volumes/volume/node'):
-            if el.find('status').text == '1':
-                up_hosts.add(el.find('hostname').text)
-    except (ParseError, AttributeError, ValueError) as e:
-        logging.info("Parsing failed to get list of up nodes of %s, "
-                     "returning empty list: %s" % (vol, e))
-
-    return list(up_hosts)
-
-
-class Volinfo(object):
-
-    def __init__(self, vol, host='localhost', prelude=[]):
-        po = Popen(prelude + ['gluster', '--xml', '--remote-host=' + host,
-                              'volume', 'info', vol],
-                   stdout=PIPE, stderr=PIPE)
-        vix = po.stdout.read()
-        po.wait()
-        po.terminate_geterr()
-        vi = XET.fromstring(vix)
-        if vi.find('opRet').text != '0':
-            if prelude:
-                via = '(via %s) ' % prelude.join(' ')
-            else:
-                via = ' '
-            raise GsyncdError('getting volume info of %s%s '
-                              'failed with errorcode %s' %
-                              (vol, via, vi.find('opErrno').text))
-        self.tree = vi
-        self.volume = vol
-        self.host = host
-
-    def get(self, elem):
-        return self.tree.findall('.//' + elem)
-
-    def is_tier(self):
-        return (self.get('typeStr')[0].text == 'Tier')
-
-    def is_hot(self, brickpath):
-        logging.debug('brickpath: ' + repr(brickpath))
-        return brickpath in self.hot_bricks
-
-    @property
-    @memoize
-    def bricks(self):
-        def bparse(b):
-            host, dirp = b.text.split(':', 2)
-            return {'host': host, 'dir': dirp}
-        return [bparse(b) for b in self.get('brick')]
-
-    @property
-    @memoize
-    def uuid(self):
-        ids = self.get('id')
-        if len(ids) != 1:
-            raise GsyncdError("volume info of %s obtained from %s: "
-                              "ambiguous uuid" % (self.volume, self.host))
-        return ids[0].text
-
-    def replica_count(self, tier, hot):
-        if (tier and hot):
-            return int(self.get('hotBricks/hotreplicaCount')[0].text)
-        elif (tier and not hot):
-            return int(self.get('coldBricks/coldreplicaCount')[0].text)
-        else:
-            return int(self.get('replicaCount')[0].text)
-
-    def disperse_count(self, tier, hot):
-        if (tier and hot):
-            # Tiering doesn't support disperse volume as hot brick,
-            # hence no xml output, so returning 0. In case, if it's
-            # supported later, we should change here.
-            return 0
-        elif (tier and not hot):
-            return int(self.get('coldBricks/colddisperseCount')[0].text)
-        else:
-            return int(self.get('disperseCount')[0].text)
-
-    @property
-    @memoize
-    def hot_bricks(self):
-        return [b.text for b in self.get('hotBricks/brick')]
-
-    def get_hot_bricks_count(self, tier):
-        if (tier):
-            return int(self.get('hotBricks/hotbrickCount')[0].text)
-        else:
-            return 0
-
-
 class Monitor(object):
 
     """class which spawns and manages gsyncd workers"""
@@ -191,7 +82,8 @@ class Monitor(object):
         # give a chance to graceful exit
         errno_wrap(os.kill, [-os.getpid(), signal.SIGTERM], [ESRCH])
 
-    def monitor(self, w, argv, cpids, agents, slave_vol, slave_host, master):
+    def monitor(self, w, argv, cpids, slave_vol, slave_host, master,
+                suuid, slavenodes):
         """the monitor loop
 
         Basic logic is a blantantly simple blunt heuristics:
@@ -211,12 +103,13 @@ class Monitor(object):
         due to the keep-alive thread)
         """
         if not self.status.get(w[0]['dir'], None):
-            self.status[w[0]['dir']] = GeorepStatus(gconf.state_file,
-                                                    w[0]['dir'])
-
-        set_monitor_status(gconf.state_file, self.ST_STARTED)
-        self.status[w[0]['dir']].set_worker_status(self.ST_INIT)
-
+            self.status[w[0]['dir']] = GeorepStatus(gconf.get("state-file"),
+                                                    w[0]['host'],
+                                                    w[0]['dir'],
+                                                    w[0]['uuid'],
+                                                    master,
+                                                    "%s::%s" % (slave_host,
+                                                                slave_vol))
         ret = 0
 
         def nwait(p, o=0):
@@ -234,7 +127,7 @@ class Monitor(object):
                     raise
 
         def exit_signalled(s):
-            """ child teminated due to receipt of SIGUSR1 """
+            """ child terminated due to receipt of SIGUSR1 """
             return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
 
         def exit_status(s):
@@ -242,77 +135,76 @@ class Monitor(object):
                 return os.WEXITSTATUS(s)
             return 1
 
-        conn_timeout = int(gconf.connection_timeout)
+        conn_timeout = gconf.get("connection-timeout")
         while ret in (0, 1):
-            remote_host = w[1]
+            remote_user, remote_host = w[1][0].split("@")
+            remote_id = w[1][1]
             # Check the status of the connected slave node
             # If the connected slave node is down then try to connect to
             # different up node.
-            m = re.match("(ssh|gluster|file):\/\/(.+)@([^:]+):(.+)",
-                         remote_host)
-            if m:
-                current_slave_host = m.group(3)
-                slave_up_hosts = get_slave_bricks_status(
-                    slave_host, slave_vol)
-
-                if current_slave_host not in slave_up_hosts:
-                    if len(slave_up_hosts) > 0:
-                        remote_host = "%s://%s@%s:%s" % (m.group(1),
-                                                         m.group(2),
-                                                         random.choice(
-                                                             slave_up_hosts),
-                                                         m.group(4))
-
-            # Spawn the worker and agent in lock to avoid fd leak
+            current_slave_host = remote_host
+            slave_up_hosts = get_up_nodes(slavenodes, gconf.get("ssh-port"))
+
+            if (current_slave_host, remote_id) not in slave_up_hosts:
+                if len(slave_up_hosts) > 0:
+                    remote_new = random.choice(slave_up_hosts)
+                    remote_host = "%s@%s" % (remote_user, remote_new[0])
+                    remote_id = remote_new[1]
+
+            # Spawn the worker in lock to avoid fd leak
             self.lock.acquire()
 
-            logging.info('-' * conn_timeout)
-            logging.info('starting gsyncd worker')
-
-            # Couple of pipe pairs for RPC communication b/w
-            # worker and changelog agent.
-
-            # read/write end for agent
-            (ra, ww) = os.pipe()
-            # read/write end for worker
-            (rw, wa) = os.pipe()
-
-            # spawn the agent process
-            apid = os.fork()
-            if apid == 0:
-                os.close(rw)
-                os.close(ww)
-                os.execv(sys.executable, argv + ['--local-path', w[0]['dir'],
-                                                 '--agent',
-                                                 '--rpc-fd',
-                                                 ','.join([str(ra), str(wa),
-                                                           str(rw), str(ww)])])
-            pr, pw = os.pipe()
+            self.status[w[0]['dir']].set_worker_status(self.ST_INIT)
+            logging.info(lf('starting gsyncd worker',
+                            brick=w[0]['dir'],
+                            slave_node=remote_host))
+
+            pr, pw = pipe()
             cpid = os.fork()
             if cpid == 0:
                 os.close(pr)
-                os.close(ra)
-                os.close(wa)
-                os.execv(sys.executable, argv + ['--feedback-fd', str(pw),
-                                                 '--local-path', w[0]['dir'],
-                                                 '--local-id',
-                                                 '.' + escape(w[0]['dir']),
-                                                 '--rpc-fd',
-                                                 ','.join([str(rw), str(ww),
-                                                           str(ra), str(wa)]),
-                                                 '--subvol-num', str(w[2])] +
-                         (['--is-hottier'] if w[3] else []) +
-                         ['--resource-remote', remote_host])
+
+                args_to_worker = argv + [
+                    'worker',
+                    rconf.args.master,
+                    rconf.args.slave,
+                    '--feedback-fd', str(pw),
+                    '--local-path', w[0]['dir'],
+                    '--local-node', w[0]['host'],
+                    '--local-node-id', w[0]['uuid'],
+                    '--slave-id', suuid,
+                    '--subvol-num', str(w[2]),
+                    '--resource-remote', remote_host,
+                    '--resource-remote-id', remote_id
+                ]
+
+                if rconf.args.config_file is not None:
+                    args_to_worker += ['-c', rconf.args.config_file]
+
+                if w[3]:
+                    args_to_worker.append("--is-hottier")
+
+                if rconf.args.debug:
+                    args_to_worker.append("--debug")
+
+                access_mount = gconf.get("access-mount")
+                if access_mount:
+                    os.execv(sys.executable, args_to_worker)
+                else:
+                    if unshare_propagation_supported():
+                        logging.debug("Worker would mount volume privately")
+                        unshare_cmd = ['unshare', '-m', '--propagation',
+                                       'private']
+                        cmd = unshare_cmd + args_to_worker
+                        os.execvp("unshare", cmd)
+                    else:
+                        logging.debug("Mount is not private. It would be lazy"
+                                      " umounted")
+                        os.execv(sys.executable, args_to_worker)
 
             cpids.add(cpid)
-            agents.add(apid)
             os.close(pw)
 
-            # close all RPC pipes in monitor
-            os.close(ra)
-            os.close(wa)
-            os.close(rw)
-            os.close(ww)
             self.lock.release()
 
             t0 = time.time()
@@ -321,53 +213,35 @@ class Monitor(object):
 
             if so:
                 ret = nwait(cpid, os.WNOHANG)
-                ret_agent = nwait(apid, os.WNOHANG)
-
-                if ret_agent is not None:
-                    # Agent is died Kill Worker
-                    logging.info("Changelog Agent died, "
-                                 "Aborting Worker(%s)" % w[0]['dir'])
-                    errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
-                    nwait(cpid)
-                    nwait(apid)
 
                 if ret is not None:
-                    logging.info("worker(%s) died before establishing "
-                                 "connection" % w[0]['dir'])
-                    nwait(apid)  # wait for agent
+                    logging.info(lf("worker died before establishing "
+                                    "connection",
+                                    brick=w[0]['dir']))
                 else:
                     logging.debug("worker(%s) connected" % w[0]['dir'])
                     while time.time() < t0 + conn_timeout:
                         ret = nwait(cpid, os.WNOHANG)
-                        ret_agent = nwait(apid, os.WNOHANG)
 
                         if ret is not None:
-                            logging.info("worker(%s) died in startup "
-                                         "phase" % w[0]['dir'])
-                            nwait(apid)  # wait for agent
-                            break
-
-                        if ret_agent is not None:
-                            # Agent is died Kill Worker
-                            logging.info("Changelog Agent died, Aborting "
-                                         "Worker(%s)" % w[0]['dir'])
-                            errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
-                            nwait(cpid)
-                            nwait(apid)
+                            logging.info(lf("worker died in startup phase",
+                                            brick=w[0]['dir']))
                             break
 
                         time.sleep(1)
             else:
-                logging.info("worker(%s) not confirmed in %d sec, "
-                             "aborting it" % (w[0]['dir'], conn_timeout))
+                logging.info(
+                    lf("Worker not confirmed after wait, aborting it. "
+                       "Gsyncd invocation on remote slave via SSH or "
+                       "gluster master mount might have hung. Please "
+                       "check the above logs for exact issue and check "
+                       "master or slave volume for errors. Restarting "
+                       "master/slave volume accordingly might help.",
+                       brick=w[0]['dir'],
+                       timeout=conn_timeout))
                 errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
-                nwait(apid)  # wait for agent
                 ret = nwait(cpid)
             if ret is None:
-                self.status[w[0]['dir']].set_worker_status(self.ST_STABLE)
-                # If worker dies, agent terminates on EOF.
-                # So lets wait for agent first.
-                nwait(apid)
                 ret = nwait(cpid)
             if exit_signalled(ret):
                 ret = 0
@@ -378,6 +252,7 @@ class Monitor(object):
                     gf_event(EVENT_GEOREP_FAULTY,
                              master_volume=master.volume,
                              master_node=w[0]['host'],
+                             master_node_id=w[0]['uuid'],
                              slave_host=slave_host,
                              slave_volume=slave_vol,
                              current_slave_host=current_slave_host,
@@ -386,104 +261,135 @@ class Monitor(object):
         self.status[w[0]['dir']].set_worker_status(self.ST_INCON)
         return ret
 
-    def multiplex(self, wspx, suuid, slave_vol, slave_host, master):
-        argv = sys.argv[:]
-        for o in ('-N', '--no-daemon', '--monitor'):
-            while o in argv:
-                argv.remove(o)
-        argv.extend(('-N', '-p', '', '--slave-id', suuid))
-        argv.insert(0, os.path.basename(sys.executable))
+    def multiplex(self, wspx, suuid, slave_vol, slave_host, master, slavenodes):
+        argv = [os.path.basename(sys.executable), sys.argv[0]]
 
         cpids = set()
-        agents = set()
         ta = []
         for wx in wspx:
             def wmon(w):
-                cpid, _ = self.monitor(w, argv, cpids, agents, slave_vol,
-                                       slave_host, master)
+                cpid, _ = self.monitor(w, argv, cpids, slave_vol,
+                                       slave_host, master, suuid, slavenodes)
                 time.sleep(1)
                 self.lock.acquire()
                 for cpid in cpids:
                     errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
-                for apid in agents:
-                    errno_wrap(os.kill, [apid, signal.SIGKILL], [ESRCH])
                 self.lock.release()
                 finalize(exval=1)
             t = Thread(target=wmon, args=[wx])
             t.start()
             ta.append(t)
+
+        # monitor status was being updated in each monitor thread. It
+        # should not be done as it can cause deadlock for a worker start.
+        # set_monitor_status uses flock to synchronize multple instances
+        # updating the file. Since each monitor thread forks worker,
+        # these processes can hold the reference to fd of status
+        # file causing deadlock to workers which starts later as flock
+        # will not be release until all references to same fd is closed.
+        # It will also cause fd leaks.
+
+        self.lock.acquire()
+        set_monitor_status(gconf.get("state-file"), self.ST_STARTED)
+        self.lock.release()
         for t in ta:
             t.join()
 
 
-def distribute(*resources):
-    master, slave = resources
-    mvol = Volinfo(master.volume, master.host)
+def distribute(master, slave):
+    if rconf.args.use_gconf_volinfo:
+        mvol = VolinfoFromGconf(master.volume, master=True)
+    else:
+        mvol = Volinfo(master.volume, master.host, master=True)
     logging.debug('master bricks: ' + repr(mvol.bricks))
     prelude = []
-    si = slave
     slave_host = None
     slave_vol = None
 
-    if isinstance(slave, SSH):
-        prelude = gconf.ssh_command.split() + [slave.remote_addr]
-        si = slave.inner_rsc
-        logging.debug('slave SSH gateway: ' + slave.remote_addr)
-    if isinstance(si, FILE):
-        sbricks = {'host': 'localhost', 'dir': si.path}
-        suuid = uuid.uuid5(uuid.NAMESPACE_URL, slave.get_url(canonical=True))
-    elif isinstance(si, GLUSTER):
-        svol = Volinfo(si.volume, slave.remote_addr.split('@')[-1])
-        sbricks = svol.bricks
-        suuid = svol.uuid
-        slave_host = slave.remote_addr.split('@')[-1]
-        slave_vol = si.volume
-
-        # save this xattr for the session delete command
-        old_stime_xattr_name = getattr(gconf, "master.stime_xattr_name", None)
-        new_stime_xattr_name = "trusted.glusterfs." + mvol.uuid + "." + \
-            svol.uuid + ".stime"
-        if not old_stime_xattr_name or \
-           old_stime_xattr_name != new_stime_xattr_name:
-            gconf.configinterface.set("master.stime_xattr_name",
-                                      new_stime_xattr_name)
-    else:
-        raise GsyncdError("unknown slave type " + slave.url)
-    logging.info('slave bricks: ' + repr(sbricks))
-    if isinstance(si, FILE):
-        slaves = [slave.url]
+    prelude = [gconf.get("ssh-command")] + \
+        gconf.get("ssh-options").split() + \
+        ["-p", str(gconf.get("ssh-port"))] + \
+        [slave.remote_addr]
+
+    logging.debug('slave SSH gateway: ' + slave.remote_addr)
+
+    if rconf.args.use_gconf_volinfo:
+        svol = VolinfoFromGconf(slave.volume, master=False)
     else:
-        slavenodes = set(b['host'] for b in sbricks)
-        if isinstance(slave, SSH) and not gconf.isolated_slave:
-            rap = SSH.parse_ssh_address(slave)
-            slaves = ['ssh://' + rap['user'] + '@' + h + ':' + si.url
-                      for h in slavenodes]
-        else:
-            slavevols = [h + ':' + si.volume for h in slavenodes]
-            if isinstance(slave, SSH):
-                slaves = ['ssh://' + rap.remote_addr + ':' + v
-                          for v in slavevols]
-            else:
-                slaves = slavevols
+        svol = Volinfo(slave.volume, "localhost", prelude, master=False)
+
+    sbricks = svol.bricks
+    suuid = svol.uuid
+    slave_host = slave.remote_addr.split('@')[-1]
+    slave_vol = slave.volume
+
+    # save this xattr for the session delete command
+    old_stime_xattr_prefix = gconf.get("stime-xattr-prefix", None)
+    new_stime_xattr_prefix = "trusted.glusterfs." + mvol.uuid + "." + \
+                             svol.uuid
+    if not old_stime_xattr_prefix or \
+       old_stime_xattr_prefix != new_stime_xattr_prefix:
+        gconf.setconfig("stime-xattr-prefix", new_stime_xattr_prefix)
+
+    logging.debug('slave bricks: ' + repr(sbricks))
+
+    slavenodes = set((b['host'], b["uuid"]) for b in sbricks)
+    rap = SSH.parse_ssh_address(slave)
+    slaves = [(rap['user'] + '@' + h[0], h[1]) for h in slavenodes]
 
     workerspex = []
     for idx, brick in enumerate(mvol.bricks):
-        if is_host_local(brick['host']):
+        if rconf.args.local_node_id == brick['uuid']:
             is_hot = mvol.is_hot(":".join([brick['host'], brick['dir']]))
             workerspex.append((brick,
                                slaves[idx % len(slaves)],
                                get_subvol_num(idx, mvol, is_hot),
                                is_hot))
-    logging.info('worker specs: ' + repr(workerspex))
-    return workerspex, suuid, slave_vol, slave_host, master
+    logging.debug('worker specs: ' + repr(workerspex))
+    return workerspex, suuid, slave_vol, slave_host, master, slavenodes
 
 
-def monitor(*resources):
+def monitor(local, remote):
     # Check if gsyncd restarted in pause state. If
     # yes, send SIGSTOP to negative of monitor pid
     # to go back to pause state.
-    if gconf.pause_on_start:
+    if rconf.args.pause_on_start:
         errno_wrap(os.kill, [-os.getpid(), signal.SIGSTOP], [ESRCH])
 
     """oh yeah, actually Monitor is used as singleton, too"""
-    return Monitor().multiplex(*distribute(*resources))
+    return Monitor().multiplex(*distribute(local, remote))
+
+
+def startup(go_daemon=True):
+    """set up logging, pidfile grabbing, daemonization"""
+    pid_file = gconf.get("pid-file")
+    if not grabpidfile():
+        sys.stderr.write("pidfile is taken, exiting.\n")
+        sys.exit(2)
+    rconf.pid_file_owned = True
+
+    if not go_daemon:
+        return
+
+    x, y = pipe()
+    cpid = os.fork()
+    if cpid:
+        os.close(x)
+        sys.exit()
+    os.close(y)
+    os.setsid()
+    dn = os.open(os.devnull, os.O_RDWR)
+    for f in (sys.stdin, sys.stdout, sys.stderr):
+        os.dup2(dn, f.fileno())
+
+    if not grabpidfile(pid_file + '.tmp'):
+        raise GsyncdError("cannot grab temporary pidfile")
+
+    os.rename(pid_file + '.tmp', pid_file)
+
+    # wait for parent to terminate
+    # so we can start up with
+    # no messing from the dirty
+    # ol' bustard
+    select((x,), (), ())
+    os.close(x)
diff --git a/geo-replication/syncdaemon/py2py3.py b/geo-replication/syncdaemon/py2py3.py
new file mode 100644
index 00000000000..f9c76e1b50a
--- /dev/null
+++ b/geo-replication/syncdaemon/py2py3.py
@@ -0,0 +1,184 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+# All python2/python3 compatibility routines
+
+import sys
+import os
+import stat
+import struct
+from syncdutils import umask
+from ctypes import create_string_buffer
+
+if sys.version_info >= (3,):
+    def pipe():
+        (r, w) = os.pipe()
+        os.set_inheritable(r, True)
+        os.set_inheritable(w, True)
+        return (r, w)
+
+    # Raw conversion of bytearray to string. Used in the cases where
+    # buffer is created by create_string_buffer which is a 8-bit char
+    # array and passed to syscalls to fetch results. Using encode/decode
+    # doesn't work as it converts to string altering the size.
+    def bytearray_to_str(byte_arr):
+        return ''.join([chr(b) for b in byte_arr])
+
+    # Raw conversion of string to bytes. This is required to convert
+    # back the string into bytearray(c char array) to use in struc
+    # pack/unpacking. Again encode/decode can't be used as it
+    # converts it alters size.
+    def str_to_bytearray(string):
+        return bytes([ord(c) for c in string])
+
+    def gr_create_string_buffer(size):
+        return create_string_buffer(b'\0', size)
+
+    def gr_query_xattr(cls, path, size, syscall, attr=None):
+        if attr:
+            return cls._query_xattr(path.encode(), size, syscall,
+                                    attr.encode())
+        else:
+            return cls._query_xattr(path.encode(), size, syscall)
+
+    def gr_lsetxattr(cls, path, attr, val):
+        return cls.libc.lsetxattr(path.encode(), attr.encode(), val,
+                                  len(val), 0)
+
+    def gr_lremovexattr(cls, path, attr):
+        return cls.libc.lremovexattr(path.encode(), attr.encode())
+
+    def gr_cl_register(libgfapi, brick, path, log_file, log_level, retries):
+        return libgfapi.gf_changelog_register(brick.encode(),
+                                              path.encode(),
+                                              log_file.encode(),
+                                              log_level, retries)
+
+    def gr_cl_done(libgfapi, clfile):
+        return libgfapi.gf_changelog_done(clfile.encode())
+
+    def gr_cl_history_changelog(libgfapi, changelog_path, start, end, num_parallel,
+                                actual_end):
+        return libgfapi.gf_history_changelog(changelog_path.encode(),
+                                             start, end, num_parallel,
+                                             actual_end)
+
+    def gr_cl_history_done(libgfapi, clfile):
+        return libgfapi.gf_history_changelog_done(clfile.encode())
+
+    # regular file
+
+    def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        return struct.pack(cls._fmt_mknod(blen),
+                           uid, gid, gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), 0, umask())
+
+    def entry_pack_reg_stat(cls, gf, bn, st):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        mo = st['mode']
+        return struct.pack(cls._fmt_mknod(blen),
+                           st['uid'], st['gid'],
+                           gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), 0, umask())
+    # mkdir
+
+    def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        return struct.pack(cls._fmt_mkdir(blen),
+                           uid, gid, gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), umask())
+    # symlink
+
+    def entry_pack_symlink(cls, gf, bn, lnk, st):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        lnk_encoded = lnk.encode()
+        llen = len(lnk_encoded)
+        return struct.pack(cls._fmt_symlink(blen, llen),
+                           st['uid'], st['gid'],
+                           gf.encode(), st['mode'], bn_encoded,
+                           lnk_encoded)
+else:
+    def pipe():
+        (r, w) = os.pipe()
+        return (r, w)
+
+    # Raw conversion of bytearray to string
+    def bytearray_to_str(byte_arr):
+        return byte_arr
+
+    # Raw conversion of string to bytearray
+    def str_to_bytearray(string):
+        return string
+
+    def gr_create_string_buffer(size):
+        return create_string_buffer('\0', size)
+
+    def gr_query_xattr(cls, path, size, syscall, attr=None):
+        if attr:
+            return cls._query_xattr(path, size, syscall, attr)
+        else:
+            return cls._query_xattr(path, size, syscall)
+
+    def gr_lsetxattr(cls, path, attr, val):
+        return cls.libc.lsetxattr(path, attr, val, len(val), 0)
+
+    def gr_lremovexattr(cls, path, attr):
+        return cls.libc.lremovexattr(path, attr)
+
+    def gr_cl_register(libgfapi, brick, path, log_file, log_level, retries):
+        return libgfapi.gf_changelog_register(brick, path, log_file,
+                                              log_level, retries)
+
+    def gr_cl_done(libgfapi, clfile):
+        return libgfapi.gf_changelog_done(clfile)
+
+    def gr_cl_history_changelog(libgfapi, changelog_path, start, end, num_parallel,
+                                actual_end):
+        return libgfapi.gf_history_changelog(changelog_path, start, end,
+                                             num_parallel, actual_end)
+
+    def gr_cl_history_done(libgfapi, clfile):
+        return libgfapi.gf_history_changelog_done(clfile)
+
+    # regular file
+
+    def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+        blen = len(bn)
+        return struct.pack(cls._fmt_mknod(blen),
+                           uid, gid, gf, mo, bn,
+                           stat.S_IMODE(mo), 0, umask())
+
+    def entry_pack_reg_stat(cls, gf, bn, st):
+        blen = len(bn)
+        mo = st['mode']
+        return struct.pack(cls._fmt_mknod(blen),
+                           st['uid'], st['gid'],
+                           gf, mo, bn,
+                           stat.S_IMODE(mo), 0, umask())
+    # mkdir
+
+    def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+        blen = len(bn)
+        return struct.pack(cls._fmt_mkdir(blen),
+                           uid, gid, gf, mo, bn,
+                           stat.S_IMODE(mo), umask())
+    # symlink
+
+    def entry_pack_symlink(cls, gf, bn, lnk, st):
+        blen = len(bn)
+        llen = len(lnk)
+        return struct.pack(cls._fmt_symlink(blen, llen),
+                           st['uid'], st['gid'],
+                           gf, st['mode'], bn, lnk)
diff --git a/geo-replication/syncdaemon/gconf.py b/geo-replication/syncdaemon/rconf.py
index 97395b41b06..ff716ee4d6d 100644
--- a/geo-replication/syncdaemon/gconf.py
+++ b/geo-replication/syncdaemon/rconf.py
@@ -9,9 +9,9 @@
 #
 
 
-class GConf(object):
+class RConf(object):
 
-    """singleton class to store globals
+    """singleton class to store runtime globals
        shared between gsyncd modules"""
 
     ssh_ctl_dir = None
@@ -21,12 +21,11 @@ class GConf(object):
     log_exit = False
     permanent_handles = []
     log_metadata = {}
-    """One variable is sufficient to track the
-       switching of worker to ACTIVE. Two variables
-       are intentionally used to track worker going
-       to PASSIVE as well mainly for debugging"""
-    active_earlier = False
-    passive_earlier = False
     mgmt_lock_fd = None
+    args = None
+    turns = 0
+    mountbroker = False
+    mount_point = None
+    mbr_umount_cmd = []
 
-gconf = GConf()
+rconf = RConf()
diff --git a/geo-replication/syncdaemon/repce.py b/geo-replication/syncdaemon/repce.py
index d7b17dda796..c622afa6373 100644
--- a/geo-replication/syncdaemon/repce.py
+++ b/geo-replication/syncdaemon/repce.py
@@ -14,30 +14,27 @@ import time
 import logging
 from threading import Condition
 try:
-    import thread
-except ImportError:
-    # py 3
     import _thread as thread
-try:
-    from Queue import Queue
 except ImportError:
-    # py 3
+    import thread
+try:
     from queue import Queue
+except ImportError:
+    from Queue import Queue
 try:
     import cPickle as pickle
 except ImportError:
-    # py 3
     import pickle
 
-from syncdutils import Thread, select
+from syncdutils import Thread, select, lf
 
-pickle_proto = -1
+pickle_proto = 2
 repce_version = 1.0
 
 
 def ioparse(i, o):
     if isinstance(i, int):
-        i = os.fdopen(i)
+        i = os.fdopen(i, 'rb')
     # rely on duck typing for recognizing
     # streams as that works uniformly
     # in py2 and py3
@@ -57,8 +54,15 @@ def send(out, *args):
 
 
 def recv(inf):
-    """load an object from input stream"""
-    return pickle.load(inf)
+    """load an object from input stream
+    python2 and python3 compatibility, inf is sys.stdin
+    and is opened as text stream by default. Hence using the
+    buffer attribute in python3
+    """
+    if hasattr(inf, "buffer"):
+        return pickle.load(inf.buffer)
+    else:
+        return pickle.load(inf)
 
 
 class RepceServer(object):
@@ -196,15 +200,17 @@ class RepceClient(object):
         """RePCe client is callabe, calling it implements a synchronous
         remote call.
 
-        We do a .push with a cbk which does a wakeup upon receiving anwser,
+        We do a .push with a cbk which does a wakeup upon receiving answer,
         then wait on the RepceJob.
         """
         rjob = self.push(
             meth, *args, **{'cbk': lambda rj, res: rj.wakeup(res)})
         exc, res = rjob.wait()
         if exc:
-            logging.error('call %s (%s) failed on peer with %s' %
-                          (repr(rjob), meth, str(type(res).__name__)))
+            logging.error(lf('call failed',
+                             call=repr(rjob),
+                             method=meth,
+                             error=str(type(res).__name__)))
             raise res
         logging.debug("call %s %s -> %s" % (repr(rjob), meth, repr(res)))
         return res
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index 9dd8988dc6d..f12c7ceaa36 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -13,272 +13,43 @@ import os
 import sys
 import stat
 import time
-import signal
 import fcntl
-import errno
 import types
 import struct
-import socket
 import logging
 import tempfile
-import threading
 import subprocess
-from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP
-from errno import EISDIR, ENOTEMPTY, ESTALE, EINVAL
-from select import error as SelectError
-import shutil
+from errno import (EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EACCES,
+                   EISDIR, ENOTEMPTY, ESTALE, EINVAL, EBUSY, EPERM)
+import errno
+
+from rconf import rconf
+import gsyncdconfig as gconf
+import libgfchangelog
 
-from gconf import gconf
 import repce
 from repce import RepceServer, RepceClient
 from master import gmaster_builder
 import syncdutils
-from syncdutils import GsyncdError, select, privileged, boolify, funcode
-from syncdutils import umask, entry2pb, gauxpfx, errno_wrap, lstat
-from syncdutils import NoStimeAvailable, PartialHistoryAvailable
-from syncdutils import ChangelogException, ChangelogHistoryNotAvailable
-from syncdutils import get_changelog_log_level
-from syncdutils import CHANGELOG_AGENT_CLIENT_VERSION
+from syncdutils import (GsyncdError, select, privileged, funcode,
+                        entry2pb, gauxpfx, errno_wrap, lstat,
+                        NoStimeAvailable, PartialHistoryAvailable,
+                        ChangelogException, ChangelogHistoryNotAvailable,
+                        get_changelog_log_level, get_rsync_version,
+                        GX_GFID_CANONICAL_LEN,
+                        gf_mount_ready, lf, Popen, sup,
+                        Xattr, matching_disk_gfid, get_gfid_from_mnt,
+                        unshare_propagation_supported, get_slv_dir_path)
 from gsyncdstatus import GeorepStatus
+from py2py3 import (pipe, str_to_bytearray, entry_pack_reg,
+                    entry_pack_reg_stat, entry_pack_mkdir,
+                    entry_pack_symlink)
 
 
-UrlRX = re.compile('\A(\w+)://([^ *?[]*)\Z')
-HostRX = re.compile('[a-zA-Z\d](?:[a-zA-Z\d.-]*[a-zA-Z\d])?', re.I)
-UserRX = re.compile("[\w!\#$%&'*+-\/=?^_`{|}~]+")
-
-
-def sup(x, *a, **kw):
-    """a rubyesque "super" for python ;)
-
-    invoke caller method in parent class with given args.
-    """
-    return getattr(super(type(x), x),
-                   sys._getframe(1).f_code.co_name)(*a, **kw)
-
-
-def desugar(ustr):
-    """transform sugared url strings to standard <scheme>://<urlbody> form
-
-    parsing logic enforces the constraint that sugared forms should contatin
-    a ':' or a '/', which ensures that sugared urls do not conflict with
-    gluster volume names.
-    """
-    m = re.match('([^:]*):(.*)', ustr)
-    if m:
-        if not m.groups()[0]:
-            return "gluster://localhost" + ustr
-        elif '@' in m.groups()[0] or re.search('[:/]', m.groups()[1]):
-            return "ssh://" + ustr
-        else:
-            return "gluster://" + ustr
-    else:
-        if ustr[0] != '/':
-            raise GsyncdError("cannot resolve sugared url '%s'" % ustr)
-        ap = os.path.normpath(ustr)
-        if ap.startswith('//'):
-            ap = ap[1:]
-        return "file://" + ap
-
-
-def gethostbyname(hnam):
-    """gethostbyname wrapper"""
-    try:
-        return socket.gethostbyname(hnam)
-    except socket.gaierror:
-        ex = sys.exc_info()[1]
-        raise GsyncdError("failed to resolve %s: %s" %
-                          (hnam, ex.strerror))
-
-
-def parse_url(ustr):
-    """instantiate an url object by scheme-to-class dispatch
-
-    The url classes taken into consideration are the ones in
-    this module whose names are full-caps.
-    """
-    m = UrlRX.match(ustr)
-    if not m:
-        ustr = desugar(ustr)
-    m = UrlRX.match(ustr)
-    if not m:
-        raise GsyncdError("malformed url")
-    sch, path = m.groups()
-    this = sys.modules[__name__]
-    if not hasattr(this, sch.upper()):
-        raise GsyncdError("unknown url scheme " + sch)
-    return getattr(this, sch.upper())(path)
-
-
-class _MetaXattr(object):
-
-    """singleton class, a lazy wrapper around the
-    libcxattr module
-
-    libcxattr (a heavy import due to ctypes) is
-    loaded only when when the single
-    instance is tried to be used.
-
-    This reduces runtime for those invocations
-    which do not need filesystem manipulation
-    (eg. for config, url parsing)
-    """
-
-    def __getattr__(self, meth):
-        from libcxattr import Xattr as LXattr
-        xmeth = [m for m in dir(LXattr) if m[0] != '_']
-        if not meth in xmeth:
-            return
-        for m in xmeth:
-            setattr(self, m, getattr(LXattr, m))
-        return getattr(self, meth)
-
-
-Xattr = _MetaXattr()
-
-
-class Popen(subprocess.Popen):
-
-    """customized subclass of subprocess.Popen with a ring
-    buffer for children error output"""
-
-    @classmethod
-    def init_errhandler(cls):
-        """start the thread which handles children's error output"""
-        cls.errstore = {}
-
-        def tailer():
-            while True:
-                errstore = cls.errstore.copy()
-                try:
-                    poe, _, _ = select(
-                        [po.stderr for po in errstore], [], [], 1)
-                except (ValueError, SelectError):
-                    # stderr is already closed wait for some time before
-                    # checking next error
-                    time.sleep(0.5)
-                    continue
-                for po in errstore:
-                    if po.stderr not in poe:
-                        continue
-                    po.lock.acquire()
-                    try:
-                        if po.on_death_row:
-                            continue
-                        la = errstore[po]
-                        try:
-                            fd = po.stderr.fileno()
-                        except ValueError:  # file is already closed
-                            time.sleep(0.5)
-                            continue
-
-                        try:
-                            l = os.read(fd, 1024)
-                        except OSError:
-                            time.sleep(0.5)
-                            continue
-
-                        if not l:
-                            continue
-                        tots = len(l)
-                        for lx in la:
-                            tots += len(lx)
-                        while tots > 1 << 20 and la:
-                            tots -= len(la.pop(0))
-                        la.append(l)
-                    finally:
-                        po.lock.release()
-        t = syncdutils.Thread(target=tailer)
-        t.start()
-        cls.errhandler = t
+ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
 
-    @classmethod
-    def fork(cls):
-        """fork wrapper that restarts errhandler thread in child"""
-        pid = os.fork()
-        if not pid:
-            cls.init_errhandler()
-        return pid
-
-    def __init__(self, args, *a, **kw):
-        """customizations for subprocess.Popen instantiation
-
-        - 'close_fds' is taken to be the default
-        - if child's stderr is chosen to be managed,
-          register it with the error handler thread
-        """
-        self.args = args
-        if 'close_fds' not in kw:
-            kw['close_fds'] = True
-        self.lock = threading.Lock()
-        self.on_death_row = False
-        self.elines = []
-        try:
-            sup(self, args, *a, **kw)
-        except:
-            ex = sys.exc_info()[1]
-            if not isinstance(ex, OSError):
-                raise
-            raise GsyncdError("""execution of "%s" failed with %s (%s)""" %
-                              (args[0], errno.errorcode[ex.errno],
-                               os.strerror(ex.errno)))
-        if kw.get('stderr') == subprocess.PIPE:
-            assert(getattr(self, 'errhandler', None))
-            self.errstore[self] = []
-
-    def errlog(self):
-        """make a log about child's failure event"""
-        filling = ""
-        if self.elines:
-            filling = ", saying:"
-        logging.error("""command "%s" returned with %s%s""" %
-                      (" ".join(self.args), repr(self.returncode), filling))
-        lp = ''
-
-        def logerr(l):
-            logging.error(self.args[0] + "> " + l)
-        for l in self.elines:
-            ls = l.split('\n')
-            ls[0] = lp + ls[0]
-            lp = ls.pop()
-            for ll in ls:
-                logerr(ll)
-        if lp:
-            logerr(lp)
-
-    def errfail(self):
-        """fail nicely if child did not terminate with success"""
-        self.errlog()
-        syncdutils.finalize(exval=1)
-
-    def terminate_geterr(self, fail_on_err=True):
-        """kill child, finalize stderr harvesting (unregister
-        from errhandler, set up .elines), fail on error if
-        asked for
-        """
-        self.lock.acquire()
-        try:
-            self.on_death_row = True
-        finally:
-            self.lock.release()
-        elines = self.errstore.pop(self)
-        if self.poll() is None:
-            self.terminate()
-            if self.poll() is None:
-                time.sleep(0.1)
-                self.kill()
-                self.wait()
-        while True:
-            if not select([self.stderr], [], [], 0.1)[0]:
-                break
-            b = os.read(self.stderr.fileno(), 1024)
-            if b:
-                elines.append(b)
-            else:
-                break
-        self.stderr.close()
-        self.elines = elines
-        if fail_on_err and self.returncode != 0:
-            self.errfail()
+slv_volume = None
+slv_host = None
 
 
 class Server(object):
@@ -295,7 +66,6 @@ class Server(object):
     NTV_FMTSTR = "!" + "B" * 19 + "II"
     FRGN_XTRA_FMT = "I"
     FRGN_FMTSTR = NTV_FMTSTR + FRGN_XTRA_FMT
-    GX_GFID_CANONICAL_LEN = 37  # canonical gfid len + '\0'
 
     # for backend gfid fetch, do not use GX_NSPACE_PFX
     GFID_XATTR = 'trusted.gfid'
@@ -305,15 +75,15 @@ class Server(object):
 
     @classmethod
     def _fmt_mknod(cls, l):
-        return "!II%dsI%dsIII" % (cls.GX_GFID_CANONICAL_LEN, l + 1)
+        return "!II%dsI%dsIII" % (GX_GFID_CANONICAL_LEN, l + 1)
 
     @classmethod
     def _fmt_mkdir(cls, l):
-        return "!II%dsI%dsII" % (cls.GX_GFID_CANONICAL_LEN, l + 1)
+        return "!II%dsI%dsII" % (GX_GFID_CANONICAL_LEN, l + 1)
 
     @classmethod
     def _fmt_symlink(cls, l1, l2):
-        return "!II%dsI%ds%ds" % (cls.GX_GFID_CANONICAL_LEN, l1 + 1, l2 + 1)
+        return "!II%dsI%ds%ds" % (GX_GFID_CANONICAL_LEN, l1 + 1, l2 + 1)
 
     def _pathguard(f):
         """decorator method that checks
@@ -325,14 +95,14 @@ class Server(object):
         fc = funcode(f)
         pi = list(fc.co_varnames).index('path')
 
-        def ff(*a):
-            path = a[pi]
+        def ff(*args):
+            path = args[pi]
             ps = path.split('/')
             if path[0] == '/' or '..' in ps:
                 raise ValueError('unsafe path')
-            a = list(a)
-            a[pi] = os.path.join(a[0].local_path, path)
-            return f(*a)
+            args = list(args)
+            args[pi] = os.path.join(args[0].local_path, path)
+            return f(*args)
         return ff
 
     @classmethod
@@ -373,23 +143,15 @@ class Server(object):
     @classmethod
     @_pathguard
     def gfid(cls, path):
-        try:
-            buf = Xattr.lgetxattr(path, cls.GFID_XATTR, 16)
+        buf = errno_wrap(Xattr.lgetxattr, [path, cls.GFID_XATTR, 16],
+                         [ENOENT], [ESTALE, ENODATA])
+        if buf == ENOENT:
+            return buf
+        else:
+            buf = str_to_bytearray(buf)
             m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(
                 ['%02x' % x for x in struct.unpack(cls.GFID_FMTSTR, buf)]))
             return '-'.join(m.groups())
-        except (IOError, OSError):
-            ex = sys.exc_info()[1]
-            if ex.errno == ENOENT:
-                return ex.errno
-            else:
-                raise
-
-    @classmethod
-    def gfid_mnt(cls, gfidpath):
-        return errno_wrap(Xattr.lgetxattr,
-                          [gfidpath, 'glusterfs.gfid.string',
-                           cls.GX_GFID_CANONICAL_LEN], [ENOENT], [ESTALE])
 
     @classmethod
     @_pathguard
@@ -477,6 +239,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'xtime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -499,6 +262,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -521,6 +285,7 @@ class Server(object):
             val = Xattr.lgetxattr(path,
                                   '.'.join([cls.GX_NSPACE, uuid, 'stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -544,6 +309,7 @@ class Server(object):
                                   '.'.join([cls.GX_NSPACE, uuid,
                                             'entry_stime']),
                                   8)
+            val = str_to_bytearray(val)
             return struct.unpack('!II', val)
         except OSError:
             ex = sys.exc_info()[1]
@@ -612,80 +378,85 @@ class Server(object):
     def entry_ops(cls, entries):
         pfx = gauxpfx()
         logging.debug('entries: %s' % repr(entries))
-        # regular file
-
-        def entry_pack_reg(gf, bn, mo, uid, gid):
-            blen = len(bn)
-            return struct.pack(cls._fmt_mknod(blen),
-                               uid, gid, gf, mo, bn,
-                               stat.S_IMODE(mo), 0, umask())
-
-        def entry_pack_reg_stat(gf, bn, st):
-            blen = len(bn)
-            mo = st['mode']
-            return struct.pack(cls._fmt_mknod(blen),
-                               st['uid'], st['gid'],
-                               gf, mo, bn,
-                               stat.S_IMODE(mo), 0, umask())
-        # mkdir
-
-        def entry_pack_mkdir(gf, bn, mo, uid, gid):
-            blen = len(bn)
-            return struct.pack(cls._fmt_mkdir(blen),
-                               uid, gid, gf, mo, bn,
-                               stat.S_IMODE(mo), umask())
-        # symlink
-
-        def entry_pack_symlink(gf, bn, lnk, st):
-            blen = len(bn)
-            llen = len(lnk)
-            return struct.pack(cls._fmt_symlink(blen, llen),
-                               st['uid'], st['gid'],
-                               gf, st['mode'], bn, lnk)
-
-        def entry_purge(entry, gfid):
+        dist_count = rconf.args.master_dist_count
+
+        def entry_purge(op, entry, gfid, e, uid, gid):
             # This is an extremely racy code and needs to be fixed ASAP.
             # The GFID check here is to be sure that the pargfid/bname
             # to be purged is the GFID gotten from the changelog.
             # (a stat(changelog_gfid) would also be valid here)
             # The race here is between the GFID check and the purge.
-            if not matching_disk_gfid(gfid, entry):
+
+            # If the entry or the gfid of the file to be deleted is not present
+            # on slave, we can ignore the unlink/rmdir
+            if isinstance(lstat(entry), int) or \
+               isinstance(lstat(os.path.join(pfx, gfid)), int):
                 return
 
-            er = errno_wrap(os.unlink, [entry], [ENOENT, ESTALE, EISDIR])
-            if isinstance(er, int):
-                if er == EISDIR:
-                    er = errno_wrap(os.rmdir, [entry], [ENOENT, ESTALE,
-                                                        ENOTEMPTY])
-                    if er == ENOTEMPTY:
-                        return er
+            if not matching_disk_gfid(gfid, entry):
+                collect_failure(e, EEXIST, uid, gid)
+                return
 
-        def collect_failure(e, cmd_ret):
+            if op == 'UNLINK':
+                er = errno_wrap(os.unlink, [entry], [ENOENT, ESTALE], [EBUSY])
+                # EISDIR is safe error, ignore. This can only happen when
+                # unlink is sent from master while fixing gfid conflicts.
+                if er != EISDIR:
+                    return er
+
+            elif op == 'RMDIR':
+                er = errno_wrap(os.rmdir, [entry], [ENOENT, ESTALE,
+                                                    ENOTEMPTY], [EBUSY])
+                if er == ENOTEMPTY:
+                    return er
+
+        def collect_failure(e, cmd_ret, uid, gid, dst=False):
+            slv_entry_info = {}
+            slv_entry_info['gfid_mismatch'] = False
+            slv_entry_info['name_mismatch'] = False
+            slv_entry_info['dst'] = dst
+            slv_entry_info['slave_isdir'] = False
+            slv_entry_info['slave_name'] = None
+            slv_entry_info['slave_gfid'] = None
             # We do this for failing fops on Slave
             # Master should be logging this
             if cmd_ret is None:
-                return
-
-            if cmd_ret == EEXIST:
-                disk_gfid = cls.gfid_mnt(e['entry'])
-                if isinstance(disk_gfid, basestring):
-                    if e['gfid'] != disk_gfid:
-                        failures.append((e, cmd_ret, disk_gfid))
-            else:
-                failures.append((e, cmd_ret))
-
-        failures = []
-
-        def matching_disk_gfid(gfid, entry):
-            disk_gfid = cls.gfid_mnt(entry)
-            if isinstance(disk_gfid, int):
                 return False
 
-            if not gfid == disk_gfid:
-                return False
+            if e.get("stat", {}):
+                # Copy actual UID/GID value back to entry stat
+                e['stat']['uid'] = uid
+                e['stat']['gid'] = gid
+
+            if cmd_ret in [EEXIST, ESTALE]:
+                if dst:
+                    en = e['entry1']
+                else:
+                    en = e['entry']
+                disk_gfid = get_gfid_from_mnt(en)
+                if isinstance(disk_gfid, str) and \
+                   e['gfid'] != disk_gfid:
+                    slv_entry_info['gfid_mismatch'] = True
+                    st = lstat(en)
+                    if not isinstance(st, int):
+                        if st and stat.S_ISDIR(st.st_mode):
+                            slv_entry_info['slave_isdir'] = True
+                            dir_name = get_slv_dir_path(slv_host, slv_volume,
+                                                        disk_gfid)
+                            slv_entry_info['slave_name'] = dir_name
+                        else:
+                            slv_entry_info['slave_isdir'] = False
+                    slv_entry_info['slave_gfid'] = disk_gfid
+                    failures.append((e, cmd_ret, slv_entry_info))
+                else:
+                    return False
+            else:
+                failures.append((e, cmd_ret, slv_entry_info))
 
             return True
 
+        failures = []
+
         def recursive_rmdir(gfid, entry, path):
             """disk_gfid check added for original path for which
             recursive_delete is called. This disk gfid check executed
@@ -699,7 +470,7 @@ class Server(object):
                 return
 
             names = []
-            names = errno_wrap(os.listdir, [path], [ENOENT], [ESTALE])
+            names = errno_wrap(os.listdir, [path], [ENOENT], [ESTALE, ENOTSUP])
             if isinstance(names, int):
                 return
 
@@ -708,7 +479,7 @@ class Server(object):
                 if not matching_disk_gfid(gfid, entry):
                     return
                 er = errno_wrap(os.remove, [fullname], [ENOENT, ESTALE,
-                                                        EISDIR])
+                                                        EISDIR], [EBUSY])
 
                 if er == EISDIR:
                     recursive_rmdir(gfid, entry, fullname)
@@ -716,22 +487,23 @@ class Server(object):
             if not matching_disk_gfid(gfid, entry):
                 return
 
-            errno_wrap(os.rmdir, [path], [ENOENT, ESTALE])
+            errno_wrap(os.rmdir, [path], [ENOENT, ESTALE], [EBUSY])
 
-        def rename_with_disk_gfid_confirmation(gfid, entry, en):
+        def rename_with_disk_gfid_confirmation(gfid, entry, en, uid, gid):
             if not matching_disk_gfid(gfid, entry):
-                logging.error("RENAME ignored: "
-                              "source entry:%s(gfid:%s) does not match with "
-                              "on-disk gfid(%s), when attempting to rename "
-                              "to %s" %
-                              (entry, gfid, cls.gfid_mnt(entry), en))
+                logging.error(lf("RENAME ignored: source entry does not match "
+                                 "with on-disk gfid",
+                                 source=entry,
+                                 gfid=gfid,
+                                 disk_gfid=get_gfid_from_mnt(entry),
+                                 target=en))
+                collect_failure(e, EEXIST, uid, gid)
                 return
 
             cmd_ret = errno_wrap(os.rename,
                                  [entry, en],
-                                 [ENOENT, EEXIST], [ESTALE])
-            collect_failure(e, cmd_ret)
-
+                                 [ENOENT, EEXIST], [ESTALE, EBUSY])
+            collect_failure(e, cmd_ret, uid, gid)
 
         for e in entries:
             blob = None
@@ -740,6 +512,12 @@ class Server(object):
             entry = e['entry']
             uid = 0
             gid = 0
+
+            # Skip entry processing if it's marked true during gfid
+            # conflict resolution
+            if e['skip_entry']:
+                continue
+
             if e.get("stat", {}):
                 # Copy UID/GID value and then reset to zero. Copied UID/GID
                 # will be used to run chown once entry is created.
@@ -752,7 +530,7 @@ class Server(object):
             if op in ['RMDIR', 'UNLINK']:
                 # Try once, if rmdir failed with ENOTEMPTY
                 # then delete recursively.
-                er = entry_purge(entry, gfid)
+                er = entry_purge(op, entry, gfid, e, uid, gid)
                 if isinstance(er, int):
                     if er == ENOTEMPTY and op == 'RMDIR':
                         # Retry if ENOTEMPTY, ESTALE
@@ -764,93 +542,178 @@ class Server(object):
                             logging.debug("Removed %s => %s/%s recursively" %
                                           (gfid, pg, bname))
                         else:
-                            logging.warn("Recursive remove %s => %s/%s"
-                                         "failed: %s" % (gfid, pg, bname,
-                                                         os.strerror(er1)))
+                            logging.warn(lf("Recursive remove failed",
+                                            gfid=gfid,
+                                            pgfid=pg,
+                                            bname=bname,
+                                            error=os.strerror(er1)))
                     else:
-                        logging.warn("Failed to remove %s => %s/%s. %s" %
-                                     (gfid, pg, bname, os.strerror(er)))
+                        logging.warn(lf("Failed to remove",
+                                        gfid=gfid,
+                                        pgfid=pg,
+                                        bname=bname,
+                                        error=os.strerror(er)))
             elif op in ['CREATE', 'MKNOD']:
                 slink = os.path.join(pfx, gfid)
                 st = lstat(slink)
                 # don't create multiple entries with same gfid
                 if isinstance(st, int):
-                    blob = entry_pack_reg(
-                        gfid, bname, e['mode'], e['uid'], e['gid'])
+                    blob = entry_pack_reg(cls, gfid, bname,
+                                          e['mode'], e['uid'], e['gid'])
+                # Self healed hardlinks are recorded as MKNOD.
+                # So if the gfid already exists, it should be
+                # processed as hard link not mknod.
+                elif op in ['MKNOD']:
+                    cmd_ret = errno_wrap(os.link,
+                                         [slink, entry],
+                                         [ENOENT, EEXIST], [ESTALE])
+                    collect_failure(e, cmd_ret, uid, gid)
             elif op == 'MKDIR':
+                en = e['entry']
                 slink = os.path.join(pfx, gfid)
                 st = lstat(slink)
                 # don't create multiple entries with same gfid
                 if isinstance(st, int):
-                    blob = entry_pack_mkdir(
-                        gfid, bname, e['mode'], e['uid'], e['gid'])
+                    blob = entry_pack_mkdir(cls, gfid, bname,
+                                            e['mode'], e['uid'], e['gid'])
+                elif (isinstance(lstat(en), int) or
+                      not matching_disk_gfid(gfid, en)):
+                    # If gfid of a directory exists on slave but path based
+                    # create is getting EEXIST. This means the directory is
+                    # renamed in master but recorded as MKDIR during hybrid
+                    # crawl. Get the directory path by reading the backend
+                    # symlink and trying to rename to new name as said by
+                    # master.
+                    logging.info(lf("Special case: rename on mkdir",
+                                    gfid=gfid, entry=repr(entry)))
+                    src_entry = get_slv_dir_path(slv_host, slv_volume, gfid)
+                    if src_entry is None:
+                        collect_failure(e, ENOENT, uid, gid)
+                    if src_entry is not None and src_entry != entry:
+                        slv_entry_info = {}
+                        slv_entry_info['gfid_mismatch'] = False
+                        slv_entry_info['name_mismatch'] = True
+                        slv_entry_info['dst'] = False
+                        slv_entry_info['slave_isdir'] = True
+                        slv_entry_info['slave_gfid'] = gfid
+                        slv_entry_info['slave_entry'] = src_entry
+
+                        failures.append((e, EEXIST, slv_entry_info))
             elif op == 'LINK':
                 slink = os.path.join(pfx, gfid)
                 st = lstat(slink)
                 if isinstance(st, int):
                     (pg, bname) = entry2pb(entry)
-                    blob = entry_pack_reg_stat(gfid, bname, e['stat'])
+                    if stat.S_ISREG(e['stat']['mode']):
+                        blob = entry_pack_reg_stat(cls, gfid, bname, e['stat'])
+                    elif stat.S_ISLNK(e['stat']['mode']):
+                        blob = entry_pack_symlink(cls, gfid, bname, e['link'],
+                                                  e['stat'])
                 else:
                     cmd_ret = errno_wrap(os.link,
                                          [slink, entry],
                                          [ENOENT, EEXIST], [ESTALE])
-                    collect_failure(e, cmd_ret)
+                    collect_failure(e, cmd_ret, uid, gid)
             elif op == 'SYMLINK':
-                blob = entry_pack_symlink(gfid, bname, e['link'], e['stat'])
-            elif op == 'RENAME':
-                en = e['entry1']
+                en = e['entry']
                 st = lstat(entry)
                 if isinstance(st, int):
+                    blob = entry_pack_symlink(cls, gfid, bname, e['link'],
+                                              e['stat'])
+                elif not matching_disk_gfid(gfid, en):
+                    collect_failure(e, EEXIST, uid, gid)
+            elif op == 'RENAME':
+                en = e['entry1']
+                # The matching disk gfid check validates two things
+                #  1. Validates name is present, return false otherwise
+                #  2. Validates gfid is same, returns false otherwise
+                # So both validations are necessary to decide src doesn't
+                # exist. We can't rely on only gfid stat as hardlink could
+                # be present and we can't rely only on name as name could
+                # exist with different gfid.
+                if not matching_disk_gfid(gfid, entry):
                     if e['stat'] and not stat.S_ISDIR(e['stat']['mode']):
-                        if stat.S_ISLNK(e['stat']['mode']) and \
-                           e['link'] is not None:
-                            (pg, bname) = entry2pb(en)
-                            blob = entry_pack_symlink(gfid, bname,
-                                                      e['link'], e['stat'])
+                        if stat.S_ISLNK(e['stat']['mode']):
+                            # src is not present, so don't sync symlink as
+                            # we don't know target. It's ok to ignore. If
+                            # it's unliked, it's fine. If it's renamed to
+                            # something else, it will be synced then.
+                            if e['link'] is not None:
+                                st1 = lstat(en)
+                                if isinstance(st1, int):
+                                    (pg, bname) = entry2pb(en)
+                                    blob = entry_pack_symlink(cls, gfid, bname,
+                                                              e['link'],
+                                                              e['stat'])
+                                elif not matching_disk_gfid(gfid, en):
+                                    collect_failure(e, EEXIST, uid, gid, True)
                         else:
-                            (pg, bname) = entry2pb(en)
-                            blob = entry_pack_reg_stat(gfid, bname, e['stat'])
+                            slink = os.path.join(pfx, gfid)
+                            st = lstat(slink)
+                            # don't create multiple entries with same gfid
+                            if isinstance(st, int):
+                                (pg, bname) = entry2pb(en)
+                                blob = entry_pack_reg_stat(cls, gfid, bname,
+                                                           e['stat'])
+                            else:
+                                cmd_ret = errno_wrap(os.link, [slink, en],
+                                                    [ENOENT, EEXIST], [ESTALE])
+                                collect_failure(e, cmd_ret, uid, gid)
                 else:
+                    st = lstat(entry)
                     st1 = lstat(en)
                     if isinstance(st1, int):
-                        rename_with_disk_gfid_confirmation(gfid, entry, en)
+                        rename_with_disk_gfid_confirmation(gfid, entry, en,
+                                                           uid, gid)
                     else:
                         if st.st_ino == st1.st_ino:
                             # we have a hard link, we can now unlink source
                             try:
-                                os.unlink(entry)
+                                errno_wrap(os.unlink, [entry],
+                                           [ENOENT, ESTALE], [EBUSY])
                             except OSError as e:
                                 if e.errno == EISDIR:
                                     try:
-                                        os.rmdir(entry)
+                                        errno_wrap(os.rmdir, [entry],
+                                                   [ENOENT, ESTALE], [EBUSY])
                                     except OSError as e:
                                         if e.errno == ENOTEMPTY:
                                             logging.error(
-                                                "Unable to delete directory "
-                                                "{0}, Both Old({1}) and New{2}"
-                                                " directories exists".format(
-                                                    entry, entry, en))
+                                                lf("Directory Rename failed. "
+                                                   "Both Old and New"
+                                                   " directories exists",
+                                                   old=entry,
+                                                   new=en))
                                         else:
                                             raise
                                 else:
                                     raise
+                        elif not matching_disk_gfid(gfid, en) and dist_count > 1:
+                            collect_failure(e, EEXIST, uid, gid, True)
                         else:
-                            rename_with_disk_gfid_confirmation(gfid, entry, en)
+                            # We are here which means matching_disk_gfid for
+                            # both source and destination has returned false
+                            # and distribution count for master vol is greater
+                            # then one. Which basically says both the source and
+                            # destination exist and not hardlinks.
+                            # So we are safe to go ahead with rename here.
+                            rename_with_disk_gfid_confirmation(gfid, entry, en,
+                                                               uid, gid)
             if blob:
                 cmd_ret = errno_wrap(Xattr.lsetxattr,
                                      [pg, 'glusterfs.gfid.newfile', blob],
-                                     [EEXIST, ENOENT],
-                                     [ESTALE, EINVAL])
-                collect_failure(e, cmd_ret)
+                                     [EEXIST, ENOENT, ESTALE],
+                                     [ESTALE, EINVAL, EBUSY])
+                collect_failure(e, cmd_ret, uid, gid)
 
                 # If UID/GID is different than zero that means we are trying
                 # create Entry with different UID/GID. Create Entry with
                 # UID:0 and GID:0, and then call chown to set UID/GID
                 if uid != 0 or gid != 0:
                     path = os.path.join(pfx, gfid)
-                    cmd_ret = errno_wrap(os.chown, [path, uid, gid], [ENOENT],
+                    cmd_ret = errno_wrap(os.lchown, [path, uid, gid], [ENOENT],
                                          [ESTALE, EINVAL])
-                collect_failure(e, cmd_ret)
+                    collect_failure(e, cmd_ret, uid, gid)
 
         return failures
 
@@ -865,17 +728,39 @@ class Server(object):
             atime = e['stat']['atime']
             mtime = e['stat']['mtime']
             go = e['go']
-            cmd_ret = errno_wrap(os.chmod, [go, mode],
-                                 [ENOENT], [ESTALE, EINVAL])
-            # This is a fail fast mechanism
-            # We do this for failing fops on Slave
-            # Master should be logging this
+            # Linux doesn't support chmod on symlink itself.
+            # It is always applied to the target file. So
+            # changelog would record target file's gfid
+            # and we are good. But 'chown' is supported on
+            # symlink file. So changelog would record symlink
+            # gfid in such cases. Since we do 'chown' 'chmod'
+            # 'utime' for each gfid recorded for metadata, and
+            # we know from changelog the metadata is on symlink's
+            # gfid or target file's gfid, we should be doing
+            # 'lchown' 'lchmod' 'utime with no-deference' blindly.
+            # But since 'lchmod' and 'utime with no de-reference' is
+            # not supported in python3, we have to rely on 'chmod'
+            # and 'utime with de-reference'. Hence avoiding 'chmod'
+            # and 'utime' if it's symlink file.
+
+            is_symlink = False
+            cmd_ret = errno_wrap(os.lchown, [go, uid, gid], [ENOENT],
+                                 [ESTALE, EINVAL])
             if isinstance(cmd_ret, int):
-                failures.append((e, cmd_ret))
                 continue
-            errno_wrap(os.chown, [go, uid, gid], [ENOENT], [ESTALE, EINVAL])
-            errno_wrap(os.utime, [go, (atime, mtime)],
-                       [ENOENT], [ESTALE, EINVAL])
+
+            is_symlink = os.path.islink(go)
+
+            if not is_symlink:
+                cmd_ret = errno_wrap(os.chmod, [go, mode],
+                                     [ENOENT, EACCES, EPERM], [ESTALE, EINVAL])
+                if isinstance(cmd_ret, int):
+                    failures.append((e, cmd_ret, "chmod"))
+
+                cmd_ret = errno_wrap(os.utime, [go, (atime, mtime)],
+                                     [ENOENT, EACCES, EPERM], [ESTALE, EINVAL])
+                if isinstance(cmd_ret, int):
+                    failures.append((e, cmd_ret, "utime"))
         return failures
 
     @classmethod
@@ -932,248 +817,273 @@ class Server(object):
         return 1.0
 
 
-class SlaveLocal(object):
+class Mounter(object):
 
-    """mix-in class to implement some factes of a slave server
+    """Abstract base class for mounter backends"""
 
-    ("mix-in" is sort of like "abstract class", ie. it's not
-    instantiated just included in the ancesty DAG. I use "mix-in"
-    to indicate that it's not used as an abstract base class,
-    rather just taken in to implement additional functionality
-    on the basis of the assumed availability of certain interfaces.)
-    """
+    def __init__(self, params):
+        self.params = params
+        self.mntpt = None
+        self.umount_cmd = []
 
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return not remote
-
-    def service_loop(self):
-        """start a RePCe server serving self's server
-
-        stop servicing if a timeout is configured and got no
-        keep-alime in that inteval
-        """
-
-        if boolify(gconf.use_rsync_xattrs) and not privileged():
-            raise GsyncdError(
-                "using rsync for extended attributes is not supported")
+    @classmethod
+    def get_glusterprog(cls):
+        gluster_cmd_dir = gconf.get("gluster-command-dir")
+        if rconf.args.subcmd == "slave":
+            gluster_cmd_dir = gconf.get("slave-gluster-command-dir")
+        return os.path.join(gluster_cmd_dir, cls.glusterprog)
+
+    def umount_l(self, d):
+        """perform lazy umount"""
+        po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        po.wait()
+        return po
 
-        repce = RepceServer(
-            self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
-        t = syncdutils.Thread(target=lambda: (repce.service_loop(),
-                                              syncdutils.finalize()))
-        t.start()
-        logging.info("slave listening")
-        if gconf.timeout and int(gconf.timeout) > 0:
-            while True:
-                lp = self.server.last_keep_alive
-                time.sleep(int(gconf.timeout))
-                if lp == self.server.last_keep_alive:
-                    logging.info(
-                        "connection inactive for %d seconds, stopping" %
-                        int(gconf.timeout))
-                    break
-        else:
-            select((), (), ())
+    @classmethod
+    def make_umount_argv(cls, d):
+        raise NotImplementedError
 
+    def make_mount_argv(self, label=None):
+        raise NotImplementedError
 
-class SlaveRemote(object):
+    def cleanup_mntpt(self, *a):
+        pass
 
-    """mix-in class to implement an interface to a remote slave"""
+    def handle_mounter(self, po):
+        po.wait()
 
-    def connect_remote(self, rargs=[], **opts):
-        """connects to a remote slave
+    def inhibit(self, label):
+        """inhibit a gluster filesystem
 
-        Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
-        which sets up the connection and set up a RePCe client to
-        communicate throuh its stdio.
+        Mount glusterfs over a temporary mountpoint,
+        change into the mount, and lazy unmount the
+        filesystem.
         """
-        slave = opts.get('slave', self.url)
-        extra_opts = []
-        so = getattr(gconf, 'session_owner', None)
-        if so:
-            extra_opts += ['--session-owner', so]
-        if boolify(gconf.use_rsync_xattrs):
-            extra_opts.append('--use-rsync-xattrs')
-        po = Popen(rargs + gconf.remote_gsyncd.split() + extra_opts +
-                   ['-N', '--listen', '--timeout', str(gconf.timeout),
-                    slave],
-                   stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                   stderr=subprocess.PIPE)
-        gconf.transport = po
-        return self.start_fd_client(po.stdout, po.stdin, **opts)
-
-    def start_fd_client(self, i, o, **opts):
-        """set up RePCe client, handshake with server
-
-        It's cut out as a separate method to let
-        subclasses hook into client startup
-        """
-        self.server = RepceClient(i, o)
-        rv = self.server.__version__()
-        exrv = {'proto': repce.repce_version, 'object': Server.version()}
-        da0 = (rv, exrv)
-        da1 = ({}, {})
-        for i in range(2):
-            for k, v in da0[i].iteritems():
-                da1[i][k] = int(v)
-        if da1[0] != da1[1]:
-            raise GsyncdError(
-                "RePCe major version mismatch: local %s, remote %s" %
-                (exrv, rv))
+        mpi, mpo = pipe()
+        mh = Popen.fork()
+        if mh:
+            # Parent
+            os.close(mpi)
+            fcntl.fcntl(mpo, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+            d = None
+            margv = self.make_mount_argv(label)
+            if self.mntpt:
+                # mntpt is determined pre-mount
+                d = self.mntpt
+                mnt_msg = d + '\0'
+                encoded_msg = mnt_msg.encode()
+                os.write(mpo, encoded_msg)
+            po = Popen(margv, **self.mountkw)
+            self.handle_mounter(po)
+            po.terminate_geterr()
+            logging.debug('auxiliary glusterfs mount in place')
+            if not d:
+                # mntpt is determined during mount
+                d = self.mntpt
+                mnt_msg = d + '\0'
+                encoded_msg = mnt_msg.encode()
+                os.write(mpo, encoded_msg)
+            encoded_msg = 'M'.encode()
+            os.write(mpo, encoded_msg)
+            t = syncdutils.Thread(target=lambda: os.chdir(d))
+            t.start()
+            tlim = rconf.starttime + gconf.get("connection-timeout")
+            while True:
+                if not t.isAlive():
+                    break
 
-    def rsync(self, files, *args, **kw):
-        """invoke rsync"""
-        if not files:
-            raise GsyncdError("no files to sync")
-        logging.debug("files: " + ", ".join(files))
-        argv = gconf.rsync_command.split() + \
-            ['-aR0', '--inplace', '--files-from=-', '--super',
-             '--stats', '--numeric-ids', '--no-implied-dirs'] + \
-            gconf.rsync_options.split() + \
-            (boolify(gconf.sync_xattrs) and ['--xattrs'] or []) + \
-            (boolify(gconf.sync_acls) and ['--acls'] or []) + \
-            ['.'] + list(args)
-
-        if gconf.log_rsync_performance:
-            # use stdout=PIPE only when log_rsync_performance enabled
-            # Else rsync will write to stdout and nobody is their
-            # to consume. If PIPE is full rsync hangs.
-            po = Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                       stderr=subprocess.PIPE)
+                if time.time() >= tlim:
+                    syncdutils.finalize(exval=1)
+                time.sleep(1)
+            os.close(mpo)
+            _, rv = syncdutils.waitpid(mh, 0)
+            if rv:
+                rv = (os.WIFEXITED(rv) and os.WEXITSTATUS(rv) or 0) - \
+                     (os.WIFSIGNALED(rv) and os.WTERMSIG(rv) or 0)
+                logging.warn(lf('stale mount possibly left behind',
+                                path=d))
+                raise GsyncdError("cleaning up temp mountpoint %s "
+                                  "failed with status %d" %
+                                  (d, rv))
         else:
-            po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        for f in files:
-            po.stdin.write(f)
-            po.stdin.write('\0')
-
-        stdout, stderr = po.communicate()
-
-        if kw.get("log_err", False):
-            for errline in stderr.strip().split("\n")[:-1]:
-                logging.error("SYNC Error(Rsync): %s" % errline)
+            rv = 0
+            try:
+                os.setsid()
+                os.close(mpo)
+                mntdata = ''
+                while True:
+                    c = os.read(mpi, 1)
+                    c = c.decode()
+                    if not c:
+                        break
+                    mntdata += c
+                if mntdata:
+                    mounted = False
+                    if mntdata[-1] == 'M':
+                        mntdata = mntdata[:-1]
+                        assert(mntdata)
+                        mounted = True
+                    assert(mntdata[-1] == '\0')
+                    mntpt = mntdata[:-1]
+                    assert(mntpt)
+
+                    umount_master = False
+                    umount_slave = False
+                    if rconf.args.subcmd == "worker" \
+                       and not unshare_propagation_supported() \
+                       and not gconf.get("access-mount"):
+                        umount_master = True
+                    if rconf.args.subcmd == "slave" \
+                       and not gconf.get("slave-access-mount"):
+                        umount_slave = True
+
+                    if mounted and (umount_master or umount_slave):
+                        po = self.umount_l(mntpt)
+                        po.terminate_geterr(fail_on_err=False)
+                        if po.returncode != 0:
+                            po.errlog()
+                            rv = po.returncode
+                        logging.debug("Lazy umount done: %s" % mntpt)
+                    if umount_master or umount_slave:
+                        self.cleanup_mntpt(mntpt)
+            except:
+                logging.exception('mount cleanup failure:')
+                rv = 200
+            os._exit(rv)
+
+        #Polling the dht.subvol.status value.
+        RETRIES = 10
+        while not gf_mount_ready():
+            if RETRIES < 0:
+                logging.error('Subvols are not up')
+                break
+            RETRIES -= 1
+            time.sleep(0.2)
 
-        if gconf.log_rsync_performance:
-            rsync_msg = []
-            for line in stdout.split("\n"):
-                if line.startswith("Number of files:") or \
-                   line.startswith("Number of regular files transferred:") or \
-                   line.startswith("Total file size:") or \
-                   line.startswith("Total transferred file size:") or \
-                   line.startswith("Literal data:") or \
-                   line.startswith("Matched data:") or \
-                   line.startswith("Total bytes sent:") or \
-                   line.startswith("Total bytes received:") or \
-                   line.startswith("sent "):
-                    rsync_msg.append(line)
-            logging.info("rsync performance: %s" % ", ".join(rsync_msg))
+        logging.debug('auxiliary glusterfs mount prepared')
 
-        return po
 
-    def tarssh(self, files, slaveurl, log_err=False):
-        """invoke tar+ssh
-        -z (compress) can be use if needed, but omitting it now
-        as it results in weird error (tar+ssh errors out (errcode: 2)
-        """
-        if not files:
-            raise GsyncdError("no files to sync")
-        logging.debug("files: " + ", ".join(files))
-        (host, rdir) = slaveurl.split(':')
-        tar_cmd = ["tar"] + \
-            ["--sparse", "-cf", "-", "--files-from", "-"]
-        ssh_cmd = gconf.ssh_command_tar.split() + \
-            ["-p", str(gconf.ssh_port)] + \
-            [host, "tar"] + \
-            ["--overwrite", "-xf", "-", "-C", rdir]
-        p0 = Popen(tar_cmd, stdout=subprocess.PIPE,
-                   stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-        p1 = Popen(ssh_cmd, stdin=p0.stdout, stderr=subprocess.PIPE)
-        for f in files:
-            p0.stdin.write(f)
-            p0.stdin.write('\n')
+class DirectMounter(Mounter):
 
-        p0.stdin.close()
-        p0.stdout.close()  # Allow p0 to receive a SIGPIPE if p1 exits.
-        # wait for tar to terminate, collecting any errors, further
-        # waiting for transfer to complete
-        _, stderr1 = p1.communicate()
+    """mounter backend which calls mount(8), umount(8) directly"""
 
-        # stdin and stdout of p0 is already closed, Reset to None and
-        # wait for child process to complete
-        p0.stdin = None
-        p0.stdout = None
-        p0.communicate()
+    mountkw = {'stderr': subprocess.PIPE, 'universal_newlines': True}
+    glusterprog = 'glusterfs'
 
-        if log_err:
-            for errline in stderr1.strip().split("\n")[:-1]:
-                logging.error("SYNC Error(Untar): %s" % errline)
-
-        return p1
-
-
-class AbstractUrl(object):
+    @staticmethod
+    def make_umount_argv(d):
+        return ['umount', '-l', d]
 
-    """abstract base class for url scheme classes"""
+    def make_mount_argv(self, label=None):
+        self.mntpt = tempfile.mkdtemp(prefix='gsyncd-aux-mount-')
+        rconf.mount_point = self.mntpt
+        return [self.get_glusterprog()] + \
+            ['--' + p for p in self.params] + [self.mntpt]
 
-    def __init__(self, path, pattern):
-        m = re.search(pattern, path)
-        if not m:
-            raise GsyncdError("malformed path")
-        self.path = path
-        return m.groups()
+    def cleanup_mntpt(self, mntpt=None):
+        if not mntpt:
+            mntpt = self.mntpt
+        errno_wrap(os.rmdir, [mntpt], [ENOENT, EBUSY])
 
-    @property
-    def scheme(self):
-        return type(self).__name__.lower()
 
-    def canonical_path(self):
-        return self.path
+class MountbrokerMounter(Mounter):
 
-    def get_url(self, canonical=False, escaped=False):
-        """format self's url in various styles"""
-        if canonical:
-            pa = self.canonical_path()
-        else:
-            pa = self.path
-        u = "://".join((self.scheme, pa))
-        if escaped:
-            u = syncdutils.escape(u)
-        return u
+    """mounter backend using the mountbroker gluster service"""
 
-    @property
-    def url(self):
-        return self.get_url()
+    mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE,
+               'universal_newlines': True}
+    glusterprog = 'gluster'
 
+    @classmethod
+    def make_cli_argv(cls):
+        return [cls.get_glusterprog()] + ['--remote-host=localhost'] + \
+            gconf.get("gluster-cli-options").split() + ['system::']
 
-class FILE(AbstractUrl, SlaveLocal, SlaveRemote):
+    @classmethod
+    def make_umount_argv(cls, d):
+        return cls.make_cli_argv() + ['umount', d, 'lazy']
 
-    """scheme class for file:// urls
+    def make_mount_argv(self, label):
+        return self.make_cli_argv() + \
+            ['mount', label, 'user-map-root=' +
+                syncdutils.getusername()] + self.params
 
-    can be used to represent a file slave server
-    on slave side, or interface to a remote file
-    file server on master side
-    """
+    def handle_mounter(self, po):
+        self.mntpt = po.stdout.readline()[:-1]
+        rconf.mount_point = self.mntpt
+        rconf.mountbroker = True
+        self.umount_cmd = self.make_cli_argv() + ['umount']
+        rconf.mbr_umount_cmd = self.umount_cmd
+        po.stdout.close()
+        sup(self, po)
+        if po.returncode != 0:
+            # if cli terminated with error due to being
+            # refused by glusterd, what it put
+            # out on stdout is a diagnostic message
+            logging.error(lf('glusterd answered', mnt=self.mntpt))
 
-    class FILEServer(Server):
 
-        """included server flavor"""
-        pass
+class GLUSTERServer(Server):
 
-    server = FILEServer
+    "server enhancements for a glusterfs backend"""
 
-    def __init__(self, path):
-        sup(self, path, '^/')
+    @classmethod
+    def _attr_unpack_dict(cls, xattr, extra_fields=''):
+        """generic volume mark fetching/parsing backed"""
+        fmt_string = cls.NTV_FMTSTR + extra_fields
+        buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
+        buf = str_to_bytearray(buf)
+        vm = struct.unpack(fmt_string, buf)
+        m = re.match(
+            '(.{8})(.{4})(.{4})(.{4})(.{12})',
+            "".join(['%02x' % x for x in vm[2:18]]))
+        uuid = '-'.join(m.groups())
+        volinfo = {'version': vm[0:2],
+                   'uuid': uuid,
+                   'retval': vm[18],
+                   'volume_mark': vm[19:21],
+                   }
+        if extra_fields:
+            return volinfo, vm[-len(extra_fields):]
+        else:
+            return volinfo
 
-    def connect(self):
-        """inhibit the resource beyond"""
-        os.chdir(self.path)
+    @classmethod
+    def foreign_volume_infos(cls):
+        """return list of valid (not expired) foreign volume marks"""
+        dict_list = []
+        xattr_list = Xattr.llistxattr_buf('.')
+        for ele in xattr_list:
+            if ele.find('.'.join([cls.GX_NSPACE, 'volume-mark', ''])) == 0:
+                d, x = cls._attr_unpack_dict(ele, cls.FRGN_XTRA_FMT)
+                now = int(time.time())
+                if x[0] > now:
+                    logging.debug("volinfo[%s] expires: %d "
+                                  "(%d sec later)" %
+                                  (d['uuid'], x[0], x[0] - now))
+                    d['timeout'] = x[0]
+                    dict_list.append(d)
+                else:
+                    try:
+                        Xattr.lremovexattr('.', ele)
+                    except OSError:
+                        pass
+        return dict_list
 
-    def rsync(self, files, log_err=False):
-        return sup(self, files, self.path, log_err=log_err)
+    @classmethod
+    def native_volume_info(cls):
+        """get the native volume mark of the underlying gluster volume"""
+        try:
+            return cls._attr_unpack_dict('.'.join([cls.GX_NSPACE,
+                                                   'volume-mark']))
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno != ENODATA:
+                raise
 
 
-class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
+class GLUSTER(object):
 
     """scheme class for gluster:// urls
 
@@ -1183,238 +1093,17 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
     (slave-ish features come from the mixins, master
     functionality is outsourced to GMaster from master)
     """
-
-    class GLUSTERServer(Server):
-
-        "server enhancements for a glusterfs backend"""
-
-        @classmethod
-        def _attr_unpack_dict(cls, xattr, extra_fields=''):
-            """generic volume mark fetching/parsing backed"""
-            fmt_string = cls.NTV_FMTSTR + extra_fields
-            buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
-            vm = struct.unpack(fmt_string, buf)
-            m = re.match(
-                '(.{8})(.{4})(.{4})(.{4})(.{12})',
-                "".join(['%02x' % x for x in vm[2:18]]))
-            uuid = '-'.join(m.groups())
-            volinfo = {'version': vm[0:2],
-                       'uuid': uuid,
-                       'retval': vm[18],
-                       'volume_mark': vm[19:21],
-                       }
-            if extra_fields:
-                return volinfo, vm[-len(extra_fields):]
-            else:
-                return volinfo
-
-        @classmethod
-        def foreign_volume_infos(cls):
-            """return list of valid (not expired) foreign volume marks"""
-            dict_list = []
-            xattr_list = Xattr.llistxattr_buf('.')
-            for ele in xattr_list:
-                if ele.find('.'.join([cls.GX_NSPACE, 'volume-mark', ''])) == 0:
-                    d, x = cls._attr_unpack_dict(ele, cls.FRGN_XTRA_FMT)
-                    now = int(time.time())
-                    if x[0] > now:
-                        logging.debug("volinfo[%s] expires: %d "
-                                      "(%d sec later)" %
-                                      (d['uuid'], x[0], x[0] - now))
-                        d['timeout'] = x[0]
-                        dict_list.append(d)
-                    else:
-                        try:
-                            Xattr.lremovexattr('.', ele)
-                        except OSError:
-                            pass
-            return dict_list
-
-        @classmethod
-        def native_volume_info(cls):
-            """get the native volume mark of the underlying gluster volume"""
-            try:
-                return cls._attr_unpack_dict('.'.join([cls.GX_NSPACE,
-                                                       'volume-mark']))
-            except OSError:
-                ex = sys.exc_info()[1]
-                if ex.errno != ENODATA:
-                    raise
-
     server = GLUSTERServer
 
-    def __init__(self, path):
-        self.host, self.volume = sup(self, path, '^(%s):(.+)' % HostRX.pattern)
-
-    def canonical_path(self):
-        return ':'.join([gethostbyname(self.host), self.volume])
-
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return not remote or \
-            (isinstance(remote, SSH) and isinstance(remote.inner_rsc, GLUSTER))
-
-    class Mounter(object):
+    def __init__(self, host, volume):
+        self.path = "%s:%s" % (host, volume)
+        self.host = host
+        self.volume = volume
 
-        """Abstract base class for mounter backends"""
-
-        def __init__(self, params):
-            self.params = params
-            self.mntpt = None
-
-        @classmethod
-        def get_glusterprog(cls):
-            return os.path.join(gconf.gluster_command_dir, cls.glusterprog)
-
-        def umount_l(self, d):
-            """perform lazy umount"""
-            po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE)
-            po.wait()
-            return po
-
-        @classmethod
-        def make_umount_argv(cls, d):
-            raise NotImplementedError
-
-        def make_mount_argv(self, *a):
-            raise NotImplementedError
-
-        def cleanup_mntpt(self, *a):
-            pass
-
-        def handle_mounter(self, po):
-            po.wait()
-
-        def inhibit(self, *a):
-            """inhibit a gluster filesystem
-
-            Mount glusterfs over a temporary mountpoint,
-            change into the mount, and lazy unmount the
-            filesystem.
-            """
-
-            mpi, mpo = os.pipe()
-            mh = Popen.fork()
-            if mh:
-                os.close(mpi)
-                fcntl.fcntl(mpo, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
-                d = None
-                margv = self.make_mount_argv(*a)
-                if self.mntpt:
-                    # mntpt is determined pre-mount
-                    d = self.mntpt
-                    os.write(mpo, d + '\0')
-                po = Popen(margv, **self.mountkw)
-                self.handle_mounter(po)
-                po.terminate_geterr()
-                logging.debug('auxiliary glusterfs mount in place')
-                if not d:
-                    # mntpt is determined during mount
-                    d = self.mntpt
-                    os.write(mpo, d + '\0')
-                os.write(mpo, 'M')
-                t = syncdutils.Thread(target=lambda: os.chdir(d))
-                t.start()
-                tlim = gconf.starttime + int(gconf.connection_timeout)
-                while True:
-                    if not t.isAlive():
-                        break
-                    if time.time() >= tlim:
-                        syncdutils.finalize(exval=1)
-                    time.sleep(1)
-                os.close(mpo)
-                _, rv = syncdutils.waitpid(mh, 0)
-                if rv:
-                    rv = (os.WIFEXITED(rv) and os.WEXITSTATUS(rv) or 0) - \
-                         (os.WIFSIGNALED(rv) and os.WTERMSIG(rv) or 0)
-                    logging.warn('stale mount possibly left behind on ' + d)
-                    raise GsyncdError("cleaning up temp mountpoint %s "
-                                      "failed with status %d" %
-                                      (d, rv))
-            else:
-                rv = 0
-                try:
-                    os.setsid()
-                    os.close(mpo)
-                    mntdata = ''
-                    while True:
-                        c = os.read(mpi, 1)
-                        if not c:
-                            break
-                        mntdata += c
-                    if mntdata:
-                        mounted = False
-                        if mntdata[-1] == 'M':
-                            mntdata = mntdata[:-1]
-                            assert(mntdata)
-                            mounted = True
-                        assert(mntdata[-1] == '\0')
-                        mntpt = mntdata[:-1]
-                        assert(mntpt)
-                        if mounted:
-                            po = self.umount_l(mntpt)
-                            po.terminate_geterr(fail_on_err=False)
-                            if po.returncode != 0:
-                                po.errlog()
-                                rv = po.returncode
-                        self.cleanup_mntpt(mntpt)
-                except:
-                    logging.exception('mount cleanup failure:')
-                    rv = 200
-                os._exit(rv)
-            logging.debug('auxiliary glusterfs mount prepared')
-
-    class DirectMounter(Mounter):
-
-        """mounter backend which calls mount(8), umount(8) directly"""
-
-        mountkw = {'stderr': subprocess.PIPE}
-        glusterprog = 'glusterfs'
-
-        @staticmethod
-        def make_umount_argv(d):
-            return ['umount', '-l', d]
-
-        def make_mount_argv(self):
-            self.mntpt = tempfile.mkdtemp(prefix='gsyncd-aux-mount-')
-            return [self.get_glusterprog()] + \
-                ['--' + p for p in self.params] + [self.mntpt]
-
-        def cleanup_mntpt(self, mntpt=None):
-            if not mntpt:
-                mntpt = self.mntpt
-            os.rmdir(mntpt)
-
-    class MountbrokerMounter(Mounter):
-
-        """mounter backend using the mountbroker gluster service"""
-
-        mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
-        glusterprog = 'gluster'
-
-        @classmethod
-        def make_cli_argv(cls):
-            return [cls.get_glusterprog()] + ['--remote-host=localhost'] + \
-                gconf.gluster_cli_options.split() + ['system::']
-
-        @classmethod
-        def make_umount_argv(cls, d):
-            return cls.make_cli_argv() + ['umount', d, 'lazy']
-
-        def make_mount_argv(self, label):
-            return self.make_cli_argv() + \
-                ['mount', label, 'user-map-root=' +
-                    syncdutils.getusername()] + self.params
-
-        def handle_mounter(self, po):
-            self.mntpt = po.stdout.readline()[:-1]
-            po.stdout.close()
-            sup(self, po)
-            if po.returncode != 0:
-                # if cli terminated with error due to being
-                # refused by glusterd, what it put
-                # out on stdout is a diagnostic message
-                logging.error('glusterd answered: %s' % self.mntpt)
+        global slv_volume
+        global slv_host
+        slv_volume = self.volume
+        slv_host = self.host
 
     def connect(self):
         """inhibit the resource beyond
@@ -1424,20 +1113,30 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         with given backend
         """
 
-        label = getattr(gconf, 'mountbroker', None)
+        logging.info("Mounting gluster volume locally...")
+        t0 = time.time()
+        label = gconf.get('mountbroker', None)
         if not label and not privileged():
             label = syncdutils.getusername()
-        mounter = label and self.MountbrokerMounter or self.DirectMounter
-        params = gconf.gluster_params.split() + \
-            (gconf.gluster_log_level and ['log-level=' +
-                                          gconf.gluster_log_level] or []) + \
-            ['log-file=' + gconf.gluster_log_file, 'volfile-server=' +
-             self.host, 'volfile-id=' + self.volume, 'client-pid=-1']
-        mounter(params).inhibit(*[l for l in [label] if l])
-
-    def connect_remote(self, *a, **kw):
-        sup(self, *a, **kw)
-        self.slavedir = "/proc/%d/cwd" % self.server.pid()
+        mounter = label and MountbrokerMounter or DirectMounter
+
+        log_file = gconf.get("gluster-log-file")
+        if rconf.args.subcmd == "slave":
+            log_file = gconf.get("slave-gluster-log-file")
+
+        log_level = gconf.get("gluster-log-level")
+        if rconf.args.subcmd == "slave":
+            log_level = gconf.get("slave-gluster-log-level")
+
+        params = gconf.get("gluster-params").split() + \
+            ['log-level=' + log_level] + \
+            ['log-file=' + log_file, 'volfile-server=' + self.host] + \
+            ['volfile-id=' + self.volume, 'client-pid=-1']
+
+        self.mounter = mounter(params)
+        self.mounter.inhibit(label)
+        logging.info(lf("Mounted gluster volume",
+                        duration="%.4f" % (time.time() - t0)))
 
     def gmaster_instantiate_tuple(self, slave):
         """return a tuple of the 'one shot' and the 'main crawl'
@@ -1446,7 +1145,7 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
                 gmaster_builder()(self, slave),
                 gmaster_builder('changeloghistory')(self, slave))
 
-    def service_loop(self, *args):
+    def service_loop(self, slave=None):
         """enter service loop
 
         - if slave given, instantiate GMaster and
@@ -1454,163 +1153,173 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
           master behavior
         - else do that's what's inherited
         """
-        if args:
-            slave = args[0]
-            if gconf.local_path:
-                class brickserver(FILE.FILEServer):
-                    local_path = gconf.local_path
-                    aggregated = self.server
-
-                    @classmethod
-                    def entries(cls, path):
-                        e = super(brickserver, cls).entries(path)
-                        # on the brick don't mess with /.glusterfs
-                        if path == '.':
-                            try:
-                                e.remove('.glusterfs')
-                                e.remove('.trashcan')
-                            except ValueError:
-                                pass
-                        return e
-
-                    @classmethod
-                    def lstat(cls, e):
-                        """ path based backend stat """
-                        return super(brickserver, cls).lstat(e)
-
-                    @classmethod
-                    def gfid(cls, e):
-                        """ path based backend gfid fetch """
-                        return super(brickserver, cls).gfid(e)
-
-                    @classmethod
-                    def linkto_check(cls, e):
-                        return super(brickserver, cls).linkto_check(e)
-                if gconf.slave_id:
-                    # define {,set_}xtime in slave, thus preempting
-                    # the call to remote, so that it takes data from
-                    # the local brick
-                    slave.server.xtime = types.MethodType(
-                        lambda _self, path, uuid: (
-                            brickserver.xtime(path,
-                                              uuid + '.' + gconf.slave_id)
-                        ),
-                        slave.server)
-                    slave.server.stime = types.MethodType(
-                        lambda _self, path, uuid: (
-                            brickserver.stime(path,
-                                              uuid + '.' + gconf.slave_id)
-                        ),
-                        slave.server)
-                    slave.server.entry_stime = types.MethodType(
-                        lambda _self, path, uuid: (
-                            brickserver.entry_stime(
-                                path,
-                                uuid + '.' + gconf.slave_id)
-                        ),
-                        slave.server)
-                    slave.server.set_stime = types.MethodType(
-                        lambda _self, path, uuid, mark: (
-                            brickserver.set_stime(path,
-                                                  uuid + '.' + gconf.slave_id,
-                                                  mark)
-                        ),
-                        slave.server)
-                    slave.server.set_entry_stime = types.MethodType(
-                        lambda _self, path, uuid, mark: (
-                            brickserver.set_entry_stime(
-                                path,
-                                uuid + '.' + gconf.slave_id,
-                                mark)
-                        ),
-                        slave.server)
-                (g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
-                g1.master.server = brickserver
-                g2.master.server = brickserver
-                g3.master.server = brickserver
-            else:
-                (g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
-                g1.master.server.aggregated = gmaster.master.server
-                g2.master.server.aggregated = gmaster.master.server
-                g3.master.server.aggregated = gmaster.master.server
-            # bad bad bad: bad way to do things like this
-            # need to make this elegant
-            # register the crawlers and start crawling
-            # g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
-            # g3 ==> changelog History
-            changelog_register_failed = False
-            (inf, ouf, ra, wa) = gconf.rpc_fd.split(',')
-            changelog_agent = RepceClient(int(inf), int(ouf))
-            status = GeorepStatus(gconf.state_file, gconf.local_path)
-            status.reset_on_worker_start()
-            rv = changelog_agent.version()
-            if int(rv) != CHANGELOG_AGENT_CLIENT_VERSION:
+        if rconf.args.subcmd == "slave":
+            if gconf.get("use-rsync-xattrs") and not privileged():
                 raise GsyncdError(
-                    "RePCe major version mismatch(changelog agent): "
-                    "local %s, remote %s" %
-                    (CHANGELOG_AGENT_CLIENT_VERSION, rv))
+                    "using rsync for extended attributes is not supported")
+
+            repce = RepceServer(
+                self.server, sys.stdin, sys.stdout, gconf.get("sync-jobs"))
+            t = syncdutils.Thread(target=lambda: (repce.service_loop(),
+                                                  syncdutils.finalize()))
+            t.start()
+            logging.info("slave listening")
+            if gconf.get("slave-timeout") and gconf.get("slave-timeout") > 0:
+                while True:
+                    lp = self.server.last_keep_alive
+                    time.sleep(gconf.get("slave-timeout"))
+                    if lp == self.server.last_keep_alive:
+                        logging.info(
+                            lf("connection inactive, stopping",
+                               timeout=gconf.get("slave-timeout")))
+                        break
+            else:
+                select((), (), ())
 
-            try:
-                workdir = g2.setup_working_dir()
-                # Register only when change_detector is not set to
-                # xsync, else agent will generate changelog files
-                # in .processing directory of working dir
-                if gconf.change_detector != 'xsync':
-                    # register with the changelog library
-                    # 9 == log level (DEBUG)
-                    # 5 == connection retries
-                    changelog_agent.init()
-                    changelog_agent.register(gconf.local_path,
-                                             workdir, gconf.changelog_log_file,
-                                             get_changelog_log_level(
-                                                 gconf.changelog_log_level),
-                                             g2.CHANGELOG_CONN_RETRIES)
-
-                register_time = int(time.time())
-                g2.register(register_time, changelog_agent, status)
-                g3.register(register_time, changelog_agent, status)
-            except ChangelogException as e:
-                logging.error("Changelog register failed, %s" % e)
-                sys.exit(1)
-
-            g1.register(status=status)
-            logging.info("Register time: %s" % register_time)
-            # oneshot: Try to use changelog history api, if not
-            # available switch to FS crawl
-            # Note: if config.change_detector is xsync then
-            # it will not use changelog history api
-            try:
-                g3.crawlwrap(oneshot=True)
-            except PartialHistoryAvailable as e:
-                logging.info('Partial history available, using xsync crawl'
-                             ' after consuming history till %s' % str(e))
-                g1.crawlwrap(oneshot=True, register_time=register_time)
-            except ChangelogHistoryNotAvailable:
-                logging.info('Changelog history not available, using xsync')
-                g1.crawlwrap(oneshot=True, register_time=register_time)
-            except NoStimeAvailable:
-                logging.info('No stime available, using xsync crawl')
-                g1.crawlwrap(oneshot=True, register_time=register_time)
-            except ChangelogException as e:
-                logging.error("Changelog History Crawl failed, %s" % e)
-                sys.exit(1)
+            return
 
-            try:
-                g2.crawlwrap()
-            except ChangelogException as e:
-                logging.error("Changelog crawl failed, %s" % e)
-                sys.exit(1)
-        else:
-            sup(self, *args)
+        class brickserver(Server):
+            local_path = rconf.args.local_path
+            aggregated = self.server
+
+            @classmethod
+            def entries(cls, path):
+                e = super(brickserver, cls).entries(path)
+                # on the brick don't mess with /.glusterfs
+                if path == '.':
+                    try:
+                        e.remove('.glusterfs')
+                        e.remove('.trashcan')
+                    except ValueError:
+                        pass
+                return e
+
+            @classmethod
+            def lstat(cls, e):
+                """ path based backend stat """
+                return super(brickserver, cls).lstat(e)
+
+            @classmethod
+            def gfid(cls, e):
+                """ path based backend gfid fetch """
+                return super(brickserver, cls).gfid(e)
+
+            @classmethod
+            def linkto_check(cls, e):
+                return super(brickserver, cls).linkto_check(e)
+
+        # define {,set_}xtime in slave, thus preempting
+        # the call to remote, so that it takes data from
+        # the local brick
+        slave.server.xtime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.xtime(path,
+                                  uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.stime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.stime(path,
+                                  uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.entry_stime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.entry_stime(
+                    path,
+                    uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.set_stime = types.MethodType(
+            lambda _self, path, uuid, mark: (
+                brickserver.set_stime(path,
+                                      uuid + '.' + rconf.args.slave_id,
+                                      mark)
+            ),
+            slave.server)
+        slave.server.set_entry_stime = types.MethodType(
+            lambda _self, path, uuid, mark: (
+                brickserver.set_entry_stime(
+                    path,
+                    uuid + '.' + rconf.args.slave_id,
+                    mark)
+            ),
+            slave.server)
+
+        (g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
+        g1.master.server = brickserver
+        g2.master.server = brickserver
+        g3.master.server = brickserver
+
+        # bad bad bad: bad way to do things like this
+        # need to make this elegant
+        # register the crawlers and start crawling
+        # g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
+        # g3 ==> changelog History
+        status = GeorepStatus(gconf.get("state-file"),
+                              rconf.args.local_node,
+                              rconf.args.local_path,
+                              rconf.args.local_node_id,
+                              rconf.args.master,
+                              rconf.args.slave)
+        status.reset_on_worker_start()
 
-    def rsync(self, files, log_err=False):
-        return sup(self, files, self.slavedir, log_err=log_err)
+        try:
+            workdir = g2.setup_working_dir()
+            # Register only when change_detector is not set to
+            # xsync, else agent will generate changelog files
+            # in .processing directory of working dir
+            if gconf.get("change-detector") != 'xsync':
+                # register with the changelog library
+                # 9 == log level (DEBUG)
+                # 5 == connection retries
+                libgfchangelog.register(rconf.args.local_path,
+                                        workdir,
+                                        gconf.get("changelog-log-file"),
+                                        get_changelog_log_level(
+                                            gconf.get("changelog-log-level")),
+                                        g2.CHANGELOG_CONN_RETRIES)
+
+            register_time = int(time.time())
+            g2.register(register_time, status)
+            g3.register(register_time, status)
+        except ChangelogException as e:
+            logging.error(lf("Changelog register failed", error=e))
+            sys.exit(1)
+
+        g1.register(status=status)
+        logging.info(lf("Register time",
+                        time=register_time))
+        # oneshot: Try to use changelog history api, if not
+        # available switch to FS crawl
+        # Note: if config.change_detector is xsync then
+        # it will not use changelog history api
+        try:
+            g3.crawlwrap(oneshot=True)
+        except PartialHistoryAvailable as e:
+            logging.info(lf('Partial history available, using xsync crawl'
+                            ' after consuming history',
+                            till=e))
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except ChangelogHistoryNotAvailable:
+            logging.info('Changelog history not available, using xsync')
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except NoStimeAvailable:
+            logging.info('No stime available, using xsync crawl')
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except ChangelogException as e:
+            logging.error(lf("Changelog History Crawl failed",
+                             error=e))
+            sys.exit(1)
 
-    def tarssh(self, files, log_err=False):
-        return sup(self, files, self.slavedir, log_err=log_err)
+        try:
+            g2.crawlwrap()
+        except ChangelogException as e:
+            logging.error(lf("Changelog crawl failed", error=e))
+            sys.exit(1)
 
 
-class SSH(AbstractUrl, SlaveRemote):
+class SSH(object):
 
     """scheme class for ssh:// urls
 
@@ -1618,13 +1327,9 @@ class SSH(AbstractUrl, SlaveRemote):
     implementing an ssh based proxy
     """
 
-    def __init__(self, path):
-        self.remote_addr, inner_url = sup(self, path,
-                                          '^((?:%s@)?%s):(.+)' %
-                                          tuple([r.pattern
-                                                 for r in (UserRX, HostRX)]))
-        self.inner_rsc = parse_url(inner_url)
-        self.volume = inner_url[1:]
+    def __init__(self, host, volume):
+        self.remote_addr = host
+        self.volume = volume
 
     @staticmethod
     def parse_ssh_address(self):
@@ -1636,35 +1341,28 @@ class SSH(AbstractUrl, SlaveRemote):
         self.remotehost = h
         return {'user': u, 'host': h}
 
-    def canonical_path(self):
-        rap = self.parse_ssh_address(self)
-        remote_addr = '@'.join([rap['user'], gethostbyname(rap['host'])])
-        return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])
-
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return False
-
-    def start_fd_client(self, *a, **opts):
-        """customizations for client startup
+    def start_fd_client(self, i, o):
+        """set up RePCe client, handshake with server
 
-        - be a no-op if we are to daemonize (client startup is deferred
-          to post-daemon stage)
-        - determine target url for rsync after consulting server
+        It's cut out as a separate method to let
+        subclasses hook into client startup
         """
-        if opts.get('deferred'):
-            return a
-        sup(self, *a)
-        ityp = type(self.inner_rsc)
-        if ityp == FILE:
-            slavepath = self.inner_rsc.path
-        elif ityp == GLUSTER:
-            slavepath = "/proc/%d/cwd" % self.server.pid()
-        else:
-            raise NotImplementedError
+        self.server = RepceClient(i, o)
+        rv = self.server.__version__()
+        exrv = {'proto': repce.repce_version, 'object': Server.version()}
+        da0 = (rv, exrv)
+        da1 = ({}, {})
+        for i in range(2):
+            for k, v in da0[i].items():
+                da1[i][k] = int(v)
+        if da1[0] != da1[1]:
+            raise GsyncdError(
+                "RePCe major version mismatch: local %s, remote %s" %
+                (exrv, rv))
+        slavepath = "/proc/%d/cwd" % self.server.pid()
         self.slaveurl = ':'.join([self.remote_addr, slavepath])
 
-    def connect_remote(self, go_daemon=None):
+    def connect_remote(self):
         """connect to inner slave url through outer ssh url
 
         Wrap the connecting utility in ssh.
@@ -1682,45 +1380,204 @@ class SSH(AbstractUrl, SlaveRemote):
         [NB. ATM gluster product does not makes use of interactive
         authentication.]
         """
-        if go_daemon == 'done':
-            return self.start_fd_client(*self.fd_pair)
-
         syncdutils.setup_ssh_ctl(tempfile.mkdtemp(prefix='gsyncd-aux-ssh-'),
                                  self.remote_addr,
-                                 self.inner_rsc.url)
-
-        deferred = go_daemon == 'postconn'
-        ret = sup(self, gconf.ssh_command.split() +
-                  ["-p", str(gconf.ssh_port)] +
-                  gconf.ssh_ctl_args + [self.remote_addr],
-                  slave=self.inner_rsc.url, deferred=deferred)
-
-        if deferred:
-            # send a message to peer so that we can wait for
-            # the answer from which we know connection is
-            # established and we can proceed with daemonization
-            # (doing that too early robs the ssh passwd prompt...)
-            # However, we'd better not start the RepceClient
-            # before daemonization (that's not preserved properly
-            # in daemon), we just do a an ad-hoc linear put/get.
-            i, o = ret
-            inf = os.fdopen(i)
-            repce.send(o, None, '__repce_version__')
-            select((inf,), (), ())
-            repce.recv(inf)
-            # hack hack hack: store a global reference to the file
-            # to save it from getting GC'd which implies closing it
-            gconf.permanent_handles.append(inf)
-            self.fd_pair = (i, o)
-            return 'should'
-
-    def rsync(self, files, log_err=False):
-        return sup(self, files, '-e',
-                   " ".join(gconf.ssh_command.split() +
-                            ["-p", str(gconf.ssh_port)] +
-                            gconf.ssh_ctl_args),
-                   *(gconf.rsync_ssh_options.split() + [self.slaveurl]),
-                   log_err=log_err)
+                                 self.volume)
+
+        logging.info("Initializing SSH connection between master and slave...")
+        t0 = time.time()
+
+        extra_opts = []
+        remote_gsyncd = gconf.get("remote-gsyncd")
+        if remote_gsyncd == "":
+            remote_gsyncd = "/nonexistent/gsyncd"
+
+        if gconf.get("use-rsync-xattrs"):
+            extra_opts.append('--use-rsync-xattrs')
+
+        args_to_slave = [gconf.get("ssh-command")] + \
+            gconf.get("ssh-options").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            rconf.ssh_ctl_args + [self.remote_addr] + \
+            [remote_gsyncd, "slave"] + \
+            extra_opts + \
+            [rconf.args.master, rconf.args.slave] + \
+            [
+                '--master-node', rconf.args.local_node,
+                '--master-node-id', rconf.args.local_node_id,
+                '--master-brick', rconf.args.local_path,
+                '--local-node', rconf.args.resource_remote,
+                '--local-node-id', rconf.args.resource_remote_id] + \
+            [
+                # Add all config arguments here, slave gsyncd will not use
+                # config file in slave side, so all overriding options should
+                # be sent as arguments
+                '--slave-timeout', str(gconf.get("slave-timeout")),
+                '--slave-log-level', gconf.get("slave-log-level"),
+                '--slave-gluster-log-level',
+                gconf.get("slave-gluster-log-level"),
+                '--slave-gluster-command-dir',
+                gconf.get("slave-gluster-command-dir"),
+                '--master-dist-count',
+                str(gconf.get("master-distribution-count"))]
+
+        if gconf.get("slave-access-mount"):
+            args_to_slave.append('--slave-access-mount')
+
+        if rconf.args.debug:
+            args_to_slave.append('--debug')
+
+        po = Popen(args_to_slave,
+                   stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE)
+        rconf.transport = po
+        self.start_fd_client(po.stdout, po.stdin)
+        logging.info(lf("SSH connection between master and slave established.",
+                        duration="%.4f" % (time.time() - t0)))
+
+    def rsync(self, files, *args, **kw):
+        """invoke rsync"""
+        if not files:
+            raise GsyncdError("no files to sync")
+        logging.debug("files: " + ", ".join(files))
+
+        extra_rsync_flags = []
+        # Performance flag, --ignore-missing-args, if rsync version is
+        # greater than 3.1.0 then include this flag.
+        if gconf.get("rsync-opt-ignore-missing-args") and \
+           get_rsync_version(gconf.get("rsync-command")) >= "3.1.0":
+            extra_rsync_flags = ["--ignore-missing-args"]
+
+        rsync_ssh_opts = [gconf.get("ssh-command")] + \
+            gconf.get("ssh-options").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            rconf.ssh_ctl_args + \
+            gconf.get("rsync-ssh-options").split()
+
+        argv = [
+            gconf.get("rsync-command"),
+            '-aR0',
+            '--inplace',
+            '--files-from=-',
+            '--super',
+            '--stats',
+            '--numeric-ids',
+            '--no-implied-dirs'
+        ]
+
+        if gconf.get("rsync-opt-existing"):
+            argv += ["--existing"]
+
+        if gconf.get("sync-xattrs"):
+            argv += ['--xattrs']
+
+        if gconf.get("sync-acls"):
+            argv += ['--acls']
+
+        argv = argv + \
+            gconf.get("rsync-options").split() + \
+            extra_rsync_flags + ['.'] + \
+            ["-e", " ".join(rsync_ssh_opts)] + \
+            [self.slaveurl]
+
+        log_rsync_performance = gconf.getr("log-rsync-performance", False)
+
+        if log_rsync_performance:
+            # use stdout=PIPE only when log_rsync_performance enabled
+            # Else rsync will write to stdout and nobody is there
+            # to consume. If PIPE is full rsync hangs.
+            po = Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, universal_newlines=True)
+        else:
+            po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
+                       universal_newlines=True)
+
+        for f in files:
+            po.stdin.write(f)
+            po.stdin.write('\0')
+
+        stdout, stderr = po.communicate()
+
+        if kw.get("log_err", False):
+            for errline in stderr.strip().split("\n")[:-1]:
+                logging.error(lf("SYNC Error",
+                                 sync_engine="Rsync",
+                                 error=errline))
+
+        if log_rsync_performance:
+            rsync_msg = []
+            for line in stdout.split("\n"):
+                if line.startswith("Number of files:") or \
+                   line.startswith("Number of regular files transferred:") or \
+                   line.startswith("Total file size:") or \
+                   line.startswith("Total transferred file size:") or \
+                   line.startswith("Literal data:") or \
+                   line.startswith("Matched data:") or \
+                   line.startswith("Total bytes sent:") or \
+                   line.startswith("Total bytes received:") or \
+                   line.startswith("sent "):
+                    rsync_msg.append(line)
+            logging.info(lf("rsync performance",
+                            data=", ".join(rsync_msg)))
+
+        return po
 
     def tarssh(self, files, log_err=False):
-        return sup(self, files, self.slaveurl, log_err=log_err)
+        """invoke tar+ssh
+        -z (compress) can be use if needed, but omitting it now
+        as it results in weird error (tar+ssh errors out (errcode: 2)
+        """
+        if not files:
+            raise GsyncdError("no files to sync")
+        logging.debug("files: " + ", ".join(files))
+        (host, rdir) = self.slaveurl.split(':')
+
+        tar_cmd = ["tar"] + \
+            ["--sparse", "-cf", "-", "--files-from", "-"]
+        ssh_cmd = gconf.get("ssh-command").split() + \
+            gconf.get("ssh-options-tar").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            [host, "tar"] + \
+            ["--overwrite", "-xf", "-", "-C", rdir]
+        p0 = Popen(tar_cmd, stdout=subprocess.PIPE,
+                   stdin=subprocess.PIPE, stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        p1 = Popen(ssh_cmd, stdin=p0.stdout, stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        for f in files:
+            p0.stdin.write(f)
+            p0.stdin.write('\n')
+
+        p0.stdin.close()
+        p0.stdout.close()  # Allow p0 to receive a SIGPIPE if p1 exits.
+
+        # stdin and stdout of p0 is already closed, Reset to None and
+        # wait for child process to complete
+        p0.stdin = None
+        p0.stdout = None
+
+        def wait_for_tar(p0):
+            _, stderr = p0.communicate()
+            if log_err:
+                for errline in stderr.strip().split("\n")[:-1]:
+                    if "No such file or directory" not in errline:
+                        logging.error(lf("SYNC Error",
+                                         sync_engine="Tarssh",
+                                         error=errline))
+
+        t = syncdutils.Thread(target=wait_for_tar, args=(p0, ))
+        # wait for tar to terminate, collecting any errors, further
+        # waiting for transfer to complete
+        t.start()
+
+        # wait for ssh process
+        _, stderr1 = p1.communicate()
+        t.join()
+
+        if log_err:
+            for errline in stderr1.strip().split("\n")[:-1]:
+                logging.error(lf("SYNC Error",
+                                 sync_engine="Tarssh",
+                                 error=errline))
+
+        return p1
diff --git a/geo-replication/syncdaemon/subcmds.py b/geo-replication/syncdaemon/subcmds.py
new file mode 100644
index 00000000000..b8508532e30
--- /dev/null
+++ b/geo-replication/syncdaemon/subcmds.py
@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+from syncdutils import lf
+import logging
+import gsyncdconfig as gconf
+
+
+ERROR_CONFIG_INVALID = 2
+ERROR_CONFIG_INVALID_VALUE = 3
+ERROR_CONFIG_NOT_CONFIGURABLE = 4
+
+
+def subcmd_monitor_status(args):
+    from gsyncdstatus import set_monitor_status
+    from rconf import rconf
+
+    set_monitor_status(gconf.get("state-file"), args.status)
+    rconf.log_exit = False
+    logging.info(lf("Monitor Status Change", status=args.status))
+
+
+def subcmd_status(args):
+    from gsyncdstatus import GeorepStatus
+
+    master_name = args.master.replace(":", "")
+    slave_data = args.slave.replace("ssh://", "")
+
+    brick_status = GeorepStatus(gconf.get("state-file"),
+                                "",
+                                args.local_path,
+                                "",
+                                master_name,
+                                slave_data,
+                                gconf.get("pid-file"))
+    checkpoint_time = gconf.get("checkpoint", 0)
+    brick_status.print_status(checkpoint_time=checkpoint_time,
+                              json_output=args.json)
+
+
+def subcmd_monitor(args):
+    import monitor
+    from resource import GLUSTER, SSH, Popen
+    go_daemon = False if args.debug else True
+
+    monitor.startup(go_daemon)
+    Popen.init_errhandler()
+    local = GLUSTER("localhost", args.master)
+    slavehost, slavevol = args.slave.split("::")
+    remote = SSH(slavehost, slavevol)
+    return monitor.monitor(local, remote)
+
+
+def subcmd_verify_spawning(args):
+    logging.info("Able to spawn gsyncd.py")
+
+
+def subcmd_worker(args):
+    import os
+    import fcntl
+
+    from resource import GLUSTER, SSH, Popen
+
+    Popen.init_errhandler()
+    fcntl.fcntl(args.feedback_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+    local = GLUSTER("localhost", args.master)
+    slave_url, slavevol = args.slave.split("::")
+    if "@" not in slave_url:
+        slavehost = args.resource_remote
+    else:
+        slavehost = "%s@%s" % (slave_url.split("@")[0], args.resource_remote)
+    remote = SSH(slavehost, slavevol)
+    remote.connect_remote()
+    local.connect()
+    logging.info("Worker spawn successful. Acknowledging back to monitor")
+    os.close(args.feedback_fd)
+    local.service_loop(remote)
+
+
+def subcmd_slave(args):
+    from resource import GLUSTER, Popen
+
+    Popen.init_errhandler()
+    slavevol = args.slave.split("::")[-1]
+    local = GLUSTER("localhost", slavevol)
+
+    local.connect()
+    local.service_loop()
+
+
+def subcmd_voluuidget(args):
+    from subprocess import Popen, PIPE
+    import xml.etree.ElementTree as XET
+
+    ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
+
+    cmd = ['gluster', '--xml', '--remote-host=' + args.host,
+           'volume', 'info', args.volname]
+
+    if args.inet6:
+        cmd.append("--inet6")
+
+    po = Popen(cmd, bufsize=0,
+               stdin=None, stdout=PIPE, stderr=PIPE,
+               universal_newlines=True)
+
+    vix, err = po.communicate()
+    if po.returncode != 0:
+        logging.info(lf("Volume info failed, unable to get "
+                        "volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=po.returncode))
+        return ""
+    vi = XET.fromstring(vix)
+    if vi.find('opRet').text != '0':
+        logging.info(lf("Unable to get volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=vi.find('opErrstr').text))
+        return ""
+
+    try:
+        voluuid = vi.find("volInfo/volumes/volume/id").text
+    except (ParseError, AttributeError, ValueError) as e:
+        logging.info(lf("Parsing failed to volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=e))
+        voluuid = ""
+
+    print(voluuid)
+
+
+def _unlink(path):
+    import os
+    from errno import ENOENT
+    from syncdutils import GsyncdError
+    import sys
+
+    try:
+        os.unlink(path)
+    except (OSError, IOError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError('Unlink error: %s' % path)
+
+
+def subcmd_delete(args):
+    import logging
+    import shutil
+    import glob
+    import sys
+    from errno import ENOENT, ENODATA
+    import struct
+
+    from syncdutils import GsyncdError, Xattr, errno_wrap
+    import gsyncdconfig as gconf
+
+    logging.info('geo-replication delete')
+    # remove the stime xattr from all the brick paths so that
+    # a re-create of a session will start sync all over again
+    stime_xattr_prefix = gconf.get('stime-xattr-prefix', None)
+
+    # Delete pid file, status file, socket file
+    cleanup_paths = []
+    cleanup_paths.append(gconf.get("pid-file"))
+
+    # Cleanup Session dir
+    try:
+        shutil.rmtree(gconf.get("georep-session-working-dir"))
+    except (IOError, OSError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError(
+                'Error while removing working dir: %s' %
+                gconf.get("georep-session-working-dir"))
+
+    # Cleanup changelog working dirs
+    try:
+        shutil.rmtree(gconf.get("working-dir"))
+    except (IOError, OSError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError(
+                'Error while removing working dir: %s' %
+                gconf.get("working-dir"))
+
+    for path in cleanup_paths:
+        # To delete temp files
+        for f in glob.glob(path + "*"):
+            _unlink(f)
+
+    if args.reset_sync_time and stime_xattr_prefix:
+        for p in args.paths:
+            if p != "":
+                # set stime to (0,0) to trigger full volume content resync
+                # to slave on session recreation
+                # look at master.py::Xcrawl   hint: zero_zero
+                errno_wrap(Xattr.lsetxattr,
+                           (p, stime_xattr_prefix + ".stime",
+                            struct.pack("!II", 0, 0)),
+                           [ENOENT, ENODATA])
+                errno_wrap(Xattr.lremovexattr,
+                           (p, stime_xattr_prefix + ".entry_stime"),
+                           [ENOENT, ENODATA])
+
+    return
+
+
+def print_config(name, value, only_value=False, use_underscore=False):
+    val = value
+    if isinstance(value, bool):
+        val = str(value).lower()
+
+    if only_value:
+        print(val)
+    else:
+        if use_underscore:
+            name = name.replace("-", "_")
+
+        print(("%s:%s" % (name, val)))
+
+
+def config_name_format(val):
+    return val.replace("_", "-")
+
+
+def subcmd_config_get(args):
+    import sys
+    import json
+
+    all_config = gconf.getall(show_defaults=args.show_defaults,
+                              show_non_configurable=True)
+    if args.name is not None:
+        val = all_config.get(config_name_format(args.name), None)
+        if val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        print_config(args.name, val["value"], only_value=args.only_value,
+                     use_underscore=args.use_underscore)
+        return
+
+    if args.json:
+        out = []
+        # Convert all values as string
+        for k in sorted(all_config):
+            v = all_config[k]
+            out.append({
+                "name": k,
+                "value": str(v["value"]),
+                "default": str(v["default"]),
+                "configurable": v["configurable"],
+                "modified": v["modified"]
+            })
+
+        print((json.dumps(out)))
+        return
+
+    for k in sorted(all_config):
+        print_config(k, all_config[k]["value"],
+                     use_underscore=args.use_underscore)
+
+
+def subcmd_config_check(args):
+    import sys
+
+    try:
+        gconf.check(config_name_format(args.name), value=args.value,
+                    with_conffile=False)
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
+    except gconf.GconfInvalidValue:
+        sys.stderr.write("Invalid config value \"%s=%s\"\n" % (args.name,
+                                                               args.value))
+        sys.exit(ERROR_CONFIG_INVALID_VALUE)
+
+
+def subcmd_config_set(args):
+    import sys
+
+    try:
+        gconf.setconfig(config_name_format(args.name), args.value)
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
+    except gconf.GconfInvalidValue:
+        sys.stderr.write("Invalid config value \"%s=%s\"\n" % (args.name,
+                                                               args.value))
+        sys.exit(ERROR_CONFIG_INVALID_VALUE)
+
+
+def subcmd_config_reset(args):
+    import sys
+
+    try:
+        gconf.resetconfig(config_name_format(args.name))
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index 5dabbeaccef..a3df103e76c 100644
--- a/geo-replication/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -15,63 +15,105 @@ import time
 import fcntl
 import shutil
 import logging
+import errno
+import threading
+import subprocess
 import socket
+from subprocess import PIPE
 from threading import Lock, Thread as baseThread
-from errno import EACCES, EAGAIN, EPIPE, ENOTCONN, ECONNABORTED
-from errno import EINTR, ENOENT, EPERM, ESTALE, errorcode
+from errno import (EACCES, EAGAIN, EPIPE, ENOTCONN, ENOMEM, ECONNABORTED,
+                   EINTR, ENOENT, ESTALE, EBUSY, ENODATA, errorcode, EIO)
 from signal import signal, SIGTERM
 import select as oselect
 from os import waitpid as owaitpid
+import xml.etree.ElementTree as XET
+from select import error as SelectError
+try:
+    from cPickle import PickleError
+except ImportError:
+    from pickle import PickleError
 
-from conf import GLUSTERFS_LIBEXECDIR
+from conf import GLUSTERFS_LIBEXECDIR, UUID_FILE
 sys.path.insert(1, GLUSTERFS_LIBEXECDIR)
 EVENTS_ENABLED = True
 try:
-    from events.eventtypes import GEOREP_FAULTY as EVENT_GEOREP_FAULTY
+    from gfevents.eventtypes import GEOREP_FAULTY as EVENT_GEOREP_FAULTY
+    from gfevents.eventtypes import GEOREP_ACTIVE as EVENT_GEOREP_ACTIVE
+    from gfevents.eventtypes import GEOREP_PASSIVE as EVENT_GEOREP_PASSIVE
+    from gfevents.eventtypes import GEOREP_CHECKPOINT_COMPLETED \
+        as EVENT_GEOREP_CHECKPOINT_COMPLETED
 except ImportError:
     # Events APIs not installed, dummy eventtypes with None
     EVENTS_ENABLED = False
     EVENT_GEOREP_FAULTY = None
+    EVENT_GEOREP_ACTIVE = None
+    EVENT_GEOREP_PASSIVE = None
+    EVENT_GEOREP_CHECKPOINT_COMPLETED = None
 
-try:
-    from cPickle import PickleError
-except ImportError:
-    # py 3
-    from pickle import PickleError
+import gsyncdconfig as gconf
+from rconf import rconf
 
-from gconf import gconf
+from hashlib import sha256 as sha256
 
-try:
-    # py 3
-    from urllib import parse as urllib
-except ImportError:
-    import urllib
-
-try:
-    from hashlib import md5 as md5
-except ImportError:
-    # py 2.4
-    from md5 import new as md5
+ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
 
 # auxiliary gfid based access prefix
 _CL_AUX_GFID_PFX = ".gfid/"
+ROOT_GFID = "00000000-0000-0000-0000-000000000001"
 GF_OP_RETRIES = 10
 
-CHANGELOG_AGENT_SERVER_VERSION = 1.0
-CHANGELOG_AGENT_CLIENT_VERSION = 1.0
+GX_GFID_CANONICAL_LEN = 37  # canonical gfid len + '\0'
+
+NodeID = None
+rsync_version = None
+unshare_mnt_propagation = None
+slv_bricks = None
+SPACE_ESCAPE_CHAR = "%20"
+NEWLINE_ESCAPE_CHAR = "%0A"
+PERCENTAGE_ESCAPE_CHAR = "%25"
+
+final_lock = Lock()
+
+def sup(x, *a, **kw):
+    """a rubyesque "super" for python ;)
+
+    invoke caller method in parent class with given args.
+    """
+    return getattr(super(type(x), x),
+                   sys._getframe(1).f_code.co_name)(*a, **kw)
 
 
 def escape(s):
     """the chosen flavor of string escaping, used all over
        to turn whatever data to creatable representation"""
-    return urllib.quote_plus(s)
+    return s.replace("/", "-").strip("-")
 
 
-def unescape(s):
-    """inverse of .escape"""
-    return urllib.unquote_plus(s)
+def escape_space_newline(s):
+    return s.replace("%", PERCENTAGE_ESCAPE_CHAR)\
+            .replace(" ", SPACE_ESCAPE_CHAR)\
+            .replace("\n", NEWLINE_ESCAPE_CHAR)
 
 
+def unescape_space_newline(s):
+    return s.replace(SPACE_ESCAPE_CHAR, " ")\
+            .replace(NEWLINE_ESCAPE_CHAR, "\n")\
+            .replace(PERCENTAGE_ESCAPE_CHAR, "%")
+
+# gf_mount_ready() returns 1 if all subvols are up, else 0
+def gf_mount_ready():
+    ret = errno_wrap(Xattr.lgetxattr,
+                     ['.', 'dht.subvol.status', 16],
+                     [ENOENT, ENOTSUP, ENODATA], [ENOMEM])
+
+    if isinstance(ret, int):
+       logging.error("failed to get the xattr value")
+       return 1
+    ret = ret.rstrip('\x00')
+    if ret == "1":
+       return 1
+    return 0
+
 def norm(s):
     if s:
         return s.replace('-', '_')
@@ -129,17 +171,26 @@ def setup_ssh_ctl(ctld, remote_addr, resource_url):
     """
     Setup GConf ssh control path parameters
     """
-    gconf.ssh_ctl_dir = ctld
+    rconf.ssh_ctl_dir = ctld
     content = "SLAVE_HOST=%s\nSLAVE_RESOURCE_URL=%s" % (remote_addr,
                                                         resource_url)
-    content_md5 = md5hex(content)
-    fname = os.path.join(gconf.ssh_ctl_dir,
-                         "%s.mft" % content_md5)
+    encoded_content = content.encode()
+    content_sha256 = sha256hex(encoded_content)
+    """
+    The length of ctl_path for ssh connection should not be > 108.
+    ssh fails with ctl_path too long if it is so. But when rsync
+    is piped to ssh, it is not taking > 90. Hence using first 32
+    bytes of hash. Hash collision doesn't matter as only one sock
+    file is created per directory.
+    """
+    content_sha256 = content_sha256[:32]
+    fname = os.path.join(rconf.ssh_ctl_dir,
+                         "%s.mft" % content_sha256)
 
-    create_manifest(fname, content)
-    ssh_ctl_path = os.path.join(gconf.ssh_ctl_dir,
-                                "%s.sock" % content_md5)
-    gconf.ssh_ctl_args = ["-oControlMaster=auto", "-S", ssh_ctl_path]
+    create_manifest(fname, encoded_content)
+    ssh_ctl_path = os.path.join(rconf.ssh_ctl_dir,
+                                "%s.sock" % content_sha256)
+    rconf.ssh_ctl_args = ["-oControlMaster=auto", "-S", ssh_ctl_path]
 
 
 def grabfile(fname, content=None):
@@ -149,7 +200,7 @@ def grabfile(fname, content=None):
     """
     # damn those messy open() mode codes
     fd = os.open(fname, os.O_CREAT | os.O_RDWR)
-    f = os.fdopen(fd, 'r+b', 0)
+    f = os.fdopen(fd, 'r+')
     try:
         fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
     except:
@@ -163,34 +214,34 @@ def grabfile(fname, content=None):
         try:
             f.truncate()
             f.write(content)
+            f.flush()
         except:
             f.close()
             raise
-    gconf.permanent_handles.append(f)
+    rconf.permanent_handles.append(f)
     return f
 
 
 def grabpidfile(fname=None, setpid=True):
     """.grabfile customization for pid files"""
     if not fname:
-        fname = gconf.pid_file
+        fname = gconf.get("pid-file")
     content = None
     if setpid:
         content = str(os.getpid()) + '\n'
     return grabfile(fname, content=content)
 
-final_lock = Lock()
-
 
-def finalize(*a, **kw):
+def finalize(*args, **kwargs):
     """all those messy final steps we go trough upon termination
 
     Do away with pidfile, ssh control dir and logging.
     """
+
     final_lock.acquire()
-    if getattr(gconf, 'pid_file', None):
-        rm_pidf = gconf.pid_file_owned
-        if gconf.cpid:
+    if gconf.get('pid_file'):
+        rm_pidf = rconf.pid_file_owned
+        if rconf.cpid:
             # exit path from parent branch of daemonization
             rm_pidf = False
             while True:
@@ -198,39 +249,50 @@ def finalize(*a, **kw):
                 if not f:
                     # child has already taken over pidfile
                     break
-                if os.waitpid(gconf.cpid, os.WNOHANG)[0] == gconf.cpid:
+                if os.waitpid(rconf.cpid, os.WNOHANG)[0] == rconf.cpid:
                     # child has terminated
                     rm_pidf = True
                     break
                 time.sleep(0.1)
         if rm_pidf:
             try:
-                os.unlink(gconf.pid_file)
+                os.unlink(rconf.pid_file)
             except:
                 ex = sys.exc_info()[1]
                 if ex.errno == ENOENT:
                     pass
                 else:
                     raise
-    if gconf.ssh_ctl_dir and not gconf.cpid:
+    if rconf.ssh_ctl_dir and not rconf.cpid:
         def handle_rm_error(func, path, exc_info):
             if exc_info[1].errno == ENOENT:
                 return
             raise exc_info[1]
 
-        shutil.rmtree(gconf.ssh_ctl_dir, onerror=handle_rm_error)
-    if getattr(gconf, 'state_socket', None):
-        try:
-            os.unlink(gconf.state_socket)
-        except:
-            if sys.exc_info()[0] == OSError:
+        shutil.rmtree(rconf.ssh_ctl_dir, onerror=handle_rm_error)
+
+    """ Unmount if not done """
+    if rconf.mount_point:
+        if rconf.mountbroker:
+            umount_cmd = rconf.mbr_umount_cmd + [rconf.mount_point, 'lazy']
+        else:
+            umount_cmd = ['umount', '-l', rconf.mount_point]
+        p0 = subprocess.Popen(umount_cmd, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+        _, errdata = p0.communicate()
+        if p0.returncode == 0:
+            try:
+                os.rmdir(rconf.mount_point)
+            except OSError:
                 pass
+        else:
+            pass
 
-    if gconf.log_exit:
+    if rconf.log_exit:
         logging.info("exiting.")
     sys.stdout.flush()
     sys.stderr.flush()
-    os._exit(kw.get('exval', 0))
+    os._exit(kwargs.get('exval', 0))
 
 
 def log_raise_exception(excont):
@@ -240,6 +302,7 @@ def log_raise_exception(excont):
     Translate some weird sounding but well understood exceptions
     into human-friendly lingo
     """
+
     is_filelog = False
     for h in logging.getLogger().handlers:
         fno = getattr(getattr(h, 'stream', None), 'fileno', None)
@@ -260,9 +323,9 @@ def log_raise_exception(excont):
             ((isinstance(exc, OSError) or isinstance(exc, IOError)) and
              exc.errno == EPIPE):
             logging.error('connection to peer is broken')
-            if hasattr(gconf, 'transport'):
-                gconf.transport.wait()
-                if gconf.transport.returncode == 127:
+            if hasattr(rconf, 'transport'):
+                rconf.transport.wait()
+                if rconf.transport.returncode == 127:
                     logging.error("getting \"No such file or directory\""
                                   "errors is most likely due to "
                                   "MISCONFIGURATION, please remove all "
@@ -276,18 +339,29 @@ def log_raise_exception(excont):
                                   "<SLAVEVOL> config remote-gsyncd "
                                   "<GSYNCD_PATH> (Example GSYNCD_PATH: "
                                   "`/usr/libexec/glusterfs/gsyncd`)")
-                gconf.transport.terminate_geterr()
+                rconf.transport.terminate_geterr()
         elif isinstance(exc, OSError) and exc.errno in (ENOTCONN,
                                                         ECONNABORTED):
-            logging.error('glusterfs session went down [%s]',
-                          errorcode[exc.errno])
+            logging.error(lf('Gluster Mount process exited',
+                             error=errorcode[exc.errno]))
+        elif isinstance(exc, OSError) and exc.errno == EIO:
+            logging.error("Getting \"Input/Output error\" "
+                          "is most likely due to "
+                          "a. Brick is down or "
+                          "b. Split brain issue.")
+            logging.error("This is expected as per design to "
+                          "keep the consistency of the file system. "
+                          "Once the above issue is resolved "
+                          "geo-replication would automatically "
+                          "proceed further.")
+            logtag = "FAIL"
         else:
             logtag = "FAIL"
         if not logtag and logging.getLogger().isEnabledFor(logging.DEBUG):
             logtag = "FULL EXCEPTION TRACE"
         if logtag:
             logging.exception(logtag + ": ")
-            sys.stderr.write("failed with %s.\n" % type(exc).__name__)
+            sys.stderr.write("failed with %s: %s.\n" % (type(exc).__name__, exc))
         excont.exval = 1
         sys.exit(excont.exval)
 
@@ -310,20 +384,20 @@ class Thread(baseThread):
       function coughs up an exception
     """
 
-    def __init__(self, *a, **kw):
-        tf = kw.get('target')
+    def __init__(self, *args, **kwargs):
+        tf = kwargs.get('target')
         if tf:
-            def twrap(*aa):
+            def twrap(*aargs):
                 excont = FreeObject(exval=0)
                 try:
-                    tf(*aa)
+                    tf(*aargs)
                 except:
                     try:
                         log_raise_exception(excont)
                     finally:
                         finalize(exval=excont.exval)
-            kw['target'] = twrap
-        baseThread.__init__(self, *a, **kw)
+            kwargs['target'] = twrap
+        baseThread.__init__(self, *args, **kwargs)
         self.setDaemon(True)
 
 
@@ -331,6 +405,33 @@ class GsyncdError(Exception):
     pass
 
 
+class _MetaXattr(object):
+
+    """singleton class, a lazy wrapper around the
+    libcxattr module
+
+    libcxattr (a heavy import due to ctypes) is
+    loaded only when when the single
+    instance is tried to be used.
+
+    This reduces runtime for those invocations
+    which do not need filesystem manipulation
+    (eg. for config, url parsing)
+    """
+
+    def __getattr__(self, meth):
+        from libcxattr import Xattr as LXattr
+        xmeth = [m for m in dir(LXattr) if m[0] != '_']
+        if meth not in xmeth:
+            return
+        for m in xmeth:
+            setattr(self, m, getattr(LXattr, m))
+        return getattr(self, meth)
+
+
+Xattr = _MetaXattr()
+
+
 def getusername(uid=None):
     if uid is None:
         uid = os.geteuid()
@@ -361,81 +462,63 @@ def boolify(s):
     lstr = s.lower()
     if lstr in true_list:
         rv = True
-    elif not lstr in false_list:
-        logging.warn("Unknown string (%s) in string to boolean conversion "
-                     "defaulting to False\n" % (s))
+    elif lstr not in false_list:
+        logging.warn(lf("Unknown string in \"string to boolean\" conversion, "
+                        "defaulting to False",
+                        str=s))
 
     return rv
 
 
-def eintr_wrap(func, exc, *a):
+def eintr_wrap(func, exc, *args):
     """
     wrapper around syscalls resilient to interrupt caused
     by signals
     """
     while True:
         try:
-            return func(*a)
+            return func(*args)
         except exc:
             ex = sys.exc_info()[1]
             if not ex.args[0] == EINTR:
                 raise
 
 
-def select(*a):
-    return eintr_wrap(oselect.select, oselect.error, *a)
+def select(*args):
+    return eintr_wrap(oselect.select, oselect.error, *args)
+
 
+def waitpid(*args):
+    return eintr_wrap(owaitpid, OSError, *args)
 
-def waitpid(*a):
-    return eintr_wrap(owaitpid, OSError, *a)
 
+def term_handler_default_hook(signum, frame):
+    finalize(signum, frame, exval=1)
 
-def set_term_handler(hook=lambda *a: finalize(*a, **{'exval': 1})):
+
+def set_term_handler(hook=term_handler_default_hook):
     signal(SIGTERM, hook)
 
 
-def is_host_local(host):
-    locaddr = False
-    for ai in socket.getaddrinfo(host, None):
-        # cf. http://github.com/gluster/glusterfs/blob/ce111f47/xlators
-        # /mgmt/glusterd/src/glusterd-utils.c#L125
-        if ai[0] == socket.AF_INET:
-            if ai[-1][0].split(".")[0] == "127":
-                locaddr = True
-                break
-        elif ai[0] == socket.AF_INET6:
-            if ai[-1][0] == "::1":
-                locaddr = True
+def get_node_uuid():
+    global NodeID
+    if NodeID is not None:
+        return NodeID
+
+    NodeID = ""
+    with open(UUID_FILE) as f:
+        for line in f:
+            if line.startswith("UUID="):
+                NodeID = line.strip().split("=")[-1]
                 break
-        else:
-            continue
-        try:
-            # use ICMP socket to avoid net.ipv4.ip_nonlocal_bind issue,
-            # cf. https://bugzilla.redhat.com/show_bug.cgi?id=890587
-            s = socket.socket(ai[0], socket.SOCK_RAW, socket.IPPROTO_ICMP)
-        except socket.error:
-            ex = sys.exc_info()[1]
-            if ex.errno != EPERM:
-                raise
-            f = None
-            try:
-                f = open("/proc/sys/net/ipv4/ip_nonlocal_bind")
-                if int(f.read()) != 0:
-                    raise GsyncdError(
-                        "non-local bind is set and not allowed to create "
-                        "raw sockets, cannot determine if %s is local" % host)
-                s = socket.socket(ai[0], socket.SOCK_DGRAM)
-            finally:
-                if f:
-                    f.close()
-        try:
-            s.bind(ai[-1])
-            locaddr = True
-            break
-        except:
-            pass
-        s.close()
-    return locaddr
+
+    if NodeID == "":
+        raise GsyncdError("Failed to get Host UUID from %s" % UUID_FILE)
+    return NodeID
+
+
+def is_host_local(host_id):
+    return host_id == get_node_uuid()
 
 
 def funcode(f):
@@ -471,8 +554,8 @@ def gauxpfx():
     return _CL_AUX_GFID_PFX
 
 
-def md5hex(s):
-    return md5(s).hexdigest()
+def sha256hex(s):
+    return sha256(s).hexdigest()
 
 
 def selfkill(sig=SIGTERM):
@@ -490,19 +573,36 @@ def errno_wrap(call, arg=[], errnos=[], retry_errnos=[]):
             ex = sys.exc_info()[1]
             if ex.errno in errnos:
                 return ex.errno
-            if not ex.errno in retry_errnos:
+            if ex.errno not in retry_errnos:
                 raise
             nr_tries += 1
             if nr_tries == GF_OP_RETRIES:
                 # probably a screwed state, cannot do much...
-                logging.warn('reached maximum retries (%s)...%s' %
-                             (repr(arg), ex))
-                return ex.errno
+                logging.warn(lf('reached maximum retries',
+                                args=repr(arg),
+                                error=ex))
+                raise
             time.sleep(0.250)  # retry the call
 
 
 def lstat(e):
-    return errno_wrap(os.lstat, [e], [ENOENT], [ESTALE])
+    return errno_wrap(os.lstat, [e], [ENOENT], [ESTALE, EBUSY])
+
+def get_gfid_from_mnt(gfidpath):
+    return errno_wrap(Xattr.lgetxattr,
+                      [gfidpath, 'glusterfs.gfid.string',
+                       GX_GFID_CANONICAL_LEN], [ENOENT], [ESTALE])
+
+
+def matching_disk_gfid(gfid, entry):
+    disk_gfid = get_gfid_from_mnt(entry)
+    if isinstance(disk_gfid, int):
+        return False
+
+    if not gfid == disk_gfid:
+        return False
+
+    return True
 
 
 class NoStimeAvailable(Exception):
@@ -523,7 +623,7 @@ class ChangelogException(OSError):
 
 def gf_event(event_type, **kwargs):
     if EVENTS_ENABLED:
-        from events.gf_event import gf_event as gfevent
+        from gfevents.gf_event import gf_event as gfevent
         gfevent(event_type, **kwargs)
 
 
@@ -542,3 +642,474 @@ class GlusterLogLevel(object):
 
 def get_changelog_log_level(lvl):
     return getattr(GlusterLogLevel, lvl, GlusterLogLevel.INFO)
+
+
+def get_master_and_slave_data_from_args(args):
+    master_name = None
+    slave_data = None
+    for arg in args:
+        if arg.startswith(":"):
+            master_name = arg.replace(":", "")
+        if "::" in arg:
+            slave_data = arg.replace("ssh://", "")
+
+    return (master_name, slave_data)
+
+def unshare_propagation_supported():
+    global unshare_mnt_propagation
+    if unshare_mnt_propagation is not None:
+        return unshare_mnt_propagation
+
+    unshare_mnt_propagation = False
+    p = subprocess.Popen(["unshare", "--help"],
+                         stderr=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         universal_newlines=True)
+    out, err = p.communicate()
+    if p.returncode == 0:
+        if "propagation" in out:
+            unshare_mnt_propagation = True
+
+    return unshare_mnt_propagation
+
+
+def get_rsync_version(rsync_cmd):
+    global rsync_version
+    if rsync_version is not None:
+        return rsync_version
+
+    rsync_version = "0"
+    p = subprocess.Popen([rsync_cmd, "--version"],
+                         stderr=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         universal_newlines=True)
+    out, err = p.communicate()
+    if p.returncode == 0:
+        rsync_version = out.split(" ", 4)[3]
+
+    return rsync_version
+
+
+def get_slv_dir_path(slv_host, slv_volume, gfid):
+    global slv_bricks
+
+    dir_path = ENOENT
+    pfx = gauxpfx()
+
+    if not slv_bricks:
+        slv_info = Volinfo(slv_volume, slv_host, master=False)
+        slv_bricks = slv_info.bricks
+    # Result of readlink would be of format as below.
+    # readlink = "../../pgfid[0:2]/pgfid[2:4]/pgfid/basename"
+    for brick in slv_bricks:
+        dir_path = errno_wrap(os.path.join,
+                              [brick['dir'],
+                               ".glusterfs", gfid[0:2],
+                               gfid[2:4],
+                               gfid], [ENOENT], [ESTALE])
+        if dir_path != ENOENT:
+            try:
+                realpath = errno_wrap(os.readlink, [dir_path],
+                                      [ENOENT], [ESTALE])
+                if not isinstance(realpath, int):
+                    realpath_parts = realpath.split('/')
+                    pargfid = realpath_parts[-2]
+                    basename = realpath_parts[-1]
+                    dir_entry = os.path.join(pfx, pargfid, basename)
+                    return dir_entry
+            except OSError:
+                # .gfid/GFID
+                gfidpath = unescape_space_newline(os.path.join(pfx, gfid))
+                realpath = errno_wrap(Xattr.lgetxattr_buf,
+                      [gfidpath, 'glusterfs.gfid2path'], [ENOENT], [ESTALE])
+                if not isinstance(realpath, int):
+                    basename = os.path.basename(realpath).rstrip('\x00')
+                    dirpath = os.path.dirname(realpath)
+                    if dirpath == "/":
+                        pargfid = ROOT_GFID
+                    else:
+                        dirpath = dirpath.strip("/")
+                        pargfid = get_gfid_from_mnt(dirpath)
+                        if isinstance(pargfid, int):
+                            return None
+                    dir_entry = os.path.join(pfx, pargfid, basename)
+                    return dir_entry
+
+    return None
+
+
+def lf(event, **kwargs):
+    """
+    Log Format helper function, log messages can be
+    easily modified to structured log format.
+    lf("Config Change", sync_jobs=4, brick=/bricks/b1) will be
+    converted as "Config Change [{brick=/bricks/b1}, {sync_jobs=4}]"
+    """
+    msgparts = []
+    for k, v in kwargs.items():
+        msgparts.append("{%s=%s}" % (k, v))
+    return "%s [%s]" % (event, ", ".join(msgparts))
+
+
+class Popen(subprocess.Popen):
+
+    """customized subclass of subprocess.Popen with a ring
+    buffer for children error output"""
+
+    @classmethod
+    def init_errhandler(cls):
+        """start the thread which handles children's error output"""
+        cls.errstore = {}
+
+        def tailer():
+            while True:
+                errstore = cls.errstore.copy()
+                try:
+                    poe, _, _ = select(
+                        [po.stderr for po in errstore], [], [], 1)
+                except (ValueError, SelectError):
+                    # stderr is already closed wait for some time before
+                    # checking next error
+                    time.sleep(0.5)
+                    continue
+                for po in errstore:
+                    if po.stderr not in poe:
+                        continue
+                    po.lock.acquire()
+                    try:
+                        if po.on_death_row:
+                            continue
+                        la = errstore[po]
+                        try:
+                            fd = po.stderr.fileno()
+                        except ValueError:  # file is already closed
+                            time.sleep(0.5)
+                            continue
+
+                        try:
+                            l = os.read(fd, 1024)
+                        except OSError:
+                            time.sleep(0.5)
+                            continue
+
+                        if not l:
+                            continue
+                        tots = len(l)
+                        for lx in la:
+                            tots += len(lx)
+                        while tots > 1 << 20 and la:
+                            tots -= len(la.pop(0))
+                        la.append(l)
+                    finally:
+                        po.lock.release()
+        t = Thread(target=tailer)
+        t.start()
+        cls.errhandler = t
+
+    @classmethod
+    def fork(cls):
+        """fork wrapper that restarts errhandler thread in child"""
+        pid = os.fork()
+        if not pid:
+            cls.init_errhandler()
+        return pid
+
+    def __init__(self, args, *a, **kw):
+        """customizations for subprocess.Popen instantiation
+
+        - 'close_fds' is taken to be the default
+        - if child's stderr is chosen to be managed,
+          register it with the error handler thread
+        """
+        self.args = args
+        if 'close_fds' not in kw:
+            kw['close_fds'] = True
+        self.lock = threading.Lock()
+        self.on_death_row = False
+        self.elines = []
+        try:
+            sup(self, args, *a, **kw)
+        except:
+            ex = sys.exc_info()[1]
+            if not isinstance(ex, OSError):
+                raise
+            raise GsyncdError("""execution of "%s" failed with %s (%s)""" %
+                              (args[0], errno.errorcode[ex.errno],
+                               os.strerror(ex.errno)))
+        if kw.get('stderr') == subprocess.PIPE:
+            assert(getattr(self, 'errhandler', None))
+            self.errstore[self] = []
+
+    def errlog(self):
+        """make a log about child's failure event"""
+        logging.error(lf("command returned error",
+                         cmd=" ".join(self.args),
+                         error=self.returncode))
+        lp = ''
+
+        def logerr(l):
+            logging.error(self.args[0] + "> " + l)
+        for l in self.elines:
+            ls = l.split('\n')
+            ls[0] = lp + ls[0]
+            lp = ls.pop()
+            for ll in ls:
+                logerr(ll)
+        if lp:
+            logerr(lp)
+
+    def errfail(self):
+        """fail nicely if child did not terminate with success"""
+        self.errlog()
+        finalize(exval=1)
+
+    def terminate_geterr(self, fail_on_err=True):
+        """kill child, finalize stderr harvesting (unregister
+        from errhandler, set up .elines), fail on error if
+        asked for
+        """
+        self.lock.acquire()
+        try:
+            self.on_death_row = True
+        finally:
+            self.lock.release()
+        elines = self.errstore.pop(self)
+        if self.poll() is None:
+            self.terminate()
+            if self.poll() is None:
+                time.sleep(0.1)
+                self.kill()
+                self.wait()
+        while True:
+            if not select([self.stderr], [], [], 0.1)[0]:
+                break
+            b = os.read(self.stderr.fileno(), 1024)
+            if b:
+                elines.append(b.decode())
+            else:
+                break
+        self.stderr.close()
+        self.elines = elines
+        if fail_on_err and self.returncode != 0:
+            self.errfail()
+
+
+def host_brick_split(value):
+    """
+    IPv6 compatible way to split and get the host
+    and brick information. Example inputs:
+    node1.example.com:/exports/bricks/brick1/brick
+    fe80::af0f:df82:844f:ef66%utun0:/exports/bricks/brick1/brick
+    """
+    parts = value.split(":")
+    brick = parts[-1]
+    hostparts = parts[0:-1]
+    return (":".join(hostparts), brick)
+
+
+class Volinfo(object):
+
+    def __init__(self, vol, host='localhost', prelude=[], master=True):
+        if master:
+            gluster_cmd_dir = gconf.get("gluster-command-dir")
+        else:
+            gluster_cmd_dir = gconf.get("slave-gluster-command-dir")
+
+        gluster_cmd = os.path.join(gluster_cmd_dir, 'gluster')
+        po = Popen(prelude + [gluster_cmd, '--xml', '--remote-host=' + host,
+                              'volume', 'info', vol],
+                   stdout=PIPE, stderr=PIPE, universal_newlines=True)
+        vix = po.stdout.read()
+        po.wait()
+        po.terminate_geterr()
+        vi = XET.fromstring(vix)
+        if vi.find('opRet').text != '0':
+            if prelude:
+                via = '(via %s) ' % prelude.join(' ')
+            else:
+                via = ' '
+            raise GsyncdError('getting volume info of %s%s '
+                              'failed with errorcode %s' %
+                              (vol, via, vi.find('opErrno').text))
+        self.tree = vi
+        self.volume = vol
+        self.host = host
+
+    def get(self, elem):
+        return self.tree.findall('.//' + elem)
+
+    def is_tier(self):
+        return (self.get('typeStr')[0].text == 'Tier')
+
+    def is_hot(self, brickpath):
+        logging.debug('brickpath: ' + repr(brickpath))
+        return brickpath in self.hot_bricks
+
+    @property
+    @memoize
+    def bricks(self):
+        def bparse(b):
+            host, dirp = host_brick_split(b.find("name").text)
+            return {'host': host, 'dir': dirp, 'uuid': b.find("hostUuid").text}
+        return [bparse(b) for b in self.get('brick')]
+
+    @property
+    @memoize
+    def uuid(self):
+        ids = self.get('id')
+        if len(ids) != 1:
+            raise GsyncdError("volume info of %s obtained from %s: "
+                              "ambiguous uuid" % (self.volume, self.host))
+        return ids[0].text
+
+    def replica_count(self, tier, hot):
+        if (tier and hot):
+            return int(self.get('hotBricks/hotreplicaCount')[0].text)
+        elif (tier and not hot):
+            return int(self.get('coldBricks/coldreplicaCount')[0].text)
+        else:
+            return int(self.get('replicaCount')[0].text)
+
+    def disperse_count(self, tier, hot):
+        if (tier and hot):
+            # Tiering doesn't support disperse volume as hot brick,
+            # hence no xml output, so returning 0. In case, if it's
+            # supported later, we should change here.
+            return 0
+        elif (tier and not hot):
+            return int(self.get('coldBricks/colddisperseCount')[0].text)
+        else:
+            return int(self.get('disperseCount')[0].text)
+
+    def distribution_count(self, tier, hot):
+        if (tier and hot):
+            return int(self.get('hotBricks/hotdistCount')[0].text)
+        elif (tier and not hot):
+            return int(self.get('coldBricks/colddistCount')[0].text)
+        else:
+            return int(self.get('distCount')[0].text)
+
+    @property
+    @memoize
+    def hot_bricks(self):
+        return [b.text for b in self.get('hotBricks/brick')]
+
+    def get_hot_bricks_count(self, tier):
+        if (tier):
+            return int(self.get('hotBricks/hotbrickCount')[0].text)
+        else:
+            return 0
+
+
+class VolinfoFromGconf(object):
+    # Glusterd will generate following config items before Geo-rep start
+    # So that Geo-rep need not run gluster commands from inside
+    # Volinfo object API/interface kept as is so that caller need not
+    # change anything except calling this instead of Volinfo()
+    #
+    # master-bricks=
+    # master-bricks=NODEID:HOSTNAME:PATH,..
+    # slave-bricks=NODEID:HOSTNAME,..
+    # master-volume-id=
+    # slave-volume-id=
+    # master-replica-count=
+    # master-disperse_count=
+    def __init__(self, vol, host='localhost', master=True):
+        self.volume = vol
+        self.host = host
+        self.master = master
+
+    def is_tier(self):
+        return False
+
+    def is_hot(self, brickpath):
+        return False
+
+    def is_uuid(self, value):
+        try:
+            uuid.UUID(value)
+            return True
+        except ValueError:
+            return False
+
+    def possible_path(self, value):
+        return "/" in value
+
+    @property
+    @memoize
+    def bricks(self):
+        pfx = "master-" if self.master else "slave-"
+        bricks_data = gconf.get(pfx + "bricks")
+        if bricks_data is None:
+            return []
+
+        bricks_data = bricks_data.split(",")
+        bricks_data = [b.strip() for b in bricks_data]
+        out = []
+        for b in bricks_data:
+            parts = b.split(":")
+            b_uuid = None
+            if self.is_uuid(parts[0]):
+                b_uuid = parts[0]
+                # Set all parts except first
+                parts = parts[1:]
+
+            if self.possible_path(parts[-1]):
+                bpath = parts[-1]
+                # Set all parts except last
+                parts = parts[0:-1]
+
+            out.append({
+                "host": ":".join(parts),   # if remaining parts are IPv6 name
+                "dir": bpath,
+                "uuid": b_uuid
+            })
+
+        return out
+
+    @property
+    @memoize
+    def uuid(self):
+        if self.master:
+            return gconf.get("master-volume-id")
+        else:
+            return gconf.get("slave-volume-id")
+
+    def replica_count(self, tier, hot):
+        return gconf.get("master-replica-count")
+
+    def disperse_count(self, tier, hot):
+        return gconf.get("master-disperse-count")
+
+    def distribution_count(self, tier, hot):
+        return gconf.get("master-distribution-count")
+
+    @property
+    @memoize
+    def hot_bricks(self):
+        return []
+
+    def get_hot_bricks_count(self, tier):
+        return 0
+
+
+def can_ssh(host, port=22):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.connect((host, port))
+        flag = True
+    except socket.error:
+        flag = False
+
+    s.close()
+    return flag
+
+
+def get_up_nodes(hosts, port):
+    # List of hosts with Hostname/IP and UUID
+    up_nodes = []
+    for h in hosts:
+        if can_ssh(h[0], port):
+            up_nodes.append(h)
+
+    return up_nodes
diff --git a/geo-replication/tests/__init__.py b/geo-replication/tests/__init__.py
index 23adbfa5171..b4648b69645 100644
--- a/geo-replication/tests/__init__.py
+++ b/geo-replication/tests/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
diff --git a/geo-replication/tests/unit/__init__.py b/geo-replication/tests/unit/__init__.py
index 23adbfa5171..b4648b69645 100644
--- a/geo-replication/tests/unit/__init__.py
+++ b/geo-replication/tests/unit/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
diff --git a/geo-replication/tests/unit/test_gsyncdstatus.py b/geo-replication/tests/unit/test_gsyncdstatus.py
index a65d659e356..9c1aa2ad4ad 100644..100755
--- a/geo-replication/tests/unit/test_gsyncdstatus.py
+++ b/geo-replication/tests/unit/test_gsyncdstatus.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
@@ -13,11 +13,11 @@ import unittest
 import os
 import urllib
 
-from syncdaemon.gstatus import GeorepStatus, set_monitor_status
-from syncdaemon.gstatus import get_default_values
-from syncdaemon.gstatus import MONITOR_STATUS, DEFAULT_STATUS
-from syncdaemon.gstatus import STATUS_VALUES, CRAWL_STATUS_VALUES
-from syncdaemon.gstatus import human_time, human_time_utc
+from syncdaemon.gstatus import (GeorepStatus, set_monitor_status,
+                                get_default_values,
+                                MONITOR_STATUS, DEFAULT_STATUS,
+                                STATUS_VALUES, CRAWL_STATUS_VALUES,
+                                human_time, human_time_utc)
 
 
 class GeorepStatusTestCase(unittest.TestCase):
diff --git a/geo-replication/tests/unit/test_syncdutils.py b/geo-replication/tests/unit/test_syncdutils.py
index 736ae274b85..ff537ab2660 100644
--- a/geo-replication/tests/unit/test_syncdutils.py
+++ b/geo-replication/tests/unit/test_syncdutils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
 # This file is part of GlusterFS.
diff --git a/glusterfs-api.pc.in b/glusterfs-api.pc.in
index 5d234543414..4a2edb7bf07 100644
--- a/glusterfs-api.pc.in
+++ b/glusterfs-api.pc.in
@@ -9,4 +9,4 @@ Description: GlusterFS API
 Version: @GFAPI_VERSION@
 Requires: @PKGCONFIG_UUID@
 Libs: -L${libdir} @GFAPI_LIBS@ -lgfapi -lglusterfs -lgfrpc -lgfxdr
-Cflags: -I${includedir}/glusterfs -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -DUSE_POSIX_ACLS=@USE_POSIX_ACLS@
+Cflags: -I${includedir} -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64 -DUSE_POSIX_ACLS=@USE_POSIX_ACLS@
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index a5b94686b86..b6d63146e14 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -9,44 +9,78 @@
 ## All argument definitions should be placed here and keep them sorted
 ##
 
-# if you wish to compile an rpm with debugging...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with debug
-%{?_with_debug:%global _with_debug --enable-debug}
+# asan
+# if you wish to compile an rpm with address sanitizer...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with asan
+%{?_with_asan:%global _with_asan --enable-asan}
 
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
+%global _with_asan %{nil}
+%endif
+
+# cmocka
 # if you wish to compile an rpm with cmocka unit testing...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with cmocka
 %{?_with_cmocka:%global _with_cmocka --enable-cmocka}
 
-# if you wish to compile an rpm without rdma support, compile like this...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without rdma
-%{?_without_rdma:%global _without_rdma --disable-ibverbs}
-
-# No RDMA Support on s390(x)
-%ifarch s390 s390x
-%global _without_rdma --disable-ibverbs
-%endif
+# debug
+# if you wish to compile an rpm with debugging...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with debug
+%{?_with_debug:%global _with_debug --enable-debug}
 
+# epoll
 # if you wish to compile an rpm without epoll...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without epoll
 %{?_without_epoll:%global _without_epoll --disable-epoll}
 
+# fusermount
 # if you wish to compile an rpm without fusermount...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without fusermount
 %{?_without_fusermount:%global _without_fusermount --disable-fusermount}
 
+# geo-rep
 # if you wish to compile an rpm without geo-replication support, compile like this...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without georeplication
 %{?_without_georeplication:%global _without_georeplication --disable-georeplication}
 
-# Disable geo-replication on EL5, as its default Python is too old
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-%global _without_georeplication --disable-georeplication
+# gnfs
+# if you wish to compile an rpm with the legacy gNFS server xlator
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with gnfs
+%{?_with_gnfs:%global _with_gnfs --enable-gnfs}
+
+# ipv6default
+# if you wish to compile an rpm with IPv6 default...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with ipv6default
+%{?_with_ipv6default:%global _with_ipv6default --with-ipv6-default}
+
+# libtirpc
+# if you wish to compile an rpm without TIRPC (i.e. use legacy glibc rpc)
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without libtirpc
+%{?_without_libtirpc:%global _without_libtirpc --without-libtirpc}
+
+# Do not use libtirpc on EL6, it does not have xdr_uint64_t() and xdr_uint32_t
+# Do not use libtirpc on EL7, it does not have xdr_sizeof()
+%if ( 0%{?rhel} && 0%{?rhel} <= 7 )
+%global _without_libtirpc --without-libtirpc
 %endif
 
+
+# ocf
 # if you wish to compile an rpm without the OCF resource agents...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without ocf
 %{?_without_ocf:%global _without_ocf --without-ocf}
 
+# server
+# if you wish to build rpms without server components, compile like this
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without server
+%{?_without_server:%global _without_server --without-server}
+
+# disable server components forcefully as rhel <= 6
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%global _without_server --without-server
+%endif
+
+# syslog
 # if you wish to build rpms without syslog logging, compile like this
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without syslog
 %{?_without_syslog:%global _without_syslog --disable-syslog}
@@ -59,24 +93,31 @@
 %global _without_syslog --disable-syslog
 %endif
 
-# if you wish to compile an rpm without the BD map support...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without bd
-%{?_without_bd:%global _without_bd --disable-bd-xlator}
+# tsan
+# if you wish to compile an rpm with thread sanitizer...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with tsan
+%{?_with_tsan:%global _with_tsan --enable-tsan}
 
-%if ( 0%{?rhel} && 0%{?rhel} < 6 || 0%{?sles_version} )
-%global _without_bd --disable-bd-xlator
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
+%global _with_tsan %{nil}
 %endif
 
-# Disable data-tiering on EL5, sqlite is too old
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-%global _without_tiering --disable-tiering
-%endif
+# valgrind
+# if you wish to compile an rpm to run all processes under valgrind...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with valgrind
+%{?_with_valgrind:%global _with_valgrind --enable-valgrind}
 
 ##-----------------------------------------------------------------------------
 ## All %%global definitions should be placed here and keep them sorted
 ##
 
-%if ( 0%{?fedora} && 0%{?fedora} > 16 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+# selinux booleans whose defalut value needs modification
+# these booleans will be consumed by "%%selinux_set_booleans" macro.
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+%global selinuxbooleans rsync_full_access=1 rsync_client=1
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
 %global _with_systemd true
 %endif
 
@@ -90,54 +131,63 @@
 %global _with_tmpfilesdir --without-tmpfilesdir
 %endif
 
-# Eventing
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
+# without server should also disable some server-only components
+%if 0%{?_without_server:1}
 %global _without_events --disable-events
+%global _without_georeplication --disable-georeplication
+%global _with_gnfs %{nil}
+%global _without_ocf --without-ocf
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+%global _usepython3 1
+%global _pythonver 3
+%else
+%global _usepython3 0
+%global _pythonver 2
 %endif
 
 # From https://fedoraproject.org/wiki/Packaging:Python#Macros
-%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
-%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
-%{!?python_sitearch: %global python_sitearch %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%{!?python2_sitelib: %global python2_sitelib %(python2 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?python2_sitearch: %global python2_sitearch %(python2 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+%global _rundir %{_localstatedir}/run
 %endif
 
 %if ( 0%{?_with_systemd:1} )
-%global _init_enable()  /bin/systemctl enable %1.service ;
-%global _init_disable() /bin/systemctl disable %1.service ;
-%global _init_restart() /bin/systemctl try-restart %1.service ;
-%global _init_start()   /bin/systemctl start %1.service ;
-%global _init_stop()    /bin/systemctl stop %1.service ;
-%global _init_install() install -D -p -m 0644 %1 %{buildroot}%{_unitdir}/%2.service ;
+%global service_start()   /bin/systemctl --quiet start %1.service || : \
+%{nil}
+%global service_stop()    /bin/systemctl --quiet stop %1.service || :\
+%{nil}
+%global service_install() install -D -p -m 0644 %1.service %{buildroot}%2 \
+%{nil}
 # can't seem to make a generic macro that works
-%global _init_glusterd   %{_unitdir}/glusterd.service
-%global _init_glusterfsd %{_unitdir}/glusterfsd.service
+%global glusterd_svcfile   %{_unitdir}/glusterd.service
+%global glusterfsd_svcfile %{_unitdir}/glusterfsd.service
+%global glusterta_svcfile %{_unitdir}/gluster-ta-volume.service
+%global glustereventsd_svcfile %{_unitdir}/glustereventsd.service
+%global glusterfssharedstorage_svcfile %{_unitdir}/glusterfssharedstorage.service
 %else
-%global _init_enable()  /sbin/chkconfig --add %1 ;
-%global _init_disable() /sbin/chkconfig --del %1 ;
-%global _init_restart() /sbin/service %1 condrestart &>/dev/null ;
-%global _init_start()   /sbin/service %1 start &>/dev/null ;
-%global _init_stop()    /sbin/service %1 stop &>/dev/null ;
-%global _init_install() install -D -p -m 0755 %1 %{buildroot}%{_sysconfdir}/init.d/%2 ;
+%global systemd_post()  /sbin/chkconfig --add %1 >/dev/null 2>&1 || : \
+%{nil}
+%global systemd_preun() /sbin/chkconfig --del %1 >/dev/null 2>&1 || : \
+%{nil}
+%global systemd_postun_with_restart() /sbin/service %1 condrestart >/dev/null 2>&1 || : \
+%{nil}
+%global service_start()   /sbin/service %1 start >/dev/null 2>&1 || : \
+%{nil}
+%global service_stop()    /sbin/service %1 stop >/dev/null 2>&1 || : \
+%{nil}
+%global service_install() install -D -p -m 0755 %1.init %{buildroot}%2 \
+%{nil}
 # can't seem to make a generic macro that works
-%global _init_glusterd   %{_sysconfdir}/init.d/glusterd
-%global _init_glusterfsd %{_sysconfdir}/init.d/glusterfsd
-%endif
-
-%if ( 0%{_for_fedora_koji_builds} )
-%if ( 0%{?_with_systemd:1} )
-%global glusterfsd_service glusterfsd.service
-%else
-%global glusterfsd_service glusterfsd.init
-%endif
+%global glusterd_svcfile   %{_sysconfdir}/init.d/glusterd
+%global glusterfsd_svcfile %{_sysconfdir}/init.d/glusterfsd
+%global glustereventsd_svcfile %{_sysconfdir}/init.d/glustereventsd
 %endif
 
 %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}}
 
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-# _sharedstatedir is not provided by RHEL5
-%global _sharedstatedir /var/lib
-%endif
-
 # We do not want to generate useless provides and requires for xlator
 # .so files to be set for glusterfs packages.
 # Filter all generated:
@@ -163,21 +213,17 @@ Summary:          Distributed File System
 Name:             glusterfs
 Version:          3.8.0
 Release:          0.1%{?prereltag:.%{prereltag}}%{?dist}
-Vendor:           Fedora Project
 %else
 Name:             @PACKAGE_NAME@
 Version:          @PACKAGE_VERSION@
 Release:          0.@PACKAGE_RELEASE@%{?dist}
-Vendor:           Gluster Community
 %endif
 License:          GPLv2 or LGPLv3+
-Group:            System Environment/Base
-URL:              http://gluster.readthedocs.io/en/latest/
+URL:              http://docs.gluster.org/
 %if ( 0%{_for_fedora_koji_builds} )
 Source0:          http://bits.gluster.org/pub/gluster/glusterfs/src/glusterfs-%{version}%{?prereltag}.tar.gz
 Source1:          glusterd.sysconfig
 Source2:          glusterfsd.sysconfig
-Source6:          rhel5-load-fuse-modules
 Source7:          glusterfsd.service
 Source8:          glusterfsd.init
 %else
@@ -186,41 +232,46 @@ Source0:          @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
 
 BuildRoot:        %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 
-%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
-BuildRequires:    python-simplejson
-%endif
+Requires(pre):    shadow-utils
 %if ( 0%{?_with_systemd:1} )
-BuildRequires:    systemd-units
+BuildRequires:    systemd
 %endif
 
-Requires:         %{name}-libs%{?_isa} = %{version}-%{release}
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libgfrpc0%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr0%{?_isa} = %{version}-%{release}
+%if ( 0%{?_with_systemd:1} )
+%{?systemd_requires}
+%endif
+%if 0%{?_with_asan:1} && !( 0%{?rhel} && 0%{?rhel} < 7 )
+BuildRequires:    libasan
+%endif
+%if 0%{?_with_tsan:1} && !( 0%{?rhel} && 0%{?rhel} < 7 )
+BuildRequires:    libtsan
+%endif
 BuildRequires:    bison flex
 BuildRequires:    gcc make libtool
 BuildRequires:    ncurses-devel readline-devel
 BuildRequires:    libxml2-devel openssl-devel
 BuildRequires:    libaio-devel libacl-devel
-BuildRequires:    python-devel
-%if ( 0%{?fedora} && 0%{?fedora} < 26 ) || ( 0%{?rhel} )
+BuildRequires:    python%{_pythonver}-devel
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
 BuildRequires:    python-ctypes
 %endif
+%if ( 0%{?_with_ipv6default:1} ) || ( 0%{!?_without_libtirpc:1} )
+BuildRequires:    libtirpc-devel
+%endif
+%if ( 0%{?fedora} && 0%{?fedora} > 27 ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+BuildRequires:    rpcgen
+%endif
 BuildRequires:    userspace-rcu-devel >= 0.7
 %if ( 0%{?rhel} && 0%{?rhel} <= 6 )
 BuildRequires:    automake
 %endif
-%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
-BuildRequires:    e2fsprogs-devel
-%else
 BuildRequires:    libuuid-devel
-%endif
 %if ( 0%{?_with_cmocka:1} )
 BuildRequires:    libcmocka-devel >= 1.0.1
 %endif
-%if ( 0%{!?_without_tiering:1} )
-BuildRequires:    sqlite-devel
-%endif
-%if ( 0%{!?_without_bd:1} )
-BuildRequires:    lvm2-devel
-%endif
 %if ( 0%{!?_without_georeplication:1} )
 BuildRequires:    libattr-devel
 %endif
@@ -229,121 +280,77 @@ BuildRequires:    libattr-devel
 BuildRequires:    firewalld
 %endif
 
-Obsoletes:        hekafs
 Obsoletes:        %{name}-common < %{version}-%{release}
 Obsoletes:        %{name}-core < %{version}-%{release}
-Obsoletes:        %{name}-ufo
+Obsoletes:        %{name}-ganesha
+Obsoletes:        %{name}-rdma < %{version}-%{release}
+%if ( 0%{!?_with_gnfs:1} )
+Obsoletes:        %{name}-gnfs < %{version}-%{release}
+%endif
 Provides:         %{name}-common = %{version}-%{release}
 Provides:         %{name}-core = %{version}-%{release}
 
 %description
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package includes the glusterfs binary, the glusterfsd daemon and the
 libglusterfs and glusterfs translator modules common to both GlusterFS server
 and client framework.
 
-%package api
-Summary:          GlusterFS api library
-Group:            System Environment/Daemons
-Requires:         %{name}%{?_isa} = %{version}-%{release}
-Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
-
-%description api
-GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
-
-This package provides the glusterfs libgfapi library.
-
-%package api-devel
-Summary:          Development Libraries
-Group:            Development/Libraries
-Requires:         %{name}%{?_isa} = %{version}-%{release}
-Requires:         %{name}-devel%{?_isa} = %{version}-%{release}
-Requires:         libacl-devel
-
-%description api-devel
-GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
-
-This package provides the api include files.
-
 %package cli
 Summary:          GlusterFS CLI
-Group:            Applications/File
-Requires:         %{name}-libs%{?_isa} = %{version}-%{release}
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libglusterd0%{?_isa} = %{version}-%{release}
 
 %description cli
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides the GlusterFS CLI application and its man page
 
-%package devel
-Summary:          Development Libraries
-Group:            Development/Libraries
-Requires:         %{name}%{?_isa} = %{version}-%{release}
-# Needed for the Glupy examples to work
-Requires:         %{name}-extra-xlators%{?_isa} = %{version}-%{release}
+%package cloudsync-plugins
+Summary:          Cloudsync Plugins
+BuildRequires:    libcurl-devel
 
-%description devel
+%description cloudsync-plugins
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
-This package provides the development libraries and include files.
+This package provides cloudsync plugins for archival feature.
 
 %package extra-xlators
 Summary:          Extra Gluster filesystem Translators
-Group:            Applications/File
 # We need python-gluster rpm for gluster module's __init__.py in Python
 # site-packages area
-Requires:         python-gluster = %{version}-%{release}
-Requires:         python
-%if ( 0%{?fedora} && 0%{?fedora} < 26 ) || ( 0%{?rhel} )
-BuildRequires:    python-ctypes
-%endif
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+Requires:         python%{_pythonver}
 
 %description extra-xlators
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides extra filesystem Translators, such as Glupy,
 for GlusterFS.
 
 %package fuse
 Summary:          Fuse client
-Group:            Applications/File
 BuildRequires:    fuse-devel
 Requires:         attr
 Requires:         psmisc
@@ -356,27 +363,46 @@ Provides:         %{name}-client = %{version}-%{release}
 
 %description fuse
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides support to FUSE based clients and inlcudes the
 glusterfs(d) binary.
 
+%if ( 0%{!?_without_server:1} )
 %package ganesha
 Summary:          NFS-Ganesha configuration
 Group:            Applications/File
 
 Requires:         %{name}-server%{?_isa} = %{version}-%{release}
-Requires:         nfs-ganesha-gluster, pcs, dbus
+Requires:         nfs-ganesha-selinux >= 2.7.6
+Requires:         nfs-ganesha-gluster >= 2.7.6
+Requires:         pcs >= 0.10.0
+Requires:         resource-agents >= 4.2.0
+Requires:         dbus
+
 %if ( 0%{?rhel} && 0%{?rhel} == 6 )
 Requires:         cman, pacemaker, corosync
 %endif
-# we need portblock resource-agent
-Requires:         %{_prefix}/lib/ocf/resource.d/portblock
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 5 )
+# we need portblock resource-agent in 3.9.5 and later.
+Requires:         net-tools
+%endif
+
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
+Requires: selinux-policy >= 3.13.1-160
+Requires(post):   policycoreutils-python
+Requires(postun): policycoreutils-python
+%else
+Requires(post):   policycoreutils-python-utils
+Requires(postun): policycoreutils-python-utils
+%endif
+%endif
 
 %description ganesha
 GlusterFS is a distributed file-system capable of scaling to several
@@ -389,101 +415,261 @@ is in user space and easily manageable.
 
 This package provides the configuration and related files for using
 NFS-Ganesha as the NFS server using GlusterFS
+%endif
 
 %if ( 0%{!?_without_georeplication:1} )
 %package geo-replication
 Summary:          GlusterFS Geo-replication
-Group:            Applications/File
 Requires:         %{name}%{?_isa} = %{version}-%{release}
 Requires:         %{name}-server%{?_isa} = %{version}-%{release}
-Requires:         python
-Requires:         python-prettytable
-%if ( 0%{?fedora} && 0%{?fedora} < 26 ) || ( 0%{?rhel} )
-BuildRequires:    python-ctypes
-%endif
-Requires:         python-gluster = %{version}-%{release}
+Requires:         python%{_pythonver}
+Requires:         python%{_pythonver}-prettytable
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+
 Requires:         rsync
+Requires:         util-linux
+# required for setting selinux bools
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+Requires(post):      policycoreutils-python-utils
+Requires(postun):    policycoreutils-python-utils
+Requires:            selinux-policy-targeted
+Requires(post):      selinux-policy-targeted
+BuildRequires:       selinux-policy-devel
+%endif
 
 %description geo-replication
 GlusterFS is a distributed file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides support to geo-replication.
 %endif
 
-%package libs
-Summary:          GlusterFS common libraries
-Group:            Applications/File
+%if ( 0%{?_with_gnfs:1} )
+%package gnfs
+Summary:          GlusterFS gNFS server
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+Requires:         nfs-utils
 
-%description libs
+%description gnfs
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
-
-This package provides the base GlusterFS libraries
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
-%package -n python-gluster
-Summary:          GlusterFS python library
-Group:            Development/Tools
-%if ( ! ( 0%{?rhel} && 0%{?rhel} < 6 || 0%{?sles_version} ) )
-# EL5 does not support noarch sub-packages
-BuildArch:        noarch
+This package provides the glusterfs legacy gNFS server xlator
 %endif
-Requires:         python
 
-%description -n python-gluster
+%package -n libglusterfs0
+Summary:          GlusterFS libglusterfs library
+Requires:         libgfrpc0%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+Provides:         %{name}-libs = %{version}-%{release}
+
+%description -n libglusterfs0
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the base libglusterfs library
+
+%package -n libglusterfs-devel
+Summary:          GlusterFS libglusterfs library
+Requires:         libgfrpc-devel%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr-devel%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+Provides:         %{name}-devel = %{version}-%{release}
+
+%description -n libglusterfs-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
-This package contains the python modules of GlusterFS and own gluster
-namespace.
+This package provides libglusterfs.so and the gluster C header files.
 
+%package -n libgfapi0
+Summary:          GlusterFS api library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-api <= %{version}-%{release}
+Provides:         %{name}-api = %{version}-%{release}
 
-%if ( 0%{!?_without_rdma:1} )
-%package rdma
-Summary:          GlusterFS rdma support for ib-verbs
-Group:            Applications/File
-BuildRequires:    libibverbs-devel
-BuildRequires:    librdmacm-devel >= 1.0.15
-Requires:         %{name}%{?_isa} = %{version}-%{release}
+%description -n libgfapi0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the glusterfs libgfapi library.
 
-%description rdma
+%package -n libgfapi-devel
+Summary:          Development Libraries
+Requires:         libglusterfs-devel%{?_isa} = %{version}-%{release}
+Requires:         libacl-devel
+Obsoletes:        %{name}-api-devel <= %{version}-%{release}
+Provides:         %{name}-api-devel = %{version}-%{release}
+
+%description -n libgfapi-devel
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfapi.so and the api C header files.
+
+%package -n libgfchangelog0
+Summary:          GlusterFS libchangelog library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfchangelog0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfchangelog library
+
+%package -n libgfchangelog-devel
+Summary:          GlusterFS libchangelog library
+Requires:         libglusterfs-devel%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfchangelog-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfchangelog.so and changelog C header files.
+
+%package -n libgfrpc0
+Summary:          GlusterFS libgfrpc0 library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfrpc0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfrpc library
+
+%package -n libgfrpc-devel
+Summary:          GlusterFS libgfrpc library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfrpc-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfrpc.so and rpc C header files.
 
-This package provides support to ib-verbs library.
+%package -n libgfxdr0
+Summary:          GlusterFS libgfxdr0 library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfxdr0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfxdr library
+
+%package -n libgfxdr-devel
+Summary:          GlusterFS libgfxdr library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfxdr-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfxdr.so.
+
+%package -n libglusterd0
+Summary:          GlusterFS libglusterd library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libglusterd0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libglusterd library
+
+%package -n python%{_pythonver}-gluster
+Summary:          GlusterFS python library
+Requires:         python%{_pythonver}
+%if ( ! %{_usepython3} )
+%{?python_provide:%python_provide python-gluster}
+Provides:         python-gluster = %{version}-%{release}
+Obsoletes:        python-gluster < 3.10
 %endif
 
+%description -n python%{_pythonver}-gluster
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package contains the python modules of GlusterFS and own gluster
+namespace.
+
 %package regression-tests
 Summary:          Development Tools
-Group:            Development/Tools
 Requires:         %{name}%{?_isa} = %{version}-%{release}
 Requires:         %{name}-fuse%{?_isa} = %{version}-%{release}
 Requires:         %{name}-server%{?_isa} = %{version}-%{release}
 ## thin provisioning support
 Requires:         lvm2 >= 2.02.89
 Requires:         perl(App::Prove) perl(Test::Harness) gcc util-linux-ng
-Requires:         python attr dbench file git libacl-devel net-tools
-Requires:         nfs-utils xfsprogs yajl
+Requires:         python%{_pythonver}
+Requires:         attr dbench file git libacl-devel net-tools
+Requires:         nfs-utils xfsprogs yajl psmisc bc
 
 %description regression-tests
 The Gluster Test Framework, is a suite of scripts used for
@@ -493,16 +679,8 @@ regression testing of Gluster.
 %package resource-agents
 Summary:          OCF Resource Agents for GlusterFS
 License:          GPLv3+
-%if ( ! ( 0%{?rhel} && 0%{?rhel} < 6 || 0%{?sles_version} ) )
-# EL5 does not support noarch sub-packages
 BuildArch:        noarch
-%endif
 # this Group handling comes from the Fedora resource-agents package
-%if ( 0%{?fedora} || 0%{?centos_version} || 0%{?rhel} )
-Group:            System Environment/Base
-%else
-Group:            Productivity/Clustering/HA
-%endif
 # for glusterd
 Requires:         %{name}-server = %{version}-%{release}
 # depending on the distribution, we need pacemaker or resource-agents
@@ -510,95 +688,119 @@ Requires:         %{_prefix}/lib/ocf/resource.d
 
 %description resource-agents
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides the resource agents which plug glusterd into
 Open Cluster Framework (OCF) compliant cluster resource managers,
 like Pacemaker.
 %endif
 
+%if ( 0%{!?_without_server:1} )
 %package server
 Summary:          Clustered file-system server
-Group:            System Environment/Daemons
 Requires:         %{name}%{?_isa} = %{version}-%{release}
 Requires:         %{name}-cli%{?_isa} = %{version}-%{release}
-Requires:         %{name}-libs%{?_isa} = %{version}-%{release}
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libgfchangelog0%{?_isa} = %{version}-%{release}
+%if ( 0%{?fedora} && 0%{?fedora} >= 30  || ( 0%{?rhel} && 0%{?rhel} >= 8 ) )
+Requires:         glusterfs-selinux >= 0.1.0-2
+%endif
 # some daemons (like quota) use a fuse-mount, glusterfsd is part of -fuse
 Requires:         %{name}-fuse%{?_isa} = %{version}-%{release}
 # self-heal daemon, rebalance, nfs-server etc. are actually clients
-Requires:         %{name}-api%{?_isa} = %{version}-%{release}
+Requires:         libgfapi0%{?_isa} = %{version}-%{release}
 Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
-# psmisc for killall, lvm2 for snapshot, and nfs-utils and
-# rpcbind/portmap for gnfs server
-Requires:         psmisc
+# lvm2 for snapshot, and nfs-utils and rpcbind/portmap for gnfs server
 Requires:         lvm2
-Requires:         nfs-utils
 %if ( 0%{?_with_systemd:1} )
-Requires(post):   systemd-units, systemd
-Requires(preun):  systemd-units
-Requires(postun): systemd-units
+%{?systemd_requires}
 %else
 Requires(post):   /sbin/chkconfig
 Requires(preun):  /sbin/service
 Requires(preun):  /sbin/chkconfig
 Requires(postun): /sbin/service
 %endif
+%if (0%{?_with_firewalld:1})
+# we install firewalld rules, so we need to have the directory owned
+%if ( 0%{!?rhel} )
+# not on RHEL because firewalld-filesystem appeared in 7.3
+# when EL7 rpm gets weak dependencies we can add a Suggests:
+Requires:         firewalld-filesystem
+%endif
+%endif
 %if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
 Requires:         rpcbind
 %else
 Requires:         portmap
 %endif
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-Obsoletes:        %{name}-geo-replication = %{version}-%{release}
-%endif
 %if ( 0%{?rhel} && 0%{?rhel} <= 6 )
 Requires:         python-argparse
 %endif
+%if ( 0%{?fedora} && 0%{?fedora} > 27 ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+Requires:         python%{_pythonver}-pyxattr
+%else
 Requires:         pyxattr
+%endif
+%if (0%{?_with_valgrind:1})
+Requires:         valgrind
+%endif
 
 %description server
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides the glusterfs server daemon.
+%endif
+
+%package thin-arbiter
+Summary:          GlusterFS thin-arbiter module
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+
+%description thin-arbiter
+This package provides a tie-breaker functionality to GlusterFS
+replicate volume. It includes translators required to provide the
+functionality, and also few other scripts required for getting the setup done.
+
+This package provides the glusterfs thin-arbiter translator.
 
 %package client-xlators
 Summary:          GlusterFS client-side translators
-Group:            Applications/File
 
 %description client-xlators
 GlusterFS is a distributed file-system capable of scaling to several
-petabytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file systems in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in user space and easily manageable.
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
 
 This package provides the translators needed on any GlusterFS client.
 
 %if ( 0%{!?_without_events:1} )
 %package events
 Summary:          GlusterFS Events
-Group:            Applications/File
 Requires:         %{name}-server%{?_isa} = %{version}-%{release}
-Requires:         python python-requests python-prettytable
-Requires:         python-gluster = %{version}-%{release}
-%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+Requires:         python%{_pythonver} python%{_pythonver}-prettytable
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
+Requires:         python-requests
+%else
+Requires:         python%{_pythonver}-requests
+%endif
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
 Requires:         python-argparse
 %endif
 %if ( 0%{?_with_systemd:1} )
-Requires(post):   systemd
+%{?systemd_requires}
 %endif
 
 %description events
@@ -608,12 +810,14 @@ GlusterFS Events
 
 %prep
 %setup -q -n %{name}-%{version}%{?prereltag}
+%if ( ! %{_usepython3} )
+echo "fixing python shebangs..."
+for f in api events extras geo-replication libglusterfs tools xlators; do
+find $f -type f -exec sed -i 's|/usr/bin/python3|/usr/bin/python2|' {} \;
+done
+%endif
 
 %build
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-CFLAGS=-DUSE_INSECURE_OPENSSL
-export CFLAGS
-%endif
 
 # RHEL6 and earlier need to manually replace config.guess and config.sub
 %if ( 0%{?rhel} && 0%{?rhel} <= 6 )
@@ -621,19 +825,23 @@ export CFLAGS
 %endif
 
 %configure \
+        %{?_with_asan} \
         %{?_with_cmocka} \
         %{?_with_debug} \
+        %{?_with_firewalld} \
+        %{?_with_gnfs} \
         %{?_with_tmpfilesdir} \
-        %{?_without_bd} \
+        %{?_with_tsan} \
+        %{?_with_valgrind} \
         %{?_without_epoll} \
+        %{?_without_events} \
         %{?_without_fusermount} \
         %{?_without_georeplication} \
-        %{?_with_firewalld} \
         %{?_without_ocf} \
-        %{?_without_rdma} \
+        %{?_without_server} \
         %{?_without_syslog} \
-        %{?_without_tiering} \
-        %{?_without_events}
+        %{?_with_ipv6default} \
+        %{?_without_libtirpc}
 
 # fix hardening and remove rpath in shlibs
 %if ( 0%{?fedora} && 0%{?fedora} > 17 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
@@ -650,9 +858,7 @@ make check
 %install
 rm -rf %{buildroot}
 make install DESTDIR=%{buildroot}
-# Install include directory
-install -p -m 0644 contrib/uuid/*.h \
-    %{buildroot}%{_includedir}/glusterfs/
+%if ( 0%{!?_without_server:1} )
 %if ( 0%{_for_fedora_koji_builds} )
 install -D -p -m 0644 %{SOURCE1} \
     %{buildroot}%{_sysconfdir}/sysconfig/glusterd
@@ -662,18 +868,12 @@ install -D -p -m 0644 %{SOURCE2} \
 install -D -p -m 0644 extras/glusterd-sysconfig \
     %{buildroot}%{_sysconfdir}/sysconfig/glusterd
 %endif
-
-%if ( 0%{_for_fedora_koji_builds} )
-%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
-install -D -p -m 0755 %{SOURCE6} \
-    %{buildroot}%{_sysconfdir}/sysconfig/modules/glusterfs-fuse.modules
-%endif
 %endif
 
 mkdir -p %{buildroot}%{_localstatedir}/log/glusterd
 mkdir -p %{buildroot}%{_localstatedir}/log/glusterfs
 mkdir -p %{buildroot}%{_localstatedir}/log/glusterfsd
-mkdir -p %{buildroot}%{_localstatedir}/run/gluster
+mkdir -p %{buildroot}%{_rundir}/gluster
 
 # Remove unwanted files from all the shared libraries
 find %{buildroot}%{_libdir} -name '*.a' -delete
@@ -696,33 +896,40 @@ https://forge.gluster.org/glusterfs-core/glusterfs/commits/v%{version}%{?prerelt
 EOM
 
 # Remove benchmarking and other unpackaged files
-%if ( 0%{?rhel} && 0%{?rhel} < 6 )
-rm -rf %{buildroot}/benchmarking
-rm -f %{buildroot}/glusterfs-mode.el
-rm -f %{buildroot}/glusterfs.vim
-%else
 # make install always puts these in %%{_defaultdocdir}/%%{name} so don't
 # use %%{_pkgdocdir}; that will be wrong on later Fedora distributions
 rm -rf %{buildroot}%{_defaultdocdir}/%{name}/benchmarking
 rm -f %{buildroot}%{_defaultdocdir}/%{name}/glusterfs-mode.el
 rm -f %{buildroot}%{_defaultdocdir}/%{name}/glusterfs.vim
-%endif
 
+%if ( 0%{!?_without_server:1} )
 # Create working directory
 mkdir -p %{buildroot}%{_sharedstatedir}/glusterd
 
 # Update configuration file to /var/lib working directory
 sed -i 's|option working-directory /etc/glusterd|option working-directory %{_sharedstatedir}/glusterd|g' \
     %{buildroot}%{_sysconfdir}/glusterfs/glusterd.vol
+%endif
 
 # Install glusterfsd .service or init.d file
+%if ( 0%{!?_without_server:1} )
 %if ( 0%{_for_fedora_koji_builds} )
-%_init_install %{glusterfsd_service} glusterfsd
+%service_install glusterfsd %{glusterfsd_svcfile}
+%endif
 %endif
 
 install -D -p -m 0644 extras/glusterfs-logrotate \
     %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs
 
+# ganesha ghosts
+%if ( 0%{!?_without_server:1} )
+mkdir -p %{buildroot}%{_sysconfdir}/ganesha
+touch %{buildroot}%{_sysconfdir}/ganesha/ganesha-ha.conf
+mkdir -p %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/
+touch %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha.conf
+touch %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha-ha.conf
+%endif
+
 %if ( 0%{!?_without_georeplication:1} )
 # geo-rep ghosts
 mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/geo-replication
@@ -731,6 +938,7 @@ install -D -p -m 0644 extras/glusterfs-georep-logrotate \
     %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs-georep
 %endif
 
+%if ( 0%{!?_without_server:1} )
 # the rest of the ghosts
 touch %{buildroot}%{_sharedstatedir}/glusterd/glusterd.info
 touch %{buildroot}%{_sharedstatedir}/glusterd/options
@@ -749,6 +957,7 @@ mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/snaps
 mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/ss_brick
 touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/nfs-server.vol
 touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+%endif
 
 find ./tests ./run-tests.sh -type f | cpio -pd %{buildroot}%{_prefix}/share/glusterfs
 
@@ -766,36 +975,59 @@ rm -rf %{buildroot}
 /sbin/ldconfig
 %if ( 0%{!?_without_syslog:1} )
 %if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
-%_init_restart rsyslog
+%systemd_postun_with_restart rsyslog
 %endif
 %endif
 exit 0
 
-%post api
-/sbin/ldconfig
+%if ( 0%{!?_without_events:1} )
+%post events
+%systemd_post glustereventsd
+%endif
 
-%if ( 0%{?rhel} == 5 )
-%post fuse
-modprobe fuse
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25 || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%post ganesha
+semanage boolean -m ganesha_use_fusefs --on
 exit 0
 %endif
+%endif
 
 %if ( 0%{!?_without_georeplication:1} )
 %post geo-replication
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+%selinux_set_booleans %{selinuxbooleans}
+%endif
 if [ $1 -ge 1 ]; then
-    %_init_restart glusterd
+    %systemd_postun_with_restart glusterd
 fi
 exit 0
 %endif
 
-%post libs
+%post -n libglusterfs0
+/sbin/ldconfig
+
+%post -n libgfapi0
+/sbin/ldconfig
+
+%post -n libgfchangelog0
+/sbin/ldconfig
+
+%post -n libgfrpc0
 /sbin/ldconfig
 
+%post -n libgfxdr0
+/sbin/ldconfig
+
+%post -n libglusterd0
+/sbin/ldconfig
+
+%if ( 0%{!?_without_server:1} )
 %post server
 # Legacy server
-%_init_enable glusterd
+%systemd_post glusterd
 %if ( 0%{_for_fedora_koji_builds} )
-%_init_enable glusterfsd
+%systemd_post glusterfsd
 %endif
 # ".cmd_log_history" is renamed to "cmd_history.log" in GlusterFS-3.7 .
 # While upgrading glusterfs-server package form GlusterFS version <= 3.6 to
@@ -837,11 +1069,7 @@ if [ -e /etc/ld.so.conf.d/glusterfs.conf ]; then
 fi
 
 %if (0%{?_with_firewalld:1})
-#reload service files if firewalld running
-if $(systemctl is-active firewalld 1>/dev/null 2>&1); then
-  #firewalld-filesystem is not available for rhel7, so command used for reload.
-  firewall-cmd  --reload 1>/dev/null 2>&1
-fi
+    %firewalld_reload
 %endif
 
 pidof -c -o %PPID -x glusterd &> /dev/null
@@ -853,202 +1081,276 @@ if [ $? -eq 0 ]; then
 
     #Cleaning leftover glusterd socket file which is created by glusterd in
     #rpm_script_t context.
-    rm -rf /var/run/glusterd.socket
+    rm -f %{_rundir}/glusterd.socket
 
     # glusterd _was_ running, we killed it, it exited after *.upgrade=on,
     # so start it again
-    %_init_start glusterd
+    %service_start glusterd
 else
     glusterd --xlator-option *.upgrade=on -N
 
     #Cleaning leftover glusterd socket file which is created by glusterd in
     #rpm_script_t context.
-    rm -rf /var/run/glusterd.socket
+    rm -f %{_rundir}/glusterd.socket
 fi
 exit 0
+%endif
+
+##-----------------------------------------------------------------------------
+## All %%pre should be placed here and keep them sorted
+##
+%pre
+getent group gluster > /dev/null || groupadd -r gluster
+getent passwd gluster > /dev/null || useradd -r -g gluster -d %{_rundir}/gluster -s /sbin/nologin -c "GlusterFS daemons" gluster
+exit 0
 
 ##-----------------------------------------------------------------------------
 ## All %%preun should be placed here and keep them sorted
 ##
+%if ( 0%{!?_without_events:1} )
+%preun events
+if [ $1 -eq 0 ]; then
+    if [ -f %glustereventsd_svcfile ]; then
+        %service_stop glustereventsd
+        %systemd_preun glustereventsd
+    fi
+fi
+exit 0
+%endif
+
+%if ( 0%{!?_without_server:1} )
 %preun server
 if [ $1 -eq 0 ]; then
-    if [ -f %_init_glusterfsd ]; then
-        %_init_stop glusterfsd
+    if [ -f %glusterfsd_svcfile ]; then
+        %service_stop glusterfsd
     fi
-    %_init_stop glusterd
-    if [ -f %_init_glusterfsd ]; then
-        %_init_disable glusterfsd
+    %service_stop glusterd
+    if [ -f %glusterfsd_svcfile ]; then
+        %systemd_preun glusterfsd
     fi
-    %_init_disable glusterd
+    %systemd_preun glusterd
 fi
 if [ $1 -ge 1 ]; then
-    if [ -f %_init_glusterfsd ]; then
-        %_init_restart glusterfsd
+    if [ -f %glusterfsd_svcfile ]; then
+        %systemd_postun_with_restart glusterfsd
     fi
-    %_init_restart glusterd
+    %systemd_postun_with_restart glusterd
 fi
 exit 0
+%endif
+
+%preun thin-arbiter
+if [ $1 -eq 0 ]; then
+    if [ -f %glusterta_svcfile ]; then
+        %service_stop gluster-ta-volume
+        %systemd_preun gluster-ta-volume
+    fi
+fi
 
 ##-----------------------------------------------------------------------------
 ## All %%postun should be placed here and keep them sorted
 ##
 %postun
-/sbin/ldconfig
 %if ( 0%{!?_without_syslog:1} )
 %if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
-%_init_restart rsyslog
+%systemd_postun_with_restart rsyslog
 %endif
 %endif
 
-%postun api
-/sbin/ldconfig
-
-%postun events
-%if ( 0%{!?_without_events:1} )
-%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
-%_init_restart glustereventsd
+%if ( 0%{!?_without_server:1} )
+%postun server
+%if (0%{?_with_firewalld:1})
+    %firewalld_reload
 %endif
+exit 0
 %endif
 
-%postun libs
-/sbin/ldconfig
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%postun ganesha
+semanage boolean -m ganesha_use_fusefs --off
+exit 0
+%endif
+%endif
 
-%postun server
-/sbin/ldconfig
-%if (0%{?_with_firewalld:1})
-#reload service files if firewalld running
-if $(systemctl is-active firewalld 1>/dev/null 2>&1); then
-    firewall-cmd  --reload
-fi
+##-----------------------------------------------------------------------------
+## All %%trigger should be placed here and keep them sorted
+##
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%trigger ganesha -- selinux-policy-targeted
+semanage boolean -m ganesha_use_fusefs --on
+exit 0
+%endif
 %endif
+
+##-----------------------------------------------------------------------------
+## All %%triggerun should be placed here and keep them sorted
+##
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%triggerun ganesha -- selinux-policy-targeted
+semanage boolean -m ganesha_use_fusefs --off
 exit 0
+%endif
+%endif
 
 ##-----------------------------------------------------------------------------
 ## All %%files should be placed here and keep them grouped
 ##
 %files
-%doc ChangeLog COPYING-GPLV2 COPYING-LGPLV3 INSTALL README.md THANKS
+%doc ChangeLog COPYING-GPLV2 COPYING-LGPLV3 INSTALL README.md THANKS COMMITMENT
 %{_mandir}/man8/*gluster*.8*
+%if ( 0%{!?_without_server:1} )
 %exclude %{_mandir}/man8/gluster.8*
-%dir %{_localstatedir}/log/glusterfs
-%if ( 0%{!?_without_rdma:1} )
-%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/rdma*
 %endif
+%dir %{_localstatedir}/log/glusterfs
+%if 0%{?!_without_server:1}
+%dir %{_datadir}/glusterfs
 %dir %{_datadir}/glusterfs/scripts
-%{_datadir}/glusterfs/scripts/post-upgrade-script-for-quota.sh
-%{_datadir}/glusterfs/scripts/pre-upgrade-script-for-quota.sh
+     %{_datadir}/glusterfs/scripts/post-upgrade-script-for-quota.sh
+     %{_datadir}/glusterfs/scripts/pre-upgrade-script-for-quota.sh
+%endif
 # xlators that are needed on the client- and on the server-side
+%dir %{_libdir}/glusterfs
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}
 %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/auth
-%{_libdir}/glusterfs/%{version}%{?prereltag}/auth/addr.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/auth/login.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/auth/addr.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/auth/login.so
 %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport
-%{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/socket.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/socket.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
 %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/error-gen.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/io-stats.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/trace.so
-%if ( ! ( 0%{?rhel} && 0%{?rhel} < 6 ) )
-# RHEL-5 based distributions have a too old openssl
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/encryption/crypt.so
-%endif
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/access-control.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/barrier.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/cdc.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changelog.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/fdl.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/gfid-access.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/read-only.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/shard.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-client.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/worm.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/meta.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-cache.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-threads.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/md-cache.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/open-behind.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/quick-read.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/read-ahead.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/readdir-ahead.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/stat-prefetch.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/write-behind.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/system/posix-acl.so
-%dir %{_localstatedir}/run/gluster
-%if 0%{?_tmpfilesdir:1}
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/error-gen.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/delay-gen.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/io-stats.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/sink.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/trace.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/access-control.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/barrier.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/cdc.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changelog.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/utime.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/gfid-access.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/namespace.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/read-only.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/shard.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-client.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/worm.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/cloudsync.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/meta.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-cache.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-threads.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/md-cache.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/open-behind.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/quick-read.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/read-ahead.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/readdir-ahead.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/stat-prefetch.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/write-behind.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/nl-cache.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/system
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/system/posix-acl.so
+%dir %attr(0775,gluster,gluster) %{_rundir}/gluster
+%if 0%{?_tmpfilesdir:1} && 0%{!?_without_server:1}
 %{_tmpfilesdir}/gluster.conf
 %endif
 
-%files api
-%exclude %{_libdir}/*.so
-# libgfapi files
-%{_libdir}/libgfapi.*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api.so
-
-%files api-devel
-%{_libdir}/pkgconfig/glusterfs-api.pc
-%{_libdir}/libgfapi.so
-%{_includedir}/glusterfs/api/*
+%if ( 0%{?_without_server:1} )
+#exclude ganesha related files
+%exclude %{_sysconfdir}/ganesha/ganesha-ha.conf.sample
+%exclude %{_libexecdir}/ganesha/*
+%exclude %{_prefix}/lib/ocf/resource.d/heartbeat/*
+%endif
 
 %files cli
 %{_sbindir}/gluster
 %{_mandir}/man8/gluster.8*
 %{_sysconfdir}/bash_completion.d/gluster
 
-%files devel
+%files cloudsync-plugins
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsyncs3.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsynccvlt.so
+
+%files -n libglusterfs-devel
 %dir %{_includedir}/glusterfs
-%{_includedir}/glusterfs/*
-%exclude %{_includedir}/glusterfs/api
-%exclude %{_libdir}/libgfapi.so
-%{_libdir}/*.so
-# Glupy Translator examples
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/glupy/debug-trace.*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/glupy/helloworld.*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/glupy/negative.*
+     %{_includedir}/glusterfs/*.h
+     %{_includedir}/glusterfs/server/*.h
+%{_libdir}/libglusterfs.so
+
+%files -n libgfapi-devel
+%dir %{_includedir}/glusterfs/api
+     %{_includedir}/glusterfs/api/*.h
+%{_libdir}/libgfapi.so
+%{_libdir}/pkgconfig/glusterfs-api.pc
+
+
+%files -n libgfchangelog-devel
+%dir %{_includedir}/glusterfs/gfchangelog
+     %{_includedir}/glusterfs/gfchangelog/*.h
+%{_libdir}/libgfchangelog.so
 %{_libdir}/pkgconfig/libgfchangelog.pc
-%if ( 0%{!?_without_tiering:1} )
-%{_libdir}/pkgconfig/libgfdb.pc
-%endif
+
+%files -n libgfrpc-devel
+%dir %{_includedir}/glusterfs/rpc
+     %{_includedir}/glusterfs/rpc/*.h
+%{_libdir}/libgfrpc.so
+
+%files -n libgfxdr-devel
+%{_libdir}/libgfxdr.so
 
 %files client-xlators
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster/*.so
-%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster/pump.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/client.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/dht2c.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster/*.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/client.so
 
 %files extra-xlators
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/encryption/rot-13.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/glupy.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/mac-compat.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/prot_client.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/prot_dht.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/prot_server.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quiesce.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/testing/features/template.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/testing/performance/symlink-cache.so
-# Glupy Python files
-%{python_sitelib}/gluster/glupy/*
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quiesce.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/playground
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/playground/template.so
 
 %files fuse
 # glusterfs is a symlink to glusterfsd, -server depends on -fuse.
 %{_sbindir}/glusterfs
 %{_sbindir}/glusterfsd
 %config(noreplace) %{_sysconfdir}/logrotate.d/glusterfs
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/fuse.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/fuse.so
 /sbin/mount.glusterfs
 %if ( 0%{!?_without_fusermount:1} )
 %{_bindir}/fusermount-glusterfs
 %endif
-%if ( 0%{_for_fedora_koji_builds} )
-%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
-%{_sysconfdir}/sysconfig/modules/glusterfs-fuse.modules
-%endif
+
+%if ( 0%{?_with_gnfs:1} && 0%{!?_without_server:1} )
+%files gnfs
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs/server.so
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs/run
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
 %endif
 
-%files ganesha
-%{_sysconfdir}/ganesha/*
-%{_libexecdir}/ganesha/*
-%{_prefix}/lib/ocf/resource.d/heartbeat/*
-%{_sharedstatedir}/glusterd/hooks/1/start/post/S31ganesha-start.sh
-%{_sharedstatedir}/glusterd/hooks/1/reset/post/S31ganesha-reset.sh
+%files thin-arbiter
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/thin-arbiter.so
+%dir %{_datadir}/glusterfs/scripts
+     %{_datadir}/glusterfs/scripts/setup-thin-arbiter.sh
+%config %{_sysconfdir}/glusterfs/thin-arbiter.vol
+
+%if ( 0%{?_with_systemd:1} )
+%{_unitdir}/gluster-ta-volume.service
+%endif
 
 %if ( 0%{!?_without_georeplication:1} )
 %files geo-replication
@@ -1056,15 +1358,25 @@ exit 0
 
 %{_sbindir}/gfind_missing_files
 %{_sbindir}/gluster-mountbroker
-%{_libexecdir}/glusterfs/gsyncd
-%{_libexecdir}/glusterfs/python/syncdaemon/*
-%{_libexecdir}/glusterfs/gverify.sh
-%{_libexecdir}/glusterfs/set_geo_rep_pem_keys.sh
-%{_libexecdir}/glusterfs/peer_gsec_create
-%{_libexecdir}/glusterfs/peer_mountbroker
-%{_libexecdir}/glusterfs/peer_mountbroker.py*
-%{_libexecdir}/glusterfs/gfind_missing_files
-%{_libexecdir}/glusterfs/peer_georep-sshkey.py*
+%dir %{_libexecdir}/glusterfs
+%dir %{_libexecdir}/glusterfs/python
+%dir %{_libexecdir}/glusterfs/python/syncdaemon
+     %{_libexecdir}/glusterfs/gsyncd
+     %{_libexecdir}/glusterfs/python/syncdaemon/*
+%dir %{_libexecdir}/glusterfs/scripts
+     %{_libexecdir}/glusterfs/scripts/get-gfid.sh
+     %{_libexecdir}/glusterfs/scripts/slave-upgrade.sh
+     %{_libexecdir}/glusterfs/scripts/gsync-upgrade.sh
+     %{_libexecdir}/glusterfs/scripts/generate-gfid-file.sh
+     %{_libexecdir}/glusterfs/scripts/gsync-sync-gfid
+     %{_libexecdir}/glusterfs/scripts/schedule_georep.py*
+     %{_libexecdir}/glusterfs/gverify.sh
+     %{_libexecdir}/glusterfs/set_geo_rep_pem_keys.sh
+     %{_libexecdir}/glusterfs/peer_gsec_create
+     %{_libexecdir}/glusterfs/peer_mountbroker
+     %{_libexecdir}/glusterfs/peer_mountbroker.py*
+     %{_libexecdir}/glusterfs/gfind_missing_files
+     %{_libexecdir}/glusterfs/peer_georep-sshkey.py*
 %{_sbindir}/gluster-georep-sshkey
 
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/geo-replication
@@ -1074,38 +1386,61 @@ exit 0
             %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create/post/S56glusterd-geo-rep-create-post.sh
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create/pre
 
-%{_datadir}/glusterfs/scripts/get-gfid.sh
-%{_datadir}/glusterfs/scripts/slave-upgrade.sh
-%{_datadir}/glusterfs/scripts/gsync-upgrade.sh
-%{_datadir}/glusterfs/scripts/generate-gfid-file.sh
-%{_datadir}/glusterfs/scripts/gsync-sync-gfid
-%{_datadir}/glusterfs/scripts/schedule_georep.py*
 %endif
 
-%files libs
-%{_libdir}/*.so.*
-%exclude %{_libdir}/libgfapi.*
-%if ( 0%{!?_without_tiering:1} )
-# libgfdb is only needed server-side
-%exclude %{_libdir}/libgfdb.*
-%endif
+%files -n libglusterfs0
+%{_libdir}/libglusterfs.so.*
+
+%files -n libgfapi0
+%{_libdir}/libgfapi.so.*
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api.so
+
+%files -n libgfchangelog0
+%{_libdir}/libgfchangelog.so.*
+
+%files -n libgfrpc0
+%{_libdir}/libgfrpc.so.*
+
+%files -n libgfxdr0
+%{_libdir}/libgfxdr.so.*
 
-%files -n python-gluster
+%files -n libglusterd0
+%{_libdir}/libglusterd.so.*
+%exclude %{_libdir}/libglusterd.so
+
+%files -n python%{_pythonver}-gluster
 # introducing glusterfs module in site packages.
 # so that all other gluster submodules can reside in the same namespace.
-%{python_sitelib}/gluster/__init__.*
-%{python_sitelib}/gluster/cliutils
-
-%if ( 0%{!?_without_rdma:1} )
-%files rdma
-%{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/rdma*
+%if ( %{_usepython3} )
+%dir %{python3_sitelib}/gluster
+     %{python3_sitelib}/gluster/__init__.*
+     %{python3_sitelib}/gluster/__pycache__
+     %{python3_sitelib}/gluster/cliutils
+%else
+%dir %{python2_sitelib}/gluster
+     %{python2_sitelib}/gluster/__init__.*
+     %{python2_sitelib}/gluster/cliutils
 %endif
 
 %files regression-tests
-%{_prefix}/share/glusterfs/run-tests.sh
-%{_prefix}/share/glusterfs/tests
-%exclude %{_prefix}/share/glusterfs/tests/basic/rpm.t
-%exclude %{_prefix}/share/glusterfs/tests/vagrant
+%dir %{_datadir}/glusterfs
+     %{_datadir}/glusterfs/run-tests.sh
+     %{_datadir}/glusterfs/tests
+%exclude %{_datadir}/glusterfs/tests/vagrant
+
+%if ( 0%{!?_without_server:1} )
+%files ganesha
+%dir %{_libexecdir}/ganesha
+%{_sysconfdir}/ganesha/ganesha-ha.conf.sample
+%{_libexecdir}/ganesha/*
+%{_prefix}/lib/ocf/resource.d/heartbeat/*
+%{_sharedstatedir}/glusterd/hooks/1/start/post/S31ganesha-start.sh
+%ghost      %attr(0644,-,-) %config(noreplace) %{_sysconfdir}/ganesha/ganesha-ha.conf
+%ghost %dir %attr(0755,-,-) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha
+%ghost      %attr(0644,-,-) %config(noreplace) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha.conf
+%ghost      %attr(0644,-,-) %config(noreplace) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha-ha.conf
+%endif
 
 %if ( 0%{!?_without_ocf:1} )
 %files resource-agents
@@ -1113,58 +1448,67 @@ exit 0
 %{_prefix}/lib/ocf/resource.d/glusterfs
 %endif
 
+%if ( 0%{!?_without_server:1} )
 %files server
 %doc extras/clear_xattrs.sh
 # sysconf
 %config(noreplace) %{_sysconfdir}/glusterfs
+%exclude %{_sysconfdir}/glusterfs/thin-arbiter.vol
 %exclude %{_sysconfdir}/glusterfs/eventsconfig.json
+%exclude %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+%exclude %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+%if ( 0%{?_with_gnfs:1} )
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs/*
+%endif
 %config(noreplace) %{_sysconfdir}/sysconfig/glusterd
 %if ( 0%{_for_fedora_koji_builds} )
 %config(noreplace) %{_sysconfdir}/sysconfig/glusterfsd
 %endif
 
 # init files
-%_init_glusterd
+%glusterd_svcfile
 %if ( 0%{_for_fedora_koji_builds} )
-%_init_glusterfsd
+%glusterfsd_svcfile
+%endif
+%if ( 0%{?_with_systemd:1} )
+%glusterfssharedstorage_svcfile
 %endif
 
 # binaries
 %{_sbindir}/glusterd
-%{_sbindir}/glfsheal
+%{_libexecdir}/glusterfs/glfsheal
+%{_sbindir}/gf_attach
+%{_sbindir}/gluster-setgfid2path
 # {_sbindir}/glusterfsd is the actual binary, but glusterfs (client) is a
 # symlink. The binary itself (and symlink) are part of the glusterfs-fuse
 # package, because glusterfs-server depends on that anyway.
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster/pump.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/arbiter.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bit-rot.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bitrot-stub.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/jbrc.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/jbr.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/dht2s.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/posix2-ds.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/experimental/posix2-mds.so
-%{_libdir}/libposix2common.so
-%if ( 0%{!?_without_tiering:1} )
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changetimerecorder.so
-%endif
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/index.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/locks.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-server.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/marker.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quota*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/trash.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/upcall.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/leases.so
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage*
-%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/decompounder.so
-%if ( 0%{!?_without_tiering:1} )
-%{_libdir}/libgfdb.so.*
-%endif
+
+# Manpages
+%{_mandir}/man8/gluster-setgfid2path.8*
+
+# xlators
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/arbiter.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bit-rot.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bitrot-stub.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/sdfs.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/index.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/locks.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix*
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-server.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/marker.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quota*
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/selinux.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/trash.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/upcall.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/leases.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt/glusterd.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage/posix.so
 
 # snap_scheduler
 %{_sbindir}/snap_scheduler.py
@@ -1177,6 +1521,12 @@ exit 0
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/bitd
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/groups
             %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/virt
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/metadata-cache
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/gluster-block
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/nl-cache
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/db-workload
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/distributed-virt
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/samba
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glusterfind
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glusterfind/.keys
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glustershd
@@ -1185,10 +1535,13 @@ exit 0
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post
             %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/disabled-quota-root-xattr-heal.sh
-            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/S10selinux-label-brick.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/S13create-subdir-mounts.sh
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre
-%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create
-%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post/S10selinux-label-brick.sh
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/pre
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/copy-file
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/copy-file/post
@@ -1196,7 +1549,8 @@ exit 0
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/post
                             %{_sharedstatedir}/glusterd/hooks/1/delete/post/S57glusterfind-delete-post
-%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre/S10selinux-del-fcontext.sh
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/post
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/pre
@@ -1218,11 +1572,7 @@ exit 0
        %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre
             %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre/S30samba-stop.sh
             %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre/S29CTDB-teardown.sh
-%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs
-%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
-%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs/run
-%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
-%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/options
+%config(noreplace) %ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/options
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/peers
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/quotad
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/scrub
@@ -1231,27 +1581,35 @@ exit 0
 %ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/vols
 
 # Extra utility script
-%{_datadir}/glusterfs/scripts/stop-all-gluster-processes.sh
+%dir %{_libexecdir}/glusterfs
+%dir %{_datadir}/glusterfs/scripts
+     %{_datadir}/glusterfs/scripts/stop-all-gluster-processes.sh
+%if ( 0%{?_with_systemd:1} )
+     %{_libexecdir}/glusterfs/mount-shared-storage.sh
+     %{_datadir}/glusterfs/scripts/control-cpu-load.sh
+     %{_datadir}/glusterfs/scripts/control-mem.sh
+%endif
 
 # Incrementalapi
-%{_libexecdir}/glusterfs/glusterfind
+     %{_libexecdir}/glusterfs/glusterfind
 %{_bindir}/glusterfind
-%{_libexecdir}/glusterfs/peer_add_secret_pub
+     %{_libexecdir}/glusterfs/peer_add_secret_pub
 
 %if ( 0%{?_with_firewalld:1} )
 %{_prefix}/lib/firewalld/services/glusterfs.xml
 %endif
-
-%{_sbindir}/gf_logdump
-%{_sbindir}/gf_recon
+# end of server files
+%endif
 
 # Events
 %if ( 0%{!?_without_events:1} )
 %files events
 %config(noreplace) %{_sysconfdir}/glusterfs/eventsconfig.json
-%dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/events
-%{_libexecdir}/glusterfs/events
-%{_libexecdir}/glusterfs/peer_eventsapi.py*
+%dir %{_sharedstatedir}/glusterd
+%dir %{_sharedstatedir}/glusterd/events
+%dir %{_libexecdir}/glusterfs
+     %{_libexecdir}/glusterfs/gfevents
+     %{_libexecdir}/glusterfs/peer_eventsapi.py*
 %{_sbindir}/glustereventsd
 %{_sbindir}/gluster-eventsapi
 %{_datadir}/glusterfs/scripts/eventsdash.py*
@@ -1263,6 +1621,130 @@ exit 0
 %endif
 
 %changelog
+* Thu May 14 2020 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- refactor, common practice, Issue #1126
+
+* Mon May 11 2020 Sunny Kumar <sunkumar@redhat.com>
+- added requires policycoreutils-python-utils on rhel8 for geo-replication
+
+* Wed Oct 9 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- remove leftover bd xlator cruft
+
+* Fri Aug 23 2019 Shwetha K Acharya <sacharya@redhat.com>
+- removed {name}-ufs from Obsoletes
+- added  "< version" for obsoletes {name}-gnfs and {name}-rdma
+
+* Mon Jul 15 2019 Jiffin Tony Thottan <jthottan@redhat.com>
+- Adding ganesha ha bits back in gluster repository
+
+* Fri Jul 12 2019 Amar Tumballi <amarts@redhat.com>
+- Remove rdma package, and mark older rdma package as 'Obsoletes'
+
+* Fri Jun 14 2019 Niels de Vos <ndevos@redhat.com>
+- always build glusterfs-cli to allow monitoring/managing from clients
+
+* Wed Mar 6 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- remove unneeded ldconfig in scriptlets
+-  reported by Igor Gnatenko in Fedora
+-   https://src.fedoraproject.org/rpms/glusterfs/pull-request/5
+
+* Mon Mar 4 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- s390x has RDMA, since around Fedora 27 and in RHEL7 since June 2016.
+
+* Tue Feb 26 2019 Ashish Pandey <aspandey@redhat.com>
+- Add thin-arbiter package
+
+* Sun Feb 24 2019 Aravinda VK <avishwan@redhat.com>
+- Renamed events package to gfevents
+
+* Thu Feb 21 2019 Jiffin Tony Thottan <jthottan@redhat.com>
+- Obsoleting gluster-gnfs package
+
+* Wed Nov 28 2018 Krutika Dhananjay <kdhananj@redhat.com>
+- Install /var/lib/glusterd/groups/distributed-virt by default
+
+* Tue Nov 13 2018 Niels de Vos <ndevos@redhat.com>
+- Add an option to build with ThreadSanitizer (TSAN)
+
+* Fri Sep 7 2018 Niels de Vos <ndevos@redhat.com>
+- Add an option to build with address sanitizer (ASAN)
+
+* Sun Jul 29 2018 Niels de Vos <ndevos@redhat.com>
+- Disable building glusterfs-resource-agents on el6 (#1609551)
+
+* Thu Feb 22 2018 Kotresh HR <khiremat@redhat.com>
+- Added util-linux as dependency to georeplication rpm (#1544382)
+
+* Thu Feb 1 2018 Niels de Vos <ndevos@redhat.com>
+- Add '--without server' option to facilitate el6 builds (#1074947)
+
+* Wed Jan 24 2018 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python-ctypes no long exists, now in python stdlib (#1538258)
+
+* Thu Jan 18 2018 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Fedora 28 glibc has removed rpc headers and rpcgen, use libtirpc
+
+* Mon Dec 25 2017 Niels de Vos <ndevos@redhat.com>
+- Fedora 28 has renamed pyxattr
+
+* Wed Sep 27 2017 Mohit Agrawal <moagrawa@redhat.com>
+- Added control-cpu-load.sh and control-mem.sh scripts to glusterfs-server section(#1496335)
+
+* Tue Aug 22 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- libibverbs-devel, librdmacm-devel -> rdma-core-devel #1483995
+
+* Thu Jul 20 2017 Aravinda VK <avishwan@redhat.com>
+- Added new tool/binary to set the gfid2path xattr on files
+
+* Thu Jul 13 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- various directories not owned by any package
+
+* Fri Jun 16 2017 Jiffin Tony Thottan <jthottan@redhat.com>
+- Add glusterfssharedstorage.service systemd file
+
+* Fri Jun 9 2017 Poornima G <pgurusid@redhat.com>
+- Install /var/lib/glusterd/groups/nl-cache by default
+
+* Wed May 10 2017 Pranith Kumar K <pkarampu@redhat.com>
+- Install /var/lib/glusterd/groups/gluster-block by default
+
+* Thu Apr 27 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- gnfs in an optional subpackage
+
+* Wed Apr 26 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- /var/run/gluster owner gluster:gluster(0775) for qemu(gfapi)
+  statedumps (#1445569)
+
+* Mon Apr 24 2017 Jiffin Tony Thottan <jhottan@redhat.com>
+- Install SELinux hook scripts that manage contexts for bricks (#1047975)
+
+* Thu Apr 20 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- firewalld-filesystem -> firewalld (#1443959)
+
+* Thu Apr 13 2017 Niels de Vos <ndevos@redhat.com>
+- the -regression-tests sub-package needs "bc" for some tests (#1442145)
+
+* Mon Mar 20 2017 Niels de Vos <ndevos@redhat.com>
+- Drop dependency on psmisc, pkill is used instead of killall (#1197308)
+
+* Thu Feb 16 2017 Niels de Vos <ndevos@redhat.com>
+- Obsolete and Provide python-gluster for upgrading from glusterfs < 3.10
+
+* Wed Feb 1 2017 Poornima G <pgurusid@redhat.com>
+- Install /var/lib/glusterd/groups/metadata-cache by default
+
+* Wed Jan 18 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python2 (versus python3) cleanup (#1414902)
+
+* Fri Jan 13 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- switch to storhaug HA
+
+* Fri Jan 6 2017 Niels de Vos <ndevos@redhat.com>
+- use macro provided by firewalld-filesystem to reload firewalld
+
+* Thu Nov 24 2016 Jiffin Tony Thottan <jhottan@redhat.com>
+- remove S31ganesha-reset.sh from hooks (#1397795)
+
 * Thu Sep 22 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
 - python-ctypes no long exists, now in python stdlib (#1378436)
 
diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am
index e8a3f99b7fa..a0a778158d8 100644
--- a/glusterfsd/src/Makefile.am
+++ b/glusterfsd/src/Makefile.am
@@ -1,21 +1,34 @@
 sbin_PROGRAMS = glusterfsd
+if WITH_SERVER
+sbin_PROGRAMS += glusterfsd gf_attach
+endif
 
 glusterfsd_SOURCES = glusterfsd.c glusterfsd-mgmt.c
 glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la ${GF_LDADD}
-
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la $(GF_LDADD) $(LIB_DL)
 glusterfsd_LDFLAGS = $(GF_LDFLAGS)
+
+gf_attach_SOURCES = gf_attach.c
+gf_attach_LDADD   = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+		    $(top_builddir)/api/src/libgfapi.la \
+		    $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		    $(top_builddir)/rpc/xdr/src/libgfxdr.la
+gf_attach_LDFLAGS = $(GF_LDFLAGS)
+
 noinst_HEADERS = glusterfsd.h glusterfsd-mem-types.h glusterfsd-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) \
 	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-DLIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\"\
 	-I$(top_srcdir)/rpc/rpc-lib/src \
 	-I$(top_srcdir)/rpc/xdr/src \
 	-I$(top_builddir)/rpc/xdr/src \
-	-I$(top_srcdir)/xlators/nfs/server/src
+	-I$(top_srcdir)/xlators/nfs/server/src \
+	-I$(top_srcdir)/xlators/protocol/server/src \
+	-I$(top_srcdir)/api/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
@@ -34,6 +47,8 @@ install-data-local:
 	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/log/glusterfs
 	$(INSTALL) -d -m 755 $(DESTDIR)$(sbindir)
 	rm -f $(DESTDIR)$(sbindir)/glusterfs
-	rm -f $(DESTDIR)$(sbindir)/glusterd
 	ln -s glusterfsd $(DESTDIR)$(sbindir)/glusterfs
+if WITH_SERVER
+	rm -f $(DESTDIR)$(sbindir)/glusterd
 	ln -s glusterfsd $(DESTDIR)$(sbindir)/glusterd
+endif
diff --git a/glusterfsd/src/gf_attach.c b/glusterfsd/src/gf_attach.c
new file mode 100644
index 00000000000..c553b0b1f61
--- /dev/null
+++ b/glusterfsd/src/gf_attach.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <glusterfs/glusterfs.h>
+#include "glfs-internal.h"
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+
+/* In seconds */
+#define CONNECT_TIMEOUT 60
+#define REPLY_TIMEOUT 120
+
+int done = 0;
+int rpc_status;
+
+pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+
+struct rpc_clnt_procedure gf_attach_actors[GLUSTERD_BRICK_MAXVALUE] = {
+    [GLUSTERD_BRICK_NULL] = {"NULL", NULL},
+    [GLUSTERD_BRICK_OP] = {"BRICK_OP", NULL},
+};
+
+struct rpc_clnt_program gf_attach_prog = {
+    .progname = "brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .proctable = gf_attach_actors,
+    .numproc = GLUSTERD_BRICK_MAXVALUE,
+};
+
+int32_t
+my_callback(struct rpc_req *req, struct iovec *iov, int count, void *frame)
+{
+    pthread_mutex_lock(&mutex);
+    rpc_status = req->rpc_status;
+    done = 1;
+    /* Signal main thread which is the only waiter */
+    pthread_cond_signal(&cond);
+    pthread_mutex_unlock(&mutex);
+    return 0;
+}
+
+/* copied from gd_syncop_submit_request */
+int
+send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op)
+{
+    int ret = -1;
+    struct timespec ts;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
+    gd1_mgmt_brick_op_req brick_req;
+    void *req = &brick_req;
+
+    brick_req.op = op;
+    brick_req.name = path;
+    brick_req.input.input_val = NULL;
+    brick_req.input.input_len = 0;
+    brick_req.dict.dict_val = NULL;
+    brick_req.dict.dict_len = 0;
+
+    req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf)
+        goto out;
+
+    iobref = iobref_new();
+    if (!iobref)
+        goto out;
+
+    iobref_add(iobref, iobuf);
+
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret == -1)
+        goto out;
+
+    iov.iov_len = ret;
+
+    /* Wait for connection */
+    timespec_now_realtime(&ts);
+    ts.tv_sec += CONNECT_TIMEOUT;
+    pthread_mutex_lock(&rpc->conn.lock);
+    {
+        while (!rpc->conn.connected)
+            if (pthread_cond_timedwait(&rpc->conn.cond, &rpc->conn.lock, &ts) ==
+                ETIMEDOUT) {
+                fprintf(stderr, "timeout waiting for RPC connection\n");
+                pthread_mutex_unlock(&rpc->conn.lock);
+                return EXIT_FAILURE;
+            }
+    }
+    pthread_mutex_unlock(&rpc->conn.lock);
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, &gf_attach_prog, op, my_callback, &iov, 1, NULL,
+                          0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    if (!ret) {
+        /* OK, wait for callback */
+        timespec_now_realtime(&ts);
+        ts.tv_sec += REPLY_TIMEOUT;
+        pthread_mutex_lock(&mutex);
+        {
+            while (!done)
+                if (pthread_cond_timedwait(&cond, &mutex, &ts) == ETIMEDOUT) {
+                    fprintf(stderr, "timeout waiting for RPC reply\n");
+                    pthread_mutex_unlock(&mutex);
+                    return EXIT_FAILURE;
+                }
+        }
+        pthread_mutex_unlock(&mutex);
+    }
+
+out:
+
+    iobref_unref(iobref);
+    iobuf_unref(iobuf);
+    if (frame)
+        STACK_DESTROY(frame->root);
+
+    if (rpc_status != 0) {
+        fprintf(stderr, "got error %d on RPC\n", rpc_status);
+        return EXIT_FAILURE;
+    }
+
+    printf("OK\n");
+    return EXIT_SUCCESS;
+}
+
+int
+usage(char *prog)
+{
+    fprintf(stderr, "Usage: %s uds_path volfile_path (to attach)\n", prog);
+    fprintf(stderr, "       %s -d uds_path brick_path (to detach)\n", prog);
+
+    return EXIT_FAILURE;
+}
+
+int
+main(int argc, char *argv[])
+{
+    glfs_t *fs;
+    struct rpc_clnt *rpc;
+    dict_t *options;
+    int ret;
+    int op = GLUSTERD_BRICK_ATTACH;
+
+    for (;;) {
+        switch (getopt(argc, argv, "d")) {
+            case 'd':
+                op = GLUSTERD_BRICK_TERMINATE;
+                break;
+            case -1:
+                goto done_parsing;
+            default:
+                return usage(argv[0]);
+        }
+    }
+done_parsing:
+    if (optind != (argc - 2)) {
+        return usage(argv[0]);
+    }
+
+    fs = glfs_new("gf-attach");
+    if (!fs) {
+        fprintf(stderr, "glfs_new failed\n");
+        return EXIT_FAILURE;
+    }
+
+    (void)glfs_set_logging(fs, "/dev/stderr", 7);
+    /*
+     * This will actually fail because we haven't defined a volume, but
+     * it will do enough initialization to get us going.
+     */
+    (void)glfs_init(fs);
+
+    options = dict_new();
+    if (!options) {
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret != 0) {
+        fprintf(stderr, "failed to set transport type\n");
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport.address-family", "unix");
+    if (ret != 0) {
+        fprintf(stderr, "failed to set address family\n");
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport.socket.connect-path", argv[optind]);
+    if (ret != 0) {
+        fprintf(stderr, "failed to set connect path\n");
+        return EXIT_FAILURE;
+    }
+
+    rpc = rpc_clnt_new(options, fs->ctx->master, "gf-attach-rpc", 0);
+    if (!rpc) {
+        fprintf(stderr, "rpc_clnt_new failed\n");
+        return EXIT_FAILURE;
+    }
+
+    if (rpc_clnt_register_notify(rpc, NULL, NULL) != 0) {
+        fprintf(stderr, "rpc_clnt_register_notify failed\n");
+        return EXIT_FAILURE;
+    }
+
+    if (rpc_clnt_start(rpc) != 0) {
+        fprintf(stderr, "rpc_clnt_start failed\n");
+        return EXIT_FAILURE;
+    }
+
+    return send_brick_req(fs->ctx->master, rpc, argv[optind + 1], op);
+}
diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 7135c0ada9e..e59b558deb0 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -10,18 +10,18 @@
 #ifndef __GLUSTERFSD_MEM_TYPES_H__
 #define __GLUSTERFSD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum gfd_mem_types_ {
-        gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
-        gfd_mt_xlator_t,
-        gfd_mt_server_cmdline_t,
-        gfd_mt_xlator_cmdline_option_t,
-        gfd_mt_char,
-        gfd_mt_call_pool_t,
-        gfd_mt_end
+    gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
+    gfd_mt_xlator_t,
+    gfd_mt_server_cmdline_t,
+    gfd_mt_xlator_cmdline_option_t,
+    gfd_mt_char,
+    gfd_mt_call_pool_t,
+    gfd_mt_end
 
 };
 #endif
diff --git a/glusterfsd/src/glusterfsd-messages.h b/glusterfsd/src/glusterfsd-messages.h
index e9c28f71263..0cdbffa71ea 100644
--- a/glusterfsd/src/glusterfsd-messages.h
+++ b/glusterfsd/src/glusterfsd-messages.h
@@ -11,107 +11,83 @@
 #ifndef _GLUSTERFSD_MESSAGES_H_
 #define _GLUSTERFSD_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
 
-#define GLFS_COMP_BASE          GLFS_MSGID_COMP_GLUSTERFSD
-#define GLFS_NUM_MESSAGES       37
-#define GLFS_MSGID_END          (GLFS_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-#define glusterfsd_msg_1 (GLFS_COMP_BASE + 1), "Could not create absolute" \
-                        " mountpoint path"
-#define glusterfsd_msg_2 (GLFS_COMP_BASE + 2), "Could not get current " \
-                        "working directory"
-#define glusterfsd_msg_3 (GLFS_COMP_BASE + 3), "failed to set mount-point" \
-                        " to options dictionary"
-#define glusterfsd_msg_4 (GLFS_COMP_BASE + 4), "failed to set dict value" \
-                        " for key %s"
-#define glusterfsd_msg_5 (GLFS_COMP_BASE + 5), "failed to set 'disable'" \
-                        " for key %s"
-#define glusterfsd_msg_6 (GLFS_COMP_BASE + 6), "failed to set 'enable'" \
-                        " for key %s"
-#define glusterfsd_msg_7 (GLFS_COMP_BASE + 7), "Not a client process, not" \
-                        " performing mount operation"
-#define glusterfsd_msg_8 (GLFS_COMP_BASE + 8), "MOUNT-POINT %s" \
-                        " initialization failed"
-#define glusterfsd_msg_9 (GLFS_COMP_BASE + 9), "loading volume file %s" \
-                        " failed"
-#define glusterfsd_msg_10 (GLFS_COMP_BASE + 10), "xlator option %s is" \
-                        " invalid"
-#define glusterfsd_msg_11 (GLFS_COMP_BASE + 11), "Fetching the volume" \
-                        " file from server..."
-#define glusterfsd_msg_12 (GLFS_COMP_BASE + 12), "volume initialization" \
-                        " failed."
-#define glusterfsd_msg_13 (GLFS_COMP_BASE + 13), "ERROR: glusterfs uuid" \
-                        " generation failed"
-#define glusterfsd_msg_14 (GLFS_COMP_BASE + 14), "ERROR: glusterfs %s" \
-                        " pool creation failed"
-#define glusterfsd_msg_15 (GLFS_COMP_BASE + 15), "ERROR: '--volfile-id' is" \
-                        " mandatory if '-s' OR '--volfile-server'" \
-                        " option is given"
-#define glusterfsd_msg_16 (GLFS_COMP_BASE + 16), "ERROR: parsing the" \
-                        " volfile failed"
-#define glusterfsd_msg_17 (GLFS_COMP_BASE + 17), "pidfile %s open failed"
-#define glusterfsd_msg_18 (GLFS_COMP_BASE + 18), "pidfile %s lock failed"
-#define glusterfsd_msg_19 (GLFS_COMP_BASE + 19), "pidfile %s unlock failed"
-#define glusterfsd_msg_20 (GLFS_COMP_BASE + 20), "pidfile %s truncation" \
-                        " failed"
-#define glusterfsd_msg_21 (GLFS_COMP_BASE + 21), "pidfile %s write failed"
-#define glusterfsd_msg_22 (GLFS_COMP_BASE + 22), "failed to execute" \
-                        " pthread_sigmask"
-#define glusterfsd_msg_23 (GLFS_COMP_BASE + 23), "failed to create pthread"
-#define glusterfsd_msg_24 (GLFS_COMP_BASE + 24), "daemonization failed"
-#define glusterfsd_msg_25 (GLFS_COMP_BASE + 25), "mount failed"
-#define glusterfsd_msg_26 (GLFS_COMP_BASE + 26), "failed to construct" \
-                        " the graph"
-#define glusterfsd_msg_27 (GLFS_COMP_BASE + 27), "fuse xlator cannot be" \
-                        " specified in volume file"
-#define glusterfsd_msg_28 (GLFS_COMP_BASE + 28), "Cannot reach volume" \
-                        " specification file"
-#define glusterfsd_msg_29 (GLFS_COMP_BASE + 29), "ERROR: glusterfs context" \
-                        " not initialized"
-#define glusterfsd_msg_30 (GLFS_COMP_BASE + 30), "Started running %s" \
-                        " version %s (args: %s)"
-#define glusterfsd_msg_31 (GLFS_COMP_BASE + 31), "Could not create new" \
-                        " sync-environment"
-#define glusterfsd_msg_32 (GLFS_COMP_BASE + 32), "received signum (%d)," \
-                        " shutting down"
-#define glusterfsd_msg_33 (GLFS_COMP_BASE + 33), "obsolete option " \
-                        "'--volfile-max-fetch-attempts or fetch-attempts' " \
-                        "was provided"
-#define glusterfsd_msg_34 (GLFS_COMP_BASE + 34), "memory accounting init" \
-                        " failed."
-#define glusterfsd_msg_35 (GLFS_COMP_BASE + 35), "rpc req buffer " \
-                        " unserialization failed."
-#define glusterfsd_msg_36 (GLFS_COMP_BASE + 36), "problem in xlator " \
-                        " loading."
-#define glusterfsd_msg_37 (GLFS_COMP_BASE + 37), "failed to get dict value"
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(
+    GLUSTERFSD, glusterfsd_msg_1, glusterfsd_msg_2, glusterfsd_msg_3,
+    glusterfsd_msg_4, glusterfsd_msg_5, glusterfsd_msg_6, glusterfsd_msg_7,
+    glusterfsd_msg_8, glusterfsd_msg_9, glusterfsd_msg_10, glusterfsd_msg_11,
+    glusterfsd_msg_12, glusterfsd_msg_13, glusterfsd_msg_14, glusterfsd_msg_15,
+    glusterfsd_msg_16, glusterfsd_msg_17, glusterfsd_msg_18, glusterfsd_msg_19,
+    glusterfsd_msg_20, glusterfsd_msg_21, glusterfsd_msg_22, glusterfsd_msg_23,
+    glusterfsd_msg_24, glusterfsd_msg_25, glusterfsd_msg_26, glusterfsd_msg_27,
+    glusterfsd_msg_28, glusterfsd_msg_29, glusterfsd_msg_30, glusterfsd_msg_31,
+    glusterfsd_msg_32, glusterfsd_msg_33, glusterfsd_msg_34, glusterfsd_msg_35,
+    glusterfsd_msg_36, glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39,
+    glusterfsd_msg_40, glusterfsd_msg_41, glusterfsd_msg_42, glusterfsd_msg_43,
+    glusterfsd_msg_029, glusterfsd_msg_041, glusterfsd_msg_042);
 
+#define glusterfsd_msg_1_STR "Could not create absolute mountpoint path"
+#define glusterfsd_msg_2_STR "Could not get current working directory"
+#define glusterfsd_msg_4_STR "failed to set mount-point to options dictionary"
+#define glusterfsd_msg_3_STR "failed to set dict value for key"
+#define glusterfsd_msg_5_STR "failed to set disable for key"
+#define glusterfsd_msg_6_STR "failed to set enable for key"
+#define glusterfsd_msg_7_STR                                                   \
+    "Not a client process, not performing mount operation"
+#define glusterfsd_msg_8_STR "MOUNT_POINT initialization failed"
+#define glusterfsd_msg_9_STR "loading volume file failed"
+#define glusterfsd_msg_10_STR "xlator option is invalid"
+#define glusterfsd_msg_11_STR "Fetching the volume file from server..."
+#define glusterfsd_msg_12_STR "volume initialization failed"
+#define glusterfsd_msg_34_STR "memory init failed"
+#define glusterfsd_msg_13_STR "ERROR: glusterfs uuid generation failed"
+#define glusterfsd_msg_14_STR "ERROR: glusterfs pool creation failed"
+#define glusterfsd_msg_15_STR                                                  \
+    "ERROR: '--volfile-id' is mandatory if '-s' OR '--volfile-server' option " \
+    "is given"
+#define glusterfsd_msg_16_STR "ERROR: parsing the volfile failed"
+#define glusterfsd_msg_33_STR                                                  \
+    "obsolete option '--volfile-max-fecth-attempts or fetch-attempts' was "    \
+    "provided"
+#define glusterfsd_msg_17_STR "pidfile open failed"
+#define glusterfsd_msg_18_STR "pidfile lock failed"
+#define glusterfsd_msg_20_STR "pidfile truncation failed"
+#define glusterfsd_msg_21_STR "pidfile write failed"
+#define glusterfsd_msg_22_STR "failed to exeute pthread_sigmask"
+#define glusterfsd_msg_23_STR "failed to create pthread"
+#define glusterfsd_msg_24_STR "daemonization failed"
+#define glusterfsd_msg_25_STR "mount failed"
+#define glusterfsd_msg_26_STR "failed to construct the graph"
+#define glusterfsd_msg_27_STR "fuse xlator cannot be specified in volume file"
+#define glusterfsd_msg_28_STR "Cannot reach volume specification file"
+#define glusterfsd_msg_29_STR "ERROR: glusterfsd context not initialized"
+#define glusterfsd_msg_43_STR                                                  \
+    "command line argument --brick-mux is valid only for brick process"
+#define glusterfsd_msg_029_STR "failed to create command line string"
+#define glusterfsd_msg_30_STR "Started running version"
+#define glusterfsd_msg_31_STR "Could not create new sync-environment"
+#define glusterfsd_msg_40_STR "No change in volfile, countinuing"
+#define glusterfsd_msg_39_STR "Unable to create/delete temporary file"
+#define glusterfsd_msg_38_STR                                                  \
+    "Not processing brick-op since volume graph is not yet active"
+#define glusterfsd_msg_35_STR "rpc req buffer unserialization failed"
+#define glusterfsd_msg_36_STR "problem in xlator loading"
+#define glusterfsd_msg_37_STR "failed to get dict value"
+#define glusterfsd_msg_41_STR "received attach request for volfile"
+#define glusterfsd_msg_42_STR "failed to unserialize xdata to dictionary"
+#define glusterfsd_msg_041_STR "can't detach. flie not found"
+#define glusterfsd_msg_042_STR                                                 \
+    "couldnot detach old graph. Aborting the reconfiguration operation"
 
 #endif /* !_GLUSTERFSD_MESSAGES_H_ */
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 00da4b0df19..eaf6796e4c3 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -13,11 +13,10 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/defaults.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
@@ -29,2390 +28,3028 @@
 #include "glusterfsd.h"
 #include "rpcsvc.h"
 #include "cli1-xdr.h"
-#include "statedump.h"
-#include "syncop.h"
-#include "xlator.h"
-#include "syscall.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/monitoring.h>
+#include "server.h"
 
 static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false;
 int need_emancipate = 0;
 
-int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
-int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
-int glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp);
-int glusterfs_graph_unknown_options (glusterfs_graph_t *graph);
-int emancipate(glusterfs_ctx_t *ctx, int ret);
+int
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx);
+int
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx);
+int
+glusterfs_process_volfp(glusterfs_ctx_t *ctx, FILE *fp);
+int
+emancipate(glusterfs_ctx_t *ctx, int ret);
+int
+glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp,
+                                   char *volfile_id, char *checksum,
+                                   dict_t *dict);
+int
+glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx,
+                                  gf_volfile_t *volfile_obj, char *checksum,
+                                  dict_t *dict);
+int
+glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp,
+                                   char *volfile_id, char *checksum,
+                                   dict_t *dict);
+int
+glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj);
+
+gf_boolean_t
+mgmt_is_multiplexed_daemon(char *name);
+
+static int
+glusterfs_volume_top_perf(const char *brick_path, dict_t *dict,
+                          gf_boolean_t write_test);
 
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfsd_ctx;
-        gf_log ("mgmt", GF_LOG_INFO, "Volume file changed");
+    ctx = glusterfsd_ctx;
+    gf_log("mgmt", GF_LOG_INFO, "Volume file changed");
 
-        glusterfs_volfile_fetch (ctx);
-        return 0;
+    glusterfs_volfile_fetch(ctx);
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_process_volfile(const char *volfile, ssize_t size, char *volfile_id,
+                     dict_t *dict)
 {
-        return 0;
-}
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    FILE *tmpfp = NULL;
+    gf_volfile_t *volfile_obj = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+    char sha256_hash[SHA256_DIGEST_LENGTH] = {
+        0,
+    };
+    int tmp_fd = -1;
+    char template[] = "/tmp/glfs.volfile.XXXXXX";
+
+    glusterfs_compute_sha256((const unsigned char *)volfile, size, sha256_hash);
+    ctx = THIS->ctx;
+    LOCK(&ctx->volfile_lock);
+    {
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(volfile_id, volfile_obj->vol_id)) {
+                if (!memcmp(sha256_hash, volfile_obj->volfile_checksum,
+                            sizeof(volfile_obj->volfile_checksum))) {
+                    UNLOCK(&ctx->volfile_lock);
+                    gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_40,
+                            NULL);
+                    goto out;
+                }
+                volfile_tmp = volfile_obj;
+                break;
+            }
+        }
 
-struct iobuf *
-glusterfs_serialize_reply (rpcsvc_request_t *req, void *arg,
-                           struct iovec *outmsg, xdrproc_t xdrproc)
-{
-        struct iobuf            *iob = NULL;
-        ssize_t                  retlen = -1;
-        ssize_t                  xdr_size = 0;
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode */
+        tmp_fd = mkstemp(template);
+        if (-1 == tmp_fd) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_39,
+                    "create template=%s", template, NULL);
+            ret = -1;
+            goto out;
+        }
 
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
+        /* Calling unlink so that when the file is closed or program
+         * terminates the temporary file is deleted.
          */
-        xdr_size = xdr_sizeof (xdrproc, arg);
-        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, xdr_size);
-        if (!iob) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
+        ret = sys_unlink(template);
+        if (ret < 0) {
+            gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_39,
+                    "delete template=%s", template, NULL);
+            ret = 0;
         }
 
-        iobuf_to_iovec (iob, outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        /* retlen is used to received the error since size_t is unsigned and we
-         * need -1 for error notification during encoding.
-         */
-        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-        if (retlen == -1) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
+        tmpfp = fdopen(tmp_fd, "w+b");
+        if (!tmpfp) {
+            ret = -1;
+            goto unlock;
         }
 
-        outmsg->iov_len = retlen;
-ret:
-        if (retlen == -1) {
-                iob = NULL;
+        fwrite(volfile, size, 1, tmpfp);
+        fflush(tmpfp);
+        if (ferror(tmpfp)) {
+            ret = -1;
+            goto unlock;
         }
 
-        return iob;
+        if (!volfile_tmp) {
+            /* There is no checksum in the list, which means simple attach
+             * the volfile
+             */
+            ret = glusterfs_process_svc_attach_volfp(ctx, tmpfp, volfile_id,
+                                                     sha256_hash, dict);
+            goto unlock;
+        }
+        ret = glusterfs_mux_volfile_reconfigure(tmpfp, ctx, volfile_obj,
+                                                sha256_hash, dict);
+        if (ret < 0) {
+            gf_msg_debug("glusterfsd-mgmt", EINVAL, "Reconfigure failed !!");
+        }
+    }
+unlock:
+    UNLOCK(&ctx->volfile_lock);
+out:
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
+    return ret;
 }
 
 int
-glusterfs_submit_reply (rpcsvc_request_t *req, void *arg,
-                        struct iovec *payload, int payloadcount,
-                        struct iobref *iobref, xdrproc_t xdrproc)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        struct iobuf           *iob        = NULL;
-        int                     ret        = -1;
-        struct iovec            rsp        = {0,};
-        char                    new_iobref = 0;
+    return 0;
+}
 
-        if (!req) {
-                GF_ASSERT (req);
-                goto out;
-        }
+struct iobuf *
+glusterfs_serialize_reply(rpcsvc_request_t *req, void *arg,
+                          struct iovec *outmsg, xdrproc_t xdrproc)
+{
+    struct iobuf *iob = NULL;
+    ssize_t retlen = -1;
+    ssize_t xdr_size = 0;
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    xdr_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, xdr_size);
+    if (!iob) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    /* retlen is used to received the error since size_t is unsigned and we
+     * need -1 for error notification during encoding.
+     */
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to encode message");
+        GF_FREE(iob);
+        goto ret;
+    }
+
+    outmsg->iov_len = retlen;
+ret:
+    if (retlen == -1) {
+        iob = NULL;
+    }
 
-        if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "out of memory");
-                        goto out;
-                }
+    return iob;
+}
 
-                new_iobref = 1;
+int
+glusterfs_submit_reply(rpcsvc_request_t *req, void *arg, struct iovec *payload,
+                       int payloadcount, struct iobref *iobref,
+                       xdrproc_t xdrproc)
+{
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iovec rsp = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req) {
+        GF_ASSERT(req);
+        goto out;
+    }
+
+    if (!iobref) {
+        iobref = iobref_new();
+        if (!iobref) {
+            gf_log(THIS->name, GF_LOG_ERROR, "out of memory");
+            goto out;
         }
 
-        iob = glusterfs_serialize_reply (req, arg, &rsp, xdrproc);
-        if (!iob) {
-                gf_log_callingfn (THIS->name, GF_LOG_ERROR, "Failed to serialize reply");
-        } else {
-                iobref_add (iobref, iob);
-        }
+        new_iobref = 1;
+    }
 
-        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
-                                     iobref);
+    iob = glusterfs_serialize_reply(req, arg, &rsp, xdrproc);
+    if (!iob) {
+        gf_log_callingfn(THIS->name, GF_LOG_ERROR, "Failed to serialize reply");
+    } else {
+        iobref_add(iobref, iob);
+    }
 
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        if (ret == -1) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Reply submission failed");
-                goto out;
-        }
+    ret = rpcsvc_submit_generic(req, &rsp, 1, payload, payloadcount, iobref);
 
-        ret = 0;
+    /* Now that we've done our job of handing the message to the RPC layer
+     * we can safely unref the iob in the hope that RPC layer must have
+     * ref'ed the iob on receiving into the txlist.
+     */
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Reply submission failed");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (iob)
-                iobuf_unref (iob);
+    if (iob)
+        iobuf_unref(iob);
 
-        if (new_iobref && iobref)
-                iobref_unref (iobref);
+    if (new_iobref && iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterfs_terminate_response_send (rpcsvc_request_t *req, int op_ret)
+glusterfs_terminate_response_send(rpcsvc_request_t *req, int op_ret)
 {
-        gd1_mgmt_brick_op_rsp   rsp = {0,};
-        dict_t                  *dict = NULL;
-        int                     ret = 0;
-
-        rsp.op_ret = op_ret;
-        rsp.op_errno = 0;
-        rsp.op_errstr = "";
-        dict = dict_new ();
-
-        if (dict)
-                ret = dict_allocate_and_serialize (dict, &rsp.output.output_val,
-                                                   &rsp.output.output_len);
-
-
-        if (ret == 0)
-                ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = 0;
+    rsp.op_errstr = "";
+    dict = dict_new();
+
+    if (dict)
+        ret = dict_allocate_and_serialize(dict, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+
+    if (ret == 0)
+        ret = glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
-        GF_FREE (rsp.output.output_val);
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    GF_FREE(rsp.output.output_val);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-glusterfs_handle_terminate (rpcsvc_request_t *req)
+glusterfs_handle_terminate(rpcsvc_request_t *req)
 {
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    ssize_t ret;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *top = NULL;
+    xlator_t *victim = NULL;
+    xlator_t *tvictim = NULL;
+    xlator_list_t **trav_p = NULL;
+    gf_boolean_t lockflag = _gf_false;
+    gf_boolean_t still_bricks_attached = _gf_false;
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ctx = glusterfsd_ctx;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        /* Find the xlator_list_t that points to our victim. */
+        if (glusterfsd_ctx->active) {
+            top = glusterfsd_ctx->active->first;
+            for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+                victim = (*trav_p)->xlator;
+                if (!victim->cleanup_starting &&
+                    strcmp(victim->name, xlator_req.name) == 0) {
+                    break;
+                }
+            }
+        }
 
-        glusterfs_terminate_response_send (req, 0);
-        cleanup_and_exit (SIGTERM);
-        return 0;
+        if (!top)
+            goto err;
+    }
+    if (!*trav_p) {
+        gf_log(THIS->name, GF_LOG_ERROR, "can't terminate %s - not found",
+               xlator_req.name);
+        /*
+         * Used to be -ENOENT.  However, the caller asked us to
+         * make sure it's down and if it's already down that's
+         * good enough.
+         */
+        glusterfs_terminate_response_send(req, 0);
+        goto err;
+    }
+
+    glusterfs_terminate_response_send(req, 0);
+    for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+        tvictim = (*trav_p)->xlator;
+        if (!tvictim->cleanup_starting &&
+            !strcmp(tvictim->name, xlator_req.name)) {
+            continue;
+        }
+        if (!tvictim->cleanup_starting) {
+            still_bricks_attached = _gf_true;
+            break;
+        }
+    }
+    if (!still_bricks_attached) {
+        gf_log(THIS->name, GF_LOG_INFO,
+               "terminating after loss of last child %s", xlator_req.name);
+        rpc_clnt_mgmt_pmap_signout(glusterfsd_ctx, xlator_req.name);
+        kill(getpid(), SIGTERM);
+    } else {
+        /* TODO cleanup sequence needs to be done properly for
+           Quota and Changelog
+        */
+        if (victim->cleanup_starting)
+            goto err;
+
+        rpc_clnt_mgmt_pmap_signout(glusterfsd_ctx, xlator_req.name);
+        victim->cleanup_starting = 1;
+
+        UNLOCK(&ctx->volfile_lock);
+        lockflag = _gf_true;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "detaching not-only"
+               " child %s",
+               xlator_req.name);
+        top->notify(top, GF_EVENT_CLEANUP, victim);
+    }
+err:
+    if (!lockflag)
+        UNLOCK(&ctx->volfile_lock);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    free(xlator_req.name);
+    xlator_req.name = NULL;
+    return 0;
 }
 
 int
-glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret,
-                                         char *msg, dict_t *output)
+glusterfs_translator_info_response_send(rpcsvc_request_t *req, int ret,
+                                        char *msg, dict_t *output)
 {
-        gd1_mgmt_brick_op_rsp    rsp = {0,};
-        gf_boolean_t             free_ptr = _gf_false;
-        GF_ASSERT (req);
-
-        rsp.op_ret = ret;
-        rsp.op_errno = 0;
-        if (ret && msg && msg[0])
-                rsp.op_errstr = msg;
-        else
-                rsp.op_errstr = "";
-
-        ret = -1;
-        if (output) {
-                ret = dict_allocate_and_serialize (output,
-                                                   &rsp.output.output_val,
-                                                   &rsp.output.output_len);
-        }
-        if (!ret)
-                free_ptr = _gf_true;
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    gf_boolean_t free_ptr = _gf_false;
+    GF_ASSERT(req);
+
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg && msg[0])
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
 
-        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        ret = 0;
-        if (free_ptr)
-                GF_FREE (rsp.output.output_val);
-        return ret;
+    ret = -1;
+    if (output) {
+        ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+    }
+    if (!ret)
+        free_ptr = _gf_true;
+
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
+    if (free_ptr)
+        GF_FREE(rsp.output.output_val);
+    return ret;
 }
 
 int
-glusterfs_xlator_op_response_send (rpcsvc_request_t *req, int op_ret,
-                                   char *msg, dict_t *output)
+glusterfs_xlator_op_response_send(rpcsvc_request_t *req, int op_ret, char *msg,
+                                  dict_t *output)
 {
-        gd1_mgmt_brick_op_rsp    rsp = {0,};
-        int                      ret = -1;
-        gf_boolean_t             free_ptr = _gf_false;
-        GF_ASSERT (req);
-
-        rsp.op_ret = op_ret;
-        rsp.op_errno = 0;
-        if (op_ret && msg && msg[0])
-                rsp.op_errstr = msg;
-        else
-                rsp.op_errstr = "";
-
-        if (output) {
-                ret = dict_allocate_and_serialize (output,
-                                                   &rsp.output.output_val,
-                                                   &rsp.output.output_len);
-        }
-        if (!ret)
-                free_ptr = _gf_true;
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    gf_boolean_t free_ptr = _gf_false;
+    GF_ASSERT(req);
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = 0;
+    if (op_ret && msg && msg[0])
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
 
-        ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    if (output) {
+        ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+    }
+    if (!ret)
+        free_ptr = _gf_true;
+
+    ret = glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
-        if (free_ptr)
-                GF_FREE (rsp.output.output_val);
+    if (free_ptr)
+        GF_FREE(rsp.output.output_val);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
+glusterfs_handle_translator_info_get(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = -1;
-        gd1_mgmt_brick_op_req   xlator_req = {0,};
-        dict_t                  *dict    = NULL;
-        xlator_t                *this = NULL;
-        gf1_cli_top_op          top_op = 0;
-        uint32_t                blk_size = 0;
-        uint32_t                blk_count = 0;
-        double                  time = 0;
-        double                  throughput = 0;
-        xlator_t                *any = NULL;
-        xlator_t                *xlator = NULL;
-        glusterfs_graph_t       *active = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        char                    msg[2048] = {0,};
-        dict_t                  *output = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &xlator_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        dict = dict_new ();
-        ret = dict_unserialize (xlator_req.input.input_val,
-                                xlator_req.input.input_len,
-                                &dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to "
-                        "unserialize req-buffer to dictionary");
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    gf1_cli_top_op top_op = 0;
+    xlator_t *any = NULL;
+    xlator_t *xlator = NULL;
+    glusterfs_graph_t *active = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char msg[2048] = {
+        0,
+    };
+    dict_t *output = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "top-op", (int32_t *)&top_op);
+    if (ret)
+        goto cont;
+    if (GF_CLI_TOP_READ_PERF == top_op) {
+        ret = glusterfs_volume_top_perf(xlator_req.name, dict, _gf_false);
+    } else if (GF_CLI_TOP_WRITE_PERF == top_op) {
+        ret = glusterfs_volume_top_perf(xlator_req.name, dict, _gf_true);
+    }
 
-        ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op);
-        if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op ||
-            GF_CLI_TOP_WRITE_PERF == top_op)) {
-                ret = dict_get_uint32 (dict, "blk-size", &blk_size);
-                if (ret)
-                        goto cont;
-                ret = dict_get_uint32 (dict, "blk-cnt", &blk_count);
-                if (ret)
-                        goto cont;
-
-                if (GF_CLI_TOP_READ_PERF == top_op) {
-                        ret = glusterfs_volume_top_read_perf
-                                (blk_size, blk_count, xlator_req.name,
-                                 &throughput, &time);
-                } else if ( GF_CLI_TOP_WRITE_PERF == top_op) {
-                        ret = glusterfs_volume_top_write_perf
-                                (blk_size, blk_count, xlator_req.name,
-                                 &throughput, &time);
-                }
-                ret = dict_set_double (dict, "time", time);
-                if (ret)
-                        goto cont;
-                ret = dict_set_double (dict, "throughput", throughput);
-                if (ret)
-                        goto cont;
-        }
 cont:
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    xlator = get_xlator_by_name(any, xlator_req.name);
+    if (!xlator) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
 
-        xlator = xlator_search_by_name (any, xlator_req.name);
+    if (strcmp(xlator->type, "debug/io-stats")) {
+        xlator = get_xlator_by_type(xlator, "debug/io-stats");
         if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          xlator_req.name);
-                goto out;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "xlator-type debug/io-stats is not loaded");
+            goto out;
         }
+    }
 
-        /*
-         * Searching by name will only get us to the decompounder translator,
-         * but we really want io-stats.  Since we know the exact relationship
-         * between these two, it's easy to get from one to the other.
-         *
-         * TBD: should this even be notify, or something else?
-         */
-        xlator = FIRST_CHILD(xlator);
-
-        output = dict_new ();
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+    output = dict_new();
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
 
 out:
-        ret = glusterfs_translator_info_response_send (req, ret, msg, output);
-
-        free (xlator_req.name);
-        free (xlator_req.input.input_val);
-        if (output)
-                dict_unref (output);
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
+
+    free(xlator_req.name);
+    free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int
-glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
-                                 char *brick_path, double *throughput,
-                                 double *time)
+static int
+glusterfs_volume_top_perf(const char *brick_path, dict_t *dict,
+                          gf_boolean_t write_test)
 {
-        int32_t                 fd = -1;
-        int32_t                 input_fd = -1;
-        char                    export_path[PATH_MAX] = {0,};
-        char                    *buf = NULL;
-        int32_t                 iter = 0;
-        int32_t                 ret = -1;
-        uint64_t                total_blks = 0;
-        struct timeval          begin, end = {0,};
-
-        GF_ASSERT (brick_path);
-        GF_ASSERT (throughput);
-        GF_ASSERT (time);
-        if (!(blk_size > 0) || ! (blk_count > 0))
-                goto out;
-
-        snprintf (export_path, sizeof (export_path), "%s/%s",
-                  brick_path, ".gf-tmp-stats-perf");
-
-        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
-        if (-1 == fd) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
-                goto out;
-        }
-
-        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
-        if (!buf) {
-                ret = -1;
-                goto out;
-        }
-
-        input_fd = open ("/dev/zero", O_RDONLY);
-        if (-1 == input_fd) {
-                ret = -1;
-                gf_log ("glusterd",GF_LOG_ERROR, "Unable to open input file");
-                goto out;
-        }
-
-        gettimeofday (&begin, NULL);
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = sys_read (input_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = sys_write (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                total_blks += ret;
-        }
+    int32_t fd = -1;
+    int32_t output_fd = -1;
+    char export_path[PATH_MAX] = {
+        0,
+    };
+    char *buf = NULL;
+    int32_t iter = 0;
+    int32_t ret = -1;
+    uint64_t total_blks = 0;
+    uint32_t blk_size;
+    uint32_t blk_count;
+    double throughput = 0;
+    double time = 0;
+    struct timeval begin, end = {
+                              0,
+                          };
+
+    GF_ASSERT(brick_path);
+
+    ret = dict_get_uint32(dict, "blk-size", &blk_size);
+    if (ret)
+        goto out;
+    ret = dict_get_uint32(dict, "blk-cnt", &blk_count);
+    if (ret)
+        goto out;
+
+    if (!(blk_size > 0) || !(blk_count > 0))
+        goto out;
+
+    buf = GF_CALLOC(1, blk_size * sizeof(*buf), gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not allocate memory");
+        goto out;
+    }
+
+    snprintf(export_path, sizeof(export_path), "%s/%s", brick_path,
+             ".gf-tmp-stats-perf");
+    fd = open(export_path, O_CREAT | O_RDWR, S_IRWXU);
+    if (-1 == fd) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+        goto out;
+    }
+
+    gettimeofday(&begin, NULL);
+    for (iter = 0; iter < blk_count; iter++) {
+        ret = sys_write(fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        total_blks += ret;
+    }
+    gettimeofday(&end, NULL);
+    if (total_blks != ((uint64_t)blk_size * blk_count)) {
+        gf_log("glusterd", GF_LOG_WARNING, "Error in write");
+        ret = -1;
+        goto out;
+    }
+
+    time = gf_tvdiff(&begin, &end);
+    throughput = total_blks / time;
+    gf_log("glusterd", GF_LOG_INFO,
+           "Throughput %.2f Mbps time %.2f secs "
+           "bytes written %" PRId64,
+           throughput, time, total_blks);
+
+    /* if it's a write test, we are done. Otherwise, we continue to the read
+     * part */
+    if (write_test == _gf_true) {
         ret = 0;
-        if (total_blks != ((uint64_t)blk_size * blk_count)) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
-                ret = -1;
-                goto out;
-        }
-
-        gettimeofday (&end, NULL);
-        *time = (end.tv_sec - begin.tv_sec) * 1e6
-                     + (end.tv_usec - begin.tv_usec);
-        *throughput = total_blks / *time;
-        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes written %"PRId64, *throughput, *time, total_blks);
+        goto out;
+    }
+
+    ret = sys_fsync(fd);
+    if (ret) {
+        gf_log("glusterd", GF_LOG_ERROR, "could not flush cache");
+        goto out;
+    }
+    ret = sys_lseek(fd, 0L, 0);
+    if (ret != 0) {
+        gf_log("glusterd", GF_LOG_ERROR, "could not seek back to start");
+        ret = -1;
+        goto out;
+    }
 
+    output_fd = open("/dev/null", O_RDWR);
+    if (-1 == output_fd) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not open output file");
+        goto out;
+    }
+
+    total_blks = 0;
+
+    gettimeofday(&begin, NULL);
+    for (iter = 0; iter < blk_count; iter++) {
+        ret = sys_read(fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        ret = sys_write(output_fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        total_blks += ret;
+    }
+    gettimeofday(&end, NULL);
+    if (total_blks != ((uint64_t)blk_size * blk_count)) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_WARNING, "Error in read");
+        goto out;
+    }
+
+    time = gf_tvdiff(&begin, &end);
+    throughput = total_blks / time;
+    gf_log("glusterd", GF_LOG_INFO,
+           "Throughput %.2f Mbps time %.2f secs "
+           "bytes read %" PRId64,
+           throughput, time, total_blks);
+    ret = 0;
 out:
-        if (fd >= 0)
-                sys_close (fd);
-        if (input_fd >= 0)
-                sys_close (input_fd);
-        GF_FREE (buf);
-        sys_unlink (export_path);
-
-        return ret;
+    if (fd >= 0)
+        sys_close(fd);
+    if (output_fd >= 0)
+        sys_close(output_fd);
+    GF_FREE(buf);
+    sys_unlink(export_path);
+    if (ret == 0) {
+        ret = dict_set_double(dict, "time", time);
+        if (ret)
+            goto end;
+        ret = dict_set_double(dict, "throughput", throughput);
+        if (ret)
+            goto end;
+    }
+end:
+    return ret;
 }
 
 int
-glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
-                                char *brick_path, double *throughput,
-                                double *time)
+glusterfs_handle_translator_op(rpcsvc_request_t *req)
 {
-        int32_t                 fd = -1;
-        int32_t                 input_fd = -1;
-        int32_t                 output_fd = -1;
-        char                    export_path[PATH_MAX] = {0,};
-        char                    *buf = NULL;
-        int32_t                 iter = 0;
-        int32_t                 ret = -1;
-        uint64_t                total_blks = 0;
-        struct timeval          begin, end = {0,};
-
-        GF_ASSERT (brick_path);
-        GF_ASSERT (throughput);
-        GF_ASSERT (time);
-        if (!(blk_size > 0) || ! (blk_count > 0))
-                goto out;
-
-        snprintf (export_path, sizeof (export_path), "%s/%s",
-                  brick_path, ".gf-tmp-stats-perf");
-        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
-        if (-1 == fd) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
-                goto out;
-        }
-
-        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
-        if (!buf) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not allocate memory");
-                goto out;
-        }
-
-        input_fd = open ("/dev/zero", O_RDONLY);
-        if (-1 == input_fd) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not open input file");
-                goto out;
-        }
-
-        output_fd = open ("/dev/null", O_RDWR);
-        if (-1 == output_fd) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not open output file");
-                goto out;
-        }
-
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = sys_read (input_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = sys_write (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    int32_t op_ret = 0;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *input = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char key[32] = {0};
+    int len;
+    char *xname = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+    int i = 0;
+    int count = 0;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    active = ctx->active;
+    if (!active) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, glusterfsd_msg_38,
+                "brick-op_no.=%d", xlator_req.op, NULL);
+        goto out;
+    }
+    any = active->first;
+    input = dict_new();
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &input);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    } else {
+        input->extra_stdfree = xlator_req.input.input_val;
+    }
+
+    ret = dict_get_int32(input, "count", &count);
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = sys_fsync (fd);
+    for (i = 0; i < count; i++) {
+        len = snprintf(key, sizeof(key), "xl-%d", i);
+        ret = dict_get_strn(input, key, len, &xname);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "could not flush cache");
-                goto out;
-        }
-        ret = sys_lseek (fd, 0L, 0);
-        if (ret != 0) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "could not seek back to start");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Couldn't get "
+                   "xlator %s ",
+                   key);
+            goto out;
         }
-        gettimeofday (&begin, NULL);
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = sys_read (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = sys_write (output_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                total_blks += ret;
-        }
-        ret = 0;
-        if (total_blks != ((uint64_t)blk_size * blk_count)) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_WARNING, "Error in read");
-                goto out;
-        }
-
-        gettimeofday (&end, NULL);
-        *time = (end.tv_sec - begin.tv_sec) * 1e6
-                + (end.tv_usec - begin.tv_usec);
-        *throughput = total_blks / *time;
-        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes read %"PRId64, *throughput, *time, total_blks);
-
+        xlator = xlator_search_by_name(any, xname);
+        if (!xlator) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "xlator %s is not "
+                   "loaded",
+                   xname);
+            goto out;
+        }
+    }
+    for (i = 0; i < count; i++) {
+        len = snprintf(key, sizeof(key), "xl-%d", i);
+        ret = dict_get_strn(input, key, len, &xname);
+        xlator = xlator_search_by_name(any, xname);
+        XLATOR_NOTIFY(ret, xlator, GF_EVENT_TRANSLATOR_OP, input, output);
+        /* If notify fails for an xlator we need to capture it but
+         * continue with the loop. */
+        if (ret)
+            op_ret = -1;
+    }
+    ret = op_ret;
 out:
-        if (fd >= 0)
-                sys_close (fd);
-        if (input_fd >= 0)
-                sys_close (input_fd);
-        if (output_fd >= 0)
-                sys_close (output_fd);
-        GF_FREE (buf);
-        sys_unlink (export_path);
-
-        return ret;
+    glusterfs_xlator_op_response_send(req, ret, "", output);
+    if (input)
+        dict_unref(input);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return 0;
 }
 
 int
-glusterfs_handle_translator_op (rpcsvc_request_t *req)
+glusterfs_handle_bitrot(rpcsvc_request_t *req)
 {
-        int32_t                  ret     = -1;
-        int32_t                  op_ret  = 0;
-        gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *input    = NULL;
-        xlator_t                 *xlator = NULL;
-        xlator_t                 *any = NULL;
-        dict_t                   *output = NULL;
-        char                     key[2048] = {0};
-        char                    *xname = NULL;
-        glusterfs_ctx_t          *ctx = NULL;
-        glusterfs_graph_t        *active = NULL;
-        xlator_t                 *this = NULL;
-        int                      i = 0;
-        int                      count = 0;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &xlator_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ctx = glusterfsd_ctx;
-        active = ctx->active;
-        any = active->first;
-        input = dict_new ();
-        ret = dict_unserialize (xlator_req.input.input_val,
-                                xlator_req.input.input_len,
-                                &input);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to "
-                        "unserialize req-buffer to dictionary");
-                goto out;
-        } else {
-                input->extra_stdfree = xlator_req.input.input_val;
-        }
-
-        ret = dict_get_int32 (input, "count", &count);
-
-        output = dict_new ();
-        if (!output) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    xlator_t *any = NULL;
+    xlator_t *this = NULL;
+    xlator_t *xlator = NULL;
+    char msg[2048] = {
+        0,
+    };
+    char xname[1024] = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    char *scrub_opt = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+
+    input = dict_new();
+    if (!input)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &input);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_35, NULL);
+        goto out;
+    }
+
+    /* Send scrubber request to bitrot xlator */
+    snprintf(xname, sizeof(xname), "%s-bit-rot-0", xlator_req.name);
+    xlator = xlator_search_by_name(any, xname);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_36, NULL);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0; i < count; i++)  {
-                snprintf (key, sizeof (key), "xl-%d", i);
-                ret = dict_get_str (input, key, &xname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Couldn't get "
-                                "xlator %s ", key);
-                        goto out;
-                }
-                xlator = xlator_search_by_name (any, xname);
-                if (!xlator) {
-                        gf_log (this->name, GF_LOG_ERROR, "xlator %s is not "
-                                "loaded", xname);
-                        goto out;
-                }
-        }
-        for (i = 0; i < count; i++)  {
-                snprintf (key, sizeof (key), "xl-%d", i);
-                ret = dict_get_str (input, key, &xname);
-                xlator = xlator_search_by_name (any, xname);
-                XLATOR_NOTIFY (xlator, GF_EVENT_TRANSLATOR_OP, input, output);
-                /* If notify fails for an xlator we need to capture it but
-                 * continue with the loop. */
-                if (ret)
-                        op_ret = -1;
-        }
-        ret = op_ret;
+    ret = dict_get_str(input, "scrub-value", &scrub_opt);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get scrub value");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_37, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (!strncmp(scrub_opt, "status", SLEN("status"))) {
+        ret = xlator->notify(xlator, GF_EVENT_SCRUB_STATUS, input, output);
+    } else if (!strncmp(scrub_opt, "ondemand", SLEN("ondemand"))) {
+        ret = xlator->notify(xlator, GF_EVENT_SCRUB_ONDEMAND, input, output);
+        if (ret == -2) {
+            snprintf(msg, sizeof(msg),
+                     "Scrubber is in "
+                     "Pause/Inactive/Running state");
+            ret = -1;
+            goto out;
+        }
+    }
 out:
-        glusterfs_xlator_op_response_send (req, ret, "", output);
-        if (input)
-                dict_unref (input);
-        if (output)
-                dict_unref (output);
-        free (xlator_req.name); //malloced by xdr
-
-        return 0;
+    glusterfs_translator_info_response_send(req, ret, msg, output);
+
+    if (input)
+        dict_unref(input);
+    free(xlator_req.input.input_val); /*malloced by xdr*/
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);
+
+    return 0;
 }
 
 int
-glusterfs_handle_bitrot (rpcsvc_request_t *req)
+glusterfs_handle_attach(rpcsvc_request_t *req)
 {
-        int32_t                  ret          = -1;
-        gd1_mgmt_brick_op_req    xlator_req   = {0,};
-        dict_t                   *input       = NULL;
-        dict_t                   *output      = NULL;
-        xlator_t                 *any         = NULL;
-        xlator_t                 *this        = NULL;
-        xlator_t                 *xlator      = NULL;
-        char                     msg[2048]    = {0,};
-        char                     xname[1024]  = {0,};
-        glusterfs_ctx_t          *ctx         = NULL;
-        glusterfs_graph_t        *active      = NULL;
-        char                     *scrub_opt   = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &xlator_req,
-                             (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-
-        if (ret < 0) {
-                /*failed to decode msg;*/
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        if (!active) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        any = active->first;
-
-        input = dict_new ();
-        if (!input)
-                goto out;
-
-        ret = dict_unserialize (xlator_req.input.input_val,
-                                xlator_req.input.input_len,
-                                &input);
-
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, glusterfsd_msg_35);
-                goto out;
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    xlator_t *nextchild = NULL;
+    glusterfs_graph_t *newgraph = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *srv_xl = NULL;
+    server_conf_t *srv_conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = this->ctx;
+    if (!ctx->cmd_args.volfile_id) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "No volfile-id provided, erroring out");
+        return -1;
+    }
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ret = 0;
+
+    if (!this->ctx->active) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "got attach for %s but no active graph", xlator_req.name);
+        goto post_unlock;
+    }
+
+    gf_log(this->name, GF_LOG_INFO, "got attach for %s", xlator_req.name);
+
+    LOCK(&ctx->volfile_lock);
+    {
+        ret = glusterfs_graph_attach(this->ctx->active, xlator_req.name,
+                                     &newgraph);
+        if (!ret && (newgraph && newgraph->first)) {
+            nextchild = newgraph->first;
+            ret = xlator_notify(nextchild, GF_EVENT_PARENT_UP, nextchild);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED,
+                        "event=ParentUp", "name=%s", nextchild->name, NULL);
+                goto unlock;
+            }
+            /* we need a protocol/server xlator as
+             * nextchild
+             */
+            srv_xl = this->ctx->active->first;
+            srv_conf = (server_conf_t *)srv_xl->private;
+            rpcsvc_autoscale_threads(this->ctx, srv_conf->rpc, 1);
         }
-
-        /* Send scrubber request to bitrot xlator */
-        snprintf (xname, sizeof (xname), "%s-bit-rot-0", xlator_req.name);
-        xlator = xlator_search_by_name (any, xname);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded", xname);
-                gf_msg (this->name, GF_LOG_ERROR, 0, glusterfsd_msg_36);
-                goto out;
-        }
-
-        output = dict_new ();
-        if (!output) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (input, "scrub-value", &scrub_opt);
         if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get scrub value");
-                gf_msg (this->name, GF_LOG_ERROR, 0, glusterfsd_msg_37);
-                ret = -1;
-                goto out;
-        }
-
-        if (!strncmp (scrub_opt, "status", strlen ("status"))) {
-                ret = xlator->notify (xlator, GF_EVENT_SCRUB_STATUS, input,
-                                      output);
-        } else if (!strncmp (scrub_opt, "ondemand", strlen ("ondemand"))) {
-                ret = xlator->notify (xlator, GF_EVENT_SCRUB_ONDEMAND, input,
-                                      output);
-                if (ret == -2) {
-                        snprintf (msg, sizeof (msg), "Scrubber is in "
-                                  "Pause/Inactive/Running state");
-                        ret = -1;
-                        goto out;
-                }
+            ret = -1;
         }
-out:
-        glusterfs_translator_info_response_send (req, ret, msg, output);
-
-        if (input)
-                dict_unref (input);
-        free (xlator_req.input.input_val); /*malloced by xdr*/
-        if (output)
-                dict_unref (output);
-        free (xlator_req.name);
-
-        return 0;
+        ret = glusterfs_translator_info_response_send(req, ret, NULL, NULL);
+        if (ret) {
+            /* Response sent back to glusterd, req is already destroyed. So
+             * resetting the ret to 0. Otherwise another response will be
+             * send from rpcsvc_check_and_reply_error. Which will lead to
+             * double resource leak.
+             */
+            ret = 0;
+        }
+    unlock:
+        UNLOCK(&ctx->volfile_lock);
+    }
+post_unlock:
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    free(xlator_req.input.input_val);
+    free(xlator_req.name);
+
+    return ret;
 }
 
 int
-glusterfs_handle_defrag (rpcsvc_request_t *req)
+glusterfs_handle_svc_attach(rpcsvc_request_t *req)
 {
-        int32_t                  ret     = -1;
-        gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *dict    = NULL;
-        xlator_t                 *xlator = NULL;
-        xlator_t                 *any = NULL;
-        dict_t                   *output = NULL;
-        char                     msg[2048] = {0};
-        glusterfs_ctx_t          *ctx = NULL;
-        glusterfs_graph_t        *active = NULL;
-        xlator_t                 *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        if (!active) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        any = active->first;
-        ret = xdr_to_generic (req->msg[0], &xlator_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (xlator_req.input.input_val,
-                                xlator_req.input.input_len,
-                                &dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to "
-                        "unserialize req-buffer to dictionary");
-                goto out;
-        }
-        xlator = xlator_search_by_name (any, xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          xlator_req.name);
-                goto out;
-        }
-
-        output = dict_new ();
-        if (!output) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = xlator->notify (xlator, GF_EVENT_VOLUME_DEFRAG, dict, output);
-
-        ret = glusterfs_translator_info_response_send (req, ret,
-                                                       msg, output);
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_41, "volfile-id=%s",
+            xlator_req.name, NULL);
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(xlator_req.dict.dict_val, xlator_req.dict.dict_len,
+                           &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, glusterfsd_msg_42, NULL);
+        goto out;
+    }
+    dict->extra_stdfree = xlator_req.dict.dict_val;
+
+    ret = 0;
+
+    ret = mgmt_process_volfile(xlator_req.input.input_val,
+                               xlator_req.input.input_len, xlator_req.name,
+                               dict);
 out:
-        if (dict)
-                dict_unref (dict);
-        free (xlator_req.input.input_val); // malloced by xdr
-        if (output)
-                dict_unref (output);
-        free (xlator_req.name); //malloced by xdr
-
-        return ret;
-
+    if (dict)
+        dict_unref(dict);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.name)
+        free(xlator_req.name);
+    glusterfs_translator_info_response_send(req, ret, NULL, NULL);
+    return 0;
 }
+
 int
-glusterfs_handle_brick_status (rpcsvc_request_t *req)
+glusterfs_handle_svc_detach(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   brick_req = {0,};
-        gd1_mgmt_brick_op_rsp   rsp = {0,};
-        glusterfs_ctx_t         *ctx = NULL;
-        glusterfs_graph_t       *active = NULL;
-        xlator_t                *this = NULL;
-        xlator_t                *any = NULL;
-        xlator_t                *xlator = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *output = NULL;
-        char                    *volname = NULL;
-        char                    *xname = NULL;
-        uint32_t                cmd = 0;
-        char                    *msg = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &brick_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        dict = dict_new ();
-        ret = dict_unserialize (brick_req.input.input_val,
-                                brick_req.input.input_len, &dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unserialize "
-                        "req-buffer to dictionary");
-                goto out;
-        }
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Couldn't get status op");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    ssize_t ret;
+    gf_volfile_t *volfile_obj = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ctx = glusterfsd_ctx;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(xlator_req.name, volfile_obj->vol_id)) {
+                volfile_tmp = volfile_obj;
+                break;
+            }
+        }
+
+        if (!volfile_tmp) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_041, "name=%s",
+                    xlator_req.name, NULL);
+            /*
+             * Used to be -ENOENT.  However, the caller asked us to
+             * make sure it's down and if it's already down that's
+             * good enough.
+             */
+            ret = 0;
+            goto out;
+        }
+        /* coverity[ORDER_REVERSAL] */
+        ret = glusterfs_process_svc_detach(ctx, volfile_tmp);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Couldn't get volname");
-                goto out;
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg("glusterfsd-mgmt", GF_LOG_ERROR, EINVAL, glusterfsd_msg_042,
+                    NULL);
+            goto out;
         }
+    }
+    UNLOCK(&ctx->volfile_lock);
+out:
+    glusterfs_terminate_response_send(req, ret);
+    free(xlator_req.name);
+    xlator_req.name = NULL;
 
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
-
-        ret = gf_asprintf (&xname, "%s-server", volname);
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                goto out;
-        }
-
-        xlator = xlator_search_by_name (any, xname);
-        if (!xlator) {
-                gf_log (this->name, GF_LOG_ERROR, "xlator %s is not loaded",
-                        xname);
-                ret = -1;
-                goto out;
-        }
-
-
-        output = dict_new ();
-        switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_MEM:
-                        ret = 0;
-                        gf_proc_dump_mem_info_to_dict (output);
-                        gf_proc_dump_mempool_info_to_dict (ctx, output);
-                        break;
-
-                case GF_CLI_STATUS_CLIENTS:
-                        ret = xlator->dumpops->priv_to_dict (xlator, output);
-                        break;
+    return 0;
+}
 
-                case GF_CLI_STATUS_INODE:
-                        ret = xlator->dumpops->inode_to_dict (xlator, output);
-                        break;
+int
+glusterfs_handle_dump_metrics(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char *filepath = NULL;
+    int fd = -1;
+    struct stat statbuf = {
+        0,
+    };
+    char *msg = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ret = -1;
+    ctx = this->ctx;
+
+    /* Infra for monitoring */
+    filepath = gf_monitor_metrics(ctx);
+    if (!filepath)
+        goto out;
+
+    fd = sys_open(filepath, O_RDONLY, 0);
+    if (fd < 0)
+        goto out;
+
+    if (sys_fstat(fd, &statbuf) < 0)
+        goto out;
+
+    if (statbuf.st_size > GF_UNIT_MB) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                "reconsider logic (%" PRId64 ")", statbuf.st_size, NULL);
+    }
+    msg = GF_CALLOC(1, (statbuf.st_size + 1), gf_common_mt_char);
+    if (!msg)
+        goto out;
+
+    ret = sys_read(fd, msg, statbuf.st_size);
+    if (ret < 0)
+        goto out;
+
+    /* Send all the data in errstr, instead of dictionary for now */
+    glusterfs_translator_info_response_send(req, 0, msg, NULL);
+
+    ret = 0;
+out:
+    if (fd >= 0)
+        sys_close(fd);
 
-                case GF_CLI_STATUS_FD:
-                        ret = xlator->dumpops->fd_to_dict (xlator, output);
-                        break;
+    GF_FREE(msg);
+    GF_FREE(filepath);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
 
-                case GF_CLI_STATUS_CALLPOOL:
-                        ret = 0;
-                        gf_proc_dump_pending_frames_to_dict (ctx->pool, output);
-                        break;
+    return ret;
+}
 
-                default:
-                        ret = -1;
-                        msg = gf_strdup ("Unknown status op");
-                        break;
-        }
-        rsp.op_ret = ret;
-        rsp.op_errno = 0;
-        if (ret && msg)
-                rsp.op_errstr = msg;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                           &rsp.output.output_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to serialize output dict to rsp");
-                goto out;
-        }
+int
+glusterfs_handle_defrag(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char msg[2048] = {0};
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+    xlator = xlator_search_by_name(any, xlator_req.name);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
 
-        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        ret = 0;
+    ret = xlator->notify(xlator, GF_EVENT_VOLUME_DEFRAG, dict, output);
 
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
 out:
-        if (dict)
-                dict_unref (dict);
-        if (output)
-                dict_unref (output);
-        free (brick_req.input.input_val);
-        GF_FREE (xname);
-        GF_FREE (msg);
-        GF_FREE (rsp.output.output_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    free(xlator_req.input.input_val);  // malloced by xdr
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return ret;
 }
-
-
 int
-glusterfs_handle_node_status (rpcsvc_request_t *req)
+glusterfs_handle_brick_status(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   node_req = {0,};
-        gd1_mgmt_brick_op_rsp   rsp = {0,};
-        glusterfs_ctx_t         *ctx = NULL;
-        glusterfs_graph_t       *active = NULL;
-        xlator_t                *any = NULL;
-        xlator_t                *node = NULL;
-        xlator_t                *subvol = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *output = NULL;
-        char                    *volname = NULL;
-        char                    *node_name = NULL;
-        char                    *subvol_name = NULL;
-        uint32_t                cmd = 0;
-        char                    *msg = NULL;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &node_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        dict = dict_new ();
-        ret = dict_unserialize (node_req.input.input_val,
-                                node_req.input.input_len, &dict);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to unserialize "
-                        "req buffer to dictionary");
-                goto out;
-        }
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get status op");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get volname");
-                goto out;
-        }
-
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
-
-        if ((cmd & GF_CLI_STATUS_NFS) != 0)
-                ret = gf_asprintf (&node_name, "%s", "nfs-server");
-        else if ((cmd & GF_CLI_STATUS_SHD) != 0)
-                ret = gf_asprintf (&node_name, "%s", "glustershd");
-        else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
-                ret = gf_asprintf (&node_name, "%s", "quotad");
-        else if ((cmd & GF_CLI_STATUS_BITD) != 0)
-                ret = gf_asprintf (&node_name, "%s", "bitd");
-        else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
-                ret = gf_asprintf (&node_name, "%s", "scrubber");
-
-        else {
-                ret = -1;
-                goto out;
-        }
-        if (ret == -1) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set node xlator name");
-                goto out;
-        }
-
-        node = xlator_search_by_name (any, node_name);
-        if (!node) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
-                        node_name);
-                goto out;
-        }
+    int ret = -1;
+    gd1_mgmt_brick_op_req brick_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+    xlator_t *server_xl = NULL;
+    xlator_t *brick_xl = NULL;
+    dict_t *dict = NULL;
+    dict_t *output = NULL;
+    uint32_t cmd = 0;
+    char *msg = NULL;
+    char *brickname = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &brick_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(brick_req.input.input_val, brick_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "req-buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Couldn't get status op");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick-name", &brickname);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Couldn't get brickname from"
+               " dict");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    if (ctx == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "ctx returned NULL");
+        ret = -1;
+        goto out;
+    }
+    if (ctx->active == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    active = ctx->active;
+    if (ctx->active->first == NULL) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "ctx->active->first "
+               "returned NULL");
+        ret = -1;
+        goto out;
+    }
+    server_xl = active->first;
 
-        if ((cmd & GF_CLI_STATUS_NFS) != 0)
-                ret = gf_asprintf (&subvol_name, "%s", volname);
-        else if ((cmd & GF_CLI_STATUS_SHD) != 0)
-                ret = gf_asprintf (&subvol_name, "%s-replicate-0", volname);
-        else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
-                ret = gf_asprintf (&subvol_name, "%s", volname);
-        else if ((cmd & GF_CLI_STATUS_BITD) != 0)
-                ret = gf_asprintf (&subvol_name, "%s", volname);
-        else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
-                ret = gf_asprintf (&subvol_name, "%s", volname);
-        else {
-                ret = -1;
-                goto out;
-        }
-        if (ret == -1) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set node xlator name");
-                goto out;
-        }
+    brick_xl = get_xlator_by_name(server_xl, brickname);
+    if (!brick_xl) {
+        gf_log(this->name, GF_LOG_ERROR, "xlator is not loaded");
+        ret = -1;
+        goto out;
+    }
+
+    output = dict_new();
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            ret = 0;
+            gf_proc_dump_mem_info_to_dict(output);
+            gf_proc_dump_mempool_info_to_dict(ctx, output);
+            break;
+
+        case GF_CLI_STATUS_CLIENTS:
+        case GF_CLI_STATUS_CLIENT_LIST:
+            ret = server_xl->dumpops->priv_to_dict(server_xl, output,
+                                                   brickname);
+            break;
+
+        case GF_CLI_STATUS_INODE:
+            ret = server_xl->dumpops->inode_to_dict(brick_xl, output);
+            break;
+
+        case GF_CLI_STATUS_FD:
+            ret = server_xl->dumpops->fd_to_dict(brick_xl, output);
+            break;
+
+        case GF_CLI_STATUS_CALLPOOL:
+            ret = 0;
+            gf_proc_dump_pending_frames_to_dict(ctx->pool, output);
+            break;
 
-        subvol = xlator_search_by_name (node, subvol_name);
-        if (!subvol) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
-                        subvol_name);
-                goto out;
-        }
+        default:
+            ret = -1;
+            msg = gf_strdup("Unknown status op");
+            break;
+    }
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg)
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
 
-        output = dict_new ();
-        switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_MEM:
-                        ret = 0;
-                        gf_proc_dump_mem_info_to_dict (output);
-                        gf_proc_dump_mempool_info_to_dict (ctx, output);
-                        break;
-
-                case GF_CLI_STATUS_CLIENTS:
-                        // clients not availbale for SHD
-                        if ((cmd & GF_CLI_STATUS_SHD) != 0)
-                                break;
-
-                        ret = dict_set_str (output, "volname", volname);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Error setting volname to dict");
-                                goto out;
-                        }
-                        ret = node->dumpops->priv_to_dict (node, output);
-                        break;
-
-                case GF_CLI_STATUS_INODE:
-                        ret = 0;
-                        inode_table_dump_to_dict (subvol->itable, "conn0",
-                                                  output);
-                        ret = dict_set_int32 (output, "conncount", 1);
-                        break;
-
-                case GF_CLI_STATUS_FD:
-                        // cannot find fd-tables in nfs-server graph
-                        // TODO: finish once found
-                        break;
-
-                case GF_CLI_STATUS_CALLPOOL:
-                        ret = 0;
-                        gf_proc_dump_pending_frames_to_dict (ctx->pool, output);
-                        break;
-
-                default:
-                        ret = -1;
-                        msg = gf_strdup ("Unknown status op");
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                        break;
-        }
-        rsp.op_ret = ret;
-        rsp.op_errno = 0;
-        if (ret && msg)
-                rsp.op_errstr = msg;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                           &rsp.output.output_len);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize output dict to rsp");
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
 
-        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        ret = 0;
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (node_req.input.input_val);
-        GF_FREE (msg);
-        GF_FREE (rsp.output.output_val);
-        GF_FREE (node_name);
-        GF_FREE (subvol_name);
-
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    if (output)
+        dict_unref(output);
+    free(brick_req.input.input_val);
+    if (brick_req.dict.dict_val)
+        free(brick_req.dict.dict_val);
+    free(brick_req.name);
+    GF_FREE(msg);
+    GF_FREE(rsp.output.output_val);
+
+    return ret;
 }
 
 int
-glusterfs_handle_nfs_profile (rpcsvc_request_t *req)
+glusterfs_handle_node_status(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   nfs_req = {0,};
-        gd1_mgmt_brick_op_rsp   rsp = {0,};
-        dict_t                  *dict = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        glusterfs_graph_t       *active = NULL;
-        xlator_t                *any = NULL;
-        xlator_t                *nfs = NULL;
-        xlator_t                *subvol = NULL;
-        char                    *volname = NULL;
-        dict_t                  *output = NULL;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &nfs_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        dict = dict_new ();
-        ret = dict_unserialize (nfs_req.input.input_val,
-                                nfs_req.input.input_len, &dict);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to "
-                        "unserialize req-buffer to dict");
-                goto out;
-        }
+    int ret = -1;
+    gd1_mgmt_brick_op_req node_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *any = NULL;
+    xlator_t *node = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *dict = NULL;
+    dict_t *output = NULL;
+    char *volname = NULL;
+    char *node_name = NULL;
+    char *subvol_name = NULL;
+    uint32_t cmd = 0;
+    char *msg = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &node_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(node_req.input.input_val, node_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "req buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get status op");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    if ((cmd & GF_CLI_STATUS_SHD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "glustershd");
+#ifdef BUILD_GNFS
+    else if ((cmd & GF_CLI_STATUS_NFS) != 0)
+        ret = gf_asprintf(&node_name, "%s", "nfs-server");
+#endif
+    else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "quotad");
+    else if ((cmd & GF_CLI_STATUS_BITD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "bitd");
+    else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
+        ret = gf_asprintf(&node_name, "%s", "scrubber");
+
+    else {
+        ret = -1;
+        goto out;
+    }
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to set node xlator name");
+        goto out;
+    }
+
+    node = xlator_search_by_name(any, node_name);
+    if (!node) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded", node_name);
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_SHD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s-replicate-0", volname);
+    else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_BITD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else {
+        ret = -1;
+        goto out;
+    }
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to set node xlator name");
+        goto out;
+    }
+
+    subvol = xlator_search_by_name(node, subvol_name);
+    if (!subvol) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               subvol_name);
+        goto out;
+    }
+
+    output = dict_new();
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            ret = 0;
+            gf_proc_dump_mem_info_to_dict(output);
+            gf_proc_dump_mempool_info_to_dict(ctx, output);
+            break;
+
+        case GF_CLI_STATUS_CLIENTS:
+            // clients not available for SHD
+            if ((cmd & GF_CLI_STATUS_SHD) != 0)
+                break;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+            ret = dict_set_str(output, "volname", volname);
+            if (ret) {
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Error setting volname to dict");
                 goto out;
-        }
+            }
+            ret = node->dumpops->priv_to_dict(node, output, NULL);
+            break;
 
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
+        case GF_CLI_STATUS_INODE:
+            ret = 0;
+            inode_table_dump_to_dict(subvol->itable, "conn0", output);
+            ret = dict_set_int32(output, "conncount", 1);
+            break;
 
-        active = ctx->active;
-        any = active->first;
+        case GF_CLI_STATUS_FD:
+            // cannot find fd-tables in nfs-server graph
+            // TODO: finish once found
+            break;
 
-        // is this needed?
-        // are problems possible by searching for subvol directly from "any"?
-        nfs = xlator_search_by_name (any, "nfs-server");
-        if (!nfs) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "xlator nfs-server is "
-                        "not loaded");
-                goto out;
-        }
-
-        subvol = xlator_search_by_name (nfs, volname);
-        if (!subvol) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "xlator %s is no loaded",
-                        volname);
-                goto out;
-        }
+        case GF_CLI_STATUS_CALLPOOL:
+            ret = 0;
+            gf_proc_dump_pending_frames_to_dict(ctx->pool, output);
+            break;
 
-        output = dict_new ();
-        ret = subvol->notify (subvol, GF_EVENT_TRANSLATOR_INFO, dict, output);
-
-        rsp.op_ret = ret;
-        rsp.op_errno = 0;
+        default:
+            ret = -1;
+            msg = gf_strdup("Unknown status op");
+            gf_log(THIS->name, GF_LOG_ERROR, "%s", msg);
+            break;
+    }
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg)
+        rsp.op_errstr = msg;
+    else
         rsp.op_errstr = "";
 
-        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                           &rsp.output.output_len);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize output dict to rsp");
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
 
-        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        ret = 0;
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
 
 out:
-        free (nfs_req.input.input_val);
-        if (dict)
-                dict_unref (dict);
-        if (output)
-                dict_unref (output);
-        GF_FREE (rsp.output.output_val);
-
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    free(node_req.input.input_val);
+    if (node_req.dict.dict_val)
+        free(node_req.dict.dict_val);
+    GF_FREE(msg);
+    GF_FREE(rsp.output.output_val);
+    GF_FREE(node_name);
+    GF_FREE(subvol_name);
+
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterfs_handle_volume_barrier_op (rpcsvc_request_t *req)
+glusterfs_handle_nfs_profile(rpcsvc_request_t *req)
 {
-        int32_t                  ret     = -1;
-        gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *dict    = NULL;
-        xlator_t                 *xlator = NULL;
-        xlator_t                 *any = NULL;
-        dict_t                   *output = NULL;
-        char                     msg[2048] = {0};
-        glusterfs_ctx_t          *ctx = NULL;
-        glusterfs_graph_t        *active = NULL;
-        xlator_t                 *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        if (!active) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int ret = -1;
+    gd1_mgmt_brick_op_req nfs_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *any = NULL;
+    xlator_t *nfs = NULL;
+    xlator_t *subvol = NULL;
+    char *volname = NULL;
+    dict_t *output = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &nfs_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(nfs_req.input.input_val, nfs_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to "
+               "unserialize req-buffer to dict");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    // is this needed?
+    // are problems possible by searching for subvol directly from "any"?
+    nfs = xlator_search_by_name(any, "nfs-server");
+    if (!nfs) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "xlator nfs-server is "
+               "not loaded");
+        goto out;
+    }
+
+    subvol = xlator_search_by_name(nfs, volname);
+    if (!subvol) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "xlator %s is no loaded", volname);
+        goto out;
+    }
 
-        any = active->first;
-        ret = xdr_to_generic (req->msg[0], &xlator_req,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    output = dict_new();
+    ret = subvol->notify(subvol, GF_EVENT_TRANSLATOR_INFO, dict, output);
 
-        ret = dict_unserialize (xlator_req.input.input_val,
-                                xlator_req.input.input_len,
-                                &dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to "
-                        "unserialize req-buffer to dictionary");
-                goto out;
-        }
-        xlator = xlator_search_by_name (any, xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          xlator_req.name);
-                goto out;
-        }
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    rsp.op_errstr = "";
 
-        output = dict_new ();
-        if (!output) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
 
-        ret = xlator->notify (xlator, GF_EVENT_VOLUME_BARRIER_OP,
-                              dict, output);
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
 
-        ret = glusterfs_translator_info_response_send (req, ret,
-                                                       msg, output);
 out:
-        if (dict)
-                dict_unref (dict);
-        free (xlator_req.input.input_val); // malloced by xdr
-        if (output)
-                dict_unref (output);
-        free (xlator_req.name); //malloced by xdr
-
-        return ret;
-
+    free(nfs_req.input.input_val);
+    if (nfs_req.dict.dict_val)
+        free(nfs_req.dict.dict_val);
+    if (dict)
+        dict_unref(dict);
+    if (output)
+        dict_unref(output);
+    GF_FREE(rsp.output.output_val);
+
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterfs_handle_barrier (rpcsvc_request_t *req)
+glusterfs_handle_volume_barrier_op(rpcsvc_request_t *req)
 {
-        int                     ret         = -1;
-        gd1_mgmt_brick_op_req   brick_req   = {0,};
-        gd1_mgmt_brick_op_rsp   brick_rsp   = {0,};
-        glusterfs_ctx_t         *ctx        = NULL;
-        glusterfs_graph_t       *active     = NULL;
-        xlator_t                *any        = NULL;
-        xlator_t                *xlator     = NULL;
-        xlator_t                *old_THIS   = NULL;
-        dict_t                  *dict       = NULL;
-        char                    name[1024]  = {0,};
-        gf_boolean_t            barrier     = _gf_true;
-        gf_boolean_t            barrier_err = _gf_false;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic(req->msg[0], &brick_req,
-                             (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
-
-        dict = dict_new();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char msg[2048] = {0};
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+    xlator = xlator_search_by_name(any, xlator_req.name);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_unserialize(brick_req.input.input_val,
-                               brick_req.input.input_len, &dict);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to unserialize "
-                        "request dictionary");
-                goto out;
-        }
+    ret = xlator->notify(xlator, GF_EVENT_VOLUME_BARRIER_OP, dict, output);
 
-        brick_rsp.op_ret = 0;
-        brick_rsp.op_errstr = ""; // initing to prevent serilaztion failures
-        old_THIS = THIS;
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
+out:
+    if (dict)
+        dict_unref(dict);
+    free(xlator_req.input.input_val);  // malloced by xdr
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return ret;
+}
 
-        /* Send barrier request to the barrier xlator */
-        snprintf (name, sizeof (name), "%s-barrier", brick_req.name);
-        xlator = xlator_search_by_name(any, name);
-        if (!xlator) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
-                        name);
-                goto out;
-        }
+int
+glusterfs_handle_barrier(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    gd1_mgmt_brick_op_req brick_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp brick_rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *top = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *old_THIS = NULL;
+    dict_t *dict = NULL;
+    gf_boolean_t barrier = _gf_true;
+    xlator_list_t *trav;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &brick_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    top = active->first;
 
-        THIS = xlator;
-        // TODO: Extend this to accept return of errnos
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_OP, dict);
-        if (ret) {
-                brick_rsp.op_ret = ret;
-                brick_rsp.op_errstr = gf_strdup ("Failed to reconfigure "
-                                                 "barrier.");
-                /* This is to invoke changelog-barrier disable if barrier
-                 * disable fails and don't invoke if barrier enable fails.
-                 */
-                barrier = dict_get_str_boolean (dict, "barrier", _gf_true);
-                if (barrier)
-                        goto submit_reply;
-                else
-                        barrier_err = _gf_true;
+    for (trav = top->children; trav; trav = trav->next) {
+        if (strcmp(trav->xlator->name, brick_req.name) == 0) {
+            break;
         }
+    }
+    if (!trav) {
+        ret = -1;
+        goto out;
+    }
+    top = trav->xlator;
 
-        /* Reset THIS so that we have it correct in case of an error below
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(brick_req.input.input_val, brick_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "request dictionary");
+        goto out;
+    }
+
+    brick_rsp.op_ret = 0;
+    brick_rsp.op_errstr = "";  // initing to prevent serilaztion failures
+    old_THIS = THIS;
+
+    /* Send barrier request to the barrier xlator */
+    xlator = get_xlator_by_type(top, "features/barrier");
+    if (!xlator) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               "features/barrier");
+        goto out;
+    }
+
+    THIS = xlator;
+    // TODO: Extend this to accept return of errnos
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_OP, dict);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "barrier notify failed");
+        brick_rsp.op_ret = ret;
+        brick_rsp.op_errstr = gf_strdup(
+            "Failed to reconfigure "
+            "barrier.");
+        /* This is to invoke changelog-barrier disable if barrier
+         * disable fails and don't invoke if barrier enable fails.
          */
-        THIS = old_THIS;
-
-        /* Send barrier request to changelog as well */
-
-        memset (name, 0, sizeof (name));
-        snprintf (name, sizeof (name), "%s-changelog", brick_req.name);
-        xlator = xlator_search_by_name(any, name);
-        if (!xlator) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
-                        name);
-                goto out;
-        }
-
-        THIS = xlator;
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_OP, dict);
-        if (ret) {
-                brick_rsp.op_ret = ret;
-                brick_rsp.op_errstr = gf_strdup ("changelog notify failed");
-                goto submit_reply;
-        }
-
-        if (barrier_err)
-                ret = -1;
+        barrier = dict_get_str_boolean(dict, "barrier", _gf_true);
+        if (barrier)
+            goto submit_reply;
+    }
+
+    /* Reset THIS so that we have it correct in case of an error below
+     */
+    THIS = old_THIS;
+
+    /* Send barrier request to changelog as well */
+    xlator = get_xlator_by_type(top, "features/changelog");
+    if (!xlator) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               "features/changelog");
+        goto out;
+    }
+
+    THIS = xlator;
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_OP, dict);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "changelog notify failed");
+        brick_rsp.op_ret = ret;
+        brick_rsp.op_errstr = gf_strdup("changelog notify failed");
+        goto submit_reply;
+    }
 
 submit_reply:
-        THIS = old_THIS;
+    THIS = old_THIS;
 
-        ret = glusterfs_submit_reply (req, &brick_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = glusterfs_submit_reply(req, &brick_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (brick_req.input.input_val);
-
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    free(brick_req.input.input_val);
+    if (brick_req.dict.dict_val)
+        free(brick_req.dict.dict_val);
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterfs_handle_rpc_msg (rpcsvc_request_t *req)
+glusterfs_handle_rpc_msg(rpcsvc_request_t *req)
 {
-        int ret = -1;
-        /* for now, nothing */
-        return ret;
+    int ret = -1;
+    /* for now, nothing */
+    return ret;
 }
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
-        [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-        [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-                                 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_event, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-        .progname  = "GlusterFS Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
-        .actors    = mgmt_cbk_actors,
-        .numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = {
-        [GF_PMAP_NULL]        = "NULL",
-        [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
-        [GF_PMAP_BRICKBYPORT] = "BRICKBYPORT",
-        [GF_PMAP_SIGNIN]      = "SIGNIN",
-        [GF_PMAP_SIGNOUT]     = "SIGNOUT",
-        [GF_PMAP_SIGNUP]      = "SIGNUP",       /* DEPRECATED - DON'T USE! */
+static char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = {
+    [GF_PMAP_NULL] = "NULL",
+    [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
+    [GF_PMAP_BRICKBYPORT] = "BRICKBYPORT",
+    [GF_PMAP_SIGNIN] = "SIGNIN",
+    [GF_PMAP_SIGNOUT] = "SIGNOUT",
+    [GF_PMAP_SIGNUP] = "SIGNUP", /* DEPRECATED - DON'T USE! */
 };
 
-
-rpc_clnt_prog_t clnt_pmap_prog = {
-        .progname  = "Gluster Portmap",
-        .prognum   = GLUSTER_PMAP_PROGRAM,
-        .progver   = GLUSTER_PMAP_VERSION,
-        .procnames = clnt_pmap_procs,
+static rpc_clnt_prog_t clnt_pmap_prog = {
+    .progname = "Gluster Portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+    .procnames = clnt_pmap_procs,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = "NULL",
-        [GF_HNDSK_SETVOLUME]    = "SETVOLUME",
-        [GF_HNDSK_GETSPEC]      = "GETSPEC",
-        [GF_HNDSK_PING]         = "PING",
-        [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-rpcsvc_actor_t glusterfs_actors[GLUSTERD_BRICK_MAXVALUE] = {
-        [GLUSTERD_BRICK_NULL]          = {"NULL",              GLUSTERD_BRICK_NULL,          glusterfs_handle_rpc_msg,             NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_TERMINATE]     = {"TERMINATE",         GLUSTERD_BRICK_TERMINATE,     glusterfs_handle_terminate,           NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_XLATOR_INFO]   = {"TRANSLATOR INFO",   GLUSTERD_BRICK_XLATOR_INFO,   glusterfs_handle_translator_info_get, NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_XLATOR_OP]     = {"TRANSLATOR OP",     GLUSTERD_BRICK_XLATOR_OP,     glusterfs_handle_translator_op,       NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_STATUS]        = {"STATUS",            GLUSTERD_BRICK_STATUS,        glusterfs_handle_brick_status,        NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_XLATOR_DEFRAG] = {"TRANSLATOR DEFRAG", GLUSTERD_BRICK_XLATOR_DEFRAG, glusterfs_handle_defrag,              NULL, 0, DRC_NA},
-        [GLUSTERD_NODE_PROFILE]        = {"NFS PROFILE",       GLUSTERD_NODE_PROFILE,        glusterfs_handle_nfs_profile,         NULL, 0, DRC_NA},
-        [GLUSTERD_NODE_STATUS]         = {"NFS STATUS",        GLUSTERD_NODE_STATUS,         glusterfs_handle_node_status,         NULL, 0, DRC_NA},
-        [GLUSTERD_VOLUME_BARRIER_OP]   = {"VOLUME BARRIER OP", GLUSTERD_VOLUME_BARRIER_OP,   glusterfs_handle_volume_barrier_op,   NULL, 0, DRC_NA},
-        [GLUSTERD_BRICK_BARRIER]       = {"BARRIER",           GLUSTERD_BRICK_BARRIER,       glusterfs_handle_barrier,             NULL, 0, DRC_NA},
-        [GLUSTERD_NODE_BITROT]         = {"BITROT",            GLUSTERD_NODE_BITROT,         glusterfs_handle_bitrot,              NULL, 0, DRC_NA},
+static rpcsvc_actor_t glusterfs_actors[GLUSTERD_BRICK_MAXVALUE] = {
+    [GLUSTERD_BRICK_NULL] = {"NULL", glusterfs_handle_rpc_msg, NULL,
+                             GLUSTERD_BRICK_NULL, DRC_NA, 0},
+    [GLUSTERD_BRICK_TERMINATE] = {"TERMINATE", glusterfs_handle_terminate, NULL,
+                                  GLUSTERD_BRICK_TERMINATE, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_INFO] = {"TRANSLATOR INFO",
+                                    glusterfs_handle_translator_info_get, NULL,
+                                    GLUSTERD_BRICK_XLATOR_INFO, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_OP] = {"TRANSLATOR OP",
+                                  glusterfs_handle_translator_op, NULL,
+                                  GLUSTERD_BRICK_XLATOR_OP, DRC_NA, 0},
+    [GLUSTERD_BRICK_STATUS] = {"STATUS", glusterfs_handle_brick_status, NULL,
+                               GLUSTERD_BRICK_STATUS, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_DEFRAG] = {"TRANSLATOR DEFRAG",
+                                      glusterfs_handle_defrag, NULL,
+                                      GLUSTERD_BRICK_XLATOR_DEFRAG, DRC_NA, 0},
+    [GLUSTERD_NODE_PROFILE] = {"NFS PROFILE", glusterfs_handle_nfs_profile,
+                               NULL, GLUSTERD_NODE_PROFILE, DRC_NA, 0},
+    [GLUSTERD_NODE_STATUS] = {"NFS STATUS", glusterfs_handle_node_status, NULL,
+                              GLUSTERD_NODE_STATUS, DRC_NA, 0},
+    [GLUSTERD_VOLUME_BARRIER_OP] = {"VOLUME BARRIER OP",
+                                    glusterfs_handle_volume_barrier_op, NULL,
+                                    GLUSTERD_VOLUME_BARRIER_OP, DRC_NA, 0},
+    [GLUSTERD_BRICK_BARRIER] = {"BARRIER", glusterfs_handle_barrier, NULL,
+                                GLUSTERD_BRICK_BARRIER, DRC_NA, 0},
+    [GLUSTERD_NODE_BITROT] = {"BITROT", glusterfs_handle_bitrot, NULL,
+                              GLUSTERD_NODE_BITROT, DRC_NA, 0},
+    [GLUSTERD_BRICK_ATTACH] = {"ATTACH", glusterfs_handle_attach, NULL,
+                               GLUSTERD_BRICK_ATTACH, DRC_NA, 0},
+
+    [GLUSTERD_DUMP_METRICS] = {"DUMP METRICS", glusterfs_handle_dump_metrics,
+                               NULL, GLUSTERD_DUMP_METRICS, DRC_NA, 0},
+
+    [GLUSTERD_SVC_ATTACH] = {"ATTACH CLIENT", glusterfs_handle_svc_attach, NULL,
+                             GLUSTERD_SVC_ATTACH, DRC_NA, 0},
+
+    [GLUSTERD_SVC_DETACH] = {"DETACH CLIENT", glusterfs_handle_svc_detach, NULL,
+                             GLUSTERD_SVC_DETACH, DRC_NA, 0},
+
 };
 
-struct rpcsvc_program glusterfs_mop_prog = {
-        .progname  = "Gluster Brick operations",
-        .prognum   = GD_BRICK_PROGRAM,
-        .progver   = GD_BRICK_VERSION,
-        .actors    = glusterfs_actors,
-        .numactors = GLUSTERD_BRICK_MAXVALUE,
-	.synctask  = _gf_true,
+static struct rpcsvc_program glusterfs_mop_prog = {
+    .progname = "Gluster Brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .actors = glusterfs_actors,
+    .numactors = GLUSTERD_BRICK_MAXVALUE,
+    .synctask = _gf_true,
 };
 
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-                     glusterfs_ctx_t *ctx,
-                     rpc_clnt_prog_t *prog, int procnum,
-                     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-        int                     ret         = -1;
-        int                     count      = 0;
-        struct iovec            iov         = {0, };
-        struct iobuf            *iobuf = NULL;
-        struct iobref           *iobref = NULL;
-        ssize_t                 xdr_size = 0;
-
-        iobref = iobref_new ();
-        if (!iobref) {
-                goto out;
-        }
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-
-                iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_pagesize (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        gf_log (THIS->name, GF_LOG_WARNING, "failed to create XDR payload");
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to create XDR payload");
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
-
-/* XXX: move these into @ctx */
-static char *oldvolfile = NULL;
-static int oldvollen = 0;
-
-
-
 int
-mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                  void *myframe)
+mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf_getspec_rsp           rsp   = {0,};
-        call_frame_t            *frame = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        int                      ret   = 0;
-        ssize_t                  size = 0;
-        FILE                    *tmpfp = NULL;
-        char                    *volfilebuf = NULL;
-
-        frame = myframe;
-        ctx = frame->this->ctx;
-
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                goto out;
-        }
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0, locked = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    char *volfile_id = NULL;
+    gf_volfile_t *volfile_obj = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+    char sha256_hash[SHA256_DIGEST_LENGTH] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/glfs.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-                ret   = -1;
-                goto out;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the 'volume file' from server");
+        ret = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        /* Server list is set by glusterd at the time of getspec */
+        ret = dict_get_str(dict, GLUSTERD_BRICK_SERVERS, &servers_list);
+        if (ret)
+            goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    size = rsp.op_ret;
+    volfile_id = frame->local;
+    if (mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) {
+        ret = mgmt_process_volfile((const char *)rsp.spec, size, volfile_id,
+                                   dict);
+        goto post_graph_mgmt;
+    }
+
+    ret = 0;
+    glusterfs_compute_sha256((const unsigned char *)rsp.spec, size,
+                             sha256_hash);
+
+    LOCK(&ctx->volfile_lock);
+    {
+        locked = 1;
+
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(volfile_id, volfile_obj->vol_id)) {
+                if (!memcmp(sha256_hash, volfile_obj->volfile_checksum,
+                            sizeof(volfile_obj->volfile_checksum))) {
+                    UNLOCK(&ctx->volfile_lock);
+                    gf_log(frame->this->name, GF_LOG_INFO,
+                           "No change in volfile,"
+                           "continuing");
+                    goto post_unlock;
+                }
+                volfile_tmp = volfile_obj;
+                break;
+            }
         }
 
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the 'volume file' from server");
-                ret = rsp.op_errno;
-                goto out;
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode */
+        tmp_fd = mkstemp(template);
+        if (-1 == tmp_fd) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(frame->this->name, GF_LOG_ERROR, 0, glusterfsd_msg_39,
+                    "create template=%s", template, NULL);
+            ret = -1;
+            goto post_unlock;
         }
 
-        ret = 0;
-        size = rsp.op_ret;
-
-        if (size == oldvollen && (memcmp (oldvolfile, rsp.spec, size) == 0)) {
-                gf_log (frame->this->name, GF_LOG_INFO,
-                        "No change in volfile, continuing");
-                goto out;
+        /* Calling unlink so that when the file is closed or program
+         * terminates the temporary file is deleted.
+         */
+        ret = sys_unlink(template);
+        if (ret < 0) {
+            gf_smsg(frame->this->name, GF_LOG_INFO, 0, glusterfsd_msg_39,
+                    "delete template=%s", template, NULL);
+            ret = 0;
         }
 
-        tmpfp = tmpfile ();
+        tmpfp = fdopen(tmp_fd, "w+b");
         if (!tmpfp) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        fwrite (rsp.spec, size, 1, tmpfp);
-        fflush (tmpfp);
-        if (ferror (tmpfp)) {
-                ret = -1;
-                goto out;
+        fwrite(rsp.spec, size, 1, tmpfp);
+        fflush(tmpfp);
+        if (ferror(tmpfp)) {
+            ret = -1;
+            goto out;
         }
 
         /*  Check if only options have changed. No need to reload the
-        *  volfile if topology hasn't changed.
-        *  glusterfs_volfile_reconfigure returns 3 possible return states
-        *  return 0          =======> reconfiguration of options has succeeded
-        *  return 1          =======> the graph has to be reconstructed and all the xlators should be inited
-        *  return -1(or -ve) =======> Some Internal Error occurred during the operation
-        */
+         *  volfile if topology hasn't changed.
+         *  glusterfs_volfile_reconfigure returns 3 possible return states
+         *  return 0          =======> reconfiguration of options has succeeded
+         *  return 1          =======> the graph has to be reconstructed and all
+         * the xlators should be inited return -1(or -ve) =======> Some Internal
+         * Error occurred during the operation
+         */
 
-        ret = glusterfs_volfile_reconfigure (oldvollen, tmpfp, ctx, oldvolfile);
+        ret = glusterfs_volfile_reconfigure(tmpfp, ctx);
         if (ret == 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "No need to re-load volfile, reconfigure done");
-                if (oldvolfile)
-                        volfilebuf = GF_REALLOC (oldvolfile, size);
-                else
-                        volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
-                if (!volfilebuf) {
-                        ret = -1;
-                        goto out;
-                }
-                oldvolfile = volfilebuf;
-                oldvollen = size;
-                memcpy (oldvolfile, rsp.spec, size);
-                goto out;
+            gf_log("glusterfsd-mgmt", GF_LOG_DEBUG,
+                   "No need to re-load volfile, reconfigure done");
+            if (!volfile_tmp) {
+                ret = -1;
+                UNLOCK(&ctx->volfile_lock);
+                gf_log("mgmt", GF_LOG_ERROR,
+                       "Graph reconfigure succeeded with out having "
+                       "checksum.");
+                goto post_unlock;
+            }
+            memcpy(volfile_tmp->volfile_checksum, sha256_hash,
+                   sizeof(volfile_tmp->volfile_checksum));
+            goto out;
         }
 
         if (ret < 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG, "Reconfigure failed !!");
-                goto out;
+            UNLOCK(&ctx->volfile_lock);
+            gf_log("glusterfsd-mgmt", GF_LOG_DEBUG, "Reconfigure failed !!");
+            goto post_unlock;
         }
 
-        ret = glusterfs_process_volfp (ctx, tmpfp);
+        ret = glusterfs_process_volfp(ctx, tmpfp);
         /* tmpfp closed */
         tmpfp = NULL;
+        tmp_fd = -1;
         if (ret)
-                goto out;
-
-        if (oldvolfile)
-                volfilebuf = GF_REALLOC (oldvolfile, size);
-        else
-                volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+            goto out;
 
-        if (!volfilebuf) {
+        if (!volfile_tmp) {
+            volfile_tmp = GF_CALLOC(1, sizeof(gf_volfile_t),
+                                    gf_common_volfile_t);
+            if (!volfile_tmp) {
                 ret = -1;
                 goto out;
-        }
-        oldvolfile = volfilebuf;
-        oldvollen = size;
-        memcpy (oldvolfile, rsp.spec, size);
-        if (!is_mgmt_rpc_reconnect) {
-                need_emancipate = 1;
-                glusterfs_mgmt_pmap_signin (ctx);
-                is_mgmt_rpc_reconnect =  _gf_true;
-        }
-
-out:
-        STACK_DESTROY (frame->root);
-
-        free (rsp.spec);
+            }
 
-        // Stop if server is running at an unsupported op-version
-        if (ENOTSUP == ret) {
-                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
-                        "op-version which is not supported");
-                cleanup_and_exit (0);
+            INIT_LIST_HEAD(&volfile_tmp->volfile_list);
+            volfile_tmp->graph = ctx->active;
+            list_add(&volfile_tmp->volfile_list, &ctx->volfile_list);
+            snprintf(volfile_tmp->vol_id, sizeof(volfile_tmp->vol_id), "%s",
+                     volfile_id);
         }
+        memcpy(volfile_tmp->volfile_checksum, sha256_hash,
+               sizeof(volfile_tmp->volfile_checksum));
+    }
+    UNLOCK(&ctx->volfile_lock);
 
-        if (ret && ctx && !ctx->active) {
-                /* Do it only for the first time */
-                /* Failed to get the volume file, something wrong,
-                   restart the process */
-                gf_log ("mgmt", GF_LOG_ERROR,
-                        "failed to fetch volume file (key:%s)",
-                        ctx->cmd_args.volfile_id);
-                cleanup_and_exit (0);
-        }
+    locked = 0;
 
+post_graph_mgmt:
+    if (!is_mgmt_rpc_reconnect) {
+        need_emancipate = 1;
+        glusterfs_mgmt_pmap_signin(ctx);
+        is_mgmt_rpc_reconnect = _gf_true;
+    }
 
-        if (tmpfp)
-                fclose (tmpfp);
+out:
 
-        return 0;
+    if (locked)
+        UNLOCK(&ctx->volfile_lock);
+post_unlock:
+    GF_FREE(frame->local);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_log("mgmt", GF_LOG_ERROR,
+               "Server is operating at an "
+               "op-version which is not supported");
+        cleanup_and_exit(0);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_log("mgmt", GF_LOG_ERROR, "failed to fetch volume file (key:%s)",
+               ctx->cmd_args.volfile_id);
+        cleanup_and_exit(0);
+    }
+
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
+
+    return 0;
 }
 
-
-int
-glusterfs_volfile_fetch (glusterfs_ctx_t *ctx)
+static int
+glusterfs_volfile_fetch_one(glusterfs_ctx_t *ctx, char *volfile_id)
 {
-        cmd_args_t       *cmd_args = NULL;
-        gf_getspec_req    req = {0, };
-        int               ret = 0;
-        call_frame_t     *frame = NULL;
-        dict_t           *dict = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        frame = create_frame (THIS, ctx->pool);
-
-        req.key = cmd_args->volfile_id;
-        req.flags = 0;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    if (!volfile_id) {
+        volfile_id = ctx->cmd_args.volfile_id;
+        if (!volfile_id) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "No volfile-id provided, erroring out");
+            return -1;
+        }
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    req.key = volfile_id;
+    req.flags = 0;
+    /*
+     * We are only storing one variable in local, hence using the same
+     * variable. If multiple local variable is required, create a struct.
+     */
+    frame->local = gf_strdup(volfile_id);
+    if (!frame->local) {
+        ret = -1;
+        goto out;
+    }
 
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version"
+               " in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version"
+               " in request dict");
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    if (cmd_args->brick_name) {
+        ret = dict_set_dynstr_with_alloc(dict, "brick_name",
+                                         cmd_args->brick_name);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Failed to set brick_name in request dict");
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to serialize dictionary");
+        goto out;
+    }
 
-        if (cmd_args->brick_name) {
-                ret = dict_set_dynstr_with_alloc (dict, "brick_name",
-                                                  cmd_args->brick_name);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Failed to set brick_name in request dict");
-                        goto out;
-                }
-        }
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+out:
+    GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
+    if (ret && frame) {
+        /* Free the frame->local fast, because we have not used memget
+         */
+        GF_FREE(frame->local);
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
 
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
-                                   (xdrproc_t)xdr_gf_getspec_req);
+    return ret;
+}
 
-out:
-        GF_FREE (req.xdata.xdata_val);
-        if (dict)
-                dict_unref (dict);
+int
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx)
+{
+    xlator_t *server_xl = NULL;
+    xlator_list_t *trav;
+    gf_volfile_t *volfile_obj = NULL;
+    int ret = 0;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        if (ctx->active &&
+            mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) {
+            list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+            {
+                ret |= glusterfs_volfile_fetch_one(ctx, volfile_obj->vol_id);
+            }
+            UNLOCK(&ctx->volfile_lock);
+            return ret;
+        }
+
+        if (ctx->active) {
+            server_xl = ctx->active->first;
+            if (strcmp(server_xl->type, "protocol/server") != 0) {
+                server_xl = NULL;
+            }
+        }
+        if (!server_xl) {
+            /* Startup (ctx->active not set) or non-server. */
+            UNLOCK(&ctx->volfile_lock);
+            return glusterfs_volfile_fetch_one(ctx, ctx->cmd_args.volfile_id);
+        }
 
-        return ret;
+        ret = 0;
+        for (trav = server_xl->children; trav; trav = trav->next) {
+            ret |= glusterfs_volfile_fetch_one(ctx, trav->xlator->volfile_id);
+        }
+    }
+    UNLOCK(&ctx->volfile_lock);
+    return ret;
 }
 
 int32_t
-mgmt_event_notify_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                  void *myframe)
+mgmt_event_notify_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        gf_event_notify_rsp      rsp   = {0,};
-        call_frame_t            *frame = NULL;
-        int                      ret   = 0;
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-                ret   = -1;
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
 
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the rsp from server");
-                ret = -1;
-                goto out;
-        }
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the rsp from server");
+        ret = -1;
+        goto out;
+    }
 out:
-        free (rsp.dict.dict_val); //malloced by xdr
-        return ret;
-
+    free(rsp.dict.dict_val);  // malloced by xdr
+    return ret;
 }
 
 int32_t
-glusterfs_rebalance_event_notify_cbk (struct rpc_req *req, struct iovec *iov,
-                                      int count, void *myframe)
+glusterfs_rebalance_event_notify_cbk(struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
 {
-        gf_event_notify_rsp      rsp   = {0,};
-        call_frame_t            *frame = NULL;
-        int                      ret   = 0;
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the rsp from server");
-                ret = -1;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the rsp from server");
+        ret = -1;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
-                ret   = -1;
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
 
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Received error (%s) from server",
-                        strerror (rsp.op_errno));
-                ret = -1;
-                goto out;
-        }
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Received error (%s) from server", strerror(rsp.op_errno));
+        ret = -1;
+        goto out;
+    }
 out:
-        free (rsp.dict.dict_val); //malloced by xdr
-
-        if (frame) {
-                STACK_DESTROY (frame->root);
-        }
+    free(rsp.dict.dict_val);  // malloced by xdr
 
-        return ret;
+    if (frame) {
+        STACK_DESTROY(frame->root);
+    }
 
+    return ret;
 }
 
 int32_t
-glusterfs_rebalance_event_notify (dict_t *dict)
+glusterfs_rebalance_event_notify(dict_t *dict)
 {
-        glusterfs_ctx_t         *ctx = NULL;
-        gf_event_notify_req      req = {0,};
-        int32_t                  ret = -1;
-        cmd_args_t              *cmd_args = NULL;
-        call_frame_t            *frame = NULL;
-
-        ctx = glusterfsd_ctx;
-        cmd_args = &ctx->cmd_args;
+    glusterfs_ctx_t *ctx = NULL;
+    gf_event_notify_req req = {
+        0,
+    };
+    int32_t ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    call_frame_t *frame = NULL;
 
-        frame = create_frame (THIS, ctx->pool);
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
 
-        req.op = GF_EN_DEFRAG_STATUS;
+    frame = create_frame(THIS, ctx->pool);
 
-        if (dict) {
-                ret = dict_set_str (dict, "volname", cmd_args->volfile_id);
-                if (ret)
-                        gf_log ("", GF_LOG_ERROR, "failed to set volname");
+    req.op = GF_EN_DEFRAG_STATUS;
 
-                ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                                   &req.dict.dict_len);
+    if (dict) {
+        ret = dict_set_str(dict, "volname", cmd_args->volfile_id);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "failed to set volname");
         }
+        ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                          &req.dict.dict_len);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "failed to serialize dict");
+        }
+    }
 
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_EVENT_NOTIFY,
-                                   glusterfs_rebalance_event_notify_cbk,
-                                   (xdrproc_t)xdr_gf_event_notify_req);
-
-        GF_FREE (req.dict.dict_val);
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_EVENT_NOTIFY,
+                              glusterfs_rebalance_event_notify_cbk,
+                              (xdrproc_t)xdr_gf_event_notify_req);
 
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-        xlator_t         *this = NULL;
-        glusterfs_ctx_t  *ctx = NULL;
-        int              ret = 0;
-        server_cmdline_t *server = NULL;
-        rpc_transport_t  *rpc_trans = NULL;
-        int              need_term = 0;
-        int              emval = 0;
-        struct dnscache6 *dnscache = NULL;
-
-        this = mydata;
-        rpc_trans = rpc->conn.trans;
-        ctx = this->ctx;
-
-        switch (event) {
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    int need_term = 0;
+    int emval = 0;
+    static int log_ctr1;
+    static int log_ctr2;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+    ctx = this->ctx;
+
+    switch (event) {
         case RPC_CLNT_DISCONNECT:
-                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                        "failed to connect with remote-host: %s (%s)",
-                        ctx->cmd_args.volfile_server, strerror (errno));
-                if (!rpc->disabled) {
-                        /*
-                         * Check if dnscache is exhausted for current server
-                         * and continue until cache is exhausted
-                         */
-                        dnscache = rpc_trans->dnscache;
-                        if (dnscache && dnscache->next) {
-                                break;
-                        }
+            if (rpc_trans->connect_failed) {
+                GF_LOG_OCCASIONALLY(log_ctr1, "glusterfsd-mgmt", GF_LOG_ERROR,
+                                    "failed to connect to remote-"
+                                    "host: %s",
+                                    ctx->cmd_args.volfile_server);
+            } else {
+                GF_LOG_OCCASIONALLY(log_ctr1, "glusterfsd-mgmt", GF_LOG_INFO,
+                                    "disconnected from remote-"
+                                    "host: %s",
+                                    ctx->cmd_args.volfile_server);
+            }
+
+            if (!rpc->disabled) {
+                /*
+                 * Check if dnscache is exhausted for current server
+                 * and continue until cache is exhausted
+                 */
+                dnscache = rpc_trans->dnscache;
+                if (dnscache && dnscache->next) {
+                    break;
                 }
-                server = ctx->cmd_args.curr_server;
-                if (server->list.next == &ctx->cmd_args.volfile_servers) {
-                        if (!ctx->active)
-                                need_term = 1;
-                        emval = ENOTCONN;
-                        gf_log("glusterfsd-mgmt", GF_LOG_INFO,
-                               "Exhausted all volfile servers");
-                        break;
+            }
+            server = ctx->cmd_args.curr_server;
+            if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                if (!ctx->active) {
+                    need_term = 1;
                 }
-                server = list_entry (server->list.next, typeof(*server), list);
-                ctx->cmd_args.curr_server = server;
-                ctx->cmd_args.volfile_server = server->volfile_server;
-
-                ret = dict_set_str (rpc_trans->options, "remote-host",
-                                    server->volfile_server);
-                if (ret != 0) {
-                        gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                                "failed to set remote-host: %s",
-                                server->volfile_server);
-                        if (!ctx->active)
-                                need_term = 1;
-                        emval = ENOTCONN;
-                        break;
+                emval = ENOTCONN;
+                GF_LOG_OCCASIONALLY(log_ctr2, "glusterfsd-mgmt", GF_LOG_INFO,
+                                    "Exhausted all volfile servers");
+                break;
+            }
+            server = list_entry(server->list.next, typeof(*server), list);
+            ctx->cmd_args.curr_server = server;
+            ctx->cmd_args.volfile_server = server->volfile_server;
+
+            ret = dict_set_str(rpc_trans->options, "remote-host",
+                               server->volfile_server);
+            if (ret != 0) {
+                gf_log("glusterfsd-mgmt", GF_LOG_ERROR,
+                       "failed to set remote-host: %s", server->volfile_server);
+                if (!ctx->active) {
+                    need_term = 1;
                 }
-                gf_log ("glusterfsd-mgmt", GF_LOG_INFO,
-                        "connecting to next volfile server %s",
-                        server->volfile_server);
+                emval = ENOTCONN;
                 break;
+            }
+            gf_log("glusterfsd-mgmt", GF_LOG_INFO,
+                   "connecting to next volfile server %s",
+                   server->volfile_server);
+            break;
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-                ret = glusterfs_volfile_fetch (ctx);
-                if (ret) {
-                        emval = ret;
-                        if (!ctx->active) {
-                                need_term = 1;
-                                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                                        "failed to fetch volume file (key:%s)",
-                                        ctx->cmd_args.volfile_id);
-                                break;
-
-                        }
+            ret = glusterfs_volfile_fetch(ctx);
+            if (ret) {
+                emval = ret;
+                if (!ctx->active) {
+                    need_term = 1;
+                    gf_log("glusterfsd-mgmt", GF_LOG_ERROR,
+                           "failed to fetch volume file (key:%s)",
+                           ctx->cmd_args.volfile_id);
+                    break;
                 }
+            }
 
-                if (is_mgmt_rpc_reconnect)
-                        glusterfs_mgmt_pmap_signin (ctx);
+            if (is_mgmt_rpc_reconnect)
+                glusterfs_mgmt_pmap_signin(ctx);
 
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        if (need_term) {
-                emancipate (ctx, emval);
-                cleanup_and_exit (1);
-        }
+    if (need_term) {
+        emancipate(ctx, emval);
+        cleanup_and_exit(1);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-glusterfs_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
-                     void *data)
+glusterfs_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                        void *data)
 {
-        if (!xl || !data) {
-                goto out;
-        }
+    if (!xl || !data) {
+        goto out;
+    }
 
-        switch (event) {
-        case RPCSVC_EVENT_ACCEPT:
-        {
-                break;
+    switch (event) {
+        case RPCSVC_EVENT_ACCEPT: {
+            break;
         }
-        case RPCSVC_EVENT_DISCONNECT:
-        {
-                break;
+        case RPCSVC_EVENT_DISCONNECT: {
+            break;
         }
 
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-glusterfs_listener_init (glusterfs_ctx_t *ctx)
+glusterfs_listener_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t              *cmd_args = NULL;
-        rpcsvc_t                *rpc = NULL;
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-
-        cmd_args = &ctx->cmd_args;
-
-        if (ctx->listener)
-                return 0;
-
-        if (!cmd_args->sock_file)
-                return 0;
-
-        ret = rpcsvc_transport_unix_options_build (&options,
-                                                   cmd_args->sock_file);
-        if (ret)
-                goto out;
-
-        rpc = rpcsvc_init (THIS, ctx, options, 8);
-        if (rpc == NULL) {
-                goto out;
-        }
-
-        ret = rpcsvc_register_notify (rpc, glusterfs_rpcsvc_notify, THIS);
-        if (ret) {
-                goto out;
-        }
-
-        ret = rpcsvc_create_listeners (rpc, options, "glusterfsd");
-        if (ret < 1) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpcsvc_program_register (rpc, &glusterfs_mop_prog);
-        if (ret) {
-                goto out;
-        }
+    cmd_args_t *cmd_args = NULL;
+    rpcsvc_t *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
 
-        ctx->listener = rpc;
+    cmd_args = &ctx->cmd_args;
 
-out:
-        return ret;
-}
+    if (ctx->listener)
+        return 0;
 
-int
-glusterfs_listener_stop (glusterfs_ctx_t *ctx)
-{
-        cmd_args_t              *cmd_args = NULL;
-        rpcsvc_t                *rpc = NULL;
-        rpcsvc_listener_t       *listener = NULL;
-        rpcsvc_listener_t       *next = NULL;
-        int                     ret = 0;
-        xlator_t                *this = NULL;
+    if (!cmd_args->sock_file)
+        return 0;
 
-        GF_ASSERT (ctx);
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        rpc = ctx->listener;
-        ctx->listener = NULL;
+    ret = rpcsvc_transport_unix_options_build(options, cmd_args->sock_file);
+    if (ret)
+        goto out;
 
-        (void) rpcsvc_program_unregister(rpc, &glusterfs_mop_prog);
+    rpc = rpcsvc_init(THIS, ctx, options, 8);
+    if (rpc == NULL) {
+        goto out;
+    }
 
-        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
-                rpcsvc_listener_destroy (listener);
-        }
+    ret = rpcsvc_register_notify(rpc, glusterfs_rpcsvc_notify, THIS);
+    if (ret) {
+        goto out;
+    }
 
-        (void) rpcsvc_unregister_notify (rpc, glusterfs_rpcsvc_notify, THIS);
+    ret = rpcsvc_create_listeners(rpc, options, "glusterfsd");
+    if (ret < 1) {
+        goto out;
+    }
 
-        GF_FREE (rpc);
+    ret = rpcsvc_program_register(rpc, &glusterfs_mop_prog, _gf_false);
+    if (ret) {
+        goto out;
+    }
 
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = sys_unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
+    ctx->listener = rpc;
 
-        if (ret) {
-                this = THIS;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
-                        "socket %s, error: %s", cmd_args->sock_file,
-                        strerror (errno));
-        }
-        return ret;
+out:
+    if (options)
+        dict_unref(options);
+    return ret;
 }
 
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-        int ret = 0;
-        switch (op)
-        {
-                case GF_EN_DEFRAG_STATUS:
-                        ret = glusterfs_rebalance_event_notify ((dict_t*) data);
-                        break;
+    int ret = 0;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            ret = glusterfs_rebalance_event_notify((dict_t *)data);
+            break;
 
-                default:
-                        gf_log ("", GF_LOG_ERROR, "Invalid op");
-                        break;
-        }
+        default:
+            gf_log("", GF_LOG_ERROR, "Invalid op");
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
+glusterfs_mgmt_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t              *cmd_args = NULL;
-        struct rpc_clnt         *rpc = NULL;
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-        int                     port = GF_DEFAULT_BASE_PORT;
-        char                    *host = NULL;
-
-        cmd_args = &ctx->cmd_args;
-        GF_VALIDATE_OR_GOTO (THIS->name, cmd_args->volfile_server, out);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    xlator_cmdline_option_t *opt = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args->volfile_server, out);
+
+    if (ctx->mgmt)
+        return 0;
 
-        if (ctx->mgmt)
-                return 0;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        if (cmd_args->volfile_server_port)
-                port = cmd_args->volfile_server_port;
+    LOCK_INIT(&ctx->volfile_lock);
 
-        host = cmd_args->volfile_server;
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
 
-        if (cmd_args->volfile_server_transport &&
-            !strcmp (cmd_args->volfile_server_transport, "unix")) {
-                ret = rpc_transport_unix_options_build (&options, host, 0);
-        } else {
-                ret = rpc_transport_inet_options_build (&options, host, port);
-        }
-        if (ret)
-                goto out;
+    host = cmd_args->volfile_server;
 
-        /* Explicitly turn on encrypted transport. */
-        if (ctx->secure_mgmt) {
-                ret = dict_set_dynstr_with_alloc
-                        (options, "transport.socket.ssl-enabled", "yes");
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "failed to set 'transport.socket.ssl-enabled' "
-                                "in options dict");
-                        goto out;
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        opt = find_xlator_option_in_cmd_args_t("address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+    if (ret)
+        goto out;
 
-                }
-        }
-
-        rpc = rpc_clnt_new (options, THIS, THIS->name, 8);
-        if (!rpc) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to create rpc clnt");
-                goto out;
-        }
-
-        ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
+    /* Explicitly turn on encrypted transport. */
+    if (ctx->secure_mgmt) {
+        ret = dict_set_dynstr_with_alloc(options,
+                                         "transport.socket.ssl-enabled", "yes");
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to register notify function");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set 'transport.socket.ssl-enabled' "
+                   "in options dict");
+            goto out;
         }
 
-        ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to register callback function");
-                goto out;
-        }
+        ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-        ctx->notify = glusterfs_mgmt_notify;
-
-        /* This value should be set before doing the 'rpc_clnt_start()' as
-           the notify function uses this variable */
-        ctx->mgmt = rpc;
-
-        ret = rpc_clnt_start (rpc);
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to create rpc clnt");
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "failed to register notify function");
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "failed to register callback function");
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-        return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
 
 static int
-mgmt_pmap_signin2_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+mgmt_pmap_signin2_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        pmap_signin_rsp  rsp   = {0,};
-        glusterfs_ctx_t *ctx   = NULL;
-        call_frame_t    *frame = NULL;
-        int              ret   = 0;
-
-        ctx = glusterfsd_ctx;
-        frame = myframe;
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
 
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    ctx = glusterfsd_ctx;
+    frame = myframe;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decode error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decode error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to register the port with glusterd");
+        ret = -1;
+        goto out;
+    }
 
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to register the port with glusterd");
-                ret = -1;
-                goto out;
-        }
+    ret = 0;
 out:
-        if (need_emancipate)
-                emancipate (ctx, ret);
-
-        STACK_DESTROY (frame->root);
-        return 0;
+    if (need_emancipate)
+        emancipate(ctx, ret);
 
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static int
-mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+mgmt_pmap_signin_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        pmap_signin_rsp  rsp   = {0,};
-        call_frame_t    *frame = NULL;
-        int              ret   = 0;
-        int              emancipate_ret  = -1;
-        pmap_signin_req  pmap_req = {0, };
-        cmd_args_t      *cmd_args = NULL;
-        glusterfs_ctx_t *ctx      = NULL;
-        char             brick_name[PATH_MAX] = {0,};
-
-        frame = myframe;
-        ctx = glusterfsd_ctx;
-        cmd_args = &ctx->cmd_args;
-
-
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decode error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to register the port with glusterd");
-                ret = -1;
-                goto out;
-        }
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    int emancipate_ret = -1;
+    pmap_signin_req pmap_req = {
+        0,
+    };
+    cmd_args_t *cmd_args = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char brick_name[PATH_MAX] = {
+        0,
+    };
+
+    frame = myframe;
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decode error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to register the port with glusterd");
+        ret = -1;
+        goto out;
+    }
 
-        if (!cmd_args->brick_port2) {
-                /* We are done with signin process */
-                emancipate_ret = 0;
-                goto out;
-        }
+    if (!cmd_args->brick_port2) {
+        /* We are done with signin process */
+        emancipate_ret = 0;
+        goto out;
+    }
 
-        snprintf (brick_name, PATH_MAX, "%s.rdma", cmd_args->brick_name);
-        pmap_req.port  = cmd_args->brick_port2;
-        pmap_req.brick = brick_name;
+    snprintf(brick_name, PATH_MAX, "%s.rdma", cmd_args->brick_name);
+    pmap_req.port = cmd_args->brick_port2;
+    pmap_req.brick = brick_name;
 
-        ret = mgmt_submit_request (&pmap_req, frame, ctx, &clnt_pmap_prog,
-                                   GF_PMAP_SIGNIN, mgmt_pmap_signin2_cbk,
-                                   (xdrproc_t)xdr_pmap_signin_req);
-        if (ret)
-                goto out;
+    ret = mgmt_submit_request(&pmap_req, frame, ctx, &clnt_pmap_prog,
+                              GF_PMAP_SIGNIN, mgmt_pmap_signin2_cbk,
+                              (xdrproc_t)xdr_pmap_signin_req);
+    if (ret)
+        goto out;
 
-        return 0;
+    return 0;
 
 out:
-        if (need_emancipate && (ret < 0 || !cmd_args->brick_port2))
-                emancipate (ctx, emancipate_ret);
+    if (need_emancipate && (ret < 0 || !cmd_args->brick_port2))
+        emancipate(ctx, emancipate_ret);
 
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 int
-glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx)
 {
-        call_frame_t     *frame                 = NULL;
-        pmap_signin_req   req                   = {0, };
-        int               ret                   = -1;
-        int               emancipate_ret        = -1;
-        cmd_args_t       *cmd_args              = NULL;
-        char              brick_name[PATH_MAX]  = {0,};
-
-        frame = create_frame (THIS, ctx->pool);
-        cmd_args = &ctx->cmd_args;
-
-        if (!cmd_args->brick_port || !cmd_args->brick_name) {
-                gf_log ("fsd-mgmt", GF_LOG_DEBUG,
-                        "portmapper signin arguments not given");
-                emancipate_ret = 0;
-                goto out;
-        }
-
-        if (cmd_args->volfile_server_transport &&
-                      !strcmp(cmd_args->volfile_server_transport, "rdma")) {
-                snprintf (brick_name, sizeof(brick_name), "%s.rdma",
-                          cmd_args->brick_name);
-                req.brick = brick_name;
-        } else
-                req.brick = cmd_args->brick_name;
-
-        req.port  = cmd_args->brick_port;
-
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
-                                   GF_PMAP_SIGNIN, mgmt_pmap_signin_cbk,
-                                   (xdrproc_t)xdr_pmap_signin_req);
-
-out:
-        if (need_emancipate && ret < 0)
-                emancipate (ctx, emancipate_ret);
-        return ret;
-}
-
-
-static int
-mgmt_pmap_signout_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
-{
-        pmap_signout_rsp  rsp   = {0,};
-        int              ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_signout_rsp);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 == rsp.op_ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to register the port with glusterd");
-                goto out;
-        }
-out:
-        return 0;
-}
-
-
-int
-glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
-{
-        int               ret = 0;
-        pmap_signout_req  req = {0, };
-        call_frame_t     *frame = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        char              brick_name[PATH_MAX]  = {0,};
-
-        frame = create_frame (THIS, ctx->pool);
-        cmd_args = &ctx->cmd_args;
-
-        if (!cmd_args->brick_port || !cmd_args->brick_name) {
-                gf_log ("fsd-mgmt", GF_LOG_DEBUG,
-                        "portmapper signout arguments not given");
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    xlator_list_t **trav_p;
+    xlator_t *top;
+    pmap_signin_req req = {
+        0,
+    };
+    int ret = -1;
+    int emancipate_ret = -1;
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args->brick_port || !cmd_args->brick_name) {
+        gf_log("fsd-mgmt", GF_LOG_DEBUG,
+               "portmapper signin arguments not given");
+        emancipate_ret = 0;
+        goto out;
+    }
+
+    req.port = cmd_args->brick_port;
+    req.pid = (int)getpid(); /* only glusterd2 consumes this */
+
+    if (ctx->active) {
+        top = ctx->active->first;
+        for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+            frame = create_frame(THIS, ctx->pool);
+            req.brick = (*trav_p)->xlator->name;
+            ret = mgmt_submit_request(&req, frame, ctx, &clnt_pmap_prog,
+                                      GF_PMAP_SIGNIN, mgmt_pmap_signin_cbk,
+                                      (xdrproc_t)xdr_pmap_signin_req);
+            if (ret < 0) {
+                gf_log(THIS->name, GF_LOG_WARNING,
+                       "failed to send sign in request; brick = %s", req.brick);
+            }
+        }
+    }
+
+    /* unfortunately, the caller doesn't care about the returned value */
 
-        if (cmd_args->volfile_server_transport &&
-                      !strcmp(cmd_args->volfile_server_transport, "rdma")) {
-                snprintf (brick_name, sizeof(brick_name), "%s.rdma",
-                          cmd_args->brick_name);
-                req.brick = brick_name;
-        } else
-                req.brick = cmd_args->brick_name;
-
-        req.port  = cmd_args->brick_port;
-        req.rdma_port = cmd_args->brick_port2;
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
-                                   GF_PMAP_SIGNOUT, mgmt_pmap_signout_cbk,
-                                   (xdrproc_t)xdr_pmap_signout_req);
 out:
-        return ret;
+    if (need_emancipate && ret < 0)
+        emancipate(ctx, emancipate_ret);
+    return ret;
 }
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 9a5edd4ddf4..dae41f33fef 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -37,9 +37,9 @@
 #include <linux/oom.h>
 #else
 #define OOM_SCORE_ADJ_MIN (-1000)
-#define OOM_SCORE_ADJ_MAX   1000
-#define OOM_DISABLE       (-17)
-#define OOM_ADJUST_MAX      15
+#define OOM_SCORE_ADJ_MAX 1000
+#define OOM_DISABLE (-17)
+#define OOM_ADJUST_MAX 15
 #endif
 #endif
 
@@ -47,1297 +47,1415 @@
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
 #include "glusterfsd-messages.h"
-#include "dict.h"
-#include "list.h"
-#include "timer.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
 #include "glusterfsd.h"
-#include "stack.h"
-#include "revision.h"
-#include "common-utils.h"
-#include "event.h"
-#include "globals.h"
-#include "statedump.h"
-#include "latency.h"
+#include <glusterfs/revision.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/latency.h>
 #include "glusterfsd-mem-types.h"
-#include "syscall.h"
-#include "call-stub.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/call-stub.h>
 #include <fnmatch.h>
 #include "rpc-clnt.h"
-#include "syncop.h"
-#include "client_t.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/client_t.h>
 #include "netgroups.h"
 #include "exports.h"
+#include <glusterfs/monitoring.h>
 
-#include "daemon.h"
-#include "tw.h"
-
+#include <glusterfs/daemon.h>
 
 /* using argp for command line parsing */
 static char gf_doc[] = "";
-static char argp_doc[] = "--volfile-server=SERVER [MOUNT-POINT]\n"       \
-        "--volfile=VOLFILE [MOUNT-POINT]";
-const char *argp_program_version = ""
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"
-        "Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com/>\n"
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
-        "It is licensed to you under your choice of the GNU Lesser\n"
-        "General Public License, version 3 or any later version (LGPLv3\n"
-        "or later), or the GNU General Public License, version 2 (GPLv2),\n"
-        "in all cases as published by the Free Software Foundation.";
+static char argp_doc[] =
+    "--volfile-server=SERVER [MOUNT-POINT]\n"
+    "--volfile=VOLFILE [MOUNT-POINT]";
+const char *argp_program_version =
+    "" PACKAGE_NAME " " PACKAGE_VERSION
+    "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION
+    "\n"
+    "Copyright (c) 2006-2016 Red Hat, Inc. "
+    "<https://www.gluster.org/>\n"
+    "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+    "It is licensed to you under your choice of the GNU Lesser\n"
+    "General Public License, version 3 or any later version (LGPLv3\n"
+    "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+    "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
-static error_t parse_opts (int32_t key, char *arg, struct argp_state *_state);
+static error_t
+parse_opts(int32_t key, char *arg, struct argp_state *_state);
 
 static struct argp_option gf_options[] = {
-        {0, 0, 0, 0, "Basic options:"},
-        {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
-         "Server to get the volume file from. Unix domain socket path when "
-         "transport type 'unix'. This option overrides --volfile option"},
-        {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
-         "File to use as VOLUME_FILE"},
-        {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
-         "File to use as VOLUME FILE"},
-
-        {"log-level", ARGP_LOG_LEVEL_KEY, "LOGLEVEL", 0,
-         "Logging severity.  Valid options are DEBUG, INFO, WARNING, ERROR, "
-         "CRITICAL, TRACE and NONE [default: INFO]"},
-        {"log-file", ARGP_LOG_FILE_KEY, "LOGFILE", 0,
-         "File to use for logging [default: "
-         DEFAULT_LOG_FILE_DIRECTORY "/" PACKAGE_NAME ".log" "]"},
-        {"logger", ARGP_LOGGER, "LOGGER", 0, "Set which logging sub-system to "
-        "log to, valid options are: gluster-log and syslog, "
-        "[default: \"gluster-log\"]"},
-        {"log-format", ARGP_LOG_FORMAT, "LOG-FORMAT", 0, "Set log format, valid"
-         " options are: no-msg-id and with-msg-id, [default: \"with-msg-id\"]"},
-        {"log-buf-size", ARGP_LOG_BUF_SIZE, "LOG-BUF-SIZE", 0, "Set logging "
-         "buffer size, [default: 5]"},
-        {"log-flush-timeout", ARGP_LOG_FLUSH_TIMEOUT, "LOG-FLUSH-TIMEOUT", 0,
-         "Set log flush timeout, [default: 2 minutes]"},
-
-        {0, 0, 0, 0, "Advanced Options:"},
-        {"volfile-server-port", ARGP_VOLFILE_SERVER_PORT_KEY, "PORT", 0,
-         "Listening port number of volfile server"},
-        {"volfile-server-transport", ARGP_VOLFILE_SERVER_TRANSPORT_KEY,
-         "TRANSPORT", 0,
-         "Transport type to get volfile from server [default: socket]"},
-        {"volfile-id", ARGP_VOLFILE_ID_KEY, "KEY", 0,
-         "'key' of the volfile to be fetched from server"},
-        {"pid-file", ARGP_PID_FILE_KEY, "PIDFILE", 0,
-         "File to use as pid file"},
-        {"socket-file", ARGP_SOCK_FILE_KEY, "SOCKFILE", 0,
-         "File to use as unix-socket"},
-        {"no-daemon", ARGP_NO_DAEMON_KEY, 0, 0,
-         "Run in foreground"},
-        {"run-id", ARGP_RUN_ID_KEY, "RUN-ID", OPTION_HIDDEN,
-         "Run ID for the process, used by scripts to keep track of process "
-         "they started, defaults to none"},
-        {"debug", ARGP_DEBUG_KEY, 0, 0,
-         "Run in debug mode.  This option sets --no-daemon, --log-level "
-         "to DEBUG and --log-file to console"},
-        {"volume-name", ARGP_VOLUME_NAME_KEY, "XLATOR-NAME", 0,
-         "Translator name to be used for MOUNT-POINT [default: top most volume "
-         "definition in VOLFILE]"},
-        {"xlator-option", ARGP_XLATOR_OPTION_KEY,"XLATOR-NAME.OPTION=VALUE", 0,
-         "Add/override an option for a translator in volume file with specified"
-         " value"},
-        {"read-only", ARGP_READ_ONLY_KEY, 0, 0,
-         "Mount the filesystem in 'read-only' mode"},
-        {"acl", ARGP_ACL_KEY, 0, 0,
-         "Mount the filesystem with POSIX ACL support"},
-        {"selinux", ARGP_SELINUX_KEY, 0, 0,
-         "Enable SELinux label (extended attributes) support on inodes"},
-        {"capability", ARGP_CAPABILITY_KEY, 0, 0,
-         "Enable Capability (extended attributes) support on inodes"},
-
-        {"print-netgroups", ARGP_PRINT_NETGROUPS, "NETGROUP-FILE", 0,
-         "Validate the netgroups file and print it out"},
-        {"print-exports", ARGP_PRINT_EXPORTS, "EXPORTS-FILE", 0,
-        "Validate the exports file and print it out"},
-
-        {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS, "0",
-         OPTION_HIDDEN, "Maximum number of attempts to fetch the volfile"},
-        {"aux-gfid-mount", ARGP_AUX_GFID_MOUNT_KEY, 0, 0,
-         "Enable access to filesystem through gfid directly"},
-        {"enable-ino32", ARGP_INODE32_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Use 32-bit inodes when mounting to workaround broken applications"
-         "that don't support 64-bit inodes"},
-        {"worm", ARGP_WORM_KEY, 0, 0,
-         "Mount the filesystem in 'worm' mode"},
-        {"mac-compat", ARGP_MAC_COMPAT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Provide stubs for attributes needed for seamless operation on Macs "
+    {0, 0, 0, 0, "Basic options:"},
+    {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
+     "Server to get the volume file from. Unix domain socket path when "
+     "transport type 'unix'. This option overrides --volfile option"},
+    {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
+     "File to use as VOLUME_FILE"},
+    {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
+     "File to use as VOLUME FILE"},
+
+    {"log-level", ARGP_LOG_LEVEL_KEY, "LOGLEVEL", 0,
+     "Logging severity.  Valid options are DEBUG, INFO, WARNING, ERROR, "
+     "CRITICAL, TRACE and NONE [default: INFO]"},
+    {"log-file", ARGP_LOG_FILE_KEY, "LOGFILE", 0,
+     "File to use for logging [default: " DEFAULT_LOG_FILE_DIRECTORY
+     "/" PACKAGE_NAME ".log"
+     "]"},
+    {"logger", ARGP_LOGGER, "LOGGER", 0,
+     "Set which logging sub-system to "
+     "log to, valid options are: gluster-log and syslog, "
+     "[default: \"gluster-log\"]"},
+    {"log-format", ARGP_LOG_FORMAT, "LOG-FORMAT", 0,
+     "Set log format, valid"
+     " options are: no-msg-id and with-msg-id, [default: \"with-msg-id\"]"},
+    {"log-buf-size", ARGP_LOG_BUF_SIZE, "LOG-BUF-SIZE", 0,
+     "Set logging "
+     "buffer size, [default: 5]"},
+    {"log-flush-timeout", ARGP_LOG_FLUSH_TIMEOUT, "LOG-FLUSH-TIMEOUT", 0,
+     "Set log flush timeout, [default: 2 minutes]"},
+
+    {0, 0, 0, 0, "Advanced Options:"},
+    {"volfile-server-port", ARGP_VOLFILE_SERVER_PORT_KEY, "PORT", 0,
+     "Listening port number of volfile server"},
+    {"volfile-server-transport", ARGP_VOLFILE_SERVER_TRANSPORT_KEY, "TRANSPORT",
+     0, "Transport type to get volfile from server [default: socket]"},
+    {"volfile-id", ARGP_VOLFILE_ID_KEY, "KEY", 0,
+     "'key' of the volfile to be fetched from server"},
+    {"pid-file", ARGP_PID_FILE_KEY, "PIDFILE", 0, "File to use as pid file"},
+    {"socket-file", ARGP_SOCK_FILE_KEY, "SOCKFILE", 0,
+     "File to use as unix-socket"},
+    {"no-daemon", ARGP_NO_DAEMON_KEY, 0, 0, "Run in foreground"},
+    {"run-id", ARGP_RUN_ID_KEY, "RUN-ID", OPTION_HIDDEN,
+     "Run ID for the process, used by scripts to keep track of process "
+     "they started, defaults to none"},
+    {"debug", ARGP_DEBUG_KEY, 0, 0,
+     "Run in debug mode.  This option sets --no-daemon, --log-level "
+     "to DEBUG and --log-file to console"},
+    {"volume-name", ARGP_VOLUME_NAME_KEY, "XLATOR-NAME", 0,
+     "Translator name to be used for MOUNT-POINT [default: top most volume "
+     "definition in VOLFILE]"},
+    {"xlator-option", ARGP_XLATOR_OPTION_KEY, "XLATOR-NAME.OPTION=VALUE", 0,
+     "Add/override an option for a translator in volume file with specified"
+     " value"},
+    {"read-only", ARGP_READ_ONLY_KEY, 0, 0,
+     "Mount the filesystem in 'read-only' mode"},
+    {"acl", ARGP_ACL_KEY, 0, 0, "Mount the filesystem with POSIX ACL support"},
+    {"selinux", ARGP_SELINUX_KEY, 0, 0,
+     "Enable SELinux label (extended attributes) support on inodes"},
+    {"capability", ARGP_CAPABILITY_KEY, 0, 0,
+     "Enable Capability (extended attributes) support on inodes"},
+    {"subdir-mount", ARGP_SUBDIR_MOUNT_KEY, "SUBDIR-PATH", 0,
+     "Mount subdirectory given [default: NULL]"},
+
+    {"print-netgroups", ARGP_PRINT_NETGROUPS, "NETGROUP-FILE", 0,
+     "Validate the netgroups file and print it out"},
+    {"print-exports", ARGP_PRINT_EXPORTS, "EXPORTS-FILE", 0,
+     "Validate the exports file and print it out"},
+    {"print-xlatordir", ARGP_PRINT_XLATORDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print xlator directory path"},
+    {"print-statedumpdir", ARGP_PRINT_STATEDUMPDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print directory path in which statedumps shall be generated"},
+    {"print-logdir", ARGP_PRINT_LOGDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print path of default log directory"},
+    {"print-libexecdir", ARGP_PRINT_LIBEXECDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print path of default libexec directory"},
+
+    {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS, "0",
+     OPTION_HIDDEN, "Maximum number of attempts to fetch the volfile"},
+    {"aux-gfid-mount", ARGP_AUX_GFID_MOUNT_KEY, 0, 0,
+     "Enable access to filesystem through gfid directly"},
+    {"enable-ino32", ARGP_INODE32_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use 32-bit inodes when mounting to workaround broken applications"
+     "that don't support 64-bit inodes"},
+    {"worm", ARGP_WORM_KEY, 0, 0, "Mount the filesystem in 'worm' mode"},
+    {"mac-compat", ARGP_MAC_COMPAT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Provide stubs for attributes needed for seamless operation on Macs "
 #ifdef GF_DARWIN_HOST_OS
-         "[default: \"on\" on client side, else \"off\"]"
+     "[default: \"on\" on client side, else \"off\"]"
 #else
-         "[default: \"off\"]"
+     "[default: \"off\"]"
 #endif
-        },
-        {"brick-name", ARGP_BRICK_NAME_KEY, "BRICK-NAME", OPTION_HIDDEN,
-         "Brick name to be registered with Gluster portmapper" },
-        {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN,
-         "Brick Port to be registered with Gluster portmapper" },
-	{"fopen-keep-cache", ARGP_FOPEN_KEEP_CACHE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-	 "Do not purge the cache on file open"},
-        {"global-timer-wheel", ARGP_GLOBAL_TIMER_WHEEL, "BOOL",
-         OPTION_ARG_OPTIONAL, "Instantiate process global timer-wheel"},
-
-        {0, 0, 0, 0, "Fuse options:"},
-        {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Use direct I/O mode in fuse kernel module"
-         " [default: \"off\" if big writes are supported, else "
-         "\"on\" for fds not opened with O_RDONLY]"},
-        {"entry-timeout", ARGP_ENTRY_TIMEOUT_KEY, "SECONDS", 0,
-         "Set entry timeout to SECONDS in fuse kernel module [default: 1]"},
-        {"negative-timeout", ARGP_NEGATIVE_TIMEOUT_KEY, "SECONDS", 0,
-         "Set negative timeout to SECONDS in fuse kernel module [default: 0]"},
-        {"attribute-timeout", ARGP_ATTRIBUTE_TIMEOUT_KEY, "SECONDS", 0,
-         "Set attribute timeout to SECONDS for inodes in fuse kernel module "
-         "[default: 1]"},
-	{"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
-	 "Set auxiliary group list timeout to SECONDS for fuse translator "
-	 "[default: 300]"},
-        {"resolve-gids", ARGP_RESOLVE_GIDS_KEY, 0, 0,
-         "Resolve all auxiliary groups in fuse translator (max 32 otherwise)"},
-	{"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
-	 "Set fuse module's background queue length to N "
-	 "[default: 64]"},
-	{"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
-	 "Set fuse module's congestion threshold to N "
-	 "[default: 48]"},
+    },
+    {"brick-name", ARGP_BRICK_NAME_KEY, "BRICK-NAME", OPTION_HIDDEN,
+     "Brick name to be registered with Gluster portmapper"},
+    {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN,
+     "Brick Port to be registered with Gluster portmapper"},
+    {"fopen-keep-cache", ARGP_FOPEN_KEEP_CACHE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Do not purge the cache on file open [default: false]"},
+    {"global-timer-wheel", ARGP_GLOBAL_TIMER_WHEEL, "BOOL", OPTION_ARG_OPTIONAL,
+     "Instantiate process global timer-wheel"},
+    {"thin-client", ARGP_THIN_CLIENT_KEY, 0, 0,
+     "Enables thin mount and connects via gfproxyd daemon"},
+    {"global-threading", ARGP_GLOBAL_THREADING_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use the global thread pool instead of io-threads"},
+    {0, 0, 0, 0, "Fuse options:"},
+    {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL|auto",
+     OPTION_ARG_OPTIONAL, "Specify direct I/O strategy [default: \"auto\"]"},
+    {"entry-timeout", ARGP_ENTRY_TIMEOUT_KEY, "SECONDS", 0,
+     "Set entry timeout to SECONDS in fuse kernel module [default: 1]"},
+    {"negative-timeout", ARGP_NEGATIVE_TIMEOUT_KEY, "SECONDS", 0,
+     "Set negative timeout to SECONDS in fuse kernel module [default: 0]"},
+    {"attribute-timeout", ARGP_ATTRIBUTE_TIMEOUT_KEY, "SECONDS", 0,
+     "Set attribute timeout to SECONDS for inodes in fuse kernel module "
+     "[default: 1]"},
+    {"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
+     "Set auxiliary group list timeout to SECONDS for fuse translator "
+     "[default: 300]"},
+    {"resolve-gids", ARGP_RESOLVE_GIDS_KEY, 0, 0,
+     "Resolve all auxiliary groups in fuse translator (max 32 otherwise)"},
+    {"lru-limit", ARGP_FUSE_LRU_LIMIT_KEY, "N", 0,
+     "Set fuse module's limit for number of inodes kept in LRU list to N "
+     "[default: 65536]"},
+    {"invalidate-limit", ARGP_FUSE_INVALIDATE_LIMIT_KEY, "N", 0,
+     "Suspend inode invalidations implied by 'lru-limit' if the number of "
+     "outstanding invalidations reaches N"},
+    {"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
+     "Set fuse module's background queue length to N "
+     "[default: 64]"},
+    {"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
+     "Set fuse module's congestion threshold to N "
+     "[default: 48]"},
 #ifdef GF_LINUX_HOST_OS
-        {"oom-score-adj", ARGP_OOM_SCORE_ADJ_KEY, "INTEGER", 0,
-         "Set oom_score_adj value for process"
-         "[default: 0]"},
+    {"oom-score-adj", ARGP_OOM_SCORE_ADJ_KEY, "INTEGER", 0,
+     "Set oom_score_adj value for process"
+     "[default: 0]"},
 #endif
-        {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
-         "client will authenticate itself with process id PID to server"},
-        {"no-root-squash", ARGP_FUSE_NO_ROOT_SQUASH_KEY, "BOOL",
-         OPTION_ARG_OPTIONAL, "disable/enable root squashing for the trusted "
-         "client"},
-        {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
-         "replace USER with root in messages"},
-        {"dump-fuse", ARGP_DUMP_FUSE_KEY, "PATH", 0,
-         "Dump fuse traffic to PATH"},
-        {"volfile-check", ARGP_VOLFILE_CHECK_KEY, 0, 0,
-         "Enable strict volume file checking"},
-        {"no-mem-accounting", ARGP_MEM_ACCOUNTING_KEY, 0, OPTION_HIDDEN,
-         "disable internal memory accounting"},
-        {"fuse-mountopts", ARGP_FUSE_MOUNTOPTS_KEY, "OPTIONS", OPTION_HIDDEN,
-         "Extra mount options to pass to FUSE"},
-        {"use-readdirp", ARGP_FUSE_USE_READDIRP_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Use readdirp mode in fuse kernel module"
-         " [default: \"yes\"]"},
-        {"secure-mgmt", ARGP_SECURE_MGMT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Override default for secure (SSL) management connections"},
-        {0, 0, 0, 0, "Miscellaneous Options:"},
-        {0, }
-};
-
-
-static struct argp argp = { gf_options, parse_opts, argp_doc, gf_doc };
-
-
-int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx);
-int glusterfs_volumes_init (glusterfs_ctx_t *ctx);
-int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
-int glusterfs_listener_init (glusterfs_ctx_t *ctx);
-int glusterfs_listener_stop (glusterfs_ctx_t *ctx);
+    {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
+     "client will authenticate itself with process id PID to server"},
+    {"no-root-squash", ARGP_FUSE_NO_ROOT_SQUASH_KEY, "BOOL",
+     OPTION_ARG_OPTIONAL,
+     "disable/enable root squashing for the trusted "
+     "client"},
+    {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
+     "replace USER with root in messages"},
+    {"dump-fuse", ARGP_DUMP_FUSE_KEY, "PATH", 0, "Dump fuse traffic to PATH"},
+    {"volfile-check", ARGP_VOLFILE_CHECK_KEY, 0, 0,
+     "Enable strict volume file checking"},
+    {"no-mem-accounting", ARGP_MEM_ACCOUNTING_KEY, 0, OPTION_HIDDEN,
+     "disable internal memory accounting"},
+    {"fuse-mountopts", ARGP_FUSE_MOUNTOPTS_KEY, "OPTIONS", OPTION_HIDDEN,
+     "Extra mount options to pass to FUSE"},
+    {"use-readdirp", ARGP_FUSE_USE_READDIRP_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use readdirp mode in fuse kernel module"
+     " [default: \"yes\"]"},
+    {"secure-mgmt", ARGP_SECURE_MGMT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Override default for secure (SSL) management connections"},
+    {"localtime-logging", ARGP_LOCALTIME_LOGGING_KEY, 0, 0,
+     "Enable localtime logging"},
+    {"process-name", ARGP_PROCESS_NAME_KEY, "PROCESS-NAME", OPTION_HIDDEN,
+     "option to specify the process type"},
+    {"event-history", ARGP_FUSE_EVENT_HISTORY_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "disable/enable fuse event-history"},
+    {"reader-thread-count", ARGP_READER_THREAD_COUNT_KEY, "INTEGER",
+     OPTION_ARG_OPTIONAL, "set fuse reader thread count"},
+    {"kernel-writeback-cache", ARGP_KERNEL_WRITEBACK_CACHE_KEY, "BOOL",
+     OPTION_ARG_OPTIONAL, "enable fuse in-kernel writeback cache"},
+    {"attr-times-granularity", ARGP_ATTR_TIMES_GRANULARITY_KEY, "NS",
+     OPTION_ARG_OPTIONAL,
+     "declare supported granularity of file attribute"
+     " times in nanoseconds"},
+    {"fuse-flush-handle-interrupt", ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY,
+     "BOOL", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
+     "handle interrupt in fuse FLUSH handler"},
+    {"auto-invalidation", ARGP_FUSE_AUTO_INVAL_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "controls whether fuse-kernel can auto-invalidate "
+     "attribute, dentry and page-cache. "
+     "Disable this only if same files/directories are not accessed across "
+     "two different mounts concurrently [default: \"on\"]"},
+    {"fuse-dev-eperm-ratelimit-ns", ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY,
+     "OPTIONS", OPTION_HIDDEN,
+     "rate limit reading from fuse device upon EPERM failure"},
+    {"brick-mux", ARGP_BRICK_MUX_KEY, 0, 0, "Enable brick mux. "},
+    {0, 0, 0, 0, "Miscellaneous Options:"},
+    {
+        0,
+    }};
+
+static struct argp argp = {gf_options, parse_opts, argp_doc, gf_doc};
+
+int
+glusterfs_pidfile_cleanup(glusterfs_ctx_t *ctx);
+int
+glusterfs_volumes_init(glusterfs_ctx_t *ctx);
+int
+glusterfs_mgmt_init(glusterfs_ctx_t *ctx);
+int
+glusterfs_listener_init(glusterfs_ctx_t *ctx);
 
+#define DICT_SET_VAL(method, dict, key, val, msgid)                            \
+    if (method(dict, key, val)) {                                              \
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, msgid, "key=%s", key);          \
+        goto err;                                                              \
+    }
 
 static int
-set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
+set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
 {
-        int              ret = 0;
-        cmd_args_t      *cmd_args = NULL;
-        char            *mount_point = NULL;
-        char            cwd[PATH_MAX] = {0,};
-
-        cmd_args = &ctx->cmd_args;
-
-        /* Check if mount-point is absolute path,
-         * if not convert to absolute path by concating with CWD
-         */
-        if (cmd_args->mount_point[0] != '/') {
-                if (getcwd (cwd, PATH_MAX) != NULL) {
-                        ret = gf_asprintf (&mount_point, "%s/%s", cwd,
-                                           cmd_args->mount_point);
-                        if (ret == -1) {
-                                gf_msg ("glusterfsd", GF_LOG_ERROR, errno,
-                                        glusterfsd_msg_1);
-                                goto err;
-                        }
-                } else {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, errno,
-                                glusterfsd_msg_2);
-                        goto err;
-                }
-        } else
-                mount_point = gf_strdup (cmd_args->mount_point);
-
-        ret = dict_set_dynstr (options, ZR_MOUNTPOINT_OPT, mount_point);
-        if (ret < 0) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_3);
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+    char *mount_point = NULL;
+    char cwd[PATH_MAX] = {
+        0,
+    };
+
+    cmd_args = &ctx->cmd_args;
+
+    /* Check if mount-point is absolute path,
+     * if not convert to absolute path by concatenating with CWD
+     */
+    if (cmd_args->mount_point[0] != '/') {
+        if (getcwd(cwd, PATH_MAX) != NULL) {
+            ret = gf_asprintf(&mount_point, "%s/%s", cwd,
+                              cmd_args->mount_point);
+            if (ret == -1) {
+                gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_1,
+                        "gf_asprintf failed", NULL);
                 goto err;
-        }
-
-        if (cmd_args->fuse_attribute_timeout >= 0) {
-                ret = dict_set_double (options, ZR_ATTR_TIMEOUT_OPT,
-                                       cmd_args->fuse_attribute_timeout);
-
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, errno, 
-                                glusterfsd_msg_4, ZR_ATTR_TIMEOUT_OPT);
-                        goto err;
-                }
-        }
-
-        if (cmd_args->fuse_entry_timeout >= 0) {
-                ret = dict_set_double (options, ZR_ENTRY_TIMEOUT_OPT,
-                                       cmd_args->fuse_entry_timeout);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                ZR_ENTRY_TIMEOUT_OPT);
-                        goto err;
-                }
-        }
-
-        if (cmd_args->fuse_negative_timeout >= 0) {
-                ret = dict_set_double (options, ZR_NEGATIVE_TIMEOUT_OPT,
-                                       cmd_args->fuse_negative_timeout);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                ZR_NEGATIVE_TIMEOUT_OPT);
-                        goto err;
-                }
-        }
-
-        if (cmd_args->client_pid_set) {
-                ret = dict_set_int32 (options, "client-pid",
-                                      cmd_args->client_pid);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "client-pid");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->uid_map_root) {
-                ret = dict_set_int32 (options, "uid-map-root",
-                                      cmd_args->uid_map_root);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "uid-map-root");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->volfile_check) {
-                ret = dict_set_int32 (options, ZR_STRICT_VOLFILE_CHECK,
-                                      cmd_args->volfile_check);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                ZR_STRICT_VOLFILE_CHECK);
-                        goto err;
-                }
-        }
-
-        if (cmd_args->dump_fuse) {
-                ret = dict_set_static_ptr (options, ZR_DUMP_FUSE,
-                                           cmd_args->dump_fuse);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                ZR_DUMP_FUSE);
-                        goto err;
-                }
-        }
-
-        if (cmd_args->acl) {
-                ret = dict_set_static_ptr (options, "acl", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "acl");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->selinux) {
-                ret = dict_set_static_ptr (options, "selinux", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "selinux");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->capability) {
-                ret = dict_set_static_ptr (options, "capability", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "capability");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->aux_gfid_mount) {
-                ret = dict_set_static_ptr (options, "virtual-gfid-access",
-                                           "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "aux-gfid-mount");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->enable_ino32) {
-                ret = dict_set_static_ptr (options, "enable-ino32", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "enable-ino32");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->read_only) {
-                ret = dict_set_static_ptr (options, "read-only", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "read-only");
-                        goto err;
-                }
-        }
-
-	switch (cmd_args->fopen_keep_cache) {
-	case GF_OPTION_ENABLE:
-		ret = dict_set_static_ptr(options, "fopen-keep-cache",
-			"on");
-		if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-				"fopen-keep-cache");
-			goto err;
-		}
-		break;
-	case GF_OPTION_DISABLE:
-		ret = dict_set_static_ptr(options, "fopen-keep-cache",
-			"off");
-		if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-				"fopen-keep-cache");
-			goto err;
-		}
-		break;
-        case GF_OPTION_DEFERRED: /* default */
+            }
+        } else {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_2,
+                    "getcwd failed", NULL);
+            goto err;
+        }
+
+    } else {
+        mount_point = gf_strdup(cmd_args->mount_point);
+    }
+    DICT_SET_VAL(dict_set_dynstr_sizen, options, ZR_MOUNTPOINT_OPT, mount_point,
+                 glusterfsd_msg_3);
+
+    if (cmd_args->fuse_attribute_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_ATTR_TIMEOUT_OPT,
+                     cmd_args->fuse_attribute_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->fuse_entry_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_ENTRY_TIMEOUT_OPT,
+                     cmd_args->fuse_entry_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->fuse_negative_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_NEGATIVE_TIMEOUT_OPT,
+                     cmd_args->fuse_negative_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->client_pid_set) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "client-pid",
+                     cmd_args->client_pid, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->uid_map_root) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "uid-map-root",
+                     cmd_args->uid_map_root, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->volfile_check) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, ZR_STRICT_VOLFILE_CHECK,
+                     cmd_args->volfile_check, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->dump_fuse) {
+        DICT_SET_VAL(dict_set_static_ptr, options, ZR_DUMP_FUSE,
+                     cmd_args->dump_fuse, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->acl) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "acl", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->selinux) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "selinux", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->capability) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "capability", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->aux_gfid_mount) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "virtual-gfid-access", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->enable_ino32) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "enable-ino32", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->read_only) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "read-only", "on",
+                     glusterfsd_msg_3);
+    }
+
+    switch (cmd_args->fopen_keep_cache) {
+        case GF_OPTION_ENABLE:
+
+            DICT_SET_VAL(dict_set_static_ptr, options, "fopen-keep-cache", "on",
+                         glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "fopen-keep-cache",
+                         "off", glusterfsd_msg_3);
+            break;
         default:
-                gf_msg_debug ("glusterfsd", 0, "fopen-keep-cache mode %d",
-                              cmd_args->fopen_keep_cache);
-                break;
-	}
-
-	if (cmd_args->gid_timeout_set) {
-		ret = dict_set_int32(options, "gid-timeout",
-			cmd_args->gid_timeout);
-		if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "gid-timeout");
-			goto err;
-		}
-	}
-
-        if (cmd_args->resolve_gids) {
-                ret = dict_set_static_ptr (options, "resolve-gids", "on");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "resolve-gids");
-                        goto err;
-                }
-        }
-
-	if (cmd_args->background_qlen) {
-		ret = dict_set_int32 (options, "background-qlen",
-                                      cmd_args->background_qlen);
-		if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "background-qlen");
-			goto err;
-		}
-	}
-	if (cmd_args->congestion_threshold) {
-		ret = dict_set_int32 (options, "congestion-threshold",
-                                      cmd_args->congestion_threshold);
-		if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "congestion-threshold");
-			goto err;
-		}
-	}
-
-        switch (cmd_args->fuse_direct_io_mode) {
+            gf_msg_debug("glusterfsd", 0, "fopen-keep-cache mode %d",
+                         cmd_args->fopen_keep_cache);
+            break;
+    }
+
+    if (cmd_args->gid_timeout_set) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "gid-timeout",
+                     cmd_args->gid_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->resolve_gids) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "resolve-gids", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->lru_limit >= 0) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "lru-limit",
+                     cmd_args->lru_limit, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->invalidate_limit >= 0) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "invalidate-limit",
+                     cmd_args->invalidate_limit, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->background_qlen) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "background-qlen",
+                     cmd_args->background_qlen, glusterfsd_msg_3);
+    }
+    if (cmd_args->congestion_threshold) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "congestion-threshold",
+                     cmd_args->congestion_threshold, glusterfsd_msg_3);
+    }
+
+    switch (cmd_args->fuse_direct_io_mode) {
         case GF_OPTION_DISABLE: /* disable */
-                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
-                                           "disable");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_5,
-                                ZR_DIRECT_IO_OPT);
-                        goto err;
-                }
-                break;
+            DICT_SET_VAL(dict_set_static_ptr, options, ZR_DIRECT_IO_OPT,
+                         "disable", glusterfsd_msg_3);
+            break;
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
-                                           "enable");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_6,
-                                ZR_DIRECT_IO_OPT);
-                        goto err;
-                }
-                break;
-        case GF_OPTION_DEFERRED: /* default */
+            DICT_SET_VAL(dict_set_static_ptr, options, ZR_DIRECT_IO_OPT,
+                         "enable", glusterfsd_msg_3);
+            break;
         default:
-                gf_msg_debug ("glusterfsd", 0, "fuse direct io type %d",
-                              cmd_args->fuse_direct_io_mode);
-                break;
-        }
+            gf_msg_debug("glusterfsd", 0, "fuse direct io type %d",
+                         cmd_args->fuse_direct_io_mode);
+            break;
+    }
 
-        switch (cmd_args->no_root_squash) {
+    switch (cmd_args->no_root_squash) {
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (options, "no-root-squash",
-                                           "enable");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_6,
-                                "no-root-squash");
-                        goto err;
-                }
-                break;
-        case GF_OPTION_DISABLE: /* disable/default */
+            DICT_SET_VAL(dict_set_static_ptr, options, "no-root-squash",
+                         "enable", glusterfsd_msg_3);
+            break;
         default:
-                ret = dict_set_static_ptr (options, "no-root-squash",
-                                           "disable");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_5,
-                                "no-root-squash");
-                        goto err;
-                }
-                gf_msg_debug ("glusterfsd", 0, "fuse no-root-squash mode %d",
-                        cmd_args->no_root_squash);
-                break;
-        }
-
-        if (!cmd_args->no_daemon_mode) {
-                ret = dict_set_static_ptr (options, "sync-to-mount",
-                                           "enable");
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "sync-mtab");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->use_readdirp) {
-                ret = dict_set_str (options, "use-readdirp",
-                                    cmd_args->use_readdirp);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                "use-readdirp");
-                        goto err;
-                }
-        }
-        ret = 0;
+            DICT_SET_VAL(dict_set_static_ptr, options, "no-root-squash",
+                         "disable", glusterfsd_msg_3);
+            gf_msg_debug("glusterfsd", 0, "fuse no-root-squash mode %d",
+                         cmd_args->no_root_squash);
+            break;
+    }
+
+    if (!cmd_args->no_daemon_mode) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "sync-to-mount", "enable",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->use_readdirp) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "use-readdirp",
+                     cmd_args->use_readdirp, glusterfsd_msg_3);
+    }
+    if (cmd_args->event_history) {
+        ret = dict_set_str(options, "event-history", cmd_args->event_history);
+        DICT_SET_VAL(dict_set_static_ptr, options, "event-history",
+                     cmd_args->event_history, glusterfsd_msg_3);
+    }
+    if (cmd_args->thin_client) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "thin-client", "on",
+                     glusterfsd_msg_3);
+    }
+    if (cmd_args->reader_thread_count) {
+        DICT_SET_VAL(dict_set_uint32, options, "reader-thread-count",
+                     cmd_args->reader_thread_count, glusterfsd_msg_3);
+    }
+
+    DICT_SET_VAL(dict_set_uint32, options, "auto-invalidation",
+                 cmd_args->fuse_auto_inval, glusterfsd_msg_3);
+
+    switch (cmd_args->kernel_writeback_cache) {
+        case GF_OPTION_ENABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "kernel-writeback-cache",
+                         "on", glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "kernel-writeback-cache",
+                         "off", glusterfsd_msg_3);
+            break;
+        default:
+            gf_msg_debug("glusterfsd", 0, "kernel-writeback-cache mode %d",
+                         cmd_args->kernel_writeback_cache);
+            break;
+    }
+    if (cmd_args->attr_times_granularity) {
+        DICT_SET_VAL(dict_set_uint32, options, "attr-times-granularity",
+                     cmd_args->attr_times_granularity, glusterfsd_msg_3);
+    }
+    switch (cmd_args->fuse_flush_handle_interrupt) {
+        case GF_OPTION_ENABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "flush-handle-interrupt",
+                         "on", glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "flush-handle-interrupt",
+                         "off", glusterfsd_msg_3);
+            break;
+        default:
+            gf_msg_debug("glusterfsd", 0, "fuse-flush-handle-interrupt mode %d",
+                         cmd_args->fuse_flush_handle_interrupt);
+            break;
+    }
+    if (cmd_args->global_threading) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "global-threading", "on",
+                     glusterfsd_msg_3);
+    }
+    if (cmd_args->fuse_dev_eperm_ratelimit_ns) {
+        DICT_SET_VAL(dict_set_uint32, options, "fuse-dev-eperm-ratelimit-ns",
+                     cmd_args->fuse_dev_eperm_ratelimit_ns, glusterfsd_msg_3);
+    }
+
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-create_fuse_mount (glusterfs_ctx_t *ctx)
+create_fuse_mount(glusterfs_ctx_t *ctx)
 {
-        int              ret = 0;
-        cmd_args_t      *cmd_args = NULL;
-        xlator_t        *master = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        if (!cmd_args->mount_point) {
-                gf_msg_trace ("glusterfsd", 0,
-                              "mount point not found, not a client process");
-                return 0;
-        }
-
-        if (ctx->process_mode != GF_CLIENT_PROCESS) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_7);
-                return -1;
-        }
-
-        master = GF_CALLOC (1, sizeof (*master),
-                            gfd_mt_xlator_t);
-        if (!master)
-                goto err;
-
-        master->name = gf_strdup ("fuse");
-        if (!master->name)
-                goto err;
-
-        if (xlator_set_type (master, "mount/fuse") == -1) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_8,
-                        cmd_args->mount_point);
-                goto err;
-        }
-
-        master->ctx      = ctx;
-        master->options  = get_new_dict ();
-        if (!master->options)
-                goto err;
-
-        ret = set_fuse_mount_options (ctx, master->options);
-        if (ret)
-                goto err;
-
-        if (cmd_args->fuse_mountopts) {
-                ret = dict_set_static_ptr (master->options, ZR_FUSE_MOUNTOPTS,
-                                           cmd_args->fuse_mountopts);
-                if (ret < 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
-                                ZR_FUSE_MOUNTOPTS);
-                        goto err;
-                }
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+    xlator_t *master = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    if (!cmd_args->mount_point) {
+        gf_msg_trace("glusterfsd", 0,
+                     "mount point not found, not a client process");
+        return 0;
+    }
+
+    if (ctx->process_mode != GF_CLIENT_PROCESS) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_7, NULL);
+        return -1;
+    }
+
+    master = GF_CALLOC(1, sizeof(*master), gfd_mt_xlator_t);
+    if (!master)
+        goto err;
+
+    master->name = gf_strdup("fuse");
+    if (!master->name)
+        goto err;
+
+    if (xlator_set_type(master, "mount/fuse") == -1) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_8,
+                "MOUNT-POINT=%s", cmd_args->mount_point, NULL);
+        goto err;
+    }
+
+    master->ctx = ctx;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
+
+    ret = set_fuse_mount_options(ctx, master->options);
+    if (ret)
+        goto err;
+
+    if (cmd_args->fuse_mountopts) {
+        ret = dict_set_static_ptr(master->options, ZR_FUSE_MOUNTOPTS,
+                                  cmd_args->fuse_mountopts);
+        if (ret < 0) {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_3,
+                    ZR_FUSE_MOUNTOPTS, NULL);
+            goto err;
         }
+    }
 
-        ret = xlator_init (master);
-        if (ret) {
-                gf_msg_debug ("glusterfsd", 0,
-                              "failed to initialize fuse translator");
-                goto err;
-        }
+    ret = xlator_init(master);
+    if (ret) {
+        gf_msg_debug("glusterfsd", 0, "failed to initialize fuse translator");
+        goto err;
+    }
 
-        ctx->master = master;
+    ctx->master = master;
 
-        return 0;
+    return 0;
 
 err:
-        if (master) {
-                xlator_destroy (master);
-        }
+    if (master) {
+        xlator_destroy(master);
+    }
 
-        return 1;
+    return 1;
 }
 
-
 static FILE *
-get_volfp (glusterfs_ctx_t *ctx)
+get_volfp(glusterfs_ctx_t *ctx)
 {
-        int          ret = 0;
-        cmd_args_t  *cmd_args = NULL;
-        FILE        *specfp = NULL;
-        struct stat  statbuf;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        ret = sys_lstat (cmd_args->volfile, &statbuf);
-        if (ret == -1) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_9,
-                        cmd_args->volfile);
-                return NULL;
-        }
-
-        if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_9,
-                        cmd_args->volfile);
-                return NULL;
-        }
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_9,
+                "volume_file=%s", cmd_args->volfile, NULL);
+        return NULL;
+    }
 
-        gf_msg_debug ("glusterfsd", 0, "loading volume file %s",
-                      cmd_args->volfile);
+    gf_msg_debug("glusterfsd", 0, "loading volume file %s", cmd_args->volfile);
 
-        return specfp;
+    return specfp;
 }
 
 static int
-gf_remember_backup_volfile_server (char *arg)
+gf_remember_backup_volfile_server(char *arg)
 {
-        glusterfs_ctx_t         *ctx = NULL;
-        cmd_args_t              *cmd_args = NULL;
-        int                      ret = -1;
-        server_cmdline_t        *server = NULL;
-
-        ctx = glusterfsd_ctx;
-        if (!ctx)
-                goto out;
-        cmd_args = &ctx->cmd_args;
-
-        if(!cmd_args)
-                goto out;
-
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            gfd_mt_server_cmdline_t);
-        if (!server)
-                goto out;
-
-        INIT_LIST_HEAD(&server->list);
-
-        server->volfile_server = gf_strdup(arg);
-
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->curr_server = server;
-        }
-
-        if (!server->volfile_server) {
-                gf_msg ("glusterfsd", GF_LOG_WARNING, 0, glusterfsd_msg_10,
-                        arg);
-                goto out;
-        }
-
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
-
-        ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+
+    ctx = glusterfsd_ctx;
+    if (!ctx)
+        goto out;
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args)
+        goto out;
+
+    ret = gf_set_volfile_server_common(
+        cmd_args, arg, GF_DEFAULT_VOLFILE_TRANSPORT, GF_DEFAULT_BASE_PORT);
+    if (ret) {
+        gf_log("glusterfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server);
-                }
-        }
-
-        return ret;
-
+    return ret;
 }
 
 static int
-gf_remember_xlator_option (char *arg)
+gf_remember_xlator_option(char *arg)
 {
-        glusterfs_ctx_t         *ctx = NULL;
-        cmd_args_t              *cmd_args  = NULL;
-        xlator_cmdline_option_t *option = NULL;
-        int                      ret = -1;
-        char                    *dot = NULL;
-        char                    *equals = NULL;
-
-        ctx = glusterfsd_ctx;
-        cmd_args = &ctx->cmd_args;
-
-        option = GF_CALLOC (1, sizeof (xlator_cmdline_option_t),
-                            gfd_mt_xlator_cmdline_option_t);
-        if (!option)
-                goto out;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    xlator_cmdline_option_t *option = NULL;
+    int ret = -1;
+    char *dot = NULL;
+    char *equals = NULL;
 
-        INIT_LIST_HEAD (&option->cmd_args);
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
 
-        dot = strchr (arg, '.');
-        if (!dot) {
-                gf_msg ("", GF_LOG_WARNING, 0, glusterfsd_msg_10, arg);
-                goto out;
-        }
+    option = GF_CALLOC(1, sizeof(xlator_cmdline_option_t),
+                       gfd_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto out;
 
-        option->volume = GF_CALLOC ((dot - arg) + 1, sizeof (char),
-                                    gfd_mt_char);
-        if (!option->volume)
-                goto out;
+    INIT_LIST_HEAD(&option->cmd_args);
 
-        strncpy (option->volume, arg, (dot - arg));
+    dot = strchr(arg, '.');
+    if (!dot) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
 
-        equals = strchr (arg, '=');
-        if (!equals) {
-                gf_msg ("", GF_LOG_WARNING, 0, glusterfsd_msg_10, arg);
-                goto out;
-        }
+    option->volume = GF_MALLOC((dot - arg) + 1, gfd_mt_char);
+    if (!option->volume)
+        goto out;
 
-        option->key = GF_CALLOC ((equals - dot) + 1, sizeof (char),
-                                 gfd_mt_char);
-        if (!option->key)
-                goto out;
+    strncpy(option->volume, arg, (dot - arg));
+    option->volume[(dot - arg)] = '\0';
 
-        strncpy (option->key, dot + 1, (equals - dot - 1));
+    equals = strchr(arg, '=');
+    if (!equals) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
 
-        if (!*(equals + 1)) {
-                gf_msg ("", GF_LOG_WARNING, 0, glusterfsd_msg_10, arg);
-                goto out;
-        }
+    option->key = GF_MALLOC((equals - dot) + 1, gfd_mt_char);
+    if (!option->key)
+        goto out;
 
-        option->value = gf_strdup (equals + 1);
+    strncpy(option->key, dot + 1, (equals - dot - 1));
+    option->key[(equals - dot - 1)] = '\0';
 
-        list_add (&option->cmd_args, &cmd_args->xlator_options);
+    if (!*(equals + 1)) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
 
-        ret = 0;
+    option->value = gf_strdup(equals + 1);
+
+    list_add(&option->cmd_args, &cmd_args->xlator_options);
+
+    ret = 0;
 out:
-        if (ret == -1) {
-                if (option) {
-                        GF_FREE (option->volume);
-                        GF_FREE (option->key);
-                        GF_FREE (option->value);
+    if (ret == -1) {
+        if (option) {
+            GF_FREE(option->volume);
+            GF_FREE(option->key);
+            GF_FREE(option->value);
 
-                        GF_FREE (option);
-                }
+            GF_FREE(option);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 #ifdef GF_LINUX_HOST_OS
 static struct oom_api_info {
-        char *oom_api_file;
-        int32_t oom_min;
-        int32_t oom_max;
+    char *oom_api_file;
+    int32_t oom_min;
+    int32_t oom_max;
 } oom_api_info[] = {
-        { "/proc/self/oom_score_adj", OOM_SCORE_ADJ_MIN, OOM_SCORE_ADJ_MAX },
-        { "/proc/self/oom_adj",       OOM_DISABLE,       OOM_ADJUST_MAX },
-        { NULL, 0, 0 }
-};
-
+    {"/proc/self/oom_score_adj", OOM_SCORE_ADJ_MIN, OOM_SCORE_ADJ_MAX},
+    {"/proc/self/oom_adj", OOM_DISABLE, OOM_ADJUST_MAX},
+    {NULL, 0, 0}};
 
 static struct oom_api_info *
-get_oom_api_info (void)
+get_oom_api_info(void)
 {
-        struct oom_api_info *api = NULL;
+    struct oom_api_info *api = NULL;
 
-        for (api = oom_api_info; api->oom_api_file; api++) {
-                if (sys_access (api->oom_api_file, F_OK) != -1) {
-                        return api;
-                }
+    for (api = oom_api_info; api->oom_api_file; api++) {
+        if (sys_access(api->oom_api_file, F_OK) != -1) {
+            return api;
         }
+    }
 
-        return NULL;
+    return NULL;
 }
 #endif
 
 static error_t
-parse_opts (int key, char *arg, struct argp_state *state)
+parse_opts(int key, char *arg, struct argp_state *state)
 {
-        cmd_args_t          *cmd_args      = NULL;
-        uint32_t             n             = 0;
+    cmd_args_t *cmd_args = NULL;
+    uint32_t n = 0;
 #ifdef GF_LINUX_HOST_OS
-        int32_t              k             = 0;
-        struct oom_api_info *api           = NULL;
+    int32_t k = 0;
+    struct oom_api_info *api = NULL;
 #endif
-        double               d             = 0.0;
-        gf_boolean_t         b             = _gf_false;
-        char                *pwd           = NULL;
-        char                 tmp_buf[2048] = {0,};
-        char                *tmp_str       = NULL;
-        char                *port_str      = NULL;
-        struct passwd       *pw            = NULL;
-        int                  ret           = 0;
-
-        cmd_args = state->input;
-
-        switch (key) {
+    double d = 0.0;
+    gf_boolean_t b = _gf_false;
+    char *pwd = NULL;
+    char *tmp_str = NULL;
+    char *port_str = NULL;
+    struct passwd *pw = NULL;
+    int ret = 0;
+
+    cmd_args = state->input;
+
+    switch (key) {
         case ARGP_VOLFILE_SERVER_KEY:
-                gf_remember_backup_volfile_server (arg);
+            gf_remember_backup_volfile_server(arg);
 
-                break;
+            break;
 
         case ARGP_READ_ONLY_KEY:
-                cmd_args->read_only = 1;
-                break;
+            cmd_args->read_only = 1;
+            break;
 
         case ARGP_ACL_KEY:
-                cmd_args->acl = 1;
-                gf_remember_xlator_option ("*-md-cache.cache-posix-acl=true");
-                break;
+            cmd_args->acl = 1;
+            gf_remember_xlator_option("*-md-cache.cache-posix-acl=true");
+            break;
 
         case ARGP_SELINUX_KEY:
-                cmd_args->selinux = 1;
-                gf_remember_xlator_option ("*-md-cache.cache-selinux=true");
-                break;
+            cmd_args->selinux = 1;
+            gf_remember_xlator_option("*-md-cache.cache-selinux=true");
+            break;
 
         case ARGP_CAPABILITY_KEY:
-                cmd_args->capability = 1;
-                break;
+            cmd_args->capability = 1;
+            break;
 
         case ARGP_AUX_GFID_MOUNT_KEY:
-                cmd_args->aux_gfid_mount = 1;
-                break;
+            cmd_args->aux_gfid_mount = 1;
+            break;
 
         case ARGP_INODE32_KEY:
-                cmd_args->enable_ino32 = 1;
-                break;
+            cmd_args->enable_ino32 = 1;
+            break;
 
         case ARGP_WORM_KEY:
-                cmd_args->worm = 1;
-                break;
+            cmd_args->worm = 1;
+            break;
 
         case ARGP_PRINT_NETGROUPS:
-                cmd_args->print_netgroups = arg;
-                break;
+            cmd_args->print_netgroups = arg;
+            break;
 
         case ARGP_PRINT_EXPORTS:
-                cmd_args->print_exports = arg;
-                break;
+            cmd_args->print_exports = arg;
+            break;
 
-        case ARGP_MAC_COMPAT_KEY:
-                if (!arg)
-                        arg = "on";
+        case ARGP_PRINT_XLATORDIR_KEY:
+            cmd_args->print_xlatordir = _gf_true;
+            break;
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->mac_compat = b;
+        case ARGP_PRINT_STATEDUMPDIR_KEY:
+            cmd_args->print_statedumpdir = _gf_true;
+            break;
 
-                        break;
-                }
+        case ARGP_PRINT_LOGDIR_KEY:
+            cmd_args->print_logdir = _gf_true;
+            break;
+
+        case ARGP_PRINT_LIBEXECDIR_KEY:
+            cmd_args->print_libexecdir = _gf_true;
+            break;
+
+        case ARGP_MAC_COMPAT_KEY:
+            if (!arg)
+                arg = "on";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->mac_compat = b;
 
-                argp_failure (state, -1, 0,
-                              "invalid value \"%s\" for mac-compat", arg);
                 break;
+            }
+
+            argp_failure(state, -1, 0, "invalid value \"%s\" for mac-compat",
+                         arg);
+            break;
 
         case ARGP_VOLUME_FILE_KEY:
-                GF_FREE (cmd_args->volfile);
-
-                if (arg[0] != '/') {
-                        pwd = getcwd (NULL, PATH_MAX);
-                        if (!pwd) {
-                               argp_failure (state, -1, errno,
-                                            "getcwd failed with error no %d",
-                                             errno);
-                               break;
-                        }
-                        snprintf (tmp_buf, 1024, "%s/%s", pwd, arg);
-                        cmd_args->volfile = gf_strdup (tmp_buf);
-                        free (pwd);
-                } else {
-                        cmd_args->volfile = gf_strdup (arg);
+            GF_FREE(cmd_args->volfile);
+
+            if (arg[0] != '/') {
+                pwd = getcwd(NULL, PATH_MAX);
+                if (!pwd) {
+                    argp_failure(state, -1, errno,
+                                 "getcwd failed with error no %d", errno);
+                    break;
                 }
+                char tmp_buf[1024];
+                snprintf(tmp_buf, sizeof(tmp_buf), "%s/%s", pwd, arg);
+                cmd_args->volfile = gf_strdup(tmp_buf);
+                free(pwd);
+            } else {
+                cmd_args->volfile = gf_strdup(arg);
+            }
 
-                break;
+            break;
 
         case ARGP_LOG_LEVEL_KEY:
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_NONE_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_NONE;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_CRITICAL_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_CRITICAL;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_ERROR_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_ERROR;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_WARNING_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_WARNING;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_INFO_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_INFO;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_DEBUG_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_DEBUG;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_TRACE_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_TRACE;
-                        break;
-                }
-
-                argp_failure (state, -1, 0, "unknown log level %s", arg);
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_NONE_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_NONE;
                 break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_CRITICAL_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_CRITICAL;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_ERROR_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_ERROR;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_WARNING_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_WARNING;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_INFO_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_INFO;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_DEBUG_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_DEBUG;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_TRACE_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_TRACE;
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown log level %s", arg);
+            break;
 
         case ARGP_LOG_FILE_KEY:
-                cmd_args->log_file = gf_strdup (arg);
-                break;
+            cmd_args->log_file = gf_strdup(arg);
+            break;
 
         case ARGP_VOLFILE_SERVER_PORT_KEY:
-                n = 0;
+            n = 0;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->volfile_server_port = n;
-                        break;
-                }
-
-                argp_failure (state, -1, 0,
-                              "unknown volfile server port %s", arg);
+            if (gf_string2uint_base10(arg, &n) == 0) {
+                cmd_args->volfile_server_port = n;
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown volfile server port %s", arg);
+            break;
 
         case ARGP_VOLFILE_SERVER_TRANSPORT_KEY:
-                cmd_args->volfile_server_transport = gf_strdup (arg);
-                break;
+            cmd_args->volfile_server_transport = gf_strdup(arg);
+            break;
 
         case ARGP_VOLFILE_ID_KEY:
-                cmd_args->volfile_id = gf_strdup (arg);
-                break;
+            cmd_args->volfile_id = gf_strdup(arg);
+            break;
+
+        case ARGP_THIN_CLIENT_KEY:
+            cmd_args->thin_client = _gf_true;
+            break;
+
+        case ARGP_BRICK_MUX_KEY:
+            cmd_args->brick_mux = _gf_true;
+            break;
 
         case ARGP_PID_FILE_KEY:
-                cmd_args->pid_file = gf_strdup (arg);
-                break;
+            cmd_args->pid_file = gf_strdup(arg);
+            break;
 
         case ARGP_SOCK_FILE_KEY:
-                cmd_args->sock_file = gf_strdup (arg);
-                break;
+            cmd_args->sock_file = gf_strdup(arg);
+            break;
 
         case ARGP_NO_DAEMON_KEY:
-                cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
-                break;
+            cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
+            break;
 
         case ARGP_RUN_ID_KEY:
-                cmd_args->run_id = gf_strdup (arg);
-                break;
+            cmd_args->run_id = gf_strdup(arg);
+            break;
 
         case ARGP_DEBUG_KEY:
-                cmd_args->debug_mode = ENABLE_DEBUG_MODE;
-                break;
+            cmd_args->debug_mode = ENABLE_DEBUG_MODE;
+            break;
         case ARGP_VOLFILE_MAX_FETCH_ATTEMPTS:
-                cmd_args->max_connect_attempts = 1;
-                break;
+            cmd_args->max_connect_attempts = 1;
+            break;
 
         case ARGP_DIRECT_IO_MODE_KEY:
-                if (!arg)
-                        arg = "on";
+            if (!arg)
+                arg = "on";
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->fuse_direct_io_mode = b;
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_direct_io_mode = b;
 
-                        break;
-                }
+                break;
+            }
 
-                argp_failure (state, -1, 0,
-                              "unknown direct I/O mode setting \"%s\"", arg);
+            if (strcmp(arg, "auto") == 0)
                 break;
 
+            argp_failure(state, -1, 0, "unknown direct I/O mode setting \"%s\"",
+                         arg);
+            break;
+
         case ARGP_FUSE_NO_ROOT_SQUASH_KEY:
-                cmd_args->no_root_squash = _gf_true;
-                break;
+            cmd_args->no_root_squash = _gf_true;
+            break;
 
         case ARGP_ENTRY_TIMEOUT_KEY:
-                d = 0.0;
-
-                gf_string2double (arg, &d);
-                if (!(d < 0.0)) {
-                        cmd_args->fuse_entry_timeout = d;
-                        break;
-                }
+            d = 0.0;
 
-                argp_failure (state, -1, 0, "unknown entry timeout %s", arg);
+            gf_string2double(arg, &d);
+            if (!(d < 0.0)) {
+                cmd_args->fuse_entry_timeout = d;
                 break;
+            }
 
-        case ARGP_NEGATIVE_TIMEOUT_KEY:
-                d = 0.0;
+            argp_failure(state, -1, 0, "unknown entry timeout %s", arg);
+            break;
 
-                ret = gf_string2double (arg, &d);
-                if ((ret == 0) && !(d < 0.0)) {
-                        cmd_args->fuse_negative_timeout = d;
-                        break;
-                }
+        case ARGP_NEGATIVE_TIMEOUT_KEY:
+            d = 0.0;
 
-                argp_failure (state, -1, 0, "unknown negative timeout %s", arg);
+            ret = gf_string2double(arg, &d);
+            if ((ret == 0) && !(d < 0.0)) {
+                cmd_args->fuse_negative_timeout = d;
                 break;
+            }
 
-        case ARGP_ATTRIBUTE_TIMEOUT_KEY:
-                d = 0.0;
+            argp_failure(state, -1, 0, "unknown negative timeout %s", arg);
+            break;
 
-                gf_string2double (arg, &d);
-                if (!(d < 0.0)) {
-                        cmd_args->fuse_attribute_timeout = d;
-                        break;
-                }
+        case ARGP_ATTRIBUTE_TIMEOUT_KEY:
+            d = 0.0;
 
-                argp_failure (state, -1, 0,
-                              "unknown attribute timeout %s", arg);
+            gf_string2double(arg, &d);
+            if (!(d < 0.0)) {
+                cmd_args->fuse_attribute_timeout = d;
                 break;
+            }
 
-        case ARGP_CLIENT_PID_KEY:
-                if (gf_string2int (arg, &cmd_args->client_pid) == 0) {
-                        cmd_args->client_pid_set = 1;
-                        break;
-                }
+            argp_failure(state, -1, 0, "unknown attribute timeout %s", arg);
+            break;
 
-                argp_failure (state, -1, 0,
-                              "unknown client pid %s", arg);
+        case ARGP_CLIENT_PID_KEY:
+            if (gf_string2int(arg, &cmd_args->client_pid) == 0) {
+                cmd_args->client_pid_set = 1;
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown client pid %s", arg);
+            break;
 
         case ARGP_USER_MAP_ROOT_KEY:
-                pw = getpwnam (arg);
-                if (pw)
-                        cmd_args->uid_map_root = pw->pw_uid;
-                else
-                        argp_failure (state, -1, 0,
-                                      "user %s does not exist", arg);
-                break;
+            pw = getpwnam(arg);
+            if (pw)
+                cmd_args->uid_map_root = pw->pw_uid;
+            else
+                argp_failure(state, -1, 0, "user %s does not exist", arg);
+            break;
 
         case ARGP_VOLFILE_CHECK_KEY:
-                cmd_args->volfile_check = 1;
-                break;
+            cmd_args->volfile_check = 1;
+            break;
 
         case ARGP_VOLUME_NAME_KEY:
-                cmd_args->volume_name = gf_strdup (arg);
-                break;
+            cmd_args->volume_name = gf_strdup(arg);
+            break;
 
         case ARGP_XLATOR_OPTION_KEY:
-                if (gf_remember_xlator_option (arg))
-                        argp_failure (state, -1, 0, "invalid xlator option  %s",
-                                      arg);
+            if (gf_remember_xlator_option(arg))
+                argp_failure(state, -1, 0, "invalid xlator option  %s", arg);
 
-                break;
+            break;
 
         case ARGP_KEY_NO_ARGS:
-                break;
+            break;
 
         case ARGP_KEY_ARG:
-                if (state->arg_num >= 1)
-                        argp_usage (state);
-
-                cmd_args->mount_point = gf_strdup (arg);
-                break;
+            if (state->arg_num >= 1)
+                argp_usage(state);
+            cmd_args->mount_point = gf_strdup(arg);
+            break;
 
         case ARGP_DUMP_FUSE_KEY:
-                cmd_args->dump_fuse = gf_strdup (arg);
-                break;
+            cmd_args->dump_fuse = gf_strdup(arg);
+            break;
         case ARGP_BRICK_NAME_KEY:
-                cmd_args->brick_name = gf_strdup (arg);
-                break;
+            cmd_args->brick_name = gf_strdup(arg);
+            break;
         case ARGP_BRICK_PORT_KEY:
-                n = 0;
-
-                port_str = strtok_r (arg, ",", &tmp_str);
-                if (gf_string2uint_base10 (port_str, &n) == 0) {
-                        cmd_args->brick_port = n;
-                        port_str = strtok_r (NULL, ",", &tmp_str);
-                        if (port_str) {
-                                if (gf_string2uint_base10 (port_str, &n) == 0)
-                                        cmd_args->brick_port2 = n;
-                                break;
-
-                                argp_failure (state, -1, 0,
-                                              "wrong brick (listen) port %s", arg);
+            n = 0;
+
+            if (arg != NULL) {
+                port_str = strtok_r(arg, ",", &tmp_str);
+                if (gf_string2uint_base10(port_str, &n) == 0) {
+                    cmd_args->brick_port = n;
+                    port_str = strtok_r(NULL, ",", &tmp_str);
+                    if (port_str) {
+                        if (gf_string2uint_base10(port_str, &n) == 0) {
+                            cmd_args->brick_port2 = n;
+                            break;
                         }
-                        break;
+                        argp_failure(state, -1, 0,
+                                     "wrong brick (listen) port %s", arg);
+                    }
+                    break;
                 }
+            }
 
-                argp_failure (state, -1, 0,
-                              "unknown brick (listen) port %s", arg);
-                break;
+            argp_failure(state, -1, 0, "unknown brick (listen) port %s", arg);
+            break;
 
         case ARGP_MEM_ACCOUNTING_KEY:
-                /* TODO: it should have got handled much earlier */
-		//gf_mem_acct_enable_set (THIS->ctx);
-                break;
+            /* TODO: it should have got handled much earlier */
+            // gf_mem_acct_enable_set (THIS->ctx);
+            break;
 
-	case ARGP_FOPEN_KEEP_CACHE_KEY:
-                if (!arg)
-                        arg = "on";
+        case ARGP_FOPEN_KEEP_CACHE_KEY:
+            if (!arg)
+                arg = "on";
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->fopen_keep_cache = b;
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fopen_keep_cache = b;
 
-                        break;
-                }
+                break;
+            }
 
-                argp_failure (state, -1, 0,
-                              "unknown cache setting \"%s\"", arg);
+            argp_failure(state, -1, 0, "unknown cache setting \"%s\"", arg);
 
-		break;
+            break;
 
         case ARGP_GLOBAL_TIMER_WHEEL:
-                cmd_args->global_timer_wheel = 1;
-                break;
+            cmd_args->global_timer_wheel = 1;
+            break;
 
-	case ARGP_GID_TIMEOUT_KEY:
-		if (!gf_string2int(arg, &cmd_args->gid_timeout)) {
-			cmd_args->gid_timeout_set = _gf_true;
-			break;
-		}
+        case ARGP_GID_TIMEOUT_KEY:
+            if (!gf_string2int(arg, &cmd_args->gid_timeout)) {
+                cmd_args->gid_timeout_set = _gf_true;
+                break;
+            }
 
-		argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
-		break;
+            argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
+            break;
 
         case ARGP_RESOLVE_GIDS_KEY:
-                cmd_args->resolve_gids = 1;
+            cmd_args->resolve_gids = 1;
+            break;
+
+        case ARGP_FUSE_LRU_LIMIT_KEY:
+            if (!gf_string2int32(arg, &cmd_args->lru_limit))
                 break;
 
-        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
-                if (!gf_string2int (arg, &cmd_args->background_qlen))
-                        break;
+            argp_failure(state, -1, 0, "unknown LRU limit option %s", arg);
+            break;
 
-                argp_failure (state, -1, 0,
-                              "unknown background qlen option %s", arg);
+        case ARGP_FUSE_INVALIDATE_LIMIT_KEY:
+            if (!gf_string2int32(arg, &cmd_args->invalidate_limit))
                 break;
-        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
-                if (!gf_string2int (arg, &cmd_args->congestion_threshold))
-                        break;
 
-                argp_failure (state, -1, 0,
-                              "unknown congestion threshold option %s", arg);
+            argp_failure(state, -1, 0, "unknown invalidate limit option %s",
+                         arg);
+            break;
+
+        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
+            if (!gf_string2int(arg, &cmd_args->background_qlen))
+                break;
+
+            argp_failure(state, -1, 0, "unknown background qlen option %s",
+                         arg);
+            break;
+        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
+            if (!gf_string2int(arg, &cmd_args->congestion_threshold))
                 break;
 
+            argp_failure(state, -1, 0, "unknown congestion threshold option %s",
+                         arg);
+            break;
+
 #ifdef GF_LINUX_HOST_OS
         case ARGP_OOM_SCORE_ADJ_KEY:
-                k = 0;
+            k = 0;
 
-                api = get_oom_api_info();
-                if (!api)
-                        goto no_oom_api;
+            api = get_oom_api_info();
+            if (!api)
+                goto no_oom_api;
 
-                if (gf_string2int (arg, &k) == 0 &&
-                    k >= api->oom_min && k <= api->oom_max) {
-                        cmd_args->oom_score_adj = gf_strdup (arg);
-                        break;
-                }
+            if (gf_string2int(arg, &k) == 0 && k >= api->oom_min &&
+                k <= api->oom_max) {
+                cmd_args->oom_score_adj = gf_strdup(arg);
+                break;
+            }
 
-                argp_failure (state, -1, 0,
-                              "unknown oom_score_adj value %s", arg);
+            argp_failure(state, -1, 0, "unknown oom_score_adj value %s", arg);
 
-no_oom_api:
-                break;
+        no_oom_api:
+            break;
 #endif
 
         case ARGP_FUSE_MOUNTOPTS_KEY:
-                cmd_args->fuse_mountopts = gf_strdup (arg);
-                break;
+            cmd_args->fuse_mountopts = gf_strdup(arg);
+            break;
 
         case ARGP_FUSE_USE_READDIRP_KEY:
-                if (!arg)
-                        arg = "yes";
-
-                if (gf_string2boolean (arg, &b) == 0) {
-                        if (b) {
-                                cmd_args->use_readdirp = "yes";
-                        } else {
-                                cmd_args->use_readdirp = "no";
-                        }
+            if (!arg)
+                arg = "yes";
 
-                        break;
+            if (gf_string2boolean(arg, &b) == 0) {
+                if (b) {
+                    cmd_args->use_readdirp = "yes";
+                } else {
+                    cmd_args->use_readdirp = "no";
                 }
 
-                argp_failure (state, -1, 0,
-                              "unknown use-readdirp setting \"%s\"", arg);
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown use-readdirp setting \"%s\"",
+                         arg);
+            break;
 
         case ARGP_LOGGER:
-                if (strcasecmp (arg, GF_LOGGER_GLUSTER_LOG) == 0)
-                        cmd_args->logger = gf_logger_glusterlog;
-                else if (strcasecmp (arg, GF_LOGGER_SYSLOG) == 0)
-                        cmd_args->logger = gf_logger_syslog;
-                else
-                        argp_failure (state, -1, 0, "unknown logger %s", arg);
+            if (strcasecmp(arg, GF_LOGGER_GLUSTER_LOG) == 0)
+                cmd_args->logger = gf_logger_glusterlog;
+            else if (strcasecmp(arg, GF_LOGGER_SYSLOG) == 0)
+                cmd_args->logger = gf_logger_syslog;
+            else
+                argp_failure(state, -1, 0, "unknown logger %s", arg);
 
-                break;
+            break;
 
         case ARGP_LOG_FORMAT:
-                if (strcasecmp (arg, GF_LOG_FORMAT_NO_MSG_ID) == 0)
-                        cmd_args->log_format = gf_logformat_traditional;
-                else if (strcasecmp (arg, GF_LOG_FORMAT_WITH_MSG_ID) == 0)
-                        cmd_args->log_format = gf_logformat_withmsgid;
-                else
-                        argp_failure (state, -1, 0, "unknown log format %s",
-                                      arg);
+            if (strcasecmp(arg, GF_LOG_FORMAT_NO_MSG_ID) == 0)
+                cmd_args->log_format = gf_logformat_traditional;
+            else if (strcasecmp(arg, GF_LOG_FORMAT_WITH_MSG_ID) == 0)
+                cmd_args->log_format = gf_logformat_withmsgid;
+            else
+                argp_failure(state, -1, 0, "unknown log format %s", arg);
 
-                break;
+            break;
 
         case ARGP_LOG_BUF_SIZE:
-                if (gf_string2uint32 (arg, &cmd_args->log_buf_size)) {
-                        argp_failure (state, -1, 0,
-                                      "unknown log buf size option %s", arg);
-                } else if (cmd_args->log_buf_size > GF_LOG_LRU_BUFSIZE_MAX) {
-                        argp_failure (state, -1, 0,
-                                      "Invalid log buf size %s. "
-                                      "Valid range: ["
-                                      GF_LOG_LRU_BUFSIZE_MIN_STR","
-                                      GF_LOG_LRU_BUFSIZE_MAX_STR"]", arg);
+            if (gf_string2uint32(arg, &cmd_args->log_buf_size)) {
+                argp_failure(state, -1, 0, "unknown log buf size option %s",
+                             arg);
+            } else if (cmd_args->log_buf_size > GF_LOG_LRU_BUFSIZE_MAX) {
+                argp_failure(state, -1, 0,
+                             "Invalid log buf size %s. "
+                             "Valid range: [" GF_LOG_LRU_BUFSIZE_MIN_STR
+                             "," GF_LOG_LRU_BUFSIZE_MAX_STR "]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_LOG_FLUSH_TIMEOUT:
+            if (gf_string2uint32(arg, &cmd_args->log_flush_timeout)) {
+                argp_failure(state, -1, 0,
+                             "unknown log flush timeout option %s", arg);
+            } else if ((cmd_args->log_flush_timeout <
+                        GF_LOG_FLUSH_TIMEOUT_MIN) ||
+                       (cmd_args->log_flush_timeout >
+                        GF_LOG_FLUSH_TIMEOUT_MAX)) {
+                argp_failure(state, -1, 0,
+                             "Invalid log flush timeout %s. "
+                             "Valid range: [" GF_LOG_FLUSH_TIMEOUT_MIN_STR
+                             "," GF_LOG_FLUSH_TIMEOUT_MAX_STR "]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_SECURE_MGMT_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->secure_mgmt = b ? 1 : 0;
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown secure-mgmt setting \"%s\"",
+                         arg);
+            break;
+
+        case ARGP_LOCALTIME_LOGGING_KEY:
+            cmd_args->localtime_logging = 1;
+            break;
+        case ARGP_PROCESS_NAME_KEY:
+            cmd_args->process_name = gf_strdup(arg);
+            break;
+        case ARGP_SUBDIR_MOUNT_KEY:
+            if (arg[0] != '/') {
+                argp_failure(state, -1, 0, "expect '/%s', provided just \"%s\"",
+                             arg, arg);
+                break;
+            }
+            cmd_args->subdir_mount = gf_strdup(arg);
+            break;
+        case ARGP_FUSE_EVENT_HISTORY_KEY:
+            if (!arg)
+                arg = "no";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                if (b) {
+                    cmd_args->event_history = "yes";
+                } else {
+                    cmd_args->event_history = "no";
                 }
 
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown event-history setting \"%s\"",
+                         arg);
+            break;
+        case ARGP_READER_THREAD_COUNT_KEY:
+            if (gf_string2uint32(arg, &cmd_args->reader_thread_count)) {
+                argp_failure(state, -1, 0,
+                             "unknown reader thread count option %s", arg);
+            } else if ((cmd_args->reader_thread_count < 1) ||
+                       (cmd_args->reader_thread_count > 64)) {
+                argp_failure(state, -1, 0,
+                             "Invalid reader thread count %s. "
+                             "Valid range: [\"1, 64\"]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_KERNEL_WRITEBACK_CACHE_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->kernel_writeback_cache = b;
 
-        case ARGP_LOG_FLUSH_TIMEOUT:
-                if (gf_string2uint32 (arg, &cmd_args->log_flush_timeout)) {
-                        argp_failure (state, -1, 0,
-                                "unknown log flush timeout option %s", arg);
-                } else if ((cmd_args->log_flush_timeout <
-                            GF_LOG_FLUSH_TIMEOUT_MIN) ||
-                           (cmd_args->log_flush_timeout >
-                            GF_LOG_FLUSH_TIMEOUT_MAX)) {
-                            argp_failure (state, -1, 0,
-                                          "Invalid log flush timeout %s. "
-                                          "Valid range: ["
-                                          GF_LOG_FLUSH_TIMEOUT_MIN_STR","
-                                          GF_LOG_FLUSH_TIMEOUT_MAX_STR"]", arg);
-                }
+                break;
+            }
+
+            argp_failure(state, -1, 0,
+                         "unknown kernel writeback cache setting \"%s\"", arg);
+            break;
+        case ARGP_ATTR_TIMES_GRANULARITY_KEY:
+            if (gf_string2uint32(arg, &cmd_args->attr_times_granularity)) {
+                argp_failure(state, -1, 0,
+                             "unknown attribute times granularity option %s",
+                             arg);
+            } else if (cmd_args->attr_times_granularity > 1000000000) {
+                argp_failure(state, -1, 0,
+                             "Invalid attribute times granularity value %s. "
+                             "Valid range: [\"0, 1000000000\"]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_flush_handle_interrupt = b;
 
                 break;
+            }
 
-        case ARGP_SECURE_MGMT_KEY:
-                if (!arg)
-                        arg = "yes";
+            argp_failure(state, -1, 0,
+                         "unknown fuse flush handle interrupt setting \"%s\"",
+                         arg);
+            break;
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->secure_mgmt = b ? 1 : 0;
-                        break;
-                }
+        case ARGP_FUSE_AUTO_INVAL_KEY:
+            if (!arg)
+                arg = "yes";
 
-                argp_failure (state, -1, 0,
-                              "unknown secure-mgmt setting \"%s\"", arg);
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_auto_inval = b;
                 break;
-	}
+            }
 
-        return 0;
+            break;
+
+        case ARGP_GLOBAL_THREADING_KEY:
+            if (!arg || (*arg == 0)) {
+                arg = "yes";
+            }
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->global_threading = b;
+                break;
+            }
+
+            argp_failure(state, -1, 0,
+                         "Invalid value for global threading \"%s\"", arg);
+            break;
+
+        case ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY:
+            if (gf_string2uint32(arg, &cmd_args->fuse_dev_eperm_ratelimit_ns)) {
+                argp_failure(state, -1, 0,
+                             "Non-numerical value for "
+                             "'fuse-dev-eperm-ratelimit-ns' option %s",
+                             arg);
+            } else if (cmd_args->fuse_dev_eperm_ratelimit_ns > 1000000000) {
+                argp_failure(state, -1, 0,
+                             "Invalid 'fuse-dev-eperm-ratelimit-ns' value %s. "
+                             "Valid range: [\"0, 1000000000\"]",
+                             arg);
+            }
+
+            break;
+    }
+    return 0;
 }
 
 gf_boolean_t
-should_call_fini (glusterfs_ctx_t *ctx, xlator_t *trav)
+should_call_fini(glusterfs_ctx_t *ctx, xlator_t *trav)
 {
-        /* There's nothing to call, so the other checks don't matter. */
-        if (!trav->fini) {
-                return _gf_false;
-        }
-
-        /* This preserves previous behavior in glusterd. */
-        if (ctx->process_mode == GF_GLUSTERD_PROCESS) {
-                return _gf_true;
-        }
+    /* There's nothing to call, so the other checks don't matter. */
+    if (!trav->fini) {
+        return _gf_false;
+    }
 
-        /* This is the only one known to be safe in glusterfsd. */
-        if (!strcmp(trav->type,"experimental/fdl")) {
-                return _gf_true;
-        }
+    /* This preserves previous behavior in glusterd. */
+    if (ctx->process_mode == GF_GLUSTERD_PROCESS) {
+        return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 void
-cleanup_and_exit (int signum)
+cleanup_and_exit(int signum)
 {
-        glusterfs_ctx_t *ctx      = NULL;
-        xlator_t        *trav     = NULL;
-
-        ctx = glusterfsd_ctx;
-
-        if (!ctx)
-                return;
-
-        /* To take or not to take the mutex here and in the other
-         * signal handler - gf_print_trace() - is the big question here.
-         *
-         * Taking mutex in signal handler would mean that if the process
-         * receives a fatal signal while another thread is holding
-         * ctx->log.log_buf_lock to perhaps log a message in _gf_msg_internal(),
-         * the offending thread hangs on the mutex lock forever without letting
-         * the process exit.
-         *
-         * On the other hand. not taking the mutex in signal handler would cause
-         * it to modify the lru_list of buffered log messages in a racy manner,
-         * corrupt the list and potentially give rise to an unending
-         * cascade of SIGSEGVs and other re-entrancy issues.
-         */
-
-        gf_log_disable_suppression_before_exit (ctx);
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *trav = NULL;
+    xlator_t *top;
+    xlator_t *victim;
+    xlator_list_t **trav_p;
 
-        gf_msg_callingfn ("", GF_LOG_WARNING, 0, glusterfsd_msg_32, signum);
+    ctx = glusterfsd_ctx;
 
-        if (ctx->cleanup_started)
-                return;
+    if (!ctx)
+        return;
 
+    /* To take or not to take the mutex here and in the other
+     * signal handler - gf_print_trace() - is the big question here.
+     *
+     * Taking mutex in signal handler would mean that if the process
+     * receives a fatal signal while another thread is holding
+     * ctx->log.log_buf_lock to perhaps log a message in _gf_msg_internal(),
+     * the offending thread hangs on the mutex lock forever without letting
+     * the process exit.
+     *
+     * On the other hand. not taking the mutex in signal handler would cause
+     * it to modify the lru_list of buffered log messages in a racy manner,
+     * corrupt the list and potentially give rise to an unending
+     * cascade of SIGSEGVs and other re-entrancy issues.
+     */
+
+    gf_log_disable_suppression_before_exit(ctx);
+
+    gf_msg_callingfn("", GF_LOG_WARNING, 0, glusterfsd_msg_32,
+                     "received signum (%d), shutting down", signum);
+
+    if (ctx->cleanup_started)
+        return;
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    {
         ctx->cleanup_started = 1;
-        glusterfs_mgmt_pmap_signout (ctx);
+
+        /* signout should be sent to all the bricks in case brick mux is enabled
+         * and multiple brick instances are attached to this process
+         */
+        if (ctx->active) {
+            top = ctx->active->first;
+            for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+                victim = (*trav_p)->xlator;
+                rpc_clnt_mgmt_pmap_signout(ctx, victim->name);
+            }
+        } else {
+            rpc_clnt_mgmt_pmap_signout(ctx, NULL);
+        }
 
         /* below part is a racy code where the rpcsvc object is freed.
          * But in another thread (epoll thread), upon poll error in the
          * socket the transports are cleaned up where again rpcsvc object
          * is accessed (which is already freed by the below function).
-         * Since the process is about to be killed dont execute the function
+         * Since the process is about to be killed don't execute the function
          * below.
          */
         /* if (ctx->listener) { */
@@ -1349,11 +1467,11 @@ cleanup_and_exit (int signum)
          * 'umount' of mount point is done properly */
         trav = ctx->master;
         if (trav && trav->fini) {
-                THIS = trav;
-                trav->fini (trav);
+            THIS = trav;
+            trav->fini(trav);
         }
 
-        glusterfs_pidfile_cleanup (ctx);
+        glusterfs_pidfile_cleanup(ctx);
 
 #if 0
         /* TODO: Properly do cleanup_and_exit(), with synchronization */
@@ -1364,296 +1482,308 @@ cleanup_and_exit (int signum)
         }
 #endif
 
-        /* call fini() of each xlator */
-
-        /*call fini for glusterd xlator */
-        /* TODO : Invoke fini for rest of the xlators */
         trav = NULL;
-        if (ctx->active)
-                trav = ctx->active->top;
-        while (trav) {
-                if (should_call_fini(ctx,trav)) {
-                        THIS = trav;
-                        trav->fini (trav);
-                }
-                trav = trav->next;
-        }
 
-        exit(signum);
+        /* previously we were releasing the cleanup mutex lock before the
+           process exit. As we are releasing the cleanup mutex lock, before
+           the process can exit some other thread which is blocked on
+           cleanup mutex lock is acquiring the cleanup mutex lock and
+           trying to acquire some resources which are already freed as a
+           part of cleanup. To avoid this, we are exiting the process without
+           releasing the cleanup mutex lock. This will not cause any lock
+           related issues as the process which acquired the lock is going down
+         */
+        /* NOTE: Only the least significant 8 bits i.e (signum & 255)
+           will be available to parent process on calling exit() */
+        exit(abs(signum));
+    }
 }
 
-
 static void
-reincarnate (int signum)
+reincarnate(int signum)
 {
-        int                 ret = 0;
-        glusterfs_ctx_t    *ctx = NULL;
-        cmd_args_t         *cmd_args = NULL;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-        ctx = glusterfsd_ctx;
-        cmd_args = &ctx->cmd_args;
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
 
-        if (cmd_args->volfile_server) {
-                gf_msg ("glusterfsd", GF_LOG_INFO, 0, glusterfsd_msg_11);
-                ret = glusterfs_volfile_fetch (ctx);
-        } else {
-                gf_msg_debug ("glusterfsd", 0,
-                              "Not reloading volume specification file"
-                              " on SIGHUP");
-        }
+    gf_msg_trace("gluster", 0, "received reincarnate request (sig:HUP)");
 
-        /* Also, SIGHUP should do logrotate */
-        gf_log_logrotate (1);
+    if (cmd_args->volfile_server) {
+        gf_smsg("glusterfsd", GF_LOG_INFO, 0, glusterfsd_msg_11, NULL);
+        ret = glusterfs_volfile_fetch(ctx);
+    }
 
-        if (ret < 0)
-                gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_12);
+    /* Also, SIGHUP should do logrotate */
+    gf_log_logrotate(1);
 
-        return;
+    if (ret < 0)
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_12, NULL);
+
+    return;
 }
 
 void
-emancipate (glusterfs_ctx_t *ctx, int ret)
+emancipate(glusterfs_ctx_t *ctx, int ret)
 {
-        /* break free from the parent */
-        if (ctx->daemon_pipe[1] != -1) {
-                sys_write (ctx->daemon_pipe[1], (void *) &ret, sizeof (ret));
-                sys_close (ctx->daemon_pipe[1]);
-                ctx->daemon_pipe[1] = -1;
-        }
+    /* break free from the parent */
+    if (ctx->daemon_pipe[1] != -1) {
+        sys_write(ctx->daemon_pipe[1], (void *)&ret, sizeof(ret));
+        sys_close(ctx->daemon_pipe[1]);
+        ctx->daemon_pipe[1] = -1;
+    }
 }
 
 static uint8_t
-gf_get_process_mode (char *exec_name)
+gf_get_process_mode(char *exec_name)
 {
-        char *dup_execname = NULL, *base = NULL;
-        uint8_t ret = 0;
+    char *dup_execname = NULL, *base = NULL;
+    uint8_t ret = 0;
 
-        dup_execname = gf_strdup (exec_name);
-        base = basename (dup_execname);
+    dup_execname = gf_strdup(exec_name);
+    base = basename(dup_execname);
 
-        if (!strncmp (base, "glusterfsd", 10)) {
-                ret = GF_SERVER_PROCESS;
-        } else if (!strncmp (base, "glusterd", 8)) {
-                ret = GF_GLUSTERD_PROCESS;
-        } else {
-                ret = GF_CLIENT_PROCESS;
-        }
+    if (!strncmp(base, "glusterfsd", 10)) {
+        ret = GF_SERVER_PROCESS;
+    } else if (!strncmp(base, "glusterd", 8)) {
+        ret = GF_GLUSTERD_PROCESS;
+    } else {
+        ret = GF_CLIENT_PROCESS;
+    }
 
-        GF_FREE (dup_execname);
+    GF_FREE(dup_execname);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t          *cmd_args = NULL;
-        struct rlimit        lim      = {0, };
-        int                  ret      = -1;
-
-        ret = xlator_mem_acct_init (THIS, gfd_mt_end);
-        if (ret != 0) {
-                gf_msg(THIS->name, GF_LOG_CRITICAL, 0, glusterfsd_msg_34);
-                return ret;
-        }
-
-        /* reset ret to -1 so that we don't need to explicitly
-         * set it in all error paths before "goto err"
-         */
-        ret = -1;
-
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_13);
-                goto out;
-        }
-
-        ctx->page_size  = 128 * GF_UNIT_KB;
-
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "iobuf");
-                goto out;
-        }
-
-        ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE,
-                                          STARTING_EVENT_THREADS);
-        if (!ctx->event_pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "event");
-                goto out;
-        }
-
-        ctx->pool = GF_CALLOC (1, sizeof (call_pool_t), gfd_mt_call_pool_t);
-        if (!ctx->pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "call");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&ctx->pool->all_frames);
-        LOCK_INIT (&ctx->pool->lock);
-
-        /* frame_mem_pool size 112 * 4k */
-        ctx->pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-        if (!ctx->pool->frame_mem_pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "frame");
-                goto out;
-        }
-        /* stack_mem_pool size 256 * 1024 */
-        ctx->pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-        if (!ctx->pool->stack_mem_pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stack");
-                goto out;
-        }
-
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-        if (!ctx->stub_mem_pool) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stub");
-                goto out;
-        }
-
-        ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-        if (!ctx->dict_pool)
-                goto out;
-
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-                                            GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-        if (!ctx->dict_pair_pool)
-                goto out;
-
-        ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-        if (!ctx->dict_data_pool)
-                goto out;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t,
-                                         GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
-        if (!ctx->logbuf_pool)
-                goto out;
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    int ret = -1;
 
-        LOCK_INIT (&ctx->lock);
-        pthread_mutex_init (&ctx->notify_lock, NULL);
-        pthread_cond_init (&ctx->notify_cond, NULL);
-
-        ctx->clienttable = gf_clienttable_alloc();
-        if (!ctx->clienttable)
-                goto out;
-
-        cmd_args = &ctx->cmd_args;
-
-        /* parsing command line arguments */
-        cmd_args->log_level = DEFAULT_LOG_LEVEL;
-        cmd_args->logger    = gf_logger_glusterlog;
-        cmd_args->log_format = gf_logformat_withmsgid;
-        cmd_args->log_buf_size = GF_LOG_LRU_BUFSIZE_DEFAULT;
-        cmd_args->log_flush_timeout = GF_LOG_FLUSH_TIMEOUT_DEFAULT;
+    if (!ctx)
+        return ret;
 
-        cmd_args->mac_compat = GF_OPTION_DISABLE;
+    ret = xlator_mem_acct_init(THIS, gfd_mt_end);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_CRITICAL, 0, glusterfsd_msg_34, NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+    ret = -1;
+
+    /* monitoring should be enabled by default */
+    ctx->measure_latency = true;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_13, NULL);
+        goto out;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "iobuf", NULL);
+        goto out;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "event", NULL);
+        goto out;
+    }
+
+    ctx->pool = GF_CALLOC(1, sizeof(call_pool_t), gfd_mt_call_pool_t);
+    if (!ctx->pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "call", NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&ctx->pool->all_frames);
+    LOCK_INIT(&ctx->pool->lock);
+
+    /* frame_mem_pool size 112 * 4k */
+    ctx->pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!ctx->pool->frame_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "frame", NULL);
+        goto out;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    ctx->pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!ctx->pool->stack_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stack", NULL);
+        goto out;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stub", NULL);
+        goto out;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto out;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto out;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto out;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto out;
+
+    pthread_mutex_init(&ctx->notify_lock, NULL);
+    pthread_mutex_init(&ctx->cleanup_lock, NULL);
+    pthread_cond_init(&ctx->notify_cond, NULL);
+
+    ctx->clienttable = gf_clienttable_alloc();
+    if (!ctx->clienttable)
+        goto out;
+
+    cmd_args = &ctx->cmd_args;
+
+    /* parsing command line arguments */
+    cmd_args->log_level = DEFAULT_LOG_LEVEL;
+    cmd_args->logger = gf_logger_glusterlog;
+    cmd_args->log_format = gf_logformat_withmsgid;
+    cmd_args->log_buf_size = GF_LOG_LRU_BUFSIZE_DEFAULT;
+    cmd_args->log_flush_timeout = GF_LOG_FLUSH_TIMEOUT_DEFAULT;
+
+    cmd_args->mac_compat = GF_OPTION_DISABLE;
 #ifdef GF_DARWIN_HOST_OS
-        /* On Darwin machines, O_APPEND is not handled,
-         * which may corrupt the data
-         */
-        cmd_args->fuse_direct_io_mode = GF_OPTION_DISABLE;
+    /* On Darwin machines, O_APPEND is not handled,
+     * which may corrupt the data
+     */
+    cmd_args->fuse_direct_io_mode = GF_OPTION_DISABLE;
 #else
-        cmd_args->fuse_direct_io_mode = GF_OPTION_DEFERRED;
+    cmd_args->fuse_direct_io_mode = GF_OPTION_DEFERRED;
 #endif
-        cmd_args->fuse_attribute_timeout = -1;
-        cmd_args->fuse_entry_timeout = -1;
-	cmd_args->fopen_keep_cache = GF_OPTION_DEFERRED;
-
-        if (ctx->mem_acct_enable)
-                cmd_args->mem_acct = 1;
-
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
-        INIT_LIST_HEAD (&cmd_args->volfile_servers);
-
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
-
-        ret = 0;
+    cmd_args->fuse_attribute_timeout = -1;
+    cmd_args->fuse_entry_timeout = -1;
+    cmd_args->fopen_keep_cache = GF_OPTION_DEFERRED;
+    cmd_args->kernel_writeback_cache = GF_OPTION_DEFERRED;
+    cmd_args->fuse_flush_handle_interrupt = GF_OPTION_DEFERRED;
+
+    if (ctx->mem_acct_enable)
+        cmd_args->mem_acct = 1;
+
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
+    INIT_LIST_HEAD(&cmd_args->volfile_servers);
+    ctx->pxl_count = 0;
+    pthread_mutex_init(&ctx->fd_lock, NULL);
+    pthread_cond_init(&ctx->fd_cond, NULL);
+    INIT_LIST_HEAD(&ctx->janitor_fds);
+
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
+
+    ret = 0;
 out:
 
-        if (ret && ctx) {
-                if (ctx->pool) {
-                        mem_pool_destroy (ctx->pool->frame_mem_pool);
-                        mem_pool_destroy (ctx->pool->stack_mem_pool);
-                }
-                GF_FREE (ctx->pool);
-                mem_pool_destroy (ctx->stub_mem_pool);
-                mem_pool_destroy (ctx->dict_pool);
-                mem_pool_destroy (ctx->dict_data_pool);
-                mem_pool_destroy (ctx->dict_pair_pool);
-                mem_pool_destroy (ctx->logbuf_pool);
+    if (ret) {
+        if (ctx->pool) {
+            mem_pool_destroy(ctx->pool->frame_mem_pool);
+            mem_pool_destroy(ctx->pool->stack_mem_pool);
         }
+        GF_FREE(ctx->pool);
+        mem_pool_destroy(ctx->stub_mem_pool);
+        mem_pool_destroy(ctx->dict_pool);
+        mem_pool_destroy(ctx->dict_data_pool);
+        mem_pool_destroy(ctx->dict_pair_pool);
+        mem_pool_destroy(ctx->logbuf_pool);
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-logging_init (glusterfs_ctx_t *ctx, const char *progpath)
+logging_init(glusterfs_ctx_t *ctx, const char *progpath)
 {
-        cmd_args_t *cmd_args = NULL;
-        int         ret = 0;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (cmd_args->log_file == NULL) {
-                ret = gf_set_log_file_path (cmd_args, ctx);
-                if (ret == -1) {
-                        fprintf (stderr, "ERROR: failed to set the log file "
-                                         "path\n");
-                        return -1;
-                }
+    if (cmd_args->log_file == NULL) {
+        ret = gf_set_log_file_path(cmd_args, ctx);
+        if (ret == -1) {
+            fprintf(stderr,
+                    "ERROR: failed to set the log file "
+                    "path\n");
+            return -1;
         }
+    }
 
-        if (cmd_args->log_ident == NULL) {
-                ret = gf_set_log_ident (cmd_args);
-                if (ret == -1) {
-                        fprintf (stderr, "ERROR: failed to set the log "
-                                         "identity\n");
-                        return -1;
-                }
+    if (cmd_args->log_ident == NULL) {
+        ret = gf_set_log_ident(cmd_args);
+        if (ret == -1) {
+            fprintf(stderr,
+                    "ERROR: failed to set the log "
+                    "identity\n");
+            return -1;
         }
+    }
 
-        /* finish log set parameters before init */
-        gf_log_set_loglevel (cmd_args->log_level);
+    /* finish log set parameters before init */
+    gf_log_set_loglevel(ctx, cmd_args->log_level);
 
-        gf_log_set_logger (cmd_args->logger);
+    gf_log_set_localtime(cmd_args->localtime_logging);
 
-        gf_log_set_logformat (cmd_args->log_format);
+    gf_log_set_logger(cmd_args->logger);
 
-        gf_log_set_log_buf_size (cmd_args->log_buf_size);
+    gf_log_set_logformat(cmd_args->log_format);
 
-        gf_log_set_log_flush_timeout (cmd_args->log_flush_timeout);
+    gf_log_set_log_buf_size(cmd_args->log_buf_size);
 
-        if (gf_log_init (ctx, cmd_args->log_file, cmd_args->log_ident) == -1) {
-                fprintf (stderr, "ERROR: failed to open logfile %s\n",
-                         cmd_args->log_file);
-                return -1;
-        }
+    gf_log_set_log_flush_timeout(cmd_args->log_flush_timeout);
 
-        /* At this point, all the logging related parameters are initialised
-         * except for the log flush timer, which will be injected post fork(2)
-         * in daemonize() . During this time, any log message that is logged
-         * will be kept buffered. And if the list that holds these messages
-         * overflows, then the same lru policy is used to drive out the least
-         * recently used message and displace it with the message just logged.
-         */
+    if (gf_log_init(ctx, cmd_args->log_file, cmd_args->log_ident) == -1) {
+        fprintf(stderr, "ERROR: failed to open logfile %s\n",
+                cmd_args->log_file);
+        return -1;
+    }
 
-        return 0;
+    /* At this point, all the logging related parameters are initialised
+     * except for the log flush timer, which will be injected post fork(2)
+     * in daemonize() . During this time, any log message that is logged
+     * will be kept buffered. And if the list that holds these messages
+     * overflows, then the same lru policy is used to drive out the least
+     * recently used message and displace it with the message just logged.
+     */
+
+    return 0;
 }
 
 void
-gf_check_and_set_mem_acct (int argc, char *argv[])
+gf_check_and_set_mem_acct(int argc, char *argv[])
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < argc; i++) {
-                if (strcmp (argv[i], "--no-mem-accounting") == 0) {
-			gf_global_mem_acct_enable_set (0);
-                        break;
-                }
+    for (i = 0; i < argc; i++) {
+        if (strcmp(argv[i], "--no-mem-accounting") == 0) {
+            gf_global_mem_acct_enable_set(0);
+            break;
         }
+    }
 }
 
 /**
@@ -1671,87 +1801,85 @@ gf_check_and_set_mem_acct (int argc, char *argv[])
  * error messages. Hence there are different return values.
  */
 int
-print_exports_file (const char *exports_file)
+print_exports_file(const char *exports_file)
 {
-        void                   *libhandle = NULL;
-        char                   *libpathfull = NULL;
-        struct exports_file    *file = NULL;
-        int                     ret = 0;
-
-        int  (*exp_file_parse)(const char *filepath,
-                               struct exports_file **expfile,
-                               struct mount3_state *ms) = NULL;
-        void (*exp_file_print)(const struct exports_file *file) = NULL;
-        void (*exp_file_deinit)(struct exports_file *ptr) = NULL;
-
-        /* XLATORDIR passed through a -D flag to GCC */
-        ret = gf_asprintf (&libpathfull, "%s/%s/server.so", XLATORDIR,
-                           "nfs");
-        if (ret < 0) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
-                ret = -1;
-                goto out;
-        }
-
-        /* Load up the library */
-        libhandle = dlopen (libpathfull, RTLD_NOW);
-        if (!libhandle) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error loading NFS server library : "
-                        "%s\n", dlerror ());
-                ret = -1;
-                goto out;
-        }
-
-        /* Load up the function */
-        exp_file_parse = dlsym (libhandle, "exp_file_parse");
-        if (!exp_file_parse) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function exp_file_parse "
-                        "in symbol.");
-                ret = -1;
-                goto out;
-        }
-
-        /* Parse the file */
-        ret = exp_file_parse (exports_file, &file, NULL);
-        if (ret < 0) {
-                ret = 1;        /* This means we failed to parse */
-                goto out;
-        }
-
-        /* Load up the function */
-        exp_file_print = dlsym (libhandle, "exp_file_print");
-        if (!exp_file_print) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function exp_file_print in symbol.");
-                ret = -1;
-                goto out;
-        }
+    void *libhandle = NULL;
+    char *libpathfull = NULL;
+    struct exports_file *file = NULL;
+    int ret = 0;
+
+    int (*exp_file_parse)(const char *filepath, struct exports_file **expfile,
+                          struct mount3_state *ms) = NULL;
+    void (*exp_file_print)(const struct exports_file *file) = NULL;
+    void (*exp_file_deinit)(struct exports_file * ptr) = NULL;
+
+    /* XLATORDIR passed through a -D flag to GCC */
+    ret = gf_asprintf(&libpathfull, "%s/%s/server.so", XLATORDIR, "nfs");
+    if (ret < 0) {
+        gf_log("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the library */
+    libhandle = dlopen(libpathfull, RTLD_NOW);
+    if (!libhandle) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error loading NFS server library : "
+               "%s\n",
+               dlerror());
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the function */
+    exp_file_parse = dlsym(libhandle, "exp_file_parse");
+    if (!exp_file_parse) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_parse "
+               "in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Parse the file */
+    ret = exp_file_parse(exports_file, &file, NULL);
+    if (ret < 0) {
+        ret = 1; /* This means we failed to parse */
+        goto out;
+    }
+
+    /* Load up the function */
+    exp_file_print = dlsym(libhandle, "exp_file_print");
+    if (!exp_file_print) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_print in symbol.");
+        ret = -1;
+        goto out;
+    }
 
-        /* Print it out to screen */
-        exp_file_print (file);
+    /* Print it out to screen */
+    exp_file_print(file);
 
-        /* Load up the function */
-        exp_file_deinit = dlsym (libhandle, "exp_file_deinit");
-        if (!exp_file_deinit) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function exp_file_deinit in lib.");
-                ret = -1;
-                goto out;
-        }
+    /* Load up the function */
+    exp_file_deinit = dlsym(libhandle, "exp_file_deinit");
+    if (!exp_file_deinit) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_deinit in lib.");
+        ret = -1;
+        goto out;
+    }
 
-        /* Free the file */
-        exp_file_deinit (file);
+    /* Free the file */
+    exp_file_deinit(file);
 
 out:
-        if (libhandle)
-                dlclose(libhandle);
-        GF_FREE (libpathfull);
-        return ret;
+    if (libhandle)
+        dlclose(libhandle);
+    GF_FREE(libpathfull);
+    return ret;
 }
 
-
 /**
  * print_netgroups_file - Print out & verify the syntax
  *                        of the netgroups file specified
@@ -1769,730 +1897,842 @@ out:
  * we want to print out a different error messages based on the ret value.
  */
 int
-print_netgroups_file (const char *netgroups_file)
+print_netgroups_file(const char *netgroups_file)
 {
-        void                   *libhandle = NULL;
-        char                   *libpathfull = NULL;
-        struct netgroups_file  *file = NULL;
-        int                     ret = 0;
-
-        struct netgroups_file  *(*ng_file_parse)(const char *file_path) = NULL;
-        void         (*ng_file_print)(const struct netgroups_file *file) = NULL;
-        void         (*ng_file_deinit)(struct netgroups_file *ptr) = NULL;
-
-        /* XLATORDIR passed through a -D flag to GCC */
-        ret = gf_asprintf (&libpathfull, "%s/%s/server.so", XLATORDIR,
-                        "nfs");
-        if (ret < 0) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
-                ret = -1;
-                goto out;
-        }
-        /* Load up the library */
-        libhandle = dlopen (libpathfull, RTLD_NOW);
-        if (!libhandle) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error loading NFS server library : %s\n", dlerror ());
-                ret = -1;
-                goto out;
-        }
-
-        /* Load up the function */
-        ng_file_parse = dlsym (libhandle, "ng_file_parse");
-        if (!ng_file_parse) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function ng_file_parse in symbol.");
-                ret = -1;
-                goto out;
-        }
-
-        /* Parse the file */
-        file = ng_file_parse (netgroups_file);
-        if (!file) {
-                ret = 1;        /* This means we failed to parse */
-                goto out;
-        }
-
-        /* Load up the function */
-        ng_file_print = dlsym (libhandle, "ng_file_print");
-        if (!ng_file_print) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function ng_file_print in symbol.");
-                ret = -1;
-                goto out;
-        }
+    void *libhandle = NULL;
+    char *libpathfull = NULL;
+    struct netgroups_file *file = NULL;
+    int ret = 0;
+
+    struct netgroups_file *(*ng_file_parse)(const char *file_path) = NULL;
+    void (*ng_file_print)(const struct netgroups_file *file) = NULL;
+    void (*ng_file_deinit)(struct netgroups_file * ptr) = NULL;
+
+    /* XLATORDIR passed through a -D flag to GCC */
+    ret = gf_asprintf(&libpathfull, "%s/%s/server.so", XLATORDIR, "nfs");
+    if (ret < 0) {
+        gf_log("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
+        ret = -1;
+        goto out;
+    }
+    /* Load up the library */
+    libhandle = dlopen(libpathfull, RTLD_NOW);
+    if (!libhandle) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error loading NFS server library : %s\n", dlerror());
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the function */
+    ng_file_parse = dlsym(libhandle, "ng_file_parse");
+    if (!ng_file_parse) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_parse in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Parse the file */
+    file = ng_file_parse(netgroups_file);
+    if (!file) {
+        ret = 1; /* This means we failed to parse */
+        goto out;
+    }
+
+    /* Load up the function */
+    ng_file_print = dlsym(libhandle, "ng_file_print");
+    if (!ng_file_print) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_print in symbol.");
+        ret = -1;
+        goto out;
+    }
 
-        /* Print it out to screen */
-        ng_file_print (file);
+    /* Print it out to screen */
+    ng_file_print(file);
 
-        /* Load up the function */
-        ng_file_deinit = dlsym (libhandle, "ng_file_deinit");
-        if (!ng_file_deinit) {
-                gf_log ("glusterfs", GF_LOG_CRITICAL,
-                        "Error finding function ng_file_deinit in lib.");
-                ret = -1;
-                goto out;
-        }
+    /* Load up the function */
+    ng_file_deinit = dlsym(libhandle, "ng_file_deinit");
+    if (!ng_file_deinit) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_deinit in lib.");
+        ret = -1;
+        goto out;
+    }
 
-        /* Free the file */
-        ng_file_deinit (file);
+    /* Free the file */
+    ng_file_deinit(file);
 
 out:
-        if (libhandle)
-                dlclose(libhandle);
-        GF_FREE (libpathfull);
-        return ret;
+    if (libhandle)
+        dlclose(libhandle);
+    GF_FREE(libpathfull);
+    return ret;
 }
 
-
 int
-parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx)
+parse_cmdline(int argc, char *argv[], glusterfs_ctx_t *ctx)
 {
-        int          process_mode = 0;
-        int          ret = 0;
-        struct stat  stbuf = {0, };
-        char         timestr[32];
-        char         tmp_logfile[1024] = { 0 };
-        char        *tmp_logfile_dyn = NULL;
-        char        *tmp_logfilebase = NULL;
-        cmd_args_t  *cmd_args = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        /* Do this before argp_parse so it can be overridden. */
-        if (sys_access (SECURE_ACCESS_FILE, F_OK) == 0) {
-                cmd_args->secure_mgmt = 1;
-        }
-
-        argp_parse (&argp, argc, argv, ARGP_IN_ORDER, NULL, cmd_args);
-        if (cmd_args->print_netgroups) {
-                /* When this option is set we don't want to do anything else
-                 * except for printing & verifying the netgroups file.
-                 */
-                ret = 0;
-                goto out;
-        }
-
-        if (cmd_args->print_exports) {
-                /* When this option is set we don't want to do anything else
-                 * except for printing & verifying the exports file.
-                  */
-                ret = 0;
-                goto out;
-        }
-
-
-        ctx->secure_mgmt = cmd_args->secure_mgmt;
-
-        if (ENABLE_DEBUG_MODE == cmd_args->debug_mode) {
-                cmd_args->log_level = GF_LOG_DEBUG;
-                cmd_args->log_file = gf_strdup ("/dev/stderr");
-                cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
-        }
-
-        process_mode = gf_get_process_mode (argv[0]);
-        ctx->process_mode = process_mode;
+    int process_mode = 0;
+    int ret = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE];
+    char tmp_logfile[1024] = {0};
+    char *tmp_logfile_dyn = NULL;
+    char *tmp_logfilebase = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int len = 0;
+    char *thin_volfileid = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    /* Do this before argp_parse so it can be overridden. */
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        cmd_args->secure_mgmt = 1;
+        ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    /* Need to set lru_limit to below 0 to indicate there was nothing
+       specified. This is needed as 0 is a valid option, and may not be
+       default value. */
+    cmd_args->lru_limit = -1;
+
+    argp_parse(&argp, argc, argv, ARGP_IN_ORDER, NULL, cmd_args);
+
+    if (cmd_args->print_xlatordir || cmd_args->print_statedumpdir ||
+        cmd_args->print_logdir || cmd_args->print_libexecdir) {
+        /* Just print, nothing else to do */
+        goto out;
+    }
+
+    if (cmd_args->print_netgroups) {
+        /* When this option is set we don't want to do anything else
+         * except for printing & verifying the netgroups file.
+         */
+        ret = 0;
+        goto out;
+    }
 
-        /* Make sure after the parsing cli, if '--volfile-server' option is
-           given, then '--volfile-id' is mandatory */
-        if (cmd_args->volfile_server && !cmd_args->volfile_id) {
-                gf_msg ("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_15);
-                ret = -1;
+    if (cmd_args->print_exports) {
+        /* When this option is set we don't want to do anything else
+         * except for printing & verifying the exports file.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    ctx->secure_mgmt = cmd_args->secure_mgmt;
+
+    if (ENABLE_DEBUG_MODE == cmd_args->debug_mode) {
+        cmd_args->log_level = GF_LOG_DEBUG;
+        cmd_args->log_file = gf_strdup("/dev/stderr");
+        cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
+    }
+
+    process_mode = gf_get_process_mode(argv[0]);
+    ctx->process_mode = process_mode;
+
+    if (cmd_args->process_name) {
+        ctx->cmd_args.process_name = cmd_args->process_name;
+    }
+    /* Make sure after the parsing cli, if '--volfile-server' option is
+       given, then '--volfile-id' is mandatory */
+    if (cmd_args->volfile_server && !cmd_args->volfile_id) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_15, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd_args->volfile_server == NULL) && (cmd_args->volfile == NULL)) {
+        if (process_mode == GF_SERVER_PROCESS)
+            cmd_args->volfile = gf_strdup(DEFAULT_SERVER_VOLFILE);
+        else if (process_mode == GF_GLUSTERD_PROCESS)
+            cmd_args->volfile = gf_strdup(DEFAULT_GLUSTERD_VOLFILE);
+        else
+            cmd_args->volfile = gf_strdup(DEFAULT_CLIENT_VOLFILE);
+
+        /* Check if the volfile exists, if not give usage output
+           and exit */
+        ret = sys_stat(cmd_args->volfile, &stbuf);
+        if (ret) {
+            gf_smsg("glusterfs", GF_LOG_CRITICAL, errno, glusterfsd_msg_16,
+                    NULL);
+            /* argp_usage (argp.) */
+            fprintf(stderr, "USAGE: %s [options] [mountpoint]\n", argv[0]);
+            goto out;
+        }
+    }
+
+    if (cmd_args->thin_client) {
+        len = strlen(cmd_args->volfile_id) + SLEN("gfproxy-client/");
+        thin_volfileid = GF_MALLOC(len + 1, gf_common_mt_char);
+        snprintf(thin_volfileid, len + 1, "gfproxy-client/%s",
+                 cmd_args->volfile_id);
+        GF_FREE(cmd_args->volfile_id);
+        cmd_args->volfile_id = thin_volfileid;
+    }
+
+    if (cmd_args->run_id) {
+        ret = sys_lstat(cmd_args->log_file, &stbuf);
+        /* If its /dev/null, or /dev/stdout, /dev/stderr,
+         * let it use the same, no need to alter
+         */
+        if (((ret == 0) &&
+             (S_ISREG(stbuf.st_mode) || S_ISLNK(stbuf.st_mode))) ||
+            (ret == -1)) {
+            /* Have separate logfile per run. */
+            gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT);
+            sprintf(tmp_logfile, "%s.%s.%d", cmd_args->log_file, timestr,
+                    getpid());
+
+            /* Create symlink to actual log file */
+            sys_unlink(cmd_args->log_file);
+
+            tmp_logfile_dyn = gf_strdup(tmp_logfile);
+            tmp_logfilebase = basename(tmp_logfile_dyn);
+            ret = sys_symlink(tmp_logfilebase, cmd_args->log_file);
+            if (ret == -1) {
+                fprintf(stderr, "ERROR: symlink of logfile failed\n");
                 goto out;
-        }
-
-        if ((cmd_args->volfile_server == NULL)
-            && (cmd_args->volfile == NULL)) {
-                if (process_mode == GF_SERVER_PROCESS)
-                        cmd_args->volfile = gf_strdup (DEFAULT_SERVER_VOLFILE);
-                else if (process_mode == GF_GLUSTERD_PROCESS)
-                        cmd_args->volfile = gf_strdup (DEFAULT_GLUSTERD_VOLFILE);
-                else
-                        cmd_args->volfile = gf_strdup (DEFAULT_CLIENT_VOLFILE);
-
-                /* Check if the volfile exists, if not give usage output
-                   and exit */
-                ret = sys_stat (cmd_args->volfile, &stbuf);
-                if (ret) {
-                        gf_msg ("glusterfs", GF_LOG_CRITICAL, errno,
-                                glusterfsd_msg_16);
-                        /* argp_usage (argp.) */
-                        fprintf (stderr, "USAGE: %s [options] [mountpoint]\n",
-                                 argv[0]);
-                        goto out;
-                }
-        }
-
-        if (cmd_args->run_id) {
-                ret = sys_lstat (cmd_args->log_file, &stbuf);
-                /* If its /dev/null, or /dev/stdout, /dev/stderr,
-                 * let it use the same, no need to alter
-                 */
-                if (((ret == 0) &&
-                     (S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode))) ||
-                    (ret == -1)) {
-                        /* Have separate logfile per run */
-                        gf_time_fmt (timestr, sizeof timestr, time (NULL),
-                                     gf_timefmt_FT);
-                        sprintf (tmp_logfile, "%s.%s.%d",
-                                 cmd_args->log_file, timestr, getpid ());
-
-                        /* Create symlink to actual log file */
-                        sys_unlink (cmd_args->log_file);
-
-                        tmp_logfile_dyn = gf_strdup (tmp_logfile);
-                        tmp_logfilebase = basename (tmp_logfile_dyn);
-                        ret = sys_symlink (tmp_logfilebase,
-                                           cmd_args->log_file);
-                        if (ret == -1) {
-                                fprintf (stderr, "ERROR: symlink of logfile failed\n");
-                                goto out;
-                        }
+            }
 
-                        GF_FREE (cmd_args->log_file);
-                        cmd_args->log_file = gf_strdup (tmp_logfile);
+            GF_FREE(cmd_args->log_file);
+            cmd_args->log_file = gf_strdup(tmp_logfile);
 
-                        GF_FREE (tmp_logfile_dyn);
-                }
+            GF_FREE(tmp_logfile_dyn);
         }
+    }
 
-        /*
-           This option was made obsolete but parsing it for backward
-           compatibility with third party applications
-         */
-        if (cmd_args->max_connect_attempts) {
-                gf_msg ("glusterfs", GF_LOG_WARNING, 0, glusterfsd_msg_33);
-        }
+    /*
+       This option was made obsolete but parsing it for backward
+       compatibility with third party applications
+     */
+    if (cmd_args->max_connect_attempts) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, 0, glusterfsd_msg_33, NULL);
+    }
 
 #ifdef GF_DARWIN_HOST_OS
-        if (cmd_args->mount_point)
-               cmd_args->mac_compat = GF_OPTION_DEFERRED;
+    if (cmd_args->mount_point)
+        cmd_args->mac_compat = GF_OPTION_DEFERRED;
 #endif
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_setup(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t  *cmd_args = NULL;
-        int          ret = -1;
-        FILE        *pidfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    FILE *pidfp = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->pid_file)
-                return 0;
+    if (!cmd_args->pid_file)
+        return 0;
 
-        pidfp = fopen (cmd_args->pid_file, "a+");
-        if (!pidfp) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_17,
-                        cmd_args->pid_file);
-                goto out;
-        }
+    pidfp = fopen(cmd_args->pid_file, "a+");
+    if (!pidfp) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_17,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        goto out;
+    }
 
-        ctx->pidfp = pidfp;
+    ctx->pidfp = pidfp;
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
-
 int
-glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_cleanup(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t      *cmd_args = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (!ctx->pidfp)
-                return 0;
+    if (!ctx->pidfp)
+        return 0;
 
-        gf_msg_trace ("glusterfsd", 0, "pidfile %s cleanup",
-                      cmd_args->pid_file);
+    gf_msg_trace("glusterfsd", 0, "pidfile %s cleanup", cmd_args->pid_file);
 
-        if (ctx->cmd_args.pid_file) {
-                sys_unlink (ctx->cmd_args.pid_file);
-                ctx->cmd_args.pid_file = NULL;
-        }
+    if (ctx->cmd_args.pid_file) {
+        GF_FREE(ctx->cmd_args.pid_file);
+        ctx->cmd_args.pid_file = NULL;
+    }
 
-        lockf (fileno (ctx->pidfp), F_ULOCK, 0);
-        fclose (ctx->pidfp);
-        ctx->pidfp = NULL;
+    lockf(fileno(ctx->pidfp), F_ULOCK, 0);
+    fclose(ctx->pidfp);
+    ctx->pidfp = NULL;
 
-        return 0;
+    return 0;
 }
 
 int
-glusterfs_pidfile_update (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_update(glusterfs_ctx_t *ctx, pid_t pid)
 {
-        cmd_args_t  *cmd_args = NULL;
-        int          ret = 0;
-        FILE        *pidfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+    FILE *pidfp = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        pidfp = ctx->pidfp;
-        if (!pidfp)
-                return 0;
+    pidfp = ctx->pidfp;
+    if (!pidfp)
+        return 0;
 
-        ret = lockf (fileno (pidfp), F_TLOCK, 0);
-        if (ret) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_18,
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = lockf(fileno(pidfp), F_TLOCK, 0);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_18,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = sys_ftruncate (fileno (pidfp), 0);
-        if (ret) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_20,
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = sys_ftruncate(fileno(pidfp), 0);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_20,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = fprintf (pidfp, "%d\n", getpid ());
-        if (ret <= 0) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = fprintf(pidfp, "%d\n", pid);
+    if (ret <= 0) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = fflush (pidfp);
-        if (ret) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = fflush(pidfp);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        gf_msg_debug ("glusterfsd", 0, "pidfile %s updated with pid %d",
-                      cmd_args->pid_file, getpid ());
+    gf_msg_debug("glusterfsd", 0, "pidfile %s updated with pid %d",
+                 cmd_args->pid_file, pid);
 
-        return 0;
+    return 0;
 }
 
-
 void *
-glusterfs_sigwaiter (void *arg)
+glusterfs_sigwaiter(void *arg)
 {
-        sigset_t  set;
-        int       ret = 0;
-        int       sig = 0;
+    sigset_t set;
+    int ret = 0;
+    int sig = 0;
+    char *file = NULL;
+
+    sigemptyset(&set);
+    sigaddset(&set, SIGINT);  /* cleanup_and_exit */
+    sigaddset(&set, SIGTERM); /* cleanup_and_exit */
+    sigaddset(&set, SIGHUP);  /* reincarnate */
+    sigaddset(&set, SIGUSR1); /* gf_proc_dump_info */
+    sigaddset(&set, SIGUSR2);
+
+    for (;;) {
+        ret = sigwait(&set, &sig);
+        if (ret)
+            continue;
 
+        switch (sig) {
+            case SIGINT:
+            case SIGTERM:
+                cleanup_and_exit(sig);
+                break;
+            case SIGHUP:
+                reincarnate(sig);
+                break;
+            case SIGUSR1:
+                gf_proc_dump_info(sig, glusterfsd_ctx);
+                break;
+            case SIGUSR2:
+                file = gf_monitor_metrics(glusterfsd_ctx);
 
-        sigemptyset (&set);
-        sigaddset (&set, SIGINT);   /* cleanup_and_exit */
-        sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
-        sigaddset (&set, SIGHUP);   /* reincarnate */
-        sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
+                /* Nothing needed to be done here */
+                GF_FREE(file);
 
-        for (;;) {
-                ret = sigwait (&set, &sig);
-                if (ret)
-                        continue;
-
-
-                switch (sig) {
-                case SIGINT:
-                case SIGTERM:
-                        cleanup_and_exit (sig);
-                        break;
-                case SIGHUP:
-                        reincarnate (sig);
-                        break;
-                case SIGUSR1:
-                        gf_proc_dump_info (sig, glusterfsd_ctx);
-                        break;
-                case SIGUSR2:
-                        gf_latency_toggle (sig, glusterfsd_ctx);
-                        break;
-                default:
-
-                        break;
-                }
+                break;
+            default:
+
+                break;
         }
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 void
-glusterfsd_print_trace (int signum)
+glusterfsd_print_trace(int signum)
 {
-	gf_print_trace (signum, glusterfsd_ctx);
+    gf_print_trace(signum, glusterfsd_ctx);
 }
 
-
 int
-glusterfs_signals_setup (glusterfs_ctx_t *ctx)
+glusterfs_signals_setup(glusterfs_ctx_t *ctx)
 {
-        sigset_t  set;
-        int       ret = 0;
-
-        sigemptyset (&set);
-
-        /* common setting for all threads */
-        signal (SIGSEGV, glusterfsd_print_trace);
-        signal (SIGABRT, glusterfsd_print_trace);
-        signal (SIGILL, glusterfsd_print_trace);
-        signal (SIGTRAP, glusterfsd_print_trace);
-        signal (SIGFPE, glusterfsd_print_trace);
-        signal (SIGBUS, glusterfsd_print_trace);
-        signal (SIGINT, cleanup_and_exit);
-        signal (SIGPIPE, SIG_IGN);
-
-        /* block these signals from non-sigwaiter threads */
-        sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
-        sigaddset (&set, SIGHUP);   /* reincarnate */
-        sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
-
-        ret = pthread_sigmask (SIG_BLOCK, &set, NULL);
-        if (ret) {
-                gf_msg ("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_22);
-                return ret;
-        }
-
-        ret = pthread_create (&ctx->sigwaiter, NULL, glusterfs_sigwaiter,
-                              (void *) &set);
-        if (ret) {
-                /*
-                  TODO:
-                  fallback to signals getting handled by other threads.
-                  setup the signal handlers
-                */
-                gf_msg ("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_23);
-                return ret;
-        }
+    sigset_t set;
+    int ret = 0;
+
+    sigemptyset(&set);
+
+    /* common setting for all threads */
+    signal(SIGSEGV, glusterfsd_print_trace);
+    signal(SIGABRT, glusterfsd_print_trace);
+    signal(SIGILL, glusterfsd_print_trace);
+    signal(SIGTRAP, glusterfsd_print_trace);
+    signal(SIGFPE, glusterfsd_print_trace);
+    signal(SIGBUS, glusterfsd_print_trace);
+    signal(SIGINT, cleanup_and_exit);
+    signal(SIGPIPE, SIG_IGN);
+
+    /* block these signals from non-sigwaiter threads */
+    sigaddset(&set, SIGTERM); /* cleanup_and_exit */
+    sigaddset(&set, SIGHUP);  /* reincarnate */
+    sigaddset(&set, SIGUSR1); /* gf_proc_dump_info */
+    sigaddset(&set, SIGUSR2);
+
+    /* Signals needed for asynchronous framework. */
+    sigaddset(&set, GF_ASYNC_SIGQUEUE);
+    sigaddset(&set, GF_ASYNC_SIGCTRL);
+
+    ret = pthread_sigmask(SIG_BLOCK, &set, NULL);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_22, NULL);
+        return ret;
+    }
 
+    ret = gf_thread_create(&ctx->sigwaiter, NULL, glusterfs_sigwaiter,
+                           (void *)&set, "sigwait");
+    if (ret) {
+        /*
+          TODO:
+          fallback to signals getting handled by other threads.
+          setup the signal handlers
+        */
+        gf_smsg("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_23, NULL);
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-daemonize (glusterfs_ctx_t *ctx)
+daemonize(glusterfs_ctx_t *ctx)
 {
-        int            ret = -1;
-        cmd_args_t    *cmd_args = NULL;
-        int            cstatus = 0;
-        int            err = 1;
-
-        cmd_args = &ctx->cmd_args;
-
-        ret = glusterfs_pidfile_setup (ctx);
-        if (ret)
-                goto out;
-
-        if (cmd_args->no_daemon_mode)
-                goto postfork;
-
-        if (cmd_args->debug_mode)
-                goto postfork;
-
-        ret = pipe (ctx->daemon_pipe);
-        if (ret) {
-                /* If pipe() fails, retain daemon_pipe[] = {-1, -1}
-                   and parent will just not wait for child status
-                */
-                ctx->daemon_pipe[0] = -1;
-                ctx->daemon_pipe[1] = -1;
-        }
-
-        ret = os_daemon_return (0, 0);
-        switch (ret) {
+    int ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int cstatus = 0;
+    int err = 1;
+    int child_pid = 0;
+
+    cmd_args = &ctx->cmd_args;
+
+    ret = glusterfs_pidfile_setup(ctx);
+    if (ret)
+        goto out;
+
+    if (cmd_args->no_daemon_mode) {
+        goto postfork;
+    }
+
+    if (cmd_args->debug_mode)
+        goto postfork;
+
+    ret = pipe(ctx->daemon_pipe);
+    if (ret) {
+        /* If pipe() fails, retain daemon_pipe[] = {-1, -1}
+           and parent will just not wait for child status
+        */
+        ctx->daemon_pipe[0] = -1;
+        ctx->daemon_pipe[1] = -1;
+    }
+
+    ret = os_daemon_return(0, 0);
+    switch (ret) {
         case -1:
-                if (ctx->daemon_pipe[0] != -1) {
-                        sys_close (ctx->daemon_pipe[0]);
-                        sys_close (ctx->daemon_pipe[1]);
-                }
+            if (ctx->daemon_pipe[0] != -1) {
+                sys_close(ctx->daemon_pipe[0]);
+                sys_close(ctx->daemon_pipe[1]);
+            }
 
-                gf_msg ("daemonize", GF_LOG_ERROR, errno, glusterfsd_msg_24);
-                goto out;
+            gf_smsg("daemonize", GF_LOG_ERROR, errno, glusterfsd_msg_24, NULL);
+            goto out;
         case 0:
-                /* child */
-                /* close read */
-                sys_close (ctx->daemon_pipe[0]);
-                break;
+            /* child */
+            /* close read */
+            sys_close(ctx->daemon_pipe[0]);
+            break;
         default:
-                /* parent */
-                /* close write */
-                sys_close (ctx->daemon_pipe[1]);
-
-                if (ctx->mnt_pid > 0) {
-                        ret = waitpid (ctx->mnt_pid, &cstatus, 0);
-                        if (!(ret == ctx->mnt_pid)) {
-                                if (WIFEXITED(cstatus)) {
-                                        err = WEXITSTATUS(cstatus);
-                                } else {
-                                        err = cstatus;
-                                }
-                                gf_msg ("daemonize", GF_LOG_ERROR, 0,
-                                        glusterfsd_msg_25);
-                                exit (err);
-                        }
+            /* parent */
+            /* close write */
+            child_pid = ret;
+            sys_close(ctx->daemon_pipe[1]);
+
+            if (ctx->mnt_pid > 0) {
+                ret = waitpid(ctx->mnt_pid, &cstatus, 0);
+                if (!(ret == ctx->mnt_pid)) {
+                    if (WIFEXITED(cstatus)) {
+                        err = WEXITSTATUS(cstatus);
+                    } else {
+                        err = cstatus;
+                    }
+                    gf_smsg("daemonize", GF_LOG_ERROR, 0, glusterfsd_msg_25,
+                            NULL);
+                    exit(err);
                 }
-                sys_read (ctx->daemon_pipe[0], (void *)&err, sizeof (err));
-                _exit (err);
-        }
+            }
+            sys_read(ctx->daemon_pipe[0], (void *)&err, sizeof(err));
+            /* NOTE: Only the least significant 8 bits i.e (err & 255)
+               will be available to parent process on calling exit() */
+            if (err)
+                _exit(abs(err));
+
+            /* Update pid in parent only for glusterd process */
+            if (ctx->process_mode == GF_GLUSTERD_PROCESS) {
+                ret = glusterfs_pidfile_update(ctx, child_pid);
+                if (ret)
+                    exit(1);
+            }
+            _exit(0);
+    }
 
 postfork:
-        ret = glusterfs_pidfile_update (ctx);
+    /* Update pid in child either process_mode is not belong to glusterd
+       or process is spawned in no daemon mode
+    */
+    if ((ctx->process_mode != GF_GLUSTERD_PROCESS) ||
+        (cmd_args->no_daemon_mode)) {
+        ret = glusterfs_pidfile_update(ctx, getpid());
         if (ret)
-                goto out;
+            goto out;
+    }
+    gf_log("glusterfs", GF_LOG_INFO, "Pid of current running process is %d",
+           getpid());
+    ret = gf_log_inject_timer_event(ctx);
 
-        ret = gf_log_inject_timer_event (ctx);
-
-        glusterfs_signals_setup (ctx);
+    glusterfs_signals_setup(ctx);
 out:
-        return ret;
+    return ret;
 }
 
-
 #ifdef GF_LINUX_HOST_OS
 static int
-set_oom_score_adj (glusterfs_ctx_t *ctx)
+set_oom_score_adj(glusterfs_ctx_t *ctx)
 {
-        int                  ret           = -1;
-        cmd_args_t          *cmd_args      =  NULL;
-        int                  fd            = -1;
-        size_t               oom_score_len =  0;
-        struct oom_api_info *api           =  NULL;
+    int ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int fd = -1;
+    size_t oom_score_len = 0;
+    struct oom_api_info *api = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->oom_score_adj)
-                goto success;
+    if (!cmd_args->oom_score_adj)
+        goto success;
 
-        api = get_oom_api_info();
-        if (!api)
-                goto out;
+    api = get_oom_api_info();
+    if (!api)
+        goto out;
 
-        fd = open (api->oom_api_file, O_WRONLY);
-        if (fd < 0)
-                goto out;
+    fd = open(api->oom_api_file, O_WRONLY);
+    if (fd < 0)
+        goto out;
 
-        oom_score_len = strlen (cmd_args->oom_score_adj);
-        if (sys_write (fd,
-                  cmd_args->oom_score_adj, oom_score_len) != oom_score_len) {
-                sys_close (fd);
-                goto out;
-        }
+    oom_score_len = strlen(cmd_args->oom_score_adj);
+    if (sys_write(fd, cmd_args->oom_score_adj, oom_score_len) !=
+        oom_score_len) {
+        sys_close(fd);
+        goto out;
+    }
 
-        if (sys_close (fd) < 0)
-                goto out;
+    if (sys_close(fd) < 0)
+        goto out;
 
 success:
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 #endif
 
-
 int
-glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp)
+glusterfs_process_volfp(glusterfs_ctx_t *ctx, FILE *fp)
 {
-        glusterfs_graph_t  *graph = NULL;
-        int                 ret = -1;
-        xlator_t           *trav = NULL;
-        int                 err = 0;
-
-        graph = glusterfs_graph_construct (fp);
-        if (!graph) {
-                gf_msg ("", GF_LOG_ERROR, 0, glusterfsd_msg_26);
-                goto out;
-        }
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
 
-        for (trav = graph->first; trav; trav = trav->next) {
-                if (strcmp (trav->type, "mount/fuse") == 0) {
-                        gf_msg ("glusterfsd", GF_LOG_ERROR, 0,
-                                glusterfsd_msg_27);
-                        goto out;
-                }
-        }
+    if (!ctx)
+        return -1;
 
-        ret = glusterfs_graph_prepare (graph, ctx);
-        if (ret) {
-                goto out;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("", GF_LOG_ERROR, 0, glusterfsd_msg_26, NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/fuse") == 0) {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_27, NULL);
+            goto out;
         }
+    }
 
-        ret = glusterfs_graph_activate (graph, ctx);
+    xlator_t *xl = graph->first;
+    if (xl && (strcmp(xl->type, "protocol/server") == 0)) {
+        (void)copy_opts_to_child(xl, FIRST_CHILD(xl), "*auth*");
+    }
 
-        if (ret) {
-                goto out;
-        }
+    ret = glusterfs_graph_prepare(graph, ctx, ctx->cmd_args.volume_name);
+    if (ret) {
+        goto out;
+    }
 
-        gf_log_dump_graph (fp, graph);
+    ret = glusterfs_graph_activate(graph, ctx);
 
-        ret = 0;
+    if (ret) {
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-        if (fp)
-                fclose (fp);
-
-        if (ret && !ctx->active) {
-                glusterfs_graph_destroy (graph);
-                /* there is some error in setting up the first graph itself */
-                err = -ret;
-                sys_write (ctx->daemon_pipe[1], (void *) &err, sizeof (err));
-                cleanup_and_exit (err);
+    if (fp)
+        fclose(fp);
+
+    if (ret) {
+        /* TODO This code makes to generic for all graphs
+           client as well as servers.For now it destroys
+           graph only for server-side xlators not for client-side
+           xlators, before destroying a graph call xlator fini for
+           xlators those call xlator_init to avoid leak
+        */
+        if (graph) {
+            xl = graph->first;
+            if ((ctx->active != graph) &&
+                (xl && !strcmp(xl->type, "protocol/server"))) {
+                /* Take dict ref for every graph xlator to avoid dict leak
+                   at the time of graph destroying
+                */
+                glusterfs_graph_fini(graph);
+                glusterfs_graph_destroy(graph);
+            }
         }
 
-        return ret;
-}
+        /* there is some error in setting up the first graph itself */
+        if (!ctx->active) {
+            emancipate(ctx, ret);
+            cleanup_and_exit(ret);
+        }
+    }
 
+    return ret;
+}
 
 int
-glusterfs_volumes_init (glusterfs_ctx_t *ctx)
+glusterfs_volumes_init(glusterfs_ctx_t *ctx)
 {
-        FILE               *fp = NULL;
-        cmd_args_t         *cmd_args = NULL;
-        int                 ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (cmd_args->sock_file) {
-                ret = glusterfs_listener_init (ctx);
-                if (ret)
-                        goto out;
-        }
+    if (cmd_args->sock_file) {
+        ret = glusterfs_listener_init(ctx);
+        if (ret)
+            goto out;
+    }
 
-        if (cmd_args->volfile_server) {
-                ret = glusterfs_mgmt_init (ctx);
-                /* return, do not emancipate() yet */
-                return ret;
-        }
+    if (cmd_args->volfile_server) {
+        ret = glusterfs_mgmt_init(ctx);
+        /* return, do not emancipate() yet */
+        return ret;
+    }
 
-        fp = get_volfp (ctx);
+    fp = get_volfp(ctx);
 
-        if (!fp) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_28);
-                ret = -1;
-                goto out;
-        }
+    if (!fp) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_28, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterfs_process_volfp (ctx, fp);
-        if (ret)
-                goto out;
+    ret = glusterfs_process_volfp(ctx, fp);
+    if (ret)
+        goto out;
 
 out:
-        emancipate (ctx, ret);
-        return ret;
+    emancipate(ctx, ret);
+    return ret;
 }
 
 /* This is the only legal global pointer  */
 glusterfs_ctx_t *glusterfsd_ctx;
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        int               ret = -1;
-        char              cmdlinestr[PATH_MAX] = {0,};
-        cmd_args_t       *cmd = NULL;
-
-	gf_check_and_set_mem_acct (argc, argv);
-
-	ctx = glusterfs_ctx_new ();
-        if (!ctx) {
-                gf_msg ("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_29);
-                return ENOMEM;
-        }
-	glusterfsd_ctx = ctx;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    char cmdlinestr[PATH_MAX] = {
+        0,
+    };
+    cmd_args_t *cmd = NULL;
+
+    gf_check_and_set_mem_acct(argc, argv);
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_29, NULL);
+        return ENOMEM;
+    }
+    glusterfsd_ctx = ctx;
+
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        return ret;
 
-        ret = glusterfs_globals_init (ctx);
-        if (ret)
-                return ret;
+    THIS->ctx = ctx;
 
-	THIS->ctx = ctx;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto out;
+    ret = parse_cmdline(argc, argv, ctx);
+    if (ret)
+        goto out;
+    cmd = &ctx->cmd_args;
 
-        ret = parse_cmdline (argc, argv, ctx);
-        if (ret)
-                goto out;
-        cmd = &ctx->cmd_args;
-        if (cmd->print_netgroups) {
-                /* If this option is set we want to print & verify the file,
-                 * set the return value (exit code in this case) and exit.
-                 */
-                ret =  print_netgroups_file (cmd->print_netgroups);
+    if (cmd->print_xlatordir) {
+        /* XLATORDIR passed through a -D flag to GCC */
+        printf("%s\n", XLATORDIR);
+        goto out;
+    }
+
+    if (cmd->print_statedumpdir) {
+        printf("%s\n", DEFAULT_VAR_RUN_DIRECTORY);
+        goto out;
+    }
+
+    if (cmd->print_logdir) {
+        printf("%s\n", DEFAULT_LOG_FILE_DIRECTORY);
+        goto out;
+    }
+
+    if (cmd->print_libexecdir) {
+        printf("%s\n", LIBEXECDIR);
+        goto out;
+    }
+
+    if (cmd->print_netgroups) {
+        /* If this option is set we want to print & verify the file,
+         * set the return value (exit code in this case) and exit.
+         */
+        ret = print_netgroups_file(cmd->print_netgroups);
+        goto out;
+    }
+
+    if (cmd->print_exports) {
+        /* If this option is set we want to print & verify the file,
+         * set the return value (exit code in this case)
+         * and exit.
+         */
+        ret = print_exports_file(cmd->print_exports);
+        goto out;
+    }
+
+    ret = logging_init(ctx, argv[0]);
+    if (ret)
+        goto out;
+
+    /* set brick_mux mode only for server process */
+    if ((ctx->process_mode != GF_SERVER_PROCESS) && cmd->brick_mux) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_43, NULL);
+        goto out;
+    }
+
+    /* log the version of glusterfs running here along with the actual
+       command line options. */
+    {
+        int i = 0;
+        int pos = 0;
+        int len = snprintf(cmdlinestr, sizeof(cmdlinestr), "%s", argv[0]);
+        for (i = 1; (i < argc) && (len > 0); i++) {
+            pos += len;
+            len = snprintf(cmdlinestr + pos, sizeof(cmdlinestr) - pos, " %s",
+                           argv[i]);
+            if ((len <= 0) || (len >= (sizeof(cmdlinestr) - pos))) {
+                gf_smsg("glusterfs", GF_LOG_ERROR, 0, glusterfsd_msg_029, NULL);
+                ret = -1;
                 goto out;
+            }
         }
+        gf_smsg(argv[0], GF_LOG_INFO, 0, glusterfsd_msg_30, "arg=%s", argv[0],
+                "version=%s", PACKAGE_VERSION, "cmdlinestr=%s", cmdlinestr,
+                NULL);
 
-        if (cmd->print_exports) {
-                /* If this option is set we want to print & verify the file,
-                 * set the return value (exit code in this case)
-                 * and exit.
-                 */
-                ret = print_exports_file (cmd->print_exports);
-                goto out;
-        }
+        ctx->cmdlinestr = gf_strdup(cmdlinestr);
+    }
 
-        ret = logging_init (ctx, argv[0]);
-        if (ret)
-                goto out;
+    gf_proc_dump_init();
 
+    ret = create_fuse_mount(ctx);
+    if (ret)
+        goto out;
 
-        /* log the version of glusterfs running here along with the actual
-           command line options. */
-        {
-                int i = 0;
-                strcpy (cmdlinestr, argv[0]);
-                for (i = 1; i < argc; i++) {
-                        strcat (cmdlinestr, " ");
-                        strncat (cmdlinestr, argv[i],
-                                 (sizeof (cmdlinestr) - 1));
-                }
-                gf_msg (argv[0], GF_LOG_INFO, 0, glusterfsd_msg_30,
-                        argv[0], PACKAGE_VERSION, cmdlinestr);
-
-		ctx->cmdlinestr = gf_strdup (cmdlinestr);
-        }
+    ret = daemonize(ctx);
+    if (ret)
+        goto out;
 
-        gf_proc_dump_init();
+    /*
+     * If we do this before daemonize, the pool-sweeper thread dies with
+     * the parent, but we want to do it as soon as possible after that in
+     * case something else depends on pool allocations.
+     */
+    mem_pools_init();
 
-        ret = create_fuse_mount (ctx);
-        if (ret)
-                goto out;
-
-        ret = daemonize (ctx);
-        if (ret)
-                goto out;
+    ret = gf_async_init(ctx);
+    if (ret < 0) {
+        goto out;
+    }
 
 #ifdef GF_LINUX_HOST_OS
-        ret = set_oom_score_adj (ctx);
-        if (ret)
-                goto out;
+    ret = set_oom_score_adj(ctx);
+    if (ret)
+        goto out;
 #endif
 
-	ctx->env = syncenv_new (0, 0, 0);
-        if (!ctx->env) {
-                gf_msg ("", GF_LOG_ERROR, 0, glusterfsd_msg_31);
-                goto out;
-        }
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        gf_smsg("", GF_LOG_ERROR, 0, glusterfsd_msg_31, NULL);
+        goto out;
+    }
 
-        /* do this _after_ deamonize() */
-        if (cmd->global_timer_wheel) {
-                ret = glusterfs_global_timer_wheel_init (ctx);
-                if (ret)
-                        goto out;
-        }
+    /* do this _after_ daemonize() */
+    if (!glusterfs_ctx_tw_get(ctx)) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterfs_volumes_init (ctx);
-        if (ret)
-                goto out;
+    ret = glusterfs_volumes_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = event_dispatch (ctx->event_pool);
+    ret = gf_event_dispatch(ctx->event_pool);
 
 out:
-//        glusterfs_ctx_destroy (ctx);
-        return ret;
+    //    glusterfs_ctx_destroy (ctx);
+    gf_async_fini();
+    return ret;
 }
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index e442bede5db..4e1413caa70 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -13,113 +13,130 @@
 #include "rpcsvc.h"
 #include "glusterd1-xdr.h"
 
-#define DEFAULT_GLUSTERD_VOLFILE              CONFDIR "/glusterd.vol"
-#define DEFAULT_CLIENT_VOLFILE                CONFDIR "/glusterfs.vol"
-#define DEFAULT_SERVER_VOLFILE                CONFDIR "/glusterfsd.vol"
+#define DEFAULT_GLUSTERD_VOLFILE CONFDIR "/glusterd.vol"
+#define DEFAULT_CLIENT_VOLFILE CONFDIR "/glusterfs.vol"
+#define DEFAULT_SERVER_VOLFILE CONFDIR "/glusterfsd.vol"
 
-#define DEFAULT_EVENT_POOL_SIZE            16384
+#define DEFAULT_EVENT_POOL_SIZE 16384
 
-#define ARGP_LOG_LEVEL_NONE_OPTION        "NONE"
-#define ARGP_LOG_LEVEL_TRACE_OPTION       "TRACE"
-#define ARGP_LOG_LEVEL_CRITICAL_OPTION    "CRITICAL"
-#define ARGP_LOG_LEVEL_ERROR_OPTION       "ERROR"
-#define ARGP_LOG_LEVEL_WARNING_OPTION     "WARNING"
-#define ARGP_LOG_LEVEL_INFO_OPTION        "INFO"
-#define ARGP_LOG_LEVEL_DEBUG_OPTION       "DEBUG"
+#define ARGP_LOG_LEVEL_NONE_OPTION "NONE"
+#define ARGP_LOG_LEVEL_TRACE_OPTION "TRACE"
+#define ARGP_LOG_LEVEL_CRITICAL_OPTION "CRITICAL"
+#define ARGP_LOG_LEVEL_ERROR_OPTION "ERROR"
+#define ARGP_LOG_LEVEL_WARNING_OPTION "WARNING"
+#define ARGP_LOG_LEVEL_INFO_OPTION "INFO"
+#define ARGP_LOG_LEVEL_DEBUG_OPTION "DEBUG"
 
-#define ENABLE_NO_DAEMON_MODE     1
-#define ENABLE_DEBUG_MODE         1
+#define ENABLE_NO_DAEMON_MODE 1
+#define ENABLE_DEBUG_MODE 1
 
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
 /* Considering 4 key/value pairs in a dictionary on an average */
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
 
-#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T     256
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
 
 enum argp_option_keys {
-        ARGP_VOLFILE_SERVER_KEY           = 's',
-        ARGP_VOLUME_FILE_KEY              = 'f',
-        ARGP_LOG_LEVEL_KEY                = 'L',
-        ARGP_LOG_FILE_KEY                 = 'l',
-        ARGP_VOLFILE_SERVER_PORT_KEY      = 131,
-        ARGP_VOLFILE_SERVER_TRANSPORT_KEY = 132,
-        ARGP_PID_FILE_KEY                 = 'p',
-        ARGP_SOCK_FILE_KEY                = 'S',
-        ARGP_NO_DAEMON_KEY                = 'N',
-        ARGP_RUN_ID_KEY                   = 'r',
-        ARGP_PRINT_NETGROUPS              = 'n',
-        ARGP_PRINT_EXPORTS                = 'e',
-        ARGP_DEBUG_KEY                    = 133,
-        ARGP_NEGATIVE_TIMEOUT_KEY         = 134,
-        ARGP_ENTRY_TIMEOUT_KEY            = 135,
-        ARGP_ATTRIBUTE_TIMEOUT_KEY        = 136,
-        ARGP_VOLUME_NAME_KEY              = 137,
-        ARGP_XLATOR_OPTION_KEY            = 138,
-        ARGP_DIRECT_IO_MODE_KEY           = 139,
+    ARGP_VOLFILE_SERVER_KEY = 's',
+    ARGP_VOLUME_FILE_KEY = 'f',
+    ARGP_LOG_LEVEL_KEY = 'L',
+    ARGP_LOG_FILE_KEY = 'l',
+    ARGP_VOLFILE_SERVER_PORT_KEY = 131,
+    ARGP_VOLFILE_SERVER_TRANSPORT_KEY = 132,
+    ARGP_PID_FILE_KEY = 'p',
+    ARGP_SOCK_FILE_KEY = 'S',
+    ARGP_NO_DAEMON_KEY = 'N',
+    ARGP_RUN_ID_KEY = 'r',
+    ARGP_PRINT_NETGROUPS = 'n',
+    ARGP_PRINT_EXPORTS = 'e',
+    ARGP_DEBUG_KEY = 133,
+    ARGP_NEGATIVE_TIMEOUT_KEY = 134,
+    ARGP_ENTRY_TIMEOUT_KEY = 135,
+    ARGP_ATTRIBUTE_TIMEOUT_KEY = 136,
+    ARGP_VOLUME_NAME_KEY = 137,
+    ARGP_XLATOR_OPTION_KEY = 138,
+    ARGP_DIRECT_IO_MODE_KEY = 139,
 #ifdef GF_DARWIN_HOST_OS
-        ARGP_NON_LOCAL_KEY                = 140,
+    ARGP_NON_LOCAL_KEY = 140,
 #endif /* DARWIN */
-        ARGP_VOLFILE_ID_KEY               = 143,
-        ARGP_VOLFILE_CHECK_KEY            = 144,
-        ARGP_VOLFILE_MAX_FETCH_ATTEMPTS   = 145,
-        ARGP_LOG_SERVER_KEY               = 146,
-        ARGP_LOG_SERVER_PORT_KEY          = 147,
-        ARGP_READ_ONLY_KEY                = 148,
-        ARGP_MAC_COMPAT_KEY               = 149,
-        ARGP_DUMP_FUSE_KEY                = 150,
-        ARGP_BRICK_NAME_KEY               = 151,
-        ARGP_BRICK_PORT_KEY               = 152,
-        ARGP_CLIENT_PID_KEY               = 153,
-        ARGP_ACL_KEY                      = 154,
-        ARGP_WORM_KEY                     = 155,
-        ARGP_USER_MAP_ROOT_KEY            = 156,
-        ARGP_MEM_ACCOUNTING_KEY           = 157,
-        ARGP_SELINUX_KEY                  = 158,
-	ARGP_FOPEN_KEEP_CACHE_KEY	  = 159,
-	ARGP_GID_TIMEOUT_KEY		  = 160,
-	ARGP_FUSE_BACKGROUND_QLEN_KEY     = 161,
-	ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
-        ARGP_INODE32_KEY                  = 163,
-	ARGP_FUSE_MOUNTOPTS_KEY		  = 164,
-        ARGP_FUSE_USE_READDIRP_KEY        = 165,
-	ARGP_AUX_GFID_MOUNT_KEY		  = 166,
-        ARGP_FUSE_NO_ROOT_SQUASH_KEY      = 167,
-        ARGP_LOGGER                       = 168,
-        ARGP_LOG_FORMAT                   = 169,
-        ARGP_LOG_BUF_SIZE                 = 170,
-        ARGP_LOG_FLUSH_TIMEOUT            = 171,
-        ARGP_SECURE_MGMT_KEY              = 172,
-        ARGP_GLOBAL_TIMER_WHEEL           = 173,
-        ARGP_RESOLVE_GIDS_KEY             = 174,
-        ARGP_CAPABILITY_KEY               = 175,
+    ARGP_VOLFILE_ID_KEY = 143,
+    ARGP_VOLFILE_CHECK_KEY = 144,
+    ARGP_VOLFILE_MAX_FETCH_ATTEMPTS = 145,
+    ARGP_LOG_SERVER_KEY = 146,
+    ARGP_LOG_SERVER_PORT_KEY = 147,
+    ARGP_READ_ONLY_KEY = 148,
+    ARGP_MAC_COMPAT_KEY = 149,
+    ARGP_DUMP_FUSE_KEY = 150,
+    ARGP_BRICK_NAME_KEY = 151,
+    ARGP_BRICK_PORT_KEY = 152,
+    ARGP_CLIENT_PID_KEY = 153,
+    ARGP_ACL_KEY = 154,
+    ARGP_WORM_KEY = 155,
+    ARGP_USER_MAP_ROOT_KEY = 156,
+    ARGP_MEM_ACCOUNTING_KEY = 157,
+    ARGP_SELINUX_KEY = 158,
+    ARGP_FOPEN_KEEP_CACHE_KEY = 159,
+    ARGP_GID_TIMEOUT_KEY = 160,
+    ARGP_FUSE_BACKGROUND_QLEN_KEY = 161,
+    ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
+    ARGP_INODE32_KEY = 163,
+    ARGP_FUSE_MOUNTOPTS_KEY = 164,
+    ARGP_FUSE_USE_READDIRP_KEY = 165,
+    ARGP_AUX_GFID_MOUNT_KEY = 166,
+    ARGP_FUSE_NO_ROOT_SQUASH_KEY = 167,
+    ARGP_LOGGER = 168,
+    ARGP_LOG_FORMAT = 169,
+    ARGP_LOG_BUF_SIZE = 170,
+    ARGP_LOG_FLUSH_TIMEOUT = 171,
+    ARGP_SECURE_MGMT_KEY = 172,
+    ARGP_GLOBAL_TIMER_WHEEL = 173,
+    ARGP_RESOLVE_GIDS_KEY = 174,
+    ARGP_CAPABILITY_KEY = 175,
 #ifdef GF_LINUX_HOST_OS
-        ARGP_OOM_SCORE_ADJ_KEY            = 176,
+    ARGP_OOM_SCORE_ADJ_KEY = 176,
 #endif
+    ARGP_LOCALTIME_LOGGING_KEY = 177,
+    ARGP_SUBDIR_MOUNT_KEY = 178,
+    ARGP_PROCESS_NAME_KEY = 179,
+    ARGP_FUSE_EVENT_HISTORY_KEY = 180,
+    ARGP_THIN_CLIENT_KEY = 181,
+    ARGP_READER_THREAD_COUNT_KEY = 182,
+    ARGP_PRINT_XLATORDIR_KEY = 183,
+    ARGP_PRINT_STATEDUMPDIR_KEY = 184,
+    ARGP_PRINT_LOGDIR_KEY = 185,
+    ARGP_KERNEL_WRITEBACK_CACHE_KEY = 186,
+    ARGP_ATTR_TIMES_GRANULARITY_KEY = 187,
+    ARGP_PRINT_LIBEXECDIR_KEY = 188,
+    ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY = 189,
+    ARGP_FUSE_LRU_LIMIT_KEY = 190,
+    ARGP_FUSE_AUTO_INVAL_KEY = 191,
+    ARGP_GLOBAL_THREADING_KEY = 192,
+    ARGP_BRICK_MUX_KEY = 193,
+    ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY = 194,
+    ARGP_FUSE_INVALIDATE_LIMIT_KEY = 195,
 };
 
-struct _gfd_vol_top_priv_t {
-        rpcsvc_request_t        *req;
-        gd1_mgmt_brick_op_req   xlator_req;
-        uint32_t                blk_count;
-        uint32_t                blk_size;
-        double                  throughput;
-        double                  time;
-        int32_t                 ret;
+struct _gfd_vol_top_priv {
+    rpcsvc_request_t *req;
+    gd1_mgmt_brick_op_req xlator_req;
+    uint32_t blk_count;
+    uint32_t blk_size;
+    double throughput;
+    double time;
+    int32_t ret;
 };
-typedef struct _gfd_vol_top_priv_t gfd_vol_top_priv_t;
+typedef struct _gfd_vol_top_priv gfd_vol_top_priv_t;
 
-int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx);
-int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
-int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
-void cleanup_and_exit (int signum);
+int
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx);
+int
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx);
+void
+cleanup_and_exit(int signum);
 
-int glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
-                                     char *brick_path, double *throughput,
-                                     double *time);
-int glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
-                                    char *brick_path, double *throughput,
-                                    double *time);
+void
+xlator_mem_cleanup(xlator_t *this);
 
 extern glusterfs_ctx_t *glusterfsd_ctx;
 #endif /* __GLUSTERFSD_H__ */
diff --git a/heal/src/Makefile.am b/heal/src/Makefile.am
index 830f9d9238b..aa18d3eff88 100644
--- a/heal/src/Makefile.am
+++ b/heal/src/Makefile.am
@@ -1,4 +1,7 @@
-sbin_PROGRAMS = glfsheal
+if WITH_SERVER
+scriptdir = $(GLUSTERFS_LIBEXECDIR)
+script_PROGRAMS = glfsheal
+endif
 
 glfsheal_SOURCES = glfs-heal.c
 
@@ -16,12 +19,9 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/rpc/xdr/src\
 	-I$(top_builddir)/rpc/xdr/src\
 	-I$(top_srcdir)/api/src\
-	-I$(top_srcdir)/contrib/argp-standalone\
-	-DDATADIR=\"$(localstatedir)\" \
-	-DSBIN_DIR=\"$(sbindir)\" \
-	$(XML_CPPFLAGS)
+	-DDATADIR=\"$(localstatedir)\"
 
-AM_CFLAGS = -Wall $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
diff --git a/heal/src/glfs-heal.c b/heal/src/glfs-heal.c
index e885dd9fe63..bf4b47f8760 100644
--- a/heal/src/glfs-heal.c
+++ b/heal/src/glfs-heal.c
@@ -12,1079 +12,1473 @@
 #include <stdlib.h>
 #include <errno.h>
 #include "glfs.h"
-#include "glfs-handles.h"
 #include "glfs-internal.h"
 #include "protocol-common.h"
-#include "syscall.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include <string.h>
-#include <time.h>
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
+#include <libgen.h>
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 
 xmlTextWriterPtr glfsh_writer;
-xmlDocPtr        glfsh_doc = NULL;
+xmlDocPtr glfsh_doc = NULL;
 #endif
 
-#define XML_RET_CHECK_AND_GOTO(ret, label)      do {            \
-                if (ret < 0) {                                  \
-                        ret = -1;                               \
-                        goto label;                             \
-                }                                               \
-                else                                            \
-                        ret = 0;                                \
-        } while (0)                                             \
+#define XML_RET_CHECK_AND_GOTO(ret, label)                                     \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        } else                                                                 \
+            ret = 0;                                                           \
+    } while (0)
 
-typedef void    (*print_status) (dict_t *, char *, uuid_t, uint64_t *,
-                 gf_boolean_t flag);
+#define MODE_XML (1 << 0)
+#define MODE_NO_LOG (1 << 1)
 
-int glfsh_heal_splitbrain_file (glfs_t *fs, xlator_t *top_subvol,
-                                loc_t *rootloc, char *file, dict_t *xattr_req);
+typedef struct num_entries {
+    uint64_t num_entries;
+    uint64_t pending_entries;
+    uint64_t spb_entries;
+    uint64_t possibly_healing_entries;
+} num_entries_t;
 
+typedef int (*print_status)(dict_t *, char *, uuid_t, num_entries_t *,
+                            gf_boolean_t flag);
+
+int
+glfsh_heal_splitbrain_file(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                           char *file, dict_t *xattr_req);
 
 typedef struct glfs_info {
-        int (*init)(void);
-        int (*print_brick_from_xl)(xlator_t *xl, loc_t *rootloc);
-        int (*print_heal_op_status)(int ret, uint64_t num_entries,
-                        char *fmt_str);
-        void (*print_heal_status)(char *path, uuid_t gfid, char *status);
-        void (*print_spb_status)(char *path, uuid_t gfid, char *status);
-        int (*end) (int op_ret, char *op_errstr);
+    int (*init)(void);
+    int (*print_brick_from_xl)(xlator_t *xl, loc_t *rootloc);
+    int (*print_heal_op_status)(int ret, uint64_t num_entries, char *fmt_str);
+    int (*print_heal_op_summary)(int ret, num_entries_t *num_entries);
+    int (*print_heal_status)(char *path, uuid_t gfid, char *status);
+    int (*print_spb_status)(char *path, uuid_t gfid, char *status);
+    int (*end)(int op_ret, char *op_errstr);
 } glfsh_info_t;
 
 glfsh_info_t *glfsh_output = NULL;
 int32_t is_xml;
 
 #define DEFAULT_HEAL_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
-#define USAGE_STR "Usage: %s <VOLNAME> [bigger-file <FILE> | "\
-                  "latest-mtime <FILE> | "\
-                  "source-brick <HOSTNAME:BRICKNAME> [<FILE>] | "\
-                  "split-brain-info]\n"
+#define USAGE_STR                                                              \
+    "Usage: %s <VOLNAME> [bigger-file <FILE> | "                               \
+    "latest-mtime <FILE> | "                                                   \
+    "source-brick <HOSTNAME:BRICKNAME> [<FILE>] | "                            \
+    "split-brain-info | info-summary] [glusterd-sock <FILE>"                   \
+    "]\n"
+
+typedef enum {
+    GLFSH_MODE_CONTINUE_ON_ERROR = 1,
+    GLFSH_MODE_EXIT_ON_FIRST_FAILURE,
+} glfsh_fail_mode_t;
 
 int
-glfsh_init ()
+glfsh_init()
 {
-        return 0;
+    return 0;
 }
 
 int
-glfsh_end (int op_ret, char *op_errstr)
+glfsh_end_op_granular_entry_heal(int op_ret, char *op_errstr)
 {
-        if (op_errstr)
-                printf ("%s\n", op_errstr);
-        return 0;
+    /* If error string is available, give it higher precedence.*/
+
+    if (op_errstr) {
+        printf("%s\n", op_errstr);
+    } else if (op_ret < 0) {
+        if (op_ret == -EAGAIN)
+            printf(
+                "One or more entries need heal. Please execute "
+                "the command again after there are no entries "
+                "to be healed\n");
+        else if (op_ret == -ENOTCONN)
+            printf(
+                "One or more bricks could be down. Please "
+                "execute the command again after bringing all "
+                "bricks online and finishing any pending "
+                "heals\n");
+        else
+            printf(
+                "Command failed - %s. Please check the logs for"
+                " more details\n",
+                strerror(-op_ret));
+    }
+    return 0;
 }
 
-void
-glfsh_print_hr_spb_status (char *path, uuid_t gfid, char *status)
+int
+glfsh_end(int op_ret, char *op_errstr)
 {
-        printf ("%s\n", path);
-        return;
+    if (op_errstr)
+        printf("%s\n", op_errstr);
+    return 0;
 }
 
-void
-glfsh_print_hr_heal_status (char *path, uuid_t gfid, char *status)
+int
+glfsh_print_hr_spb_status(char *path, uuid_t gfid, char *status)
 {
-        printf ("%s%s\n", path, status);
+    printf("%s\n", path);
+    fflush(stdout);
+    return 0;
 }
 
-#if (HAVE_LIB_XML)
+int
+glfsh_no_print_hr_status(char *path, uuid_t gfid, char *status)
+{
+    return 0;
+}
 
 int
-glfsh_xml_init ()
+glfsh_print_hr_heal_status(char *path, uuid_t gfid, char *status)
 {
-        int     ret     = -1;
-        glfsh_writer = xmlNewTextWriterDoc (&glfsh_doc, 0);
-        if (glfsh_writer == NULL) {
-                return -1;
-        }
+    printf("%s%s\n", path, status);
+    fflush(stdout);
+    return 0;
+}
+
+#if (HAVE_LIB_XML)
 
-        ret = xmlTextWriterStartDocument (glfsh_writer, "1.0", "UTF-8",
-                        "yes");
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        /* <cliOutput> */
-        ret = xmlTextWriterStartElement (glfsh_writer,
-                        (xmlChar *)"cliOutput");
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        /* <healInfo> */
-        xmlTextWriterStartElement (glfsh_writer,
-                        (xmlChar *)"healInfo");
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-        /* <bricks> */
-        xmlTextWriterStartElement (glfsh_writer,
-                        (xmlChar *)"bricks");
+int
+glfsh_xml_init()
+{
+    int ret = -1;
+    glfsh_writer = xmlNewTextWriterDoc(&glfsh_doc, 0);
+    if (glfsh_writer == NULL) {
+        return -1;
+    }
+
+    ret = xmlTextWriterStartDocument(glfsh_writer, "1.0", "UTF-8", "yes");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* <cliOutput> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"cliOutput");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* <healInfo> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"healInfo");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    /* <bricks> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"bricks");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    xmlTextWriterFlush(glfsh_writer);
 xml_out:
-        return ret;
+    return ret;
 }
 
 int
-glfsh_xml_end (int op_ret, char *op_errstr)
+glfsh_xml_end(int op_ret, char *op_errstr)
 {
-        int                     ret             = -1;
-        int                     op_errno        = 0;
-        gf_boolean_t            alloc           = _gf_false;
-
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                op_ret = -1;
-                if (op_errstr == NULL) {
-                        op_errstr = gf_strdup (strerror (op_errno));
-                        alloc = _gf_true;
-                }
-        } else {
-                op_errstr = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    gf_boolean_t alloc = _gf_false;
+
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+        if (op_errstr == NULL) {
+            op_errstr = gf_strdup(strerror(op_errno));
+            alloc = _gf_true;
         }
-
-        /* </bricks> */
-        ret = xmlTextWriterEndElement (glfsh_writer);
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        /* </healInfo> */
-        ret = xmlTextWriterEndElement (glfsh_writer);
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        ret = xmlTextWriterWriteFormatElement (glfsh_writer,
-                        (xmlChar *)"opRet", "%d", op_ret);
-
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        ret = xmlTextWriterWriteFormatElement (glfsh_writer,
-                        (xmlChar *)"opErrno",
-                        "%d", op_errno);
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-        if (op_errstr)
-                ret = xmlTextWriterWriteFormatElement (glfsh_writer,
-                                (xmlChar *)"opErrstr",
-                                "%s", op_errstr);
-        else
-                ret = xmlTextWriterWriteFormatElement (glfsh_writer,
-                                (xmlChar *)"opErrstr",
-                                "%s", "");
-        ret = xmlTextWriterEndDocument (glfsh_writer);
-        XML_RET_CHECK_AND_GOTO (ret, xml_out);
-
-
-        /* Dump xml document to stdout and pretty format it */
-        xmlSaveFormatFileEnc ("-", glfsh_doc, "UTF-8", 1);
-
-        xmlFreeTextWriter (glfsh_writer);
-        xmlFreeDoc (glfsh_doc);
+    } else {
+        op_errstr = NULL;
+    }
+
+    /* </bricks> */
+    ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* </healInfo> */
+    ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    ret = xmlTextWriterWriteFormatElement(glfsh_writer, (xmlChar *)"opRet",
+                                          "%d", op_ret);
+
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    ret = xmlTextWriterWriteFormatElement(glfsh_writer, (xmlChar *)"opErrno",
+                                          "%d", op_errno);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    if (op_errstr)
+        ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"opErrstr", "%s", op_errstr);
+    else
+        ret = xmlTextWriterWriteFormatElement(glfsh_writer,
+                                              (xmlChar *)"opErrstr", "%s", "");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    ret = xmlTextWriterEndDocument(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* Dump xml document to stdout and pretty format it */
+    xmlSaveFormatFileEnc("-", glfsh_doc, "UTF-8", 1);
+
+    xmlFreeTextWriter(glfsh_writer);
+    xmlFreeDoc(glfsh_doc);
 xml_out:
-        if (alloc)
-                GF_FREE (op_errstr);
-        return ret;
+    if (alloc)
+        GF_FREE(op_errstr);
+    return ret;
 }
 
 int
-glfsh_print_xml_heal_op_status (int ret, uint64_t num_entries, char *fmt_str)
+glfsh_print_xml_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
 {
-        if (ret < 0 && num_entries == 0) {
-                xmlTextWriterWriteFormatElement (glfsh_writer,
-                                (xmlChar *)"status",
-                                "%s", strerror (-ret));
-                if (fmt_str) {
-                        xmlTextWriterWriteFormatElement (glfsh_writer,
-                                        (xmlChar *)"numberOfEntries",
-                                        "-");
-                }
-                goto out;
-        } else if (ret == 0) {
-                xmlTextWriterWriteFormatElement (glfsh_writer,
-                                                 (xmlChar *)"status",
-                                                 "%s", "Connected");
+    int x_ret = 0;
+    if (ret < 0 && num_entries == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", strerror(-ret));
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"numberOfEntries", "-");
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
         }
-
-        if (ret < 0) {
-                if (fmt_str) {
-                        xmlTextWriterWriteFormatElement (glfsh_writer,
-                                        (xmlChar *)"status",
-                                        "Failed to process entries completely. "
-                                        "(%s)%s %"PRIu64"", strerror (-ret),
-                                        fmt_str,
-                                        num_entries);
-                }
-        } else {
-                if (fmt_str)
-                        xmlTextWriterWriteFormatElement (glfsh_writer,
-                                        (xmlChar *)"numberOfEntries",
-                                        "%"PRIu64"", num_entries);
+        goto out;
+    } else if (ret == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", "Connected");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+
+    if (ret < 0) {
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"status",
+                "Failed to process entries completely. "
+                "(%s)%s %" PRIu64 "",
+                strerror(-ret), fmt_str, num_entries);
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
+        }
+    } else {
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"numberOfEntries", "%" PRIu64 "",
+                num_entries);
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
         }
+    }
 out:
-        return xmlTextWriterEndElement (glfsh_writer);
+    if (x_ret >= 0) {
+        x_ret = xmlTextWriterEndElement(glfsh_writer);
+        if (x_ret >= 0) {
+            xmlTextWriterFlush(glfsh_writer);
+            x_ret = 0;
+        } else {
+            x_ret = -1;
+        }
+    }
+    return x_ret;
 }
 
-void
-glfsh_print_xml_file_status (char *path, uuid_t gfid, char *status)
+int
+glfsh_print_xml_heal_op_summary(int ret, num_entries_t *num_entries)
 {
-        xmlTextWriterStartElement (glfsh_writer, (xmlChar *)"file");
-        xmlTextWriterWriteFormatAttribute (glfsh_writer, (xmlChar *)"gfid",
-                                           "%s", uuid_utoa (gfid));
-        xmlTextWriterWriteFormatString (glfsh_writer, "%s", path);
-        xmlTextWriterEndElement (glfsh_writer);
-        return;
+    int x_ret = 0;
+
+    if (ret < 0 && num_entries == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", strerror(-ret));
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"totalNumberOfEntries", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInHealPending", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInSplitBrain", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesPossiblyHealing", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        goto out;
+    } else if (ret == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", "Connected");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+
+    if (ret < 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status",
+            "Failed to process entries"
+            " completely. "
+            "(%s)totalNumberOfEntries%" PRIu64 "",
+            strerror(-ret), num_entries->num_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    } else {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"totalNumberOfEntries", "%" PRIu64 "",
+            num_entries->num_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInHealPending",
+            "%" PRIu64 "", num_entries->pending_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInSplitBrain",
+            "%" PRIu64 "", num_entries->spb_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesPossiblyHealing",
+            "%" PRIu64 "", num_entries->possibly_healing_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+out:
+    if (x_ret >= 0) {
+        x_ret = xmlTextWriterEndElement(glfsh_writer);
+    }
+    return x_ret;
 }
 
 int
-glfsh_print_xml_brick_from_xl (xlator_t *xl, loc_t *rootloc)
+glfsh_print_xml_file_status(char *path, uuid_t gfid, char *status)
 {
-        char    *remote_host = NULL;
-        char    *remote_subvol = NULL;
-        char    *uuid = NULL;
-        int     ret = 0;
-        int     x_ret = 0;
-
-        ret = dict_get_str (xl->options, "remote-host", &remote_host);
-        if (ret < 0)
-                goto print;
-
-        ret = dict_get_str (xl->options, "remote-subvolume", &remote_subvol);
-        if (ret < 0)
-                goto print;
-        ret = syncop_getxattr (xl, rootloc, &xl->options,
-                        GF_XATTR_NODE_UUID_KEY, NULL, NULL);
-        if (ret < 0)
-                goto print;
+    int x_ret = 0;
+
+    x_ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterWriteFormatAttribute(glfsh_writer, (xmlChar *)"gfid",
+                                              "%s", uuid_utoa(gfid));
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterWriteFormatString(glfsh_writer, "%s", path);
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    xmlTextWriterFlush(glfsh_writer);
+out:
+    return x_ret;
+}
 
-        ret = dict_get_str (xl->options, GF_XATTR_NODE_UUID_KEY, &uuid);
-        if (ret < 0)
-                goto print;
+int
+glfsh_print_xml_brick_from_xl(xlator_t *xl, loc_t *rootloc)
+{
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+    char *uuid = NULL;
+    int ret = 0;
+    int x_ret = 0;
+
+    ret = dict_get_str(xl->options, "remote-host", &remote_host);
+    if (ret < 0)
+        goto print;
+
+    ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+    if (ret < 0)
+        goto print;
+    ret = syncop_getxattr(xl, rootloc, &xl->options, GF_XATTR_NODE_UUID_KEY,
+                          NULL, NULL);
+    if (ret < 0)
+        goto print;
+
+    ret = dict_get_str(xl->options, GF_XATTR_NODE_UUID_KEY, &uuid);
+    if (ret < 0)
+        goto print;
 print:
 
-        x_ret = xmlTextWriterStartElement (glfsh_writer, (xmlChar *)"brick");
-        XML_RET_CHECK_AND_GOTO (x_ret, xml_out);
-        x_ret = xmlTextWriterWriteFormatAttribute (glfsh_writer,
-                        (xmlChar *)"hostUuid", "%s", uuid?uuid:"-");
-        XML_RET_CHECK_AND_GOTO (x_ret, xml_out);
-
-        x_ret = xmlTextWriterWriteFormatElement (glfsh_writer,
-                        (xmlChar *)"name", "%s:%s",
-                        remote_host ? remote_host : "-",
-                        remote_subvol ? remote_subvol : "-");
-        XML_RET_CHECK_AND_GOTO (x_ret, xml_out);
+    x_ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"brick");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+    x_ret = xmlTextWriterWriteFormatAttribute(
+        glfsh_writer, (xmlChar *)"hostUuid", "%s", uuid ? uuid : "-");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+
+    x_ret = xmlTextWriterWriteFormatElement(
+        glfsh_writer, (xmlChar *)"name", "%s:%s",
+        remote_host ? remote_host : "-", remote_subvol ? remote_subvol : "-");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+    xmlTextWriterFlush(glfsh_writer);
 xml_out:
-        return ret;
+    return ret;
 }
 #endif
 
 int
-glfsh_link_inode_update_loc (loc_t *loc, struct iatt *iattr)
+glfsh_link_inode_update_loc(loc_t *loc, struct iatt *iattr)
 {
-        inode_t       *link_inode = NULL;
-        int           ret = -1;
+    inode_t *link_inode = NULL;
+    int ret = -1;
 
-        link_inode = inode_link (loc->inode, NULL, NULL, iattr);
-        if (link_inode == NULL)
-                goto out;
+    link_inode = inode_link(loc->inode, NULL, NULL, iattr);
+    if (link_inode == NULL)
+        goto out;
 
-        inode_unref (loc->inode);
-        loc->inode = link_inode;
-        ret = 0;
+    inode_unref(loc->inode);
+    loc->inode = link_inode;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glfsh_print_hr_heal_op_status (int ret, uint64_t num_entries, char *fmt_str)
+glfsh_no_print_hr_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
 {
-        if (ret < 0 && num_entries == 0) {
-                printf ("Status: %s\n", strerror (-ret));
-                if (fmt_str)
-                        printf ("%s -\n", fmt_str);
-                goto out;
-        } else if (ret == 0) {
-                printf ("Status: Connected\n");
-        }
+    return 0;
+}
 
-        if (ret < 0) {
-                if (fmt_str)
-                        printf ("Status: Failed to process entries completely. "
-                                "(%s)\n%s %"PRIu64"\n",
-                         strerror (-ret), fmt_str, num_entries);
-        } else {
-                if (fmt_str)
-                        printf ("%s %"PRIu64"\n", fmt_str, num_entries);
-        }
+int
+glfsh_print_hr_heal_op_summary(int ret, num_entries_t *num_entries)
+{
+    if (ret < 0 && num_entries->num_entries == 0) {
+        printf("Status: %s\n", strerror(-ret));
+        printf("Total Number of entries: -\n");
+        printf("Number of entries in heal pending: -\n");
+        printf("Number of entries in split-brain: -\n");
+        printf("Number of entries possibly healing: -\n");
+        goto out;
+    } else if (ret == 0) {
+        printf("Status: Connected\n");
+    }
+
+    if (ret < 0) {
+        printf(
+            "Status: Failed to process entries completely. "
+            "(%s)\nTotal Number of entries: %" PRIu64 "\n",
+            strerror(-ret), num_entries->num_entries);
+    } else {
+        printf("Total Number of entries: %" PRIu64 "\n",
+               num_entries->num_entries);
+        printf("Number of entries in heal pending: %" PRIu64 "\n",
+               num_entries->pending_entries);
+        printf("Number of entries in split-brain: %" PRIu64 "\n",
+               num_entries->spb_entries);
+        printf("Number of entries possibly healing: %" PRIu64 "\n",
+               num_entries->possibly_healing_entries);
+    }
 out:
-        printf ("\n");
-        return 0;
+    printf("\n");
+    fflush(stdout);
+    return 0;
 }
 
 int
-glfsh_print_heal_op_status (int ret, uint64_t num_entries,
-                            gf_xl_afr_op_t heal_op)
+glfsh_print_hr_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
 {
-        char *fmt_str = NULL;
-
-        if (heal_op == GF_SHD_OP_INDEX_SUMMARY)
-                fmt_str = "Number of entries:";
-        else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES)
-                fmt_str = "Number of entries in split-brain:";
-        else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK)
-                fmt_str = "Number of healed entries:";
+    if (ret < 0 && num_entries == 0) {
+        printf("Status: %s\n", strerror(-ret));
+        if (fmt_str)
+            printf("%s -\n", fmt_str);
+        goto out;
+    } else if (ret == 0) {
+        printf("Status: Connected\n");
+    }
+
+    if (ret < 0) {
+        if (fmt_str)
+            printf(
+                "Status: Failed to process entries completely. "
+                "(%s)\n%s %" PRIu64 "\n",
+                strerror(-ret), fmt_str, num_entries);
+    } else {
+        if (fmt_str)
+            printf("%s %" PRIu64 "\n", fmt_str, num_entries);
+    }
+out:
+    printf("\n");
+    return 0;
+}
 
-        return glfsh_output->print_heal_op_status (ret, num_entries, fmt_str);
+int
+glfsh_print_info_summary(int ret, num_entries_t *num_entries)
+{
+    return glfsh_output->print_heal_op_summary(ret, num_entries);
 }
 
 int
-glfsh_get_index_dir_loc (loc_t *rootloc, xlator_t *xl, loc_t *dirloc,
-                         int32_t *op_errno, char *vgfid)
+glfsh_print_heal_op_status(int ret, uint64_t num_entries,
+                           gf_xl_afr_op_t heal_op)
 {
-        void      *index_gfid = NULL;
-        int       ret = 0;
-        dict_t    *xattr = NULL;
-        struct iatt   iattr = {0};
-        struct iatt   parent = {0};
-
-        ret = syncop_getxattr (xl, rootloc, &xattr, vgfid, NULL, NULL);
-        if (ret < 0) {
-                *op_errno = -ret;
-                goto out;
-        }
+    char *fmt_str = NULL;
 
-        ret = dict_get_ptr (xattr, vgfid, &index_gfid);
-        if (ret < 0) {
-                *op_errno = EINVAL;
-                goto out;
-        }
+    if (heal_op == GF_SHD_OP_INDEX_SUMMARY)
+        fmt_str = "Number of entries:";
+    else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES)
+        fmt_str = "Number of entries in split-brain:";
+    else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK)
+        fmt_str = "Number of healed entries:";
 
-        gf_uuid_copy (dirloc->gfid, index_gfid);
-        dirloc->path = "";
-        dirloc->inode = inode_new (rootloc->inode->table);
-        ret = syncop_lookup (xl, dirloc, &iattr, &parent, NULL, NULL);
-        dirloc->path = NULL;
-        if (ret < 0) {
-                *op_errno = -ret;
-                goto out;
-        }
-        ret = glfsh_link_inode_update_loc (dirloc, &iattr);
-        if (ret)
-                goto out;
-        glfs_loc_touchup (dirloc);
+    return glfsh_output->print_heal_op_status(ret, num_entries, fmt_str);
+}
 
-        ret = 0;
+int
+glfsh_get_index_dir_loc(loc_t *rootloc, xlator_t *xl, loc_t *dirloc,
+                        int32_t *op_errno, char *vgfid)
+{
+    void *index_gfid = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    struct iatt iattr = {0};
+    struct iatt parent = {0};
+
+    ret = syncop_getxattr(xl, rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret < 0) {
+        *op_errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret < 0) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_uuid_copy(dirloc->gfid, index_gfid);
+    dirloc->path = "";
+    dirloc->inode = inode_new(rootloc->inode->table);
+    ret = syncop_lookup(xl, dirloc, &iattr, &parent, NULL, NULL);
+    dirloc->path = NULL;
+    if (ret < 0) {
+        *op_errno = -ret;
+        goto out;
+    }
+    ret = glfsh_link_inode_update_loc(dirloc, &iattr);
+    if (ret)
+        goto out;
+
+    ret = glfs_loc_touchup(dirloc);
+    if (ret < 0) {
+        *op_errno = errno;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (xattr)
-                dict_unref (xattr);
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
-static xlator_t*
-_get_ancestor (xlator_t *xl, gf_xl_afr_op_t heal_op)
+static xlator_t *
+_get_ancestor(xlator_t *xl, gf_xl_afr_op_t heal_op)
 {
-        static char *replica_xl[] = {"cluster/replicate", NULL};
-        static char *heal_xls[] = {"cluster/replicate", "cluster/disperse",
-                                         NULL};
-        char **ancestors = NULL;
+    static char *replica_xl[] = {"cluster/replicate", NULL};
+    static char *heal_xls[] = {"cluster/replicate", "cluster/disperse", NULL};
+    char **ancestors = NULL;
 
-        if (heal_op == GF_SHD_OP_INDEX_SUMMARY)
-                ancestors = heal_xls;
-        else
-                ancestors = replica_xl;
+    if (heal_op == GF_SHD_OP_INDEX_SUMMARY || heal_op == GF_SHD_OP_HEAL_SUMMARY)
+        ancestors = heal_xls;
+    else
+        ancestors = replica_xl;
 
-        if (!xl || !xl->parents)
-                return NULL;
+    if (!xl || !xl->parents)
+        return NULL;
 
-        while (xl->parents) {
-                xl = xl->parents->xlator;
-                if (!xl)
-                        break;
-                if (gf_get_index_by_elem (ancestors, xl->type) != -1)
-                        return xl;
-        }
+    while (xl->parents) {
+        xl = xl->parents->xlator;
+        if (!xl)
+            break;
+        if (gf_get_index_by_elem(ancestors, xl->type) != -1)
+            return xl;
+    }
 
-        return NULL;
+    return NULL;
 }
 
 int
-glfsh_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+glfsh_index_purge(xlator_t *subvol, inode_t *inode, char *name)
 {
-        loc_t loc = {0, };
-        int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-        loc.parent = inode_ref (inode);
-        loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-        ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-void
-glfsh_print_spb_status (dict_t *dict, char *path, uuid_t gfid,
-                        uint64_t *num_entries, gf_boolean_t flag)
+int
+glfsh_print_summary_status(dict_t *dict, char *path, uuid_t gfid,
+                           num_entries_t *num_entries, gf_boolean_t flag)
 {
-        int   ret    = 0;
-        gf_boolean_t    pending         = _gf_false;
-        gf_boolean_t    split_b         = _gf_false;
-        char            *value          = NULL;
-        char            gfid_str[64]    = {0};
+    int ret = 0;
+    char *value = NULL;
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret)
+        goto out;
+
+    if ((!strcmp(value, "heal")) || (!strcmp(value, "heal-pending"))) {
+        (num_entries->pending_entries)++;
+    } else if ((!strcmp(value, "split-brain")) ||
+               (!strcmp(value, "split-brain-pending"))) {
+        (num_entries->spb_entries)++;
+    } else if ((!strcmp(value, "possibly-healing-pending")) ||
+               (!strcmp(value, "possibly-healing"))) {
+        (num_entries->possibly_healing_entries)++;
+    } else {
+        goto out;
+    }
+    (num_entries->num_entries)++;
+out:
+    return ret;
+}
 
-        ret = dict_get_str (dict, "heal-info", &value);
-        if (ret)
-                return;
+int
+glfsh_print_spb_status(dict_t *dict, char *path, uuid_t gfid,
+                       num_entries_t *num_entries, gf_boolean_t flag)
+{
+    int ret = 0;
+    gf_boolean_t pending = _gf_false;
+    gf_boolean_t split_b = _gf_false;
+    char *value = NULL;
+    char gfid_str[64] = {0};
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret)
+        return 0;
 
-        if (!strcmp (value, "split-brain")) {
-                split_b = _gf_true;
-        } else if (!strcmp (value, "split-brain-pending")) {
-                split_b = _gf_true;
-                pending = _gf_true;
+    if (!strcmp(value, "split-brain")) {
+        split_b = _gf_true;
+    } else if (!strcmp(value, "split-brain-pending")) {
+        split_b = _gf_true;
+        pending = _gf_true;
+    }
+    /* Consider the entry only iff :
+     * 1) The dir being processed is not indices/dirty, indicated by
+     *    flag == _gf_false
+     * 2) The dir being processed is indices/dirty but the entry also
+     *    exists in indices/xattrop dir and has already been processed.
+     */
+    if (split_b) {
+        if (!flag || (flag && !pending)) {
+            (num_entries->num_entries)++;
+            glfsh_output->print_spb_status(
+                path ? path : uuid_utoa_r(gfid, gfid_str), gfid, NULL);
         }
-        /* Consider the entry only iff :
-         * 1) The dir being processed is not indices/dirty, indicated by
-         *    flag == _gf_false
-         * 2) The dir being processed is indices/dirty but the entry also
-         *    exists in indices/xattrop dir and has already been processed.
-         */
-        if (split_b) {
-                if (!flag || (flag && !pending)) {
-                        (*num_entries)++;
-                        glfsh_output->print_spb_status (path ? path :
-                                                uuid_utoa_r (gfid, gfid_str),
-                                                gfid, NULL);
-                }
-        }
-        return;
+    }
+    return 0;
 }
 
-void
-glfsh_print_heal_status (dict_t *dict, char *path, uuid_t gfid,
-                         uint64_t *num_entries, gf_boolean_t ignore_dirty)
+int
+glfsh_print_heal_status(dict_t *dict, char *path, uuid_t gfid,
+                        num_entries_t *num_entries, gf_boolean_t ignore_dirty)
 {
-        int             ret             = 0;
-        gf_boolean_t    pending         = _gf_false;
-        char            *status         = NULL;
-        char            *value          = NULL;
-        char            gfid_str[64]    = {0};
-
-        ret = dict_get_str (dict, "heal-info", &value);
-        if (ret || (!strcmp (value, "no-heal")))
-                return;
-
-        if (!strcmp (value, "heal")) {
-                ret = gf_asprintf (&status, " ");
-                if (ret < 0)
-                        goto out;
-        } else if (!strcmp (value, "possibly-healing")) {
-                ret = gf_asprintf (&status,
-                                   " - Possibly undergoing heal\n");
-                if (ret < 0)
-                        goto out;
-        } else if (!strcmp (value, "split-brain")) {
-                ret = gf_asprintf (&status, " - Is in split-brain\n");
-                if (ret < 0)
-                        goto out;
-        } else if (!strcmp (value, "heal-pending")) {
-                pending = _gf_true;
-                ret = gf_asprintf (&status, " ");
-                if (ret < 0)
-                        goto out;
-        } else if (!strcmp (value, "split-brain-pending")) {
-                pending = _gf_true;
-                ret = gf_asprintf (&status, " - Is in split-brain\n");
-                if (ret < 0)
-                        goto out;
-        } else if (!strcmp (value, "possibly-healing-pending")) {
-                pending = _gf_true;
-                ret = gf_asprintf (&status,
-                                   " - Possibly undergoing heal\n");
-                if (ret < 0)
-                        goto out;
-        }
+    int ret = 0;
+    gf_boolean_t pending = _gf_false;
+    char *status = NULL;
+    char *value = NULL;
+    char gfid_str[64] = {0};
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret || (!strcmp(value, "no-heal")))
+        return 0;
+
+    if (!strcmp(value, "heal")) {
+        ret = gf_asprintf(&status, " ");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "possibly-healing")) {
+        ret = gf_asprintf(&status, " - Possibly undergoing heal");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "split-brain")) {
+        ret = gf_asprintf(&status, " - Is in split-brain");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "heal-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " ");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "split-brain-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " - Is in split-brain");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "possibly-healing-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " - Possibly undergoing heal");
+        if (ret < 0)
+            goto out;
+    }
 out:
-        /* If ignore_dirty is set, it means indices/dirty directory is
-         * being processed. Ignore the entry if it also exists in
-         * indices/xattrop.
-         * Boolean pending is set to true if the entry also exists in
-         * indices/xattrop directory.
-         */
-        if (ignore_dirty) {
-                if (pending) {
-                        GF_FREE (status);
-                        status = NULL;
-                        return;
-                }
+    /* If ignore_dirty is set, it means indices/dirty directory is
+     * being processed. Ignore the entry if it also exists in
+     * indices/xattrop.
+     * Boolean pending is set to true if the entry also exists in
+     * indices/xattrop directory.
+     */
+    if (ignore_dirty) {
+        if (pending) {
+            GF_FREE(status);
+            status = NULL;
+            return 0;
         }
-        if (ret == -1)
-                status = NULL;
+    }
+    if (ret == -1)
+        status = NULL;
 
-        (*num_entries)++;
-        glfsh_output->print_heal_status (path ? path :
-                                         uuid_utoa_r (gfid, gfid_str),
-                                         gfid,
-                                         status ? status : "");
+    (num_entries->num_entries)++;
+    glfsh_output->print_heal_status(path ? path : uuid_utoa_r(gfid, gfid_str),
+                                    gfid, status ? status : "");
 
-        GF_FREE (status);
-        return;
+    GF_FREE(status);
+    return 0;
 }
 
-static int
-glfsh_heal_entries (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                    gf_dirent_t *entries,  uint64_t *offset,
-                    uint64_t *num_entries, dict_t *xattr_req) {
-
-        gf_dirent_t      *entry          = NULL;
-        gf_dirent_t      *tmp            = NULL;
-        int               ret            = 0;
-        char              file[64]      = {0};
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                *offset = entry->d_off;
-                if ((strcmp (entry->d_name, ".") == 0) ||
-                    (strcmp (entry->d_name, "..") == 0))
-                        continue;
-                memset (file, 0, sizeof(file));
-                snprintf (file, sizeof(file), "gfid:%s", entry->d_name);
-                ret = glfsh_heal_splitbrain_file (fs, top_subvol, rootloc, file,
-                                                 xattr_req);
-                if (ret)
-                        continue;
-                (*num_entries)++;
-        }
+int
+glfsh_heal_status_boolean(dict_t *dict, char *path, uuid_t gfid,
+                          num_entries_t *num_entries, gf_boolean_t ignore_dirty)
+{
+    int ret = 0;
+    char *value = NULL;
 
-        return ret;
+    ret = dict_get_str(dict, "heal-info", &value);
+    if ((!ret) && (!strcmp(value, "no-heal")))
+        return 0;
+    else
+        return -1;
+}
+
+static void
+glfsh_heal_entries(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                   gf_dirent_t *entries, uint64_t *offset,
+                   num_entries_t *num_entries, dict_t *xattr_req)
+{
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    int ret = 0;
+    char file[64] = {0};
+
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        *offset = entry->d_off;
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+        snprintf(file, sizeof(file), "gfid:%s", entry->d_name);
+        ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file,
+                                         xattr_req);
+        if (ret)
+            continue;
+        (num_entries->num_entries)++;
+    }
 }
 
 static int
-glfsh_process_entries (xlator_t *xl, fd_t *fd, gf_dirent_t *entries,
-                       uint64_t *offset, uint64_t *num_entries,
-                       print_status glfsh_print_status,
-                       gf_boolean_t ignore_dirty)
+glfsh_process_entries(xlator_t *xl, fd_t *fd, gf_dirent_t *entries,
+                      uint64_t *offset, num_entries_t *num_entries,
+                      print_status glfsh_print_status,
+                      gf_boolean_t ignore_dirty, glfsh_fail_mode_t mode)
 {
-        gf_dirent_t      *entry = NULL;
-        gf_dirent_t      *tmp = NULL;
-        int              ret = 0;
-        char            *path = NULL;
-        uuid_t          gfid = {0};
-        xlator_t        *this = NULL;
-        dict_t          *dict = NULL;
-        loc_t           loc   = {0,};
-        this = THIS;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                *offset = entry->d_off;
-                if ((strcmp (entry->d_name, ".") == 0) ||
-                    (strcmp (entry->d_name, "..") == 0))
-                        continue;
-
-                if (dict) {
-                        dict_unref (dict);
-                        dict = NULL;
-                }
-                gf_uuid_clear (gfid);
-                GF_FREE (path);
-                path = NULL;
-
-                gf_uuid_parse (entry->d_name, gfid);
-                gf_uuid_copy (loc.gfid, gfid);
-                ret = syncop_getxattr (this, &loc, &dict, GF_HEAL_INFO, NULL,
-                                       NULL);
-                if (ret)
-                        continue;
-
-                ret = syncop_gfid_to_path (this->itable, xl, gfid, &path);
-
-                if (ret == -ENOENT || ret == -ESTALE) {
-                        glfsh_index_purge (xl, fd->inode, entry->d_name);
-                        ret = 0;
-                        continue;
-                }
-                if (dict)
-                        glfsh_print_status (dict, path, gfid,
-                                            num_entries, ignore_dirty);
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    int ret = 0;
+    int print_status = 0;
+    char *path = NULL;
+    uuid_t gfid = {0};
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    loc_t loc = {
+        0,
+    };
+    this = THIS;
+
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        *offset = entry->d_off;
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+
+        if (dict) {
+            dict_unref(dict);
+            dict = NULL;
+        }
+        gf_uuid_clear(gfid);
+        GF_FREE(path);
+        path = NULL;
+
+        gf_uuid_parse(entry->d_name, gfid);
+        gf_uuid_copy(loc.gfid, gfid);
+        ret = syncop_getxattr(this, &loc, &dict, GF_HEAL_INFO, NULL, NULL);
+        if (ret) {
+            if ((mode != GLFSH_MODE_CONTINUE_ON_ERROR) && (ret == -ENOTCONN))
+                goto out;
+            else
+                continue;
+        }
+
+        ret = syncop_gfid_to_path(this->itable, xl, gfid, &path);
+
+        if (ret == -ENOENT || ret == -ESTALE) {
+            glfsh_index_purge(xl, fd->inode, entry->d_name);
+            ret = 0;
+            continue;
         }
-        ret = 0;
-        GF_FREE (path);
         if (dict) {
-                dict_unref (dict);
-                dict = NULL;
+            print_status = glfsh_print_status(dict, path, gfid, num_entries,
+                                              ignore_dirty);
+            if ((print_status) && (mode != GLFSH_MODE_CONTINUE_ON_ERROR)) {
+                ret = -EAGAIN;
+                goto out;
+            }
         }
-        return ret;
+    }
+    ret = 0;
+out:
+    GF_FREE(path);
+    if (dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return ret;
 }
 
 static int
-glfsh_crawl_directory (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                       xlator_t *readdir_xl, fd_t *fd, loc_t *loc,
-                       dict_t *xattr_req, uint64_t *num_entries,
-                       gf_boolean_t ignore)
+glfsh_crawl_directory(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                      xlator_t *readdir_xl, fd_t *fd, loc_t *loc,
+                      dict_t *xattr_req, num_entries_t *num_entries,
+                      gf_boolean_t ignore)
 {
-        uint64_t        offset = 0;
-        gf_dirent_t     entries;
-        int             ret = 0;
-        gf_boolean_t    free_entries = _gf_false;
-        int             heal_op = -1;
-
-        INIT_LIST_HEAD (&entries.list);
-        ret = dict_get_int32 (xattr_req, "heal-op", &heal_op);
-        if (ret)
-                return ret;
+    int ret = 0;
+    int heal_op = -1;
+    uint64_t offset = 0;
+    gf_dirent_t entries;
+    gf_boolean_t free_entries = _gf_false;
+    glfsh_fail_mode_t mode = GLFSH_MODE_CONTINUE_ON_ERROR;
+
+    INIT_LIST_HEAD(&entries.list);
+    ret = dict_get_int32(xattr_req, "heal-op", &heal_op);
+    if (ret)
+        return ret;
 
-        while (1) {
-                ret = syncop_readdir (readdir_xl, fd, 131072, offset, &entries,
-                                      NULL, NULL);
-                if (ret <= 0)
-                        break;
-                ret = 0;
-                free_entries = _gf_true;
-
-                if (list_empty (&entries.list))
-                        goto out;
-
-                if (heal_op == GF_SHD_OP_INDEX_SUMMARY) {
-                        ret = glfsh_process_entries (readdir_xl, fd,
-                                                     &entries, &offset,
-                                                     num_entries,
-                                                     glfsh_print_heal_status,
-                                                     ignore);
-                        if (ret < 0)
-                                goto out;
-                } else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES) {
-                        ret = glfsh_process_entries (readdir_xl, fd,
-                                                     &entries, &offset,
-                                                     num_entries,
-                                                     glfsh_print_spb_status,
-                                                     ignore);
-                        if (ret < 0)
-                                goto out;
-                } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
-                        ret = glfsh_heal_entries (fs, top_subvol, rootloc,
-                                                  &entries, &offset,
-                                                  num_entries, xattr_req);
-                }
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-        }
+    if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        mode = GLFSH_MODE_EXIT_ON_FIRST_FAILURE;
+
+    while (1) {
+        ret = syncop_readdir(readdir_xl, fd, 131072, offset, &entries, NULL,
+                             NULL);
+        if (ret <= 0)
+            break;
         ret = 0;
+        free_entries = _gf_true;
+
+        if (list_empty(&entries.list))
+            goto out;
+
+        if (heal_op == GF_SHD_OP_INDEX_SUMMARY) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_heal_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_spb_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_HEAL_SUMMARY) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_summary_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
+            glfsh_heal_entries(fs, top_subvol, rootloc, &entries, &offset,
+                               num_entries, xattr_req);
+        } else if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_heal_status_boolean,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        }
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+    }
+    ret = 0;
 out:
-        if (free_entries)
-                gf_dirent_free (&entries);
-        return ret;
+    if (free_entries)
+        gf_dirent_free(&entries);
+    return ret;
 }
 
 static int
-glfsh_print_brick_from_xl (xlator_t *xl, loc_t *rootloc)
+glfsh_no_print_brick_from_xl(xlator_t *xl, loc_t *rootloc)
 {
-        char    *remote_host = NULL;
-        char    *remote_subvol = NULL;
-        int     ret = 0;
+    return 0;
+}
 
-        ret = dict_get_str (xl->options, "remote-host", &remote_host);
-        if (ret < 0)
-                goto out;
+static int
+glfsh_print_brick_from_xl(xlator_t *xl, loc_t *rootloc)
+{
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+    int ret = 0;
 
-        ret = dict_get_str (xl->options, "remote-subvolume", &remote_subvol);
-        if (ret < 0)
-                goto out;
+    ret = dict_get_str(xl->options, "remote-host", &remote_host);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+    if (ret < 0)
+        goto out;
 out:
-        if (ret < 0)
-                printf ("Brick - Not able to get brick information\n");
-        else
-                printf ("Brick %s:%s\n", remote_host, remote_subvol);
-        return ret;
+    if (ret < 0)
+        printf("Brick - Not able to get brick information\n");
+    else
+        printf("Brick %s:%s\n", remote_host, remote_subvol);
+    return ret;
 }
 
 int
-glfsh_print_pending_heals_type (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                                xlator_t *xl, gf_xl_afr_op_t heal_op,
-                                dict_t *xattr_req, char *vgfid,
-                                uint64_t *num_entries)
+glfsh_print_pending_heals_type(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                               xlator_t *xl, gf_xl_afr_op_t heal_op,
+                               dict_t *xattr_req, char *vgfid,
+                               num_entries_t *num_entries)
 {
-        int ret = 0;
-        loc_t   dirloc = {0};
-        fd_t    *fd = NULL;
-        int32_t op_errno = 0;
-        gf_boolean_t ignore = _gf_false;
-
-        if (!strcmp(vgfid, GF_XATTROP_DIRTY_GFID))
-                ignore = _gf_true;
-
-        ret = glfsh_get_index_dir_loc (rootloc, xl, &dirloc, &op_errno,
-                                       vgfid);
-        if (ret < 0) {
-                if (op_errno == ESTALE || op_errno == ENOENT ||
-                    op_errno == ENOTSUP)
-                        ret = 0;
-                else
-                        ret = -op_errno;
-                goto out;
-        }
-
-        ret = syncop_dirfd (xl, &dirloc, &fd, GF_CLIENT_PID_GLFS_HEAL);
-        if (ret)
-                goto out;
-
-        ret = glfsh_crawl_directory (fs, top_subvol, rootloc, xl, fd, &dirloc,
-                                     xattr_req, num_entries, ignore);
-        if (fd)
-                fd_unref (fd);
+    int ret = 0;
+    loc_t dirloc = {0};
+    fd_t *fd = NULL;
+    int32_t op_errno = 0;
+    gf_boolean_t ignore = _gf_false;
+
+    if (!strcmp(vgfid, GF_XATTROP_DIRTY_GFID))
+        ignore = _gf_true;
+
+    ret = glfsh_get_index_dir_loc(rootloc, xl, &dirloc, &op_errno, vgfid);
+    if (ret < 0) {
+        if (op_errno == ESTALE || op_errno == ENOENT || op_errno == ENOTSUP)
+            ret = 0;
+        else
+            ret = -op_errno;
+        goto out;
+    }
+
+    ret = syncop_dirfd(xl, &dirloc, &fd, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret)
+        goto out;
+
+    ret = glfsh_crawl_directory(fs, top_subvol, rootloc, xl, fd, &dirloc,
+                                xattr_req, num_entries, ignore);
+    if (fd)
+        fd_unref(fd);
 out:
-        loc_wipe (&dirloc);
-        return ret;
+    loc_wipe(&dirloc);
+    return ret;
 }
 
 int
-glfsh_print_pending_heals (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                           xlator_t *xl, gf_xl_afr_op_t heal_op, gf_boolean_t
-                           is_parent_replicate)
+glfsh_print_pending_heals(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                          xlator_t *xl, gf_xl_afr_op_t heal_op,
+                          gf_boolean_t is_parent_replicate)
 {
-        int ret = 0;
-        uint64_t count = 0, total = 0;
+    int ret = 0;
+    num_entries_t num_entries = {
+        0,
+    };
+    num_entries_t total = {
+        0,
+    };
+
+    dict_t *xattr_req = NULL;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op", heal_op);
+    if (ret)
+        goto out;
+
+    if ((!is_parent_replicate) &&
+        ((heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||
+         (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE))) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = glfsh_output->print_brick_from_xl(xl, rootloc);
+    if (ret < 0)
+        goto out;
+
+    ret = glfsh_print_pending_heals_type(fs, top_subvol, rootloc, xl, heal_op,
+                                         xattr_req, GF_XATTROP_INDEX_GFID,
+                                         &num_entries);
+
+    if (ret < 0 && heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        goto out;
+
+    total.num_entries += num_entries.num_entries;
+    total.pending_entries += num_entries.pending_entries;
+    total.spb_entries += num_entries.spb_entries;
+    total.possibly_healing_entries += num_entries.possibly_healing_entries;
+    num_entries.num_entries = 0;
+    num_entries.pending_entries = 0;
+    num_entries.spb_entries = 0;
+    num_entries.possibly_healing_entries = 0;
+    if (ret == -ENOTCONN)
+        goto out;
+
+    if (is_parent_replicate) {
+        ret = glfsh_print_pending_heals_type(
+            fs, top_subvol, rootloc, xl, heal_op, xattr_req,
+            GF_XATTROP_DIRTY_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        total.pending_entries += num_entries.pending_entries;
+        total.spb_entries += num_entries.spb_entries;
+        total.possibly_healing_entries += num_entries.possibly_healing_entries;
+    }
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (heal_op == GF_SHD_OP_HEAL_SUMMARY) {
+        glfsh_print_info_summary(ret, &total);
+    } else {
+        glfsh_print_heal_op_status(ret, total.num_entries, heal_op);
+    }
+    return ret;
+}
 
-        dict_t *xattr_req = NULL;
+static int
+glfsh_set_heal_options(glfs_t *fs, gf_xl_afr_op_t heal_op)
+{
+    int ret = 0;
 
-        xattr_req = dict_new();
-        if (!xattr_req)
-                goto out;
-        ret = dict_set_int32 (xattr_req, "heal-op", heal_op);
-        if (ret)
-                goto out;
+    ret = glfs_set_xlator_option(fs, "*-replicate-*",
+                                 "background-self-heal-count", "0");
+    if (ret)
+        goto out;
 
-        ret = glfsh_output->print_brick_from_xl (xl, rootloc);
-        if (ret < 0)
-                goto out;
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "halo-enabled", "off");
+    if (ret)
+        goto out;
 
-        ret = glfsh_print_pending_heals_type (fs, top_subvol, rootloc, xl,
-                                              heal_op, xattr_req,
-                                              GF_XATTROP_INDEX_GFID, &count);
-        total += count;
-        count = 0;
-        if (ret == -ENOTCONN)
-                goto out;
+    if ((heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) &&
+        (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) &&
+        (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME))
+        return 0;
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "data-self-heal", "on");
+    if (ret)
+        goto out;
 
-        if (is_parent_replicate) {
-                ret = glfsh_print_pending_heals_type (fs, top_subvol,
-                                                      rootloc, xl,
-                                                      heal_op, xattr_req,
-                                                      GF_XATTROP_DIRTY_GFID,
-                                                      &count);
-                total += count;
-        }
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        glfsh_print_heal_op_status (ret, total, heal_op);
-        return ret;
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "metadata-self-heal",
+                                 "on");
+    if (ret)
+        goto out;
+
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "entry-self-heal", "on");
 
+out:
+    return ret;
 }
 
 static int
-glfsh_validate_volume (xlator_t *xl, gf_xl_afr_op_t heal_op)
+glfsh_validate_volume(xlator_t *xl, gf_xl_afr_op_t heal_op)
 {
-        xlator_t        *heal_xl = NULL;
-        int             ret = -1;
-
-        while (xl->next)
-                xl = xl->next;
-
-        while (xl) {
-                if (strcmp (xl->type, "protocol/client") == 0) {
-                        heal_xl = _get_ancestor (xl, heal_op);
-                        if (heal_xl) {
-                                ret = 0;
-                                break;
-                        }
-                }
-
-                xl = xl->prev;
+    xlator_t *heal_xl = NULL;
+    int ret = -1;
+
+    while (xl->next)
+        xl = xl->next;
+
+    while (xl) {
+        if (strcmp(xl->type, "protocol/client") == 0) {
+            heal_xl = _get_ancestor(xl, heal_op);
+            if (heal_xl) {
+                ret = 0;
+                break;
+            }
         }
 
-        return ret;
+        xl = xl->prev;
+    }
+
+    return ret;
 }
 
-static xlator_t*
-_brick_path_to_client_xlator (xlator_t *top_subvol, char *hostname,
-                              char *brickpath)
+static xlator_t *
+_brick_path_to_client_xlator(xlator_t *top_subvol, char *hostname,
+                             char *brickpath)
 {
-        int ret             = 0;
-        xlator_t *xl        = NULL;
-        char *remote_host   = NULL;
-        char *remote_subvol = NULL;
-
-        xl = top_subvol;
-
-        while (xl->next)
-                xl = xl->next;
-
-        while (xl) {
-                if (!strcmp (xl->type, "protocol/client")) {
-                        ret = dict_get_str (xl->options, "remote-host",
-                                                    &remote_host);
-                        if (ret < 0)
-                                goto out;
-                        ret = dict_get_str (xl->options,
-                                            "remote-subvolume", &remote_subvol);
-                        if (ret < 0)
-                                goto out;
-                        if (!strcmp (hostname, remote_host) &&
-                            !strcmp (brickpath, remote_subvol))
-                                return xl;
-                }
-                xl = xl->prev;
+    int ret = 0;
+    xlator_t *xl = NULL;
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+
+    xl = top_subvol;
+
+    while (xl->next)
+        xl = xl->next;
+
+    while (xl) {
+        if (!strcmp(xl->type, "protocol/client")) {
+            ret = dict_get_str(xl->options, "remote-host", &remote_host);
+            if (ret < 0)
+                goto out;
+            ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+            if (ret < 0)
+                goto out;
+            if (!strcmp(hostname, remote_host) &&
+                !strcmp(brickpath, remote_subvol))
+                return xl;
         }
+        xl = xl->prev;
+    }
 
 out:
-        return NULL;
+    return NULL;
 }
 
-
 int
-glfsh_gather_heal_info (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                        gf_xl_afr_op_t heal_op)
+glfsh_gather_heal_info(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                       gf_xl_afr_op_t heal_op)
 {
-        xlator_t  *xl       = NULL;
-        xlator_t  *heal_xl   = NULL;
-        xlator_t  *old_THIS = NULL;
-
-        xl = top_subvol;
-        while (xl->next)
-                xl = xl->next;
-        while (xl) {
-                if (strcmp (xl->type, "protocol/client") == 0) {
-                        heal_xl = _get_ancestor (xl, heal_op);
-                        if (heal_xl) {
-                                old_THIS = THIS;
-                                THIS = heal_xl;
-                                glfsh_print_pending_heals (fs, top_subvol,
-                                                           rootloc, xl,
-                                                           heal_op,
-                                                           !strcmp
-                                                           (heal_xl->type,
-                                                           "cluster/replicate"));
-                                THIS = old_THIS;
-                        }
-                }
-
-                xl = xl->prev;
+    int ret = 0;
+    xlator_t *xl = NULL;
+    xlator_t *heal_xl = NULL;
+    xlator_t *old_THIS = NULL;
+
+    xl = top_subvol;
+    while (xl->next)
+        xl = xl->next;
+    while (xl) {
+        if (strcmp(xl->type, "protocol/client") == 0 &&
+            !strstr(xl->name, "-ta-")) {
+            heal_xl = _get_ancestor(xl, heal_op);
+            if (heal_xl) {
+                old_THIS = THIS;
+                THIS = heal_xl;
+                ret = glfsh_print_pending_heals(
+                    fs, top_subvol, rootloc, xl, heal_op,
+                    !strcmp(heal_xl->type, "cluster/replicate"));
+                THIS = old_THIS;
+
+                if ((ret < 0) &&
+                    (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE))
+                    goto out;
+            }
         }
 
-        return 0;
+        xl = xl->prev;
+    }
+
+out:
+    if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        ret = 0;
+
+    return ret;
 }
 
 int
-_validate_directory (dict_t *xattr_req, char *file)
+_validate_directory(dict_t *xattr_req, char *file)
 {
-        int heal_op = -1;
-        int ret = 0;
-
-        ret = dict_get_int32 (xattr_req, "heal-op", &heal_op);
-        if (ret)
-                return ret;
-
-        if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) {
-                printf ("'bigger-file' not a valid option for directories.\n");
-                ret = -1;
-        } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
-                printf ("'source-brick' option used on a directory (%s). "
-                        "Performing conservative merge.\n", file);
-        }
+    int heal_op = -1;
+    int ret = 0;
 
+    ret = dict_get_int32(xattr_req, "heal-op", &heal_op);
+    if (ret)
         return ret;
+
+    if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) {
+        printf("'bigger-file' not a valid option for directories.\n");
+        ret = -1;
+    } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
+        printf(
+            "'source-brick' option used on a directory (%s). "
+            "Performing conservative merge.\n",
+            file);
+    }
+
+    return ret;
 }
 
 int
-glfsh_heal_splitbrain_file (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+glfsh_heal_splitbrain_file(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
                            char *file, dict_t *xattr_req)
 {
-        int          ret        = -1;
-        int          reval      = 0;
-        loc_t        loc        = {0, };
-        char        *path       = NULL;
-        char        *filename   = NULL;
-        struct iatt  iatt       = {0, };
-        xlator_t    *xl         = top_subvol;
-        dict_t      *xattr_rsp  = NULL;
-        char        *sh_fail_msg = NULL;
-        int32_t      op_errno   = 0;
-
-        if (!strncmp (file, "gfid:", 5)) {
-                filename = gf_strdup(file);
-                path = strtok (filename, ":");
-                path = strtok (NULL, ";");
-                gf_uuid_parse (path, loc.gfid);
-                loc.path = gf_strdup (uuid_utoa (loc.gfid));
-                loc.inode = inode_new (rootloc->inode->table);
-                ret = syncop_lookup (xl, &loc, &iatt, 0, xattr_req, &xattr_rsp);
-                if (ret) {
-                        op_errno = -ret;
-                        printf ("Lookup failed on %s:%s.\n", file,
-                                strerror(op_errno));
-                        goto out;
-                }
-        } else {
-                if (file[0] != '/') {
-                        printf ("<FILE> must be absolute path w.r.t. the "
-                                "volume, starting with '/'\n");
-                        ret = -1;
-                        goto out;
-                }
-retry:
-                ret = glfs_resolve (fs, xl, file, &loc, &iatt, reval);
-                ESTALE_RETRY (ret, errno, reval, &loc, retry);
-                if (ret) {
-                        printf("Lookup failed on %s:%s\n",
-                               file, strerror (errno));
-                        goto out;
-                }
+    int ret = -1;
+    int reval = 0;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    char *filename = NULL;
+    char *filename1 = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *xl = top_subvol;
+    dict_t *xattr_rsp = NULL;
+    char *sh_fail_msg = NULL;
+    char *gfid_heal_msg = NULL;
+    int32_t op_errno = 0;
+    gf_boolean_t flag = _gf_false;
+
+    if (!strncmp(file, "gfid:", 5)) {
+        filename = gf_strdup(file);
+        if (!filename) {
+            printf("Error allocating memory to filename\n");
+            goto out;
         }
-
-        if (iatt.ia_type == IA_IFDIR) {
-                ret = _validate_directory (xattr_req, file);
-                if (ret)
-                        goto out;
+        path = strtok(filename, ":");
+        path = strtok(NULL, ";");
+        gf_uuid_parse(path, loc.gfid);
+        loc.path = gf_strdup(uuid_utoa(loc.gfid));
+        if (!loc.path) {
+            printf("Error allocating memory to path\n");
+            goto out;
+        }
+        loc.inode = inode_new(rootloc->inode->table);
+        if (!loc.inode) {
+            printf("Error getting inode\n");
+            goto out;
         }
-        ret = syncop_getxattr (xl, &loc, &xattr_rsp, GF_AFR_HEAL_SBRAIN,
-                               xattr_req, NULL);
+        ret = syncop_lookup(xl, &loc, &iatt, 0, xattr_req, &xattr_rsp);
         if (ret) {
-                op_errno = -ret;
-                printf ("Healing %s failed:%s.\n", file, strerror(op_errno));
-                goto out;
+            op_errno = -ret;
+            printf("Lookup failed on %s:%s.\n", file, strerror(op_errno));
+            goto out;
+        }
+    } else {
+        if (file[0] != '/') {
+            printf(
+                "<FILE> must be absolute path w.r.t. the "
+                "volume, starting with '/'\n");
+            ret = -1;
+            goto out;
+        }
+        path1 = gf_strdup(file);
+        if (!path1) {
+            printf("Error allocating memory to path\n");
+            ret = -1;
+            goto out;
+        }
+        path2 = gf_strdup(file);
+        if (!path2) {
+            printf("Error allocating memory to path\n");
+            ret = -1;
+            goto out;
+        }
+        path = dirname(path1);
+        filename1 = basename(path2);
+    retry1:
+        ret = glfs_resolve(fs, xl, path, &loc, &iatt, reval);
+        ESTALE_RETRY(ret, errno, reval, &loc, retry1);
+        if (ret) {
+            printf("Lookup failed on %s:%s\n", path, strerror(errno));
+            goto out;
+        }
+        GF_FREE((char *)loc.path);
+        loc.path = gf_strdup(file);
+        if (!loc.path) {
+            printf("Error allocating memory for path\n");
+            ret = -1;
+            goto out;
+        }
+        loc.parent = inode_unref(loc.parent);
+        loc.parent = inode_ref(loc.inode);
+        loc.inode = inode_unref(loc.inode);
+        loc.inode = inode_new(rootloc->inode->table);
+        if (!loc.inode) {
+            printf("Error getting inode\n");
+            ret = -1;
+            goto out;
         }
-        ret = dict_get_str (xattr_rsp, "sh-fail-msg", &sh_fail_msg);
+        loc.name = filename1;
+        gf_uuid_copy(loc.pargfid, loc.gfid);
+        gf_uuid_clear(loc.gfid);
+
+        ret = syncop_lookup(xl, &loc, &iatt, 0, xattr_req, &xattr_rsp);
+        if (ret) {
+            op_errno = -ret;
+            printf("Lookup failed on %s:%s.\n", file, strerror(op_errno));
+            flag = _gf_true;
+        }
+
+        ret = dict_get_str(xattr_rsp, "gfid-heal-msg", &gfid_heal_msg);
         if (!ret) {
-                printf ("Healing %s failed: %s.\n", file, sh_fail_msg);
-                ret = -1;
-                goto out;
+            printf("%s for file %s\n", gfid_heal_msg, file);
+            loc_wipe(&loc);
+            goto out;
         }
-        printf ("Healed %s.\n", file);
-        ret = 0;
+        if (flag)
+            goto out;
+
+        reval = 0;
+        loc_wipe(&loc);
+        memset(&iatt, 0, sizeof(iatt));
+
+    retry2:
+        ret = glfs_resolve(fs, xl, file, &loc, &iatt, reval);
+        ESTALE_RETRY(ret, errno, reval, &loc, retry2);
+        if (ret) {
+            printf("Lookup failed on %s:%s\n", file, strerror(errno));
+            goto out;
+        }
+    }
+
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = _validate_directory(xattr_req, file);
+        if (ret)
+            goto out;
+    }
+    ret = syncop_getxattr(xl, &loc, &xattr_rsp, GF_AFR_HEAL_SBRAIN, xattr_req,
+                          NULL);
+    if (ret) {
+        op_errno = -ret;
+        printf("Healing %s failed:%s.\n", file, strerror(op_errno));
+        goto out;
+    }
+    ret = dict_get_str(xattr_rsp, "sh-fail-msg", &sh_fail_msg);
+    if (!ret) {
+        printf("Healing %s failed: %s.\n", file, sh_fail_msg);
+        ret = -1;
+        goto out;
+    }
+    printf("Healed %s.\n", file);
+    ret = 0;
 out:
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-        return ret;
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+    if (path1)
+        GF_FREE(path1);
+    if (path2)
+        GF_FREE(path2);
+    if (filename)
+        GF_FREE(filename);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-glfsh_heal_from_brick_type (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
-                            char *hostname, char *brickpath, xlator_t *client,
-                            dict_t *xattr_req, char *vgfid,
-                            uint64_t *num_entries)
+glfsh_heal_from_brick_type(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                           char *hostname, char *brickpath, xlator_t *client,
+                           dict_t *xattr_req, char *vgfid,
+                           num_entries_t *num_entries)
 {
-        fd_t     *fd        = NULL;
-        loc_t     dirloc    = {0};
-        int32_t   op_errno  = 0;
-        int       ret       = -1;
-
-        ret = glfsh_get_index_dir_loc (rootloc, client, &dirloc,
-                                       &op_errno, vgfid);
-        if (ret < 0) {
-                if (op_errno == ESTALE || op_errno == ENOENT)
-                        ret = 0;
-                else
-                        ret = -op_errno;
-                goto out;
-        }
-
-        ret = syncop_dirfd (client, &dirloc, &fd,
-                            GF_CLIENT_PID_GLFS_HEAL);
-        if (ret)
-                goto out;
-        ret = glfsh_crawl_directory (fs, top_subvol, rootloc, client,
-                                     fd, &dirloc, xattr_req, num_entries,
-                                     _gf_false);
-        if (fd)
-                fd_unref (fd);
+    fd_t *fd = NULL;
+    loc_t dirloc = {0};
+    int32_t op_errno = 0;
+    int ret = -1;
+
+    ret = glfsh_get_index_dir_loc(rootloc, client, &dirloc, &op_errno, vgfid);
+    if (ret < 0) {
+        if (op_errno == ESTALE || op_errno == ENOENT)
+            ret = 0;
+        else
+            ret = -op_errno;
+        goto out;
+    }
+
+    ret = syncop_dirfd(client, &dirloc, &fd, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret)
+        goto out;
+    ret = glfsh_crawl_directory(fs, top_subvol, rootloc, client, fd, &dirloc,
+                                xattr_req, num_entries, _gf_false);
+    if (fd)
+        fd_unref(fd);
 out:
-        loc_wipe (&dirloc);
-        return ret;
+    loc_wipe(&dirloc);
+    return ret;
 }
 
 int
-glfsh_heal_from_brick (glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+glfsh_heal_from_brick(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
                       char *hostname, char *brickpath, char *file)
 {
-        int       ret       = -1;
-        uint64_t   count     = 0, total = 0;
-        dict_t   *xattr_req = NULL;
-        xlator_t *client    = NULL;
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+    xlator_t *client = NULL;
+    num_entries_t num_entries = {
+        0,
+    };
+    num_entries_t total = {
+        0,
+    };
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op",
+                         GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret)
+        goto out;
+    client = _brick_path_to_client_xlator(top_subvol, hostname, brickpath);
+    if (!client) {
+        printf("\"%s:%s\"- No such brick available in the volume.\n", hostname,
+               brickpath);
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(xattr_req, "child-name", client->name);
+    if (ret)
+        goto out;
+    if (file)
+        ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file,
+                                         xattr_req);
+    else {
+        ret = glfsh_heal_from_brick_type(fs, top_subvol, rootloc, hostname,
+                                         brickpath, client, xattr_req,
+                                         GF_XATTROP_INDEX_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        num_entries.num_entries = 0;
+        if (ret == -ENOTCONN)
+            goto out;
 
-        xattr_req = dict_new();
-        if (!xattr_req)
-                goto out;
-        ret = dict_set_int32 (xattr_req, "heal-op",
-                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-        if (ret)
-                goto out;
-        client = _brick_path_to_client_xlator (top_subvol, hostname, brickpath);
-        if (!client) {
-                printf("\"%s:%s\"- No such brick available in the volume.\n",
-                       hostname, brickpath);
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str (xattr_req, "child-name", client->name);
-        if (ret)
-                goto out;
-        if (file)
-                ret = glfsh_heal_splitbrain_file (fs, top_subvol, rootloc, file,
-                                                  xattr_req);
-        else {
-                ret = glfsh_heal_from_brick_type (fs, top_subvol, rootloc,
-                                                  hostname, brickpath,
-                                                  client, xattr_req,
-                                                  GF_XATTROP_INDEX_GFID,
-                                                  &count);
-                total += count;
-                count = 0;
-                if (ret == -ENOTCONN)
-                        goto out;
-
-                ret = glfsh_heal_from_brick_type (fs, top_subvol, rootloc,
-                                                  hostname, brickpath,
-                                                  client, xattr_req,
-                                                  GF_XATTROP_DIRTY_GFID,
-                                                  &count);
-                total += count;
-                if (ret < 0)
-                        goto out;
-        }
+        ret = glfsh_heal_from_brick_type(fs, top_subvol, rootloc, hostname,
+                                         brickpath, client, xattr_req,
+                                         GF_XATTROP_DIRTY_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        if (ret < 0)
+            goto out;
+    }
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        if (!file)
-                glfsh_print_heal_op_status (ret, total,
-                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (!file)
+        glfsh_print_heal_op_status(ret, total.num_entries,
+                                   GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
 
-        return ret;
+    return ret;
 }
 
 int
-glfsh_heal_from_bigger_file_or_mtime (glfs_t *fs, xlator_t *top_subvol,
-                                      loc_t *rootloc, char *file,
-                                      gf_xl_afr_op_t heal_op)
+glfsh_heal_from_bigger_file_or_mtime(glfs_t *fs, xlator_t *top_subvol,
+                                     loc_t *rootloc, char *file,
+                                     gf_xl_afr_op_t heal_op)
 {
-
-        int ret = -1;
-        dict_t *xattr_req = NULL;
-
-        xattr_req = dict_new();
-        if (!xattr_req)
-                goto out;
-        ret = dict_set_int32 (xattr_req, "heal-op", heal_op);
-        if (ret)
-                goto out;
-        ret = glfsh_heal_splitbrain_file (fs, top_subvol, rootloc, file,
-                                         xattr_req);
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op", heal_op);
+    if (ret)
+        goto out;
+    ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file, xattr_req);
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        return ret;
+    if (xattr_req)
+        dict_unref(xattr_req);
+    return ret;
 }
 
 static void
-cleanup (glfs_t *fs)
+cleanup(glfs_t *fs)
 {
-        if (!fs)
-                return;
+    if (!fs)
+        return;
 #if 0
         /* glfs fini path is still racy and crashing the program. Since
-         * this program any way has to die, we are not gonna call fini
+         * this program any way has to die, we are not going to call fini
          * in the released versions. i.e. final builds. For all
          * internal testing lets enable this so that glfs_fini code
          * path becomes stable. */
@@ -1092,222 +1486,308 @@ cleanup (glfs_t *fs)
 #endif
 }
 
-
 glfsh_info_t glfsh_human_readable = {
-        .init = glfsh_init,
-        .print_brick_from_xl = glfsh_print_brick_from_xl,
-        .print_heal_op_status = glfsh_print_hr_heal_op_status,
-        .print_heal_status = glfsh_print_hr_heal_status,
-        .print_spb_status = glfsh_print_hr_spb_status,
-        .end = glfsh_end
-};
+    .init = glfsh_init,
+    .print_brick_from_xl = glfsh_print_brick_from_xl,
+    .print_heal_op_status = glfsh_print_hr_heal_op_status,
+    .print_heal_op_summary = glfsh_print_hr_heal_op_summary,
+    .print_heal_status = glfsh_print_hr_heal_status,
+    .print_spb_status = glfsh_print_hr_spb_status,
+    .end = glfsh_end};
+
+glfsh_info_t glfsh_no_print = {
+    .init = glfsh_init,
+    .print_brick_from_xl = glfsh_no_print_brick_from_xl,
+    .print_heal_op_status = glfsh_no_print_hr_heal_op_status,
+    .print_heal_status = glfsh_no_print_hr_status,
+    .print_spb_status = glfsh_no_print_hr_status,
+    .end = glfsh_end_op_granular_entry_heal};
 
 #if (HAVE_LIB_XML)
 glfsh_info_t glfsh_xml_output = {
-        .init = glfsh_xml_init,
-        .print_brick_from_xl = glfsh_print_xml_brick_from_xl,
-        .print_heal_op_status = glfsh_print_xml_heal_op_status,
-        .print_heal_status = glfsh_print_xml_file_status,
-        .print_spb_status = glfsh_print_xml_file_status,
-        .end = glfsh_xml_end
-};
+    .init = glfsh_xml_init,
+    .print_brick_from_xl = glfsh_print_xml_brick_from_xl,
+    .print_heal_op_status = glfsh_print_xml_heal_op_status,
+    .print_heal_op_summary = glfsh_print_xml_heal_op_summary,
+    .print_heal_status = glfsh_print_xml_file_status,
+    .print_spb_status = glfsh_print_xml_file_status,
+    .end = glfsh_xml_end};
 #endif
 
+static void
+parse_flags(int *argc, char **argv, int *flags)
+{
+    int i = 0;
+    char *opt = NULL;
+    int count = 0;
+
+    for (i = 0; i < *argc; i++) {
+        opt = strtail(argv[i], "--");
+        if (!opt)
+            continue;
+        if (strcmp(opt, "nolog") == 0) {
+            *flags |= MODE_NO_LOG;
+            count++;
+        } else if (strcmp(opt, "xml") == 0) {
+            *flags |= MODE_XML;
+            count++;
+        }
+    }
+    *argc = *argc - count;
+}
+
 int
-main (int argc, char **argv)
+main(int argc, char **argv)
 {
-        glfs_t    *fs = NULL;
-        int        ret = 0;
-        char      *volname = NULL;
-        xlator_t  *top_subvol = NULL;
-        loc_t     rootloc = {0};
-        char      logfilepath[PATH_MAX] = {0};
-        char      *hostname = NULL;
-        char      *path = NULL;
-        char      *file = NULL;
-        char      *op_errstr = NULL;
-        gf_xl_afr_op_t heal_op = -1;
-
-        if (argc < 2) {
-                printf (USAGE_STR, argv[0]);
-                ret = -1;
-                goto out;
+    glfs_t *fs = NULL;
+    int ret = 0;
+    char *volname = NULL;
+    xlator_t *top_subvol = NULL;
+    loc_t rootloc = {0};
+    char logfilepath[PATH_MAX] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *file = NULL;
+    char *op_errstr = NULL;
+    char *socket_filepath = NULL;
+    gf_xl_afr_op_t heal_op = -1;
+    gf_loglevel_t log_level = GF_LOG_INFO;
+    int flags = 0;
+
+    if (argc < 2) {
+        printf(USAGE_STR, argv[0]);
+        ret = -1;
+        goto out;
+    } else if (argc >= 4) {
+        if (!strcmp(argv[argc - 2], "glusterd-sock")) {
+            socket_filepath = argv[argc - 1];
+            argc = argc - 2;
         }
+    }
+    volname = argv[1];
+
+    parse_flags(&argc, argv, &flags);
+    if (flags & MODE_NO_LOG)
+        log_level = GF_LOG_NONE;
+    if (flags & MODE_XML)
+        is_xml = 1;
 
-        volname = argv[1];
-        switch (argc) {
+    switch (argc) {
         case 2:
-                heal_op = GF_SHD_OP_INDEX_SUMMARY;
-                break;
+            heal_op = GF_SHD_OP_INDEX_SUMMARY;
+            break;
         case 3:
-                if (!strcmp (argv[2], "split-brain-info")) {
-                        heal_op = GF_SHD_OP_SPLIT_BRAIN_FILES;
-                } else if (!strcmp (argv[2], "xml")) {
-                        heal_op = GF_SHD_OP_INDEX_SUMMARY;
-                        is_xml = 1;
-                } else {
-                        printf (USAGE_STR, argv[0]);
-                        ret = -1;
-                        goto out;
-                }
-                break;
+            if (!strcmp(argv[2], "split-brain-info")) {
+                heal_op = GF_SHD_OP_SPLIT_BRAIN_FILES;
+            } else if (!strcmp(argv[2], "granular-entry-heal-op")) {
+                heal_op = GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE;
+            } else if (!strcmp(argv[2], "info-summary")) {
+                heal_op = GF_SHD_OP_HEAL_SUMMARY;
+            } else {
+                printf(USAGE_STR, argv[0]);
+                ret = -1;
+                goto out;
+            }
+            break;
         case 4:
-                if ((!strcmp (argv[2], "split-brain-info"))
-                                && (!strcmp (argv[3], "xml"))) {
-                        heal_op = GF_SHD_OP_SPLIT_BRAIN_FILES;
-                        is_xml = 1;
-                } else if (!strcmp (argv[2], "bigger-file")) {
-                        heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE;
-                        file = argv[3];
-                } else if (!strcmp (argv[2], "latest-mtime")) {
-                        heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME;
-                        file = argv[3];
-                } else if (!strcmp (argv[2], "source-brick")) {
-                        heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
-                        hostname = strtok (argv[3], ":");
-                        path = strtok (NULL, ":");
-                } else {
-                        printf (USAGE_STR, argv[0]);
-                        ret = -1;
-                        goto out;
-                }
-                break;
+            if (!strcmp(argv[2], "bigger-file")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE;
+                file = argv[3];
+            } else if (!strcmp(argv[2], "latest-mtime")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME;
+                file = argv[3];
+            } else if (!strcmp(argv[2], "source-brick")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
+                hostname = strtok(argv[3], ":");
+                path = strtok(NULL, ":");
+            } else {
+                printf(USAGE_STR, argv[0]);
+                ret = -1;
+                goto out;
+            }
+            break;
         case 5:
-                if (!strcmp (argv[2], "source-brick")) {
-                        heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
-                        hostname = strtok (argv[3], ":");
-                        path = strtok (NULL, ":");
-                        file = argv[4];
-                } else {
-                        printf (USAGE_STR, argv[0]);
-                        ret = -1;
-                        goto out;
-                }
-                break;
-        default:
-                printf (USAGE_STR, argv[0]);
+            if (!strcmp(argv[2], "source-brick")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
+                hostname = strtok(argv[3], ":");
+                path = strtok(NULL, ":");
+                file = argv[4];
+            } else {
+                printf(USAGE_STR, argv[0]);
                 ret = -1;
                 goto out;
-        }
+            }
+            break;
+        default:
+            printf(USAGE_STR, argv[0]);
+            ret = -1;
+            goto out;
+    }
 
-        glfsh_output = &glfsh_human_readable;
-        if (is_xml) {
+    glfsh_output = &glfsh_human_readable;
+    if (is_xml) {
 #if (HAVE_LIB_XML)
-                glfsh_output = &glfsh_xml_output;
+        if ((heal_op == GF_SHD_OP_INDEX_SUMMARY) ||
+            (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES) ||
+            (heal_op == GF_SHD_OP_HEAL_SUMMARY)) {
+            glfsh_output = &glfsh_xml_output;
+        } else {
+            printf(USAGE_STR, argv[0]);
+            ret = -1;
+            goto out;
+        }
 #else
-                /*No point doing anything, just fail the command*/
-                exit (EXIT_FAILURE);
+        /*No point doing anything, just fail the command*/
+        exit(EXIT_FAILURE);
 #endif
-
-        }
-
-        ret = glfsh_output->init ();
-        if (ret)
-                exit (EXIT_FAILURE);
-
-        fs = glfs_new (volname);
-        if (!fs) {
-                ret = -errno;
-                gf_asprintf (&op_errstr, "Not able to initialize volume '%s'",
-                             volname);
-                goto out;
-        }
-
-        if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
-                fs->ctx->secure_mgmt = 1;
-        }
-
-        ret = glfs_set_volfile_server (fs, "unix", DEFAULT_GLUSTERD_SOCKFILE, 0);
-        if (ret) {
-                ret = -errno;
-                gf_asprintf (&op_errstr, "Setting the volfile server failed, "
-                             "%s", strerror (errno));
-                goto out;
-        }
-        snprintf (logfilepath, sizeof (logfilepath),
-                  DEFAULT_HEAL_LOG_FILE_DIRECTORY"/glfsheal-%s.log", volname);
-        ret = glfs_set_logging(fs, logfilepath, GF_LOG_INFO);
-        if (ret < 0) {
-                ret = -errno;
-                gf_asprintf (&op_errstr, "Failed to set the log file path, "
-                             "%s", strerror (errno));
-                goto out;
-        }
-
-        ret = glfs_init (fs);
-        if (ret < 0) {
-                ret = -errno;
-                if (errno == ENOENT) {
-                        gf_asprintf (&op_errstr, "Volume %s does not exist",
-                                     volname);
-                } else {
-                        gf_asprintf (&op_errstr, "%s: Not able to fetch "
-                                     "volfile from glusterd", volname);
-                }
-                goto out;
-        }
-
-        top_subvol = glfs_active_subvol (fs);
-        if (!top_subvol) {
-                ret = -errno;
-                if (errno == ENOTCONN) {
-                        gf_asprintf (&op_errstr, "Volume %s is not started "
-                                                 "(Or) All the bricks are not "
-                                                 "running.", volname);
-                }
-                else {
-                        gf_asprintf (&op_errstr, "%s: Not able to mount the "
-                                             "volume, %s", volname,
-                                             strerror (errno));
-                }
-                goto out;
+    }
+
+    if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        glfsh_output = &glfsh_no_print;
+
+    ret = glfsh_output->init();
+    if (ret)
+        exit(EXIT_FAILURE);
+
+    fs = glfs_new(volname);
+    if (!fs) {
+        ret = -errno;
+        gf_asprintf(&op_errstr, "Not able to initialize volume '%s'", volname);
+        goto out;
+    }
+
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+    if (socket_filepath != NULL) {
+        ret = glfs_set_volfile_server(fs, "unix", socket_filepath, 0);
+    } else {
+        ret = glfs_set_volfile_server(fs, "unix", DEFAULT_GLUSTERD_SOCKFILE, 0);
+    }
+    if (ret) {
+        ret = -errno;
+        gf_asprintf(&op_errstr,
+                    "Setting the volfile server failed, "
+                    "%s",
+                    strerror(errno));
+        goto out;
+    }
+
+    ret = glfsh_set_heal_options(fs, heal_op);
+    if (ret) {
+        printf("Setting xlator heal options failed, %s\n", strerror(errno));
+        goto out;
+    }
+    snprintf(logfilepath, sizeof(logfilepath),
+             DEFAULT_HEAL_LOG_FILE_DIRECTORY "/glfsheal-%s.log", volname);
+    ret = glfs_set_logging(fs, logfilepath, log_level);
+    if (ret < 0) {
+        ret = -errno;
+        gf_asprintf(&op_errstr,
+                    "Failed to set the log file path, "
+                    "%s",
+                    strerror(errno));
+        goto out;
+    }
+
+    ret = glfs_setfspid(fs, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret) {
+        printf("Setting client pid failed, %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = glfs_init(fs);
+    if (ret < 0) {
+        ret = -errno;
+        if (errno == ENOENT) {
+            gf_asprintf(&op_errstr, "Volume %s does not exist", volname);
+        } else {
+            gf_asprintf(&op_errstr,
+                        "%s: Not able to fetch "
+                        "volfile from glusterd",
+                        volname);
         }
-
-        ret = glfsh_validate_volume (top_subvol, heal_op);
-        if (ret < 0) {
-                ret = -EINVAL;
-                gf_asprintf (&op_errstr, "Volume %s is not of type %s", volname,
-                                     (heal_op == GF_SHD_OP_INDEX_SUMMARY) ?
-                                     "replicate/disperse":"replicate");
-                goto out;
+        goto out;
+    }
+
+    top_subvol = glfs_active_subvol(fs);
+    if (!top_subvol) {
+        ret = -errno;
+        if (errno == ENOTCONN) {
+            gf_asprintf(&op_errstr,
+                        "Volume %s is not started "
+                        "(Or) All the bricks are not "
+                        "running.",
+                        volname);
+        } else {
+            gf_asprintf(&op_errstr,
+                        "%s: Not able to mount the "
+                        "volume, %s",
+                        volname, strerror(errno));
         }
-        rootloc.inode = inode_ref (top_subvol->itable->root);
-        glfs_loc_touchup (&rootloc);
-
-        switch (heal_op) {
+        goto out;
+    }
+
+    char *var_str = (heal_op == GF_SHD_OP_INDEX_SUMMARY ||
+                     heal_op == GF_SHD_OP_HEAL_SUMMARY)
+                        ? "replicate/disperse"
+                        : "replicate";
+
+    ret = glfsh_validate_volume(top_subvol, heal_op);
+    if (ret < 0) {
+        ret = -EINVAL;
+        gf_asprintf(&op_errstr,
+                    "This command is supported "
+                    "for only volumes of %s type. Volume %s "
+                    "is not of type %s",
+                    var_str, volname, var_str);
+        goto out;
+    }
+    rootloc.inode = inode_ref(top_subvol->itable->root);
+    ret = glfs_loc_touchup(&rootloc);
+    if (ret < 0) {
+        ret = -errno;
+        goto out;
+    }
+
+    switch (heal_op) {
         case GF_SHD_OP_INDEX_SUMMARY:
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-                ret = glfsh_gather_heal_info (fs, top_subvol, &rootloc,
-                                              heal_op);
-                break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_HEAL_SUMMARY:
+            ret = glfsh_gather_heal_info(fs, top_subvol, &rootloc, heal_op);
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
         case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-                ret = glfsh_heal_from_bigger_file_or_mtime (fs, top_subvol,
-                                                   &rootloc, file, heal_op);
-                        break;
+            ret = glfsh_heal_from_bigger_file_or_mtime(fs, top_subvol, &rootloc,
+                                                       file, heal_op);
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = glfsh_heal_from_brick (fs, top_subvol, &rootloc,
-                                             hostname, path, file);
-                break;
+            ret = glfsh_heal_from_brick(fs, top_subvol, &rootloc, hostname,
+                                        path, file);
+            break;
         default:
-                ret = -EINVAL;
-                break;
-        }
-
-        glfsh_output->end (ret, NULL);
-        loc_wipe (&rootloc);
-        glfs_subvol_done (fs, top_subvol);
-        cleanup (fs);
-
-        return ret;
+            ret = -EINVAL;
+            break;
+    }
+
+    glfsh_output->end(ret, NULL);
+    if (ret < 0)
+        ret = -ret;
+    loc_wipe(&rootloc);
+    glfs_subvol_done(fs, top_subvol);
+    cleanup(fs);
+
+    return ret;
 out:
-        if (fs && top_subvol)
-                glfs_subvol_done (fs, top_subvol);
-        loc_wipe (&rootloc);
-        cleanup (fs);
-        if (glfsh_output)
-                glfsh_output->end (ret, op_errstr);
-        if (op_errstr)
-                GF_FREE (op_errstr);
-        return ret;
+    if (fs && top_subvol)
+        glfs_subvol_done(fs, top_subvol);
+    loc_wipe(&rootloc);
+    cleanup(fs);
+    if (glfsh_output)
+        glfsh_output->end(ret, op_errstr);
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    return ret;
 }
diff --git a/libgfchangelog.pc.in b/libgfchangelog.pc.in
index 91c85e6f1f8..79eac2ad2d3 100644
--- a/libgfchangelog.pc.in
+++ b/libgfchangelog.pc.in
@@ -9,4 +9,4 @@ Description: GlusterFS Changelog Consumer Library
 Version: @LIBGFCHANGELOG_VERSION@
 Requires: @PKGCONFIG_UUID@
 Libs: -L${libdir} -lgfchangelog -lglusterfs
-Cflags: -I${includedir}/glusterfs/gfchangelog -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64
+Cflags: -I${includedir} -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
diff --git a/libgfdb.pc.in b/libgfdb.pc.in
deleted file mode 100644
index 945685cde98..00000000000
--- a/libgfdb.pc.in
+++ /dev/null
@@ -1,12 +0,0 @@
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-
-Name: libgfdb
-Description: GlusterFS Database Library
-Version: @LIBGFDB_VERSION@
-Libs: -L${libdir} -lgfchangedb -lglusterfs
-Cflags: -I${includedir}/glusterfs/gfdb
-Requires: sqlite3 @PKGCONFIG_UUID@
diff --git a/xlators/experimental/dht2/dht2-client/Makefile.am b/libglusterd/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/dht2/dht2-client/Makefile.am
+++ b/libglusterd/Makefile.am
diff --git a/libglusterd/src/Makefile.am b/libglusterd/src/Makefile.am
new file mode 100644
index 00000000000..684d2bac96b
--- /dev/null
+++ b/libglusterd/src/Makefile.am
@@ -0,0 +1,31 @@
+libglusterd_la_CFLAGS = $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+	-DDATADIR=\"$(localstatedir)\"
+
+libglusterd_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-DXLATORPARENTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)\" \
+	-DXXH_NAMESPACE=GF_ -D__USE_LARGEFILE64 \
+	-I$(CONTRIBDIR)/rbtree \
+	-I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \
+	-DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel \
+	-I$(CONTRIBDIR)/xxhash \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src/
+
+libglusterd_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS)
+libglusterd_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) $(GF_LDFLAGS) \
+	-export-symbols $(top_srcdir)/libglusterd/src/libglusterd.sym
+
+lib_LTLIBRARIES = libglusterd.la
+
+libglusterd_la_SOURCES = gd-common-utils.c
+
+libglusterd_la_HEADERS = gd-common-utils.h
+
+libglusterd_ladir = $(includedir)/glusterfs
+
+noinst_HEADERS = gd-common-utils.h
+
+EXTRA_DIST = libglusterd.sym
+
+CLEANFILES =
diff --git a/libglusterd/src/gd-common-utils.c b/libglusterd/src/gd-common-utils.c
new file mode 100644
index 00000000000..243fab215e6
--- /dev/null
+++ b/libglusterd/src/gd-common-utils.c
@@ -0,0 +1,78 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "gd-common-utils.h"
+#include "cli1-xdr.h"
+
+int
+get_vol_type(int type, int dist_count, int brick_count)
+{
+    if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
+        (dist_count < brick_count))
+        type = type + GF_CLUSTER_TYPE_MAX - 1;
+
+    return type;
+}
+
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val)
+{
+    switch (mem_num) {
+        case 0:
+            return (sts_val->node);
+        case 1:
+            return (sts_val->master);
+        case 2:
+            return (sts_val->brick);
+        case 3:
+            return (sts_val->slave_user);
+        case 4:
+            return (sts_val->slave);
+        case 5:
+            return (sts_val->slave_node);
+        case 6:
+            return (sts_val->worker_status);
+        case 7:
+            return (sts_val->crawl_status);
+        case 8:
+            return (sts_val->last_synced);
+        case 9:
+            return (sts_val->entry);
+        case 10:
+            return (sts_val->data);
+        case 11:
+            return (sts_val->meta);
+        case 12:
+            return (sts_val->failures);
+        case 13:
+            return (sts_val->checkpoint_time);
+        case 14:
+            return (sts_val->checkpoint_completed);
+        case 15:
+            return (sts_val->checkpoint_completion_time);
+        case 16:
+            return (sts_val->brick_host_uuid);
+        case 17:
+            return (sts_val->last_synced_utc);
+        case 18:
+            return (sts_val->checkpoint_time_utc);
+        case 19:
+            return (sts_val->checkpoint_completion_time_utc);
+        case 20:
+            return (sts_val->slavekey);
+        case 21:
+            return (sts_val->session_slave);
+        default:
+            goto out;
+    }
+
+out:
+    return NULL;
+}
diff --git a/libglusterd/src/gd-common-utils.h b/libglusterd/src/gd-common-utils.h
new file mode 100644
index 00000000000..b9bb4f956fe
--- /dev/null
+++ b/libglusterd/src/gd-common-utils.h
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GD_COMMON_UTILS_H
+#define _GD_COMMON_UTILS_H
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stddef.h>
+
+#include "protocol-common.h"
+#include "rpcsvc.h"
+
+int
+get_vol_type(int type, int dist_count, int brick_count);
+
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val);
+
+#endif /* _GD_COMMON_UTILS_H */
diff --git a/libglusterd/src/libglusterd.sym b/libglusterd/src/libglusterd.sym
new file mode 100644
index 00000000000..45969a87c12
--- /dev/null
+++ b/libglusterd/src/libglusterd.sym
@@ -0,0 +1,2 @@
+get_vol_type
+get_struct_variable
diff --git a/libglusterfs/Makefile.am b/libglusterfs/Makefile.am
index 8e5a4a0ccbf..d471a3f9243 100644
--- a/libglusterfs/Makefile.am
+++ b/libglusterfs/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = src src/gfdb
+SUBDIRS = src
 
 CLEANFILES = 
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 20163da769c..385e8ef4600 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -6,13 +6,16 @@ libglusterfs_la_CFLAGS = $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
 libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
 	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
 	-DXLATORPARENTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)\" \
-	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
-	-I$(top_srcdir)/rpc/rpc-lib/src/ -I$(CONTRIBDIR)/rbtree \
+	-DXXH_NAMESPACE=GF_ -D__USE_LARGEFILE64 \
+	-I$(CONTRIBDIR)/rbtree \
 	-I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \
-	-DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel
+	-DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel \
+	-I$(CONTRIBDIR)/xxhash
 
-libglusterfs_la_LIBADD = @LEXLIB@ $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS)
-libglusterfs_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION)
+libglusterfs_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS) $(LIB_DL) \
+	$(URCU_LIBS) $(URCU_CDS_LIBS)
+libglusterfs_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) $(GF_LDFLAGS) \
+	-export-symbols $(top_srcdir)/libglusterfs/src/libglusterfs.sym
 
 lib_LTLIBRARIES = libglusterfs.la
 libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
@@ -28,32 +31,42 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
 	circ-buff.c event-history.c gidcache.c ctx.c client_t.c event-poll.c \
 	event-epoll.c syncop-utils.c cluster-syncop.c refcount.c \
 	$(CONTRIBDIR)/libgen/basename_r.c \
-	$(CONTRIBDIR)/libgen/dirname_r.c $(CONTRIBDIR)/stdlib/gf_mkostemp.c \
+	$(CONTRIBDIR)/libgen/dirname_r.c \
 	strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \
 	$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
-	$(CONTRIBDIR)/timer-wheel/find_last_bit.c tw.c default-args.c locking.c \
-	compound-fop-utils.c throttle-tbf.c
+	$(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \
+	$(CONTRIBDIR)/xxhash/xxhash.c \
+	throttle-tbf.c monitoring.c async.c
 
 nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c
 nodist_libglusterfs_la_HEADERS = y.tab.h
 
 BUILT_SOURCES = graph.lex.c defaults.c eventtypes.h
 
-libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \
-	dict.h glusterfs.h hashfn.h timespec.h logging.h xlator.h \
-	stack.h timer.h list.h inode.h call-stub.h compat.h fd.h \
-	revision.h compat-errno.h event.h mem-pool.h byte-order.h \
-	gf-dirent.h locking.h syscall.h iobuf.h globals.h statedump.h \
-	checksum.h daemon.h store.h rbthash.h iatt.h latency.h \
-	mem-types.h syncop.h cluster-syncop.h graph-utils.h trie.h \
-	refcount.h run.h options.h lkowner.h fd-lk.h circ-buff.h \
-	event-history.h gidcache.h client_t.h glusterfs-acl.h \
-	glfs-message-id.h template-component-messages.h strfd.h \
-	syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \
-	lvm-defaults.h quota-common-utils.h rot-buffs.h \
-	compat-uuid.h upcall-utils.h throttle-tbf.h events.h\
-	compound-fop-utils.h
+libglusterfs_la_HEADERS = glusterfs/common-utils.h glusterfs/defaults.h \
+	glusterfs/default-args.h glusterfs/dict.h glusterfs/glusterfs.h \
+	glusterfs/hashfn.h glusterfs/timespec.h glusterfs/logging.h \
+	glusterfs/xlator.h glusterfs/stack.h glusterfs/timer.h glusterfs/list.h \
+	glusterfs/inode.h glusterfs/call-stub.h glusterfs/compat.h glusterfs/fd.h \
+	glusterfs/revision.h glusterfs/compat-errno.h glusterfs/gf-event.h \
+	glusterfs/mem-pool.h glusterfs/byte-order.h glusterfs/gf-dirent.h \
+	glusterfs/locking.h glusterfs/syscall.h glusterfs/iobuf.h \
+	glusterfs/globals.h glusterfs/statedump.h glusterfs/checksum.h \
+	glusterfs/daemon.h glusterfs/store.h glusterfs/rbthash.h glusterfs/iatt.h \
+	glusterfs/latency.h glusterfs/mem-types.h glusterfs/syncop.h \
+	glusterfs/cluster-syncop.h glusterfs/graph-utils.h glusterfs/trie.h \
+	glusterfs/refcount.h glusterfs/run.h glusterfs/options.h \
+	glusterfs/lkowner.h glusterfs/fd-lk.h glusterfs/circ-buff.h \
+	glusterfs/event-history.h glusterfs/gidcache.h glusterfs/client_t.h \
+	glusterfs/glusterfs-acl.h glusterfs/glfs-message-id.h \
+	glusterfs/template-component-messages.h glusterfs/strfd.h \
+	glusterfs/syncop-utils.h glusterfs/parse-utils.h \
+	glusterfs/libglusterfs-messages.h glusterfs/lvm-defaults.h \
+	glusterfs/quota-common-utils.h glusterfs/rot-buffs.h \
+	glusterfs/compat-uuid.h glusterfs/upcall-utils.h glusterfs/throttle-tbf.h \
+	glusterfs/events.h glusterfs/atomic.h glusterfs/monitoring.h \
+	glusterfs/async.h glusterfs/glusterfs-fops.h
 
 libglusterfs_ladir = $(includedir)/glusterfs
 
@@ -61,18 +74,12 @@ noinst_HEADERS = unittest/unittest.h \
 	$(CONTRIBDIR)/rbtree/rb.h \
 	$(CONTRIBDIR)/mount/mntent_compat.h \
 	$(CONTRIBDIR)/libexecinfo/execinfo_compat.h \
-	$(CONTRIBDIR)/timer-wheel/timer-wheel.h
-
-if !HAVE_LIBUUID
-# FIXME: unbundle libuuid, see compat-uuid.h.
-libglusterfs_la_SOURCES += $(CONTRIBDIR)/uuid/clear.c \
-	$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
-	$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
-	$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
-	$(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c \
-	$(CONTRIBDIR)/uuid/unpack.c
-endif
-
+	$(CONTRIBDIR)/timer-wheel/timer-wheel.h \
+	$(CONTRIBDIR)/xxhash/xxhash.h \
+	$(CONTRIBDIR)/userspace-rcu/wfcqueue.h \
+	$(CONTRIBDIR)/userspace-rcu/wfstack.h \
+	$(CONTRIBDIR)/userspace-rcu/static-wfcqueue.h \
+	$(CONTRIBDIR)/userspace-rcu/static-wfstack.h
 
 eventtypes.h: $(top_srcdir)/events/eventskeygen.py
 	$(PYTHON) $(top_srcdir)/events/eventskeygen.py C_HEADER
@@ -83,7 +90,7 @@ endif
 
 libgfchangelog_HEADERS = changelog.h
 
-EXTRA_DIST = graph.l graph.y defaults-tmpl.c
+EXTRA_DIST = graph.l graph.y defaults-tmpl.c libglusterfs.sym
 
 graph.lex.c: graph.l y.tab.h
 	$(LEX) -Pgraphyy -t $(srcdir)/graph.l > $@
@@ -95,7 +102,8 @@ y.tab.h: graph.y
 defaults.c: defaults-tmpl.c generator.py gen-defaults.py
 	$(PYTHON) $(srcdir)/gen-defaults.py $(srcdir)/defaults-tmpl.c > $@
 
-CLEANFILES = $(nodist_libglusterfs_la_SOURCES) $(nodist_libglusterfs_la_HEADERS)
+CLEANFILES = $(nodist_libglusterfs_la_SOURCES) \
+	$(nodist_libglusterfs_la_HEADERS) *.pyc
 
 if UNITTEST
 CLEANFILES += *.gcda *.gcno *_xunit.xml
diff --git a/libglusterfs/src/async.c b/libglusterfs/src/async.c
new file mode 100644
index 00000000000..1d6cfa374b6
--- /dev/null
+++ b/libglusterfs/src/async.c
@@ -0,0 +1,720 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc <https://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* To implement an efficient thread pool with minimum contention we have used
+ * the following ideas:
+ *
+ *    - The queue of jobs has been implemented using a Wait-Free queue provided
+ *      by the userspace-rcu library. This queue requires a mutex when multiple
+ *      consumers can be extracting items from it concurrently, but the locked
+ *      region is very small, which minimizes the chances of contention. To
+ *      further minimize contention, the number of active worker threads that
+ *      are accessing the queue is dynamically adjusted so that we always have
+ *      the minimum required amount of workers contending for the queue. Adding
+ *      new items can be done with a single atomic operation, without locks.
+ *
+ *    - All queue management operations, like creating more threads, enabling
+ *      sleeping ones, etc. are done by a single thread. This makes it possible
+ *      to manage all scaling related information and workers lists without
+ *      locks. This functionality is implemented as a role that can be assigned
+ *      to any of the worker threads, which avoids that some lengthy operations
+ *      could interfere with this task.
+ *
+ *    - Management is based on signals. We used signals for management tasks to
+ *      avoid multiple system calls for each request (with signals we can wait
+ *      for multiple events and get some additional data for each request in a
+ *      single call, instead of first polling and then reading).
+ *
+ * TODO: There are some other changes that can take advantage of this new
+ *       thread pool.
+ *
+ *          - Use this thread pool as the core threading model for synctasks. I
+ *            think this would improve synctask performance because I think we
+ *            currently have some contention there for some workloads.
+ *
+ *          - Implement a per thread timer that will allow adding and removing
+ *            timers without using mutexes.
+ *
+ *          - Integrate with userspace-rcu library in QSBR mode, allowing
+ *            other portions of code to be implemented using RCU-based
+ *            structures with a extremely fast read side without contention.
+ *
+ *          - Integrate I/O into the thread pool so that the thread pool is
+ *            able to efficiently manage all loads and scale dynamically. This
+ *            could make it possible to minimize context switching when serving
+ *            requests from fuse or network.
+ *
+ *          - Dynamically scale the number of workers based on system load.
+ *            This will make it possible to reduce contention when system is
+ *            heavily loaded, improving performance under these circumstances
+ *            (or minimizing performance loss). This will also make it possible
+ *            that gluster can coexist with other processes that also consume
+ *            CPU, with minimal interference from each other.
+ */
+
+#include <unistd.h>
+#include <pthread.h>
+#include <errno.h>
+
+#include "glusterfs/list.h"
+#include "glusterfs/mem-types.h"
+#include "glusterfs/async.h"
+
+/* These macros wrap a simple system/library call to check the returned error
+ * and log a message in case of failure. */
+#define GF_ASYNC_CHECK(_func, _args...)                                        \
+    ({                                                                         \
+        int32_t __async_error = -_func(_args);                                 \
+        if (caa_unlikely(__async_error != 0)) {                                \
+            gf_async_error(__async_error, #_func "() failed.");                \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+#define GF_ASYNC_CHECK_ERRNO(_func, _args...)                                  \
+    ({                                                                         \
+        int32_t __async_error = _func(_args);                                  \
+        if (caa_unlikely(__async_error < 0)) {                                 \
+            __async_error = -errno;                                            \
+            gf_async_error(__async_error, #_func "() failed.");                \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+/* These macros are used when, based on POSIX documentation, the function
+ * should never fail under the conditions we are using it. So any unexpected
+ * error will be handled as a fatal event. It probably means a critical bug
+ * or memory corruption. In both cases we consider that stopping the process
+ * is safer (otherwise it could cause more corruption with unknown effects
+ * that could be worse). */
+#define GF_ASYNC_CANTFAIL(_func, _args...)                                     \
+    do {                                                                       \
+        int32_t __async_error = -_func(_args);                                 \
+        if (caa_unlikely(__async_error != 0)) {                                \
+            gf_async_fatal(__async_error, #_func "() failed");                 \
+        }                                                                      \
+    } while (0)
+
+#define GF_ASYNC_CANTFAIL_ERRNO(_func, _args...)                               \
+    ({                                                                         \
+        int32_t __async_error = _func(_args);                                  \
+        if (caa_unlikely(__async_error < 0)) {                                 \
+            __async_error = -errno;                                            \
+            gf_async_fatal(__async_error, #_func "() failed");                 \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+/* TODO: for now we allocate a static array of workers. There's an issue if we
+ *       try to use dynamic memory since these workers are initialized very
+ *       early in the process startup and it seems that sometimes not all is
+ *       ready to use dynamic memory. */
+static gf_async_worker_t gf_async_workers[GF_ASYNC_MAX_THREADS];
+
+/* This is the only global variable needed to manage the entire framework. */
+gf_async_control_t gf_async_ctrl = {};
+
+static __thread gf_async_worker_t *gf_async_current_worker = NULL;
+
+/* The main function of the worker threads. */
+static void *
+gf_async_worker(void *arg);
+
+static void
+gf_async_sync_init(void)
+{
+    GF_ASYNC_CANTFAIL(pthread_barrier_init, &gf_async_ctrl.sync, NULL, 2);
+}
+
+static void
+gf_async_sync_now(void)
+{
+    int32_t ret;
+
+    ret = pthread_barrier_wait(&gf_async_ctrl.sync);
+    if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
+        GF_ASYNC_CANTFAIL(pthread_barrier_destroy, &gf_async_ctrl.sync);
+        ret = 0;
+    }
+    if (caa_unlikely(ret != 0)) {
+        gf_async_fatal(-ret, "pthread_barrier_wait() failed");
+    }
+}
+
+static void
+gf_async_sigmask_empty(sigset_t *mask)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigemptyset, mask);
+}
+
+static void
+gf_async_sigmask_add(sigset_t *mask, int32_t signal)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigaddset, mask, signal);
+}
+
+static void
+gf_async_sigmask_set(int32_t mode, sigset_t *mask, sigset_t *old)
+{
+    GF_ASYNC_CANTFAIL(pthread_sigmask, mode, mask, old);
+}
+
+static void
+gf_async_sigaction(int32_t signum, const struct sigaction *action,
+                   struct sigaction *old)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigaction, signum, action, old);
+}
+
+static int32_t
+gf_async_sigwait(sigset_t *set)
+{
+    int32_t ret, signum;
+
+    do {
+        ret = sigwait(set, &signum);
+    } while (caa_unlikely((ret < 0) && (errno == EINTR)));
+
+    if (caa_unlikely(ret < 0)) {
+        ret = -errno;
+        gf_async_fatal(ret, "sigwait() failed");
+    }
+
+    return signum;
+}
+
+static int32_t
+gf_async_sigtimedwait(sigset_t *set, struct timespec *timeout)
+{
+    int32_t ret;
+
+    do {
+        ret = sigtimedwait(set, NULL, timeout);
+    } while (caa_unlikely((ret < 0) && (errno == EINTR)));
+    if (caa_unlikely(ret < 0)) {
+        ret = -errno;
+        /* EAGAIN means that the timeout has expired, so we allow this error.
+         * Any other error shouldn't happen. */
+        if (caa_unlikely(ret != -EAGAIN)) {
+            gf_async_fatal(ret, "sigtimedwait() failed");
+        }
+        ret = 0;
+    }
+
+    return ret;
+}
+
+static void
+gf_async_sigbroadcast(int32_t signum)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(kill, gf_async_ctrl.pid, signum);
+}
+
+static void
+gf_async_signal_handler(int32_t signum)
+{
+    /* We should never handle a signal in this function. */
+    gf_async_fatal(-EBUSY,
+                   "Unexpected processing of signal %d through a handler.",
+                   signum);
+}
+
+static void
+gf_async_signal_setup(void)
+{
+    struct sigaction action;
+
+    /* We configure all related signals so that we can detect threads using an
+     * invalid signal mask that doesn't block our critical signal. */
+    memset(&action, 0, sizeof(action));
+    action.sa_handler = gf_async_signal_handler;
+
+    gf_async_sigaction(GF_ASYNC_SIGCTRL, &action, &gf_async_ctrl.handler_ctrl);
+
+    gf_async_sigaction(GF_ASYNC_SIGQUEUE, &action,
+                       &gf_async_ctrl.handler_queue);
+}
+
+static void
+gf_async_signal_restore(void)
+{
+    /* Handlers we have previously changed are restored back to their original
+     * value. */
+
+    if (gf_async_ctrl.handler_ctrl.sa_handler != gf_async_signal_handler) {
+        gf_async_sigaction(GF_ASYNC_SIGCTRL, &gf_async_ctrl.handler_ctrl, NULL);
+    }
+
+    if (gf_async_ctrl.handler_queue.sa_handler != gf_async_signal_handler) {
+        gf_async_sigaction(GF_ASYNC_SIGQUEUE, &gf_async_ctrl.handler_queue,
+                           NULL);
+    }
+}
+
+static void
+gf_async_signal_flush(void)
+{
+    struct timespec delay;
+
+    delay.tv_sec = 0;
+    delay.tv_nsec = 0;
+
+    /* We read all pending signals so that they don't trigger once the signal
+     * mask of some thread is changed. */
+    while (gf_async_sigtimedwait(&gf_async_ctrl.sigmask_ctrl, &delay) > 0) {
+    }
+    while (gf_async_sigtimedwait(&gf_async_ctrl.sigmask_queue, &delay) > 0) {
+    }
+}
+
+static int32_t
+gf_async_thread_create(pthread_t *thread, int32_t id, void *data)
+{
+    int32_t ret;
+
+    ret = gf_thread_create(thread, NULL, gf_async_worker, data,
+                           GF_ASYNC_THREAD_NAME "%u", id);
+    if (caa_unlikely(ret < 0)) {
+        /* TODO: gf_thread_create() should return a more specific error
+         *       code. */
+        return -ENOMEM;
+    }
+
+    return 0;
+}
+
+static void
+gf_async_thread_wait(pthread_t thread)
+{
+    /* TODO: this is a blocking call executed inside one of the workers of the
+     *       thread pool. This is bad, but this is only executed once we have
+     *       received a notification from the thread that it's terminating, so
+     *       this should return almost immediately. However, to be more robust
+     *       it would be better to use pthread_timedjoin_np() (or even a call
+     *       to pthread_tryjoin_np() followed by a delayed recheck if it
+     *       fails), but they are not portable. We should see how to do this
+     *       in other platforms. */
+    GF_ASYNC_CANTFAIL(pthread_join, thread, NULL);
+}
+
+static int32_t
+gf_async_worker_create(void)
+{
+    struct cds_wfs_node *node;
+    gf_async_worker_t *worker;
+    uint32_t counts, running, max;
+    int32_t ret;
+
+    node = __cds_wfs_pop_blocking(&gf_async_ctrl.available);
+    if (caa_unlikely(node == NULL)) {
+        /* There are no more available workers. We have all threads running. */
+        return 1;
+    }
+    cds_wfs_node_init(node);
+
+    ret = 1;
+
+    counts = uatomic_read(&gf_async_ctrl.counts);
+    max = uatomic_read(&gf_async_ctrl.max_threads);
+    running = GF_ASYNC_COUNT_RUNNING(counts);
+    if (running < max) {
+        uatomic_add(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(1, 0));
+
+        worker = caa_container_of(node, gf_async_worker_t, stack);
+
+        ret = gf_async_thread_create(&worker->thread, worker->id, worker);
+        if (caa_likely(ret >= 0)) {
+            return 0;
+        }
+
+        uatomic_add(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(-1, 0));
+    }
+
+    cds_wfs_push(&gf_async_ctrl.available, node);
+
+    return ret;
+}
+
+static void
+gf_async_worker_enable(void)
+{
+    /* This will wake one of the spare workers. If all workers are busy now,
+     * the signal will be queued so that the first one that completes its
+     * work will become the leader. */
+    gf_async_sigbroadcast(GF_ASYNC_SIGCTRL);
+
+    /* We have consumed a spare worker. We create another one for future
+     * needs. */
+    gf_async_worker_create();
+}
+
+static void
+gf_async_worker_wait(void)
+{
+    int32_t signum;
+
+    signum = gf_async_sigwait(&gf_async_ctrl.sigmask_ctrl);
+    if (caa_unlikely(signum != GF_ASYNC_SIGCTRL)) {
+        gf_async_fatal(-EINVAL, "Worker received an unexpected signal (%d)",
+                       signum);
+    }
+}
+
+static void
+gf_async_leader_wait(void)
+{
+    int32_t signum;
+
+    signum = gf_async_sigwait(&gf_async_ctrl.sigmask_queue);
+    if (caa_unlikely(signum != GF_ASYNC_SIGQUEUE)) {
+        gf_async_fatal(-EINVAL, "Leader received an unexpected signal (%d)",
+                       signum);
+    }
+}
+
+static void
+gf_async_run(struct cds_wfcq_node *node)
+{
+    gf_async_t *async;
+
+    /* We've just got work from the queue. Process it. */
+    async = caa_container_of(node, gf_async_t, queue);
+    /* TODO: remove dependency from THIS and xl. */
+    THIS = async->xl;
+    async->cbk(async->xl, async);
+}
+
+static void
+gf_async_worker_run(void)
+{
+    struct cds_wfcq_node *node;
+
+    do {
+        /* We keep executing jobs from the queue while it's not empty. Note
+         * that while we do this, we are ignoring any stop request. That's
+         * fine, since we need to process our own 'join' messages to fully
+         * terminate all threads. Note that normal jobs should have already
+         * completed once a stop request is received. */
+        node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                         &gf_async_ctrl.queue.tail);
+        if (node != NULL) {
+            gf_async_run(node);
+        }
+    } while (node != NULL);
+
+    /* TODO: I've tried to keep the worker looking at the queue for some small
+     *       amount of time in a busy loop to see if more jobs come soon. With
+     *       this I attempted to avoid the overhead of signal management if
+     *       jobs come fast enough. However experimental results seem to
+     *       indicate that doing this, CPU utilization grows and performance
+     *       is actually reduced. We need to see if that's because I used bad
+     *       parameters or it's really better to do it as it's done now. */
+}
+
+static void
+gf_async_leader_run(void)
+{
+    struct cds_wfcq_node *node;
+
+    node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                     &gf_async_ctrl.queue.tail);
+    while (caa_unlikely(node == NULL)) {
+        gf_async_leader_wait();
+
+        node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                         &gf_async_ctrl.queue.tail);
+    }
+
+    /* Activate the next available worker thread. It will become the new
+     * leader. */
+    gf_async_worker_enable();
+
+    gf_async_run(node);
+}
+
+static uint32_t
+gf_async_stop_check(gf_async_worker_t *worker)
+{
+    uint32_t counts, old, running, max;
+
+    /* First we check if we should stop without doing any costly atomic
+     * operation. */
+    old = uatomic_read(&gf_async_ctrl.counts);
+    max = uatomic_read(&gf_async_ctrl.max_threads);
+    running = GF_ASYNC_COUNT_RUNNING(old);
+    while (running > max) {
+        /* There are too many threads. We try to stop the current worker. */
+        counts = uatomic_cmpxchg(&gf_async_ctrl.counts, old,
+                                 old + GF_ASYNC_COUNTS(-1, 1));
+        if (old != counts) {
+            /* Another thread has just updated the counts. We need to retry. */
+            old = counts;
+            running = GF_ASYNC_COUNT_RUNNING(old);
+
+            continue;
+        }
+
+        running--;
+        worker->running = false;
+    }
+
+    return running;
+}
+
+static void
+gf_async_stop_all(xlator_t *xl, gf_async_t *async)
+{
+    if (gf_async_stop_check(gf_async_current_worker) > 0) {
+        /* There are more workers running. We propagate the stop request to
+         * them. */
+        gf_async(async, xl, gf_async_stop_all);
+    }
+}
+
+static void
+gf_async_join(xlator_t *xl, gf_async_t *async)
+{
+    gf_async_worker_t *worker;
+
+    worker = caa_container_of(async, gf_async_worker_t, async);
+
+    gf_async_thread_wait(worker->thread);
+
+    cds_wfs_push(&gf_async_ctrl.available, &worker->stack);
+}
+
+static void
+gf_async_terminate(gf_async_worker_t *worker)
+{
+    uint32_t counts;
+
+    counts = uatomic_add_return(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(0, -1));
+    if (counts == 0) {
+        /* This is the termination of the last worker thread. We need to
+         * synchronize the main thread that is waiting for all workers to
+         * finish. */
+        gf_async_ctrl.sync_thread = worker->thread;
+
+        gf_async_sync_now();
+    } else {
+        /* Force someone else to join this thread to release resources. */
+        gf_async(&worker->async, THIS, gf_async_join);
+    }
+}
+
+static void *
+gf_async_worker(void *arg)
+{
+    gf_async_worker_t *worker;
+
+    worker = (gf_async_worker_t *)arg;
+    gf_async_current_worker = worker;
+
+    worker->running = true;
+    do {
+        /* This thread does nothing until someone enables it to become a
+         * leader. */
+        gf_async_worker_wait();
+
+        /* This thread is now a leader. It will process jobs from the queue
+         * and, if necessary, enable another worker and transfer leadership
+         * to it. */
+        gf_async_leader_run();
+
+        /* This thread is not a leader anymore. It will continue processing
+         * queued jobs until it becomes empty. */
+        gf_async_worker_run();
+
+        /* Stop the current thread if there are too many threads running. */
+        gf_async_stop_check(worker);
+    } while (worker->running);
+
+    gf_async_terminate(worker);
+
+    return NULL;
+}
+
+static void
+gf_async_cleanup(void)
+{
+    /* We do some basic initialization of the global variable 'gf_async_ctrl'
+     * so that it's put into a relatively consistent state. */
+
+    gf_async_ctrl.enabled = false;
+
+    gf_async_ctrl.pid = 0;
+    gf_async_sigmask_empty(&gf_async_ctrl.sigmask_ctrl);
+    gf_async_sigmask_empty(&gf_async_ctrl.sigmask_queue);
+
+    /* This is used to later detect if the handler of these signals have been
+     * changed or not. */
+    gf_async_ctrl.handler_ctrl.sa_handler = gf_async_signal_handler;
+    gf_async_ctrl.handler_queue.sa_handler = gf_async_signal_handler;
+
+    gf_async_ctrl.table = NULL;
+    gf_async_ctrl.max_threads = 0;
+    gf_async_ctrl.counts = 0;
+}
+
+void
+gf_async_fini(void)
+{
+    gf_async_t async;
+
+    if (uatomic_read(&gf_async_ctrl.counts) != 0) {
+        /* We ensure that all threads will quit on the next check. */
+        gf_async_ctrl.max_threads = 0;
+
+        /* Send the stop request to the thread pool. This will cause the
+         * execution of gf_async_stop_all() by one of the worker threads which,
+         * eventually, will terminate all worker threads. */
+        gf_async(&async, THIS, gf_async_stop_all);
+
+        /* We synchronize here with the last thread. */
+        gf_async_sync_now();
+
+        /* We have just synchronized with the latest thread. Now just wait for
+         * it to terminate. */
+        gf_async_thread_wait(gf_async_ctrl.sync_thread);
+
+        gf_async_signal_flush();
+    }
+
+    gf_async_signal_restore();
+
+    gf_async_cleanup();
+}
+
+void
+gf_async_adjust_threads(int32_t threads)
+{
+    if (threads == 0) {
+        /* By default we allow a maximum of 2 * #cores worker threads. This
+         * value is to try to accommodate threads that will do some I/O. Having
+         * more threads than cores we can keep CPU busy even if some threads
+         * are blocked for I/O. In the most efficient case, we can have #cores
+         * computing threads and #cores blocked threads on I/O. However this is
+         * hard to achieve because we can end with more than #cores computing
+         * threads, which won't provide a real benefit and will increase
+         * contention.
+         *
+         * TODO: implement a more intelligent dynamic maximum based on CPU
+         *       usage and/or system load. */
+        threads = sysconf(_SC_NPROCESSORS_ONLN) * 2;
+        if (threads < 0) {
+            /* If we can't get the current number of processors, we pick a
+             * random number. */
+            threads = 16;
+        }
+    }
+    if (threads > GF_ASYNC_MAX_THREADS) {
+        threads = GF_ASYNC_MAX_THREADS;
+    }
+    uatomic_set(&gf_async_ctrl.max_threads, threads);
+}
+
+int32_t
+gf_async_init(glusterfs_ctx_t *ctx)
+{
+    sigset_t set;
+    gf_async_worker_t *worker;
+    uint32_t i;
+    int32_t ret;
+    bool running;
+
+    gf_async_cleanup();
+
+    if (!ctx->cmd_args.global_threading ||
+        (ctx->process_mode == GF_GLUSTERD_PROCESS)) {
+        return 0;
+    }
+
+    /* At the init time, the maximum number of threads has not yet been
+     * configured. We use a small starting value that will be layer dynamically
+     * adjusted when ctx->config.max_threads is updated. */
+    gf_async_adjust_threads(GF_ASYNC_SPARE_THREADS + 1);
+
+    gf_async_ctrl.pid = getpid();
+
+    __cds_wfs_init(&gf_async_ctrl.available);
+    cds_wfcq_init(&gf_async_ctrl.queue.head, &gf_async_ctrl.queue.tail);
+
+    gf_async_sync_init();
+
+    /* TODO: it would be cleaner to use dynamic memory, but at this point some
+     *       memory management resources are not yet initialized. */
+    gf_async_ctrl.table = gf_async_workers;
+
+    /* We keep all workers in a stack. It will be used when a new thread needs
+     * to be created. */
+    for (i = GF_ASYNC_MAX_THREADS; i > 0; i--) {
+        worker = &gf_async_ctrl.table[i - 1];
+
+        worker->id = i - 1;
+        cds_wfs_node_init(&worker->stack);
+        cds_wfs_push(&gf_async_ctrl.available, &worker->stack);
+    }
+
+    /* Prepare the signal mask for regular workers and the leader. */
+    gf_async_sigmask_add(&gf_async_ctrl.sigmask_ctrl, GF_ASYNC_SIGCTRL);
+    gf_async_sigmask_add(&gf_async_ctrl.sigmask_queue, GF_ASYNC_SIGQUEUE);
+
+    /* TODO: this is needed to block our special signals in the current thread
+     *       and all children that it starts. It would be cleaner to do it when
+     *       signals are initialized, but there doesn't seem to be a unique
+     *       place to do that, so for now we do it here. */
+    gf_async_sigmask_empty(&set);
+    gf_async_sigmask_add(&set, GF_ASYNC_SIGCTRL);
+    gf_async_sigmask_add(&set, GF_ASYNC_SIGQUEUE);
+    gf_async_sigmask_set(SIG_BLOCK, &set, NULL);
+
+    /* Configure the signal handlers. This is mostly for safety, not really
+     * needed, but it doesn't hurt. Note that the caller must ensure that the
+     * signals we need to run are already blocked in any thread already
+     * started. Otherwise this won't work. */
+    gf_async_signal_setup();
+
+    running = false;
+
+    /* We start the spare workers + 1 for the leader. */
+    for (i = 0; i < GF_ASYNC_SPARE_THREADS; i++) {
+        ret = gf_async_worker_create();
+        if (caa_unlikely(ret < 0)) {
+            /* This is the initial start up so we enforce that the spare
+             * threads are created. If this fails at the beginning, it's very
+             * unlikely that the async workers could do its job, so we abort
+             * the initialization. */
+            goto out;
+        }
+
+        /* Once the first thread is started, we can enable it to become the
+         * initial leader. */
+        if ((ret == 0) && !running) {
+            running = true;
+            gf_async_worker_enable();
+        }
+    }
+
+    if (caa_unlikely(!running)) {
+        gf_async_fatal(-ENOMEM, "No worker thread has started");
+    }
+
+    gf_async_ctrl.enabled = true;
+
+    ret = 0;
+
+out:
+    if (ret < 0) {
+        gf_async_error(ret, "Unable to initialize the thread pool.");
+        gf_async_fini();
+    }
+
+    return ret;
+}
diff --git a/libglusterfs/src/byte-order.h b/libglusterfs/src/byte-order.h
deleted file mode 100644
index 4101db2c71d..00000000000
--- a/libglusterfs/src/byte-order.h
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _BYTE_ORDER_H
-#define _BYTE_ORDER_H
-
-#include <inttypes.h>
-
-#define LS1 0x00ffU
-#define MS1 0xff00U
-#define LS2 0x0000ffffU
-#define MS2 0xffff0000U
-#define LS4 0x00000000ffffffffULL
-#define MS4 0xffffffff00000000ULL
-
-
-static uint16_t (*hton16) (uint16_t);
-static uint32_t (*hton32) (uint32_t);
-static uint64_t (*hton64) (uint64_t);
-
-#define ntoh16 hton16
-#define ntoh32 hton32
-#define ntoh64 hton64
-
-static uint16_t (*htole16) (uint16_t);
-static uint32_t (*htole32) (uint32_t);
-static uint64_t (*htole64) (uint64_t);
-
-#define letoh16 htole16
-#define letoh32 htole32
-#define letoh64 htole64
-
-static uint16_t (*htobe16) (uint16_t);
-static uint32_t (*htobe32) (uint32_t);
-static uint64_t (*htobe64) (uint64_t);
-
-#define betoh16 htobe16
-#define betoh32 htobe32
-#define betoh64 htobe64
-
-
-#define do_swap2(x) (((x&LS1) << 8)|(((x&MS1) >> 8)))
-#define do_swap4(x) ((do_swap2(x&LS2) << 16)|(do_swap2((x&MS2) >> 16)))
-#define do_swap8(x) ((do_swap4(x&LS4) << 32)|(do_swap4((x&MS4) >> 32)))
-
-
-static inline uint16_t
-__swap16 (uint16_t x)
-{
-	return do_swap2(x);
-}
-
-
-static inline uint32_t
-__swap32 (uint32_t x)
-{
-	return do_swap4(x);
-}
-
-
-static inline uint64_t
-__swap64 (uint64_t x)
-{
-	return do_swap8(x);
-}
-
-
-static inline uint16_t
-__noswap16 (uint16_t x)
-{
-	return x;
-}
-
-
-static inline uint32_t
-__noswap32 (uint32_t x)
-{
-	return x;
-}
-
-
-static inline uint64_t
-__noswap64 (uint64_t x)
-{
-	return x;
-}
-
-
-static inline uint16_t
-__byte_order_n16 (uint16_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-                /* cpu is be */
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton16 (i);
-}
-
-
-static inline uint32_t
-__byte_order_n32 (uint32_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-                /* cpu is be */
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton32 (i);
-}
-
-
-static inline uint64_t
-__byte_order_n64 (uint64_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-                /* cpu is be */
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton64 (i);
-}
-
-
-static uint16_t (*hton16) (uint16_t) = __byte_order_n16;
-static uint32_t (*hton32) (uint32_t) = __byte_order_n32;
-static uint64_t (*hton64) (uint64_t) = __byte_order_n64;
-
-
-static inline uint16_t
-__byte_order_le16 (uint16_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htole16 = __noswap16;
-		htole32 = __noswap32;
-		htole64 = __noswap64;
-	} else {
-                /* cpu is be */
-		htole16 = __swap16;
-		htole32 = __swap32;
-		htole64 = __swap64;
-	}
-
-	return htole16 (i);
-}
-
-
-static inline uint32_t
-__byte_order_le32 (uint32_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htole16 = __noswap16;
-		htole32 = __noswap32;
-		htole64 = __noswap64;
-	} else {
-                /* cpu is be */
-		htole16 = __swap16;
-		htole32 = __swap32;
-		htole64 = __swap64;
-	}
-
-	return htole32 (i);
-}
-
-
-static inline uint64_t
-__byte_order_le64 (uint64_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htole16 = __noswap16;
-		htole32 = __noswap32;
-		htole64 = __noswap64;
-	} else {
-                /* cpu is be */
-		htole16 = __swap16;
-		htole32 = __swap32;
-		htole64 = __swap64;
-	}
-
-	return htole64 (i);
-}
-
-
-static uint16_t (*htole16) (uint16_t) = __byte_order_le16;
-static uint32_t (*htole32) (uint32_t) = __byte_order_le32;
-static uint64_t (*htole64) (uint64_t) = __byte_order_le64;
-
-
-static inline uint16_t
-__byte_order_be16 (uint16_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htobe16 = __swap16;
-		htobe32 = __swap32;
-		htobe64 = __swap64;
-	} else {
-                /* cpu is be */
-		htobe16 = __noswap16;
-		htobe32 = __noswap32;
-		htobe64 = __noswap64;
-	}
-
-	return htobe16 (i);
-}
-
-
-static inline uint32_t
-__byte_order_be32 (uint32_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htobe16 = __swap16;
-		htobe32 = __swap32;
-		htobe64 = __swap64;
-	} else {
-                /* cpu is be */
-		htobe16 = __noswap16;
-		htobe32 = __noswap32;
-		htobe64 = __noswap64;
-	}
-
-	return htobe32 (i);
-}
-
-
-static inline uint64_t
-__byte_order_be64 (uint64_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-                /* cpu is le */
-		htobe16 = __swap16;
-		htobe32 = __swap32;
-		htobe64 = __swap64;
-	} else {
-                /* cpu is be */
-		htobe16 = __noswap16;
-		htobe32 = __noswap32;
-		htobe64 = __noswap64;
-	}
-
-	return htobe64 (i);
-}
-
-
-static uint16_t (*htobe16) (uint16_t) = __byte_order_be16;
-static uint32_t (*htobe32) (uint32_t) = __byte_order_be32;
-static uint64_t (*htobe64) (uint64_t) = __byte_order_be64;
-
-
-
-#endif /* _BYTE_ORDER_H */
diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c
index 9ed2493e80b..ee84f08acd4 100644
--- a/libglusterfs/src/call-stub.c
+++ b/libglusterfs/src/call-stub.c
@@ -11,2530 +11,2457 @@
 #include <openssl/md5.h>
 #include <inttypes.h>
 
-#include "call-stub.h"
-#include "mem-types.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/call-stub.h"
+#include "glusterfs/mem-types.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 static call_stub_t *
-stub_new (call_frame_t *frame,
-          char wind,
-          glusterfs_fop_t fop)
+stub_new(call_frame_t *frame, const char wind, const glusterfs_fop_t fop)
 {
-        call_stub_t *new = NULL;
+    call_stub_t *new = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", frame, out);
 
-        new = mem_get0 (frame->this->ctx->stub_mem_pool);
-        GF_VALIDATE_OR_GOTO ("call-stub", new, out);
+    new = mem_get0(frame->this->ctx->stub_mem_pool);
+    GF_VALIDATE_OR_GOTO("call-stub", new, out);
 
-        new->frame = frame;
-        new->wind = wind;
-        new->fop = fop;
-        new->stub_mem_pool = frame->this->ctx->stub_mem_pool;
-        INIT_LIST_HEAD (&new->list);
+    new->frame = frame;
+    new->wind = wind;
+    new->fop = fop;
+    new->stub_mem_pool = frame->this->ctx->stub_mem_pool;
+    INIT_LIST_HEAD(&new->list);
 
-        INIT_LIST_HEAD (&new->args_cbk.entries);
+    INIT_LIST_HEAD(&new->args_cbk.entries);
 out:
-        return new;
+    return new;
 }
 
-
 call_stub_t *
-fop_lookup_stub (call_frame_t *frame, fop_lookup_t fn, loc_t *loc,
-                 dict_t *xdata)
+fop_lookup_stub(call_frame_t *frame, fop_lookup_t fn, loc_t *loc, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_LOOKUP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LOOKUP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.lookup = fn;
-        args_lookup_store (&stub->args, loc, xdata);
+    stub->fn.lookup = fn;
+    args_lookup_store(&stub->args, loc, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame, fop_lookup_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent)
+fop_lookup_cbk_stub(call_frame_t *frame, fop_lookup_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode, struct iatt *buf,
+                    dict_t *xdata, struct iatt *postparent)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_LOOKUP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_LOOKUP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.lookup = fn;
-        args_lookup_cbk_store (&stub->args_cbk, op_ret, op_errno, inode,
-                               buf, xdata, postparent);
+    stub->fn_cbk.lookup = fn;
+    args_lookup_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf, xdata,
+                          postparent);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_stat_stub (call_frame_t *frame, fop_stat_t fn,
-               loc_t *loc, dict_t *xdata)
+fop_stat_stub(call_frame_t *frame, fop_stat_t fn, loc_t *loc, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_STAT);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_STAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.stat = fn;
-        args_stat_store (&stub->args, loc, xdata);
+    stub->fn.stat = fn;
+    args_stat_store(&stub->args, loc, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame, fop_stat_cbk_t fn,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *buf, dict_t *xdata)
+fop_stat_cbk_stub(call_frame_t *frame, fop_stat_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_STAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_STAT);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.stat = fn;
-        args_stat_cbk_store (&stub->args_cbk, op_ret, op_errno, buf,
-                             xdata);
+    stub->fn_cbk.stat = fn;
+    args_stat_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fstat_stub (call_frame_t *frame, fop_fstat_t fn,
-                fd_t *fd, dict_t *xdata)
+fop_fstat_stub(call_frame_t *frame, fop_fstat_t fn, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 1, GF_FOP_FSTAT);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSTAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fstat = fn;
-        args_fstat_store (&stub->args, fd, xdata);
+    stub->fn.fstat = fn;
+    args_fstat_store(&stub->args, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_fstat_cbk_stub(call_frame_t *frame, fop_fstat_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_FSTAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.fstat = fn;
+    args_fstat_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame, fop_fstat_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *buf, dict_t *xdata)
+fop_truncate_stub(call_frame_t *frame, fop_truncate_t fn, loc_t *loc, off_t off,
+                  dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FSTAT);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_TRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fstat = fn;
-        args_fstat_cbk_store (&stub->args_cbk, op_ret, op_errno, buf,
-                              xdata);
+    stub->fn.truncate = fn;
+    args_truncate_store(&stub->args, loc, off, xdata);
 out:
-        return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_truncate_cbk_stub(call_frame_t *frame, fop_truncate_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_TRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.truncate = fn;
+    args_truncate_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_truncate_stub (call_frame_t *frame, fop_truncate_t fn,
-                   loc_t *loc, off_t off, dict_t *xdata)
+fop_ftruncate_stub(call_frame_t *frame, fop_ftruncate_t fn, fd_t *fd, off_t off,
+                   dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 1, GF_FOP_FTRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_TRUNCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.ftruncate = fn;
+    args_ftruncate_store(&stub->args, fd, off, xdata);
 
-        stub->fn.truncate = fn;
-        args_truncate_store (&stub->args, loc, off, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame, fop_truncate_cbk_t fn,
+fop_ftruncate_cbk_stub(call_frame_t *frame, fop_ftruncate_cbk_t fn,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		       struct iatt *postbuf, dict_t *xdata)
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FTRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_TRUNCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.ftruncate = fn;
+    args_ftruncate_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
-        stub->fn_cbk.truncate = fn;
-        args_truncate_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                                 prebuf, postbuf, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame, fop_ftruncate_t fn,
-                    fd_t *fd, off_t off, dict_t *xdata)
+fop_access_stub(call_frame_t *frame, fop_access_t fn, loc_t *loc, int32_t mask,
+                dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FTRUNCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn.ftruncate = fn;
-        args_ftruncate_store (&stub->args, fd, off, xdata);
+    stub = stub_new(frame, 1, GF_FOP_ACCESS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn.access = fn;
+    args_access_store(&stub->args, loc, mask, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame, fop_ftruncate_cbk_t fn,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-			struct iatt *postbuf, dict_t *xdata)
+fop_access_cbk_stub(call_frame_t *frame, fop_access_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_FTRUNCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.ftruncate = fn;
-        args_ftruncate_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                                  prebuf, postbuf, xdata);
+    stub = stub_new(frame, 0, GF_FOP_ACCESS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn_cbk.access = fn;
+    args_access_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_access_stub (call_frame_t *frame, fop_access_t fn,
-                 loc_t *loc, int32_t mask, dict_t *xdata)
+fop_readlink_stub(call_frame_t *frame, fop_readlink_t fn, loc_t *loc,
+                  size_t size, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_ACCESS);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_READLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.access = fn;
-        args_access_store (&stub->args, loc, mask, xdata);
+    stub->fn.readlink = fn;
+    args_readlink_store(&stub->args, loc, size, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame, fop_access_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_readlink_cbk_stub(call_frame_t *frame, fop_readlink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, const char *path,
+                      struct iatt *stbuf, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_ACCESS);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_READLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.access = fn;
-        args_access_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.readlink = fn;
+    args_readlink_cbk_store(&stub->args_cbk, op_ret, op_errno, path, stbuf,
+                            xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readlink_stub (call_frame_t *frame, fop_readlink_t fn,
-                   loc_t *loc, size_t size, dict_t *xdata)
+fop_mknod_stub(call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
+               dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_READLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_MKNOD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.readlink = fn;
-        args_readlink_store (&stub->args, loc, size, xdata);
+    stub->fn.mknod = fn;
+    args_mknod_store(&stub->args, loc, mode, rdev, umask, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame, fop_readlink_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       const char *path, struct iatt *stbuf, dict_t *xdata)
+fop_mknod_cbk_stub(call_frame_t *frame, fop_mknod_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_READLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_MKNOD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.readlink = fn;
-        args_readlink_cbk_store (&stub->args_cbk, op_ret, op_errno, path,
-                                 stbuf, xdata);
+    stub->fn_cbk.mknod = fn;
+    args_mknod_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                         preparent, postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
-                dev_t rdev, mode_t umask, dict_t *xdata)
+fop_mkdir_stub(call_frame_t *frame, fop_mkdir_t fn, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_MKNOD);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_MKDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.mknod = fn;
-        args_mknod_store (&stub->args, loc, mode, rdev, umask, xdata);
+    stub->fn.mkdir = fn;
+    args_mkdir_store(&stub->args, loc, mode, umask, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame, fop_mknod_cbk_t fn, int32_t op_ret,
-                    int32_t op_errno, inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-		    dict_t *xdata)
+fop_mkdir_cbk_stub(call_frame_t *frame, fop_mkdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_MKNOD);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_MKDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.mknod = fn;
-        args_mknod_cbk_store (&stub->args_cbk, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    stub->fn_cbk.mkdir = fn;
+    args_mkdir_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                         preparent, postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
-                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+fop_unlink_stub(call_frame_t *frame, fop_unlink_t fn, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 1, GF_FOP_UNLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_MKDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.unlink = fn;
+    args_unlink_store(&stub->args, loc, xflag, xdata);
 
-        stub->fn.mkdir = fn;
-        args_mkdir_store (&stub->args, loc, mode, umask, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame, fop_mkdir_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
+fop_unlink_cbk_stub(call_frame_t *frame, fop_unlink_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *preparent,
                     struct iatt *postparent, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_MKDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_UNLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.mkdir = fn;
-        args_mkdir_cbk_store (&stub->args_cbk, op_ret, op_errno, inode,
-                              buf, preparent, postparent, xdata);
+    stub->fn_cbk.unlink = fn;
+    args_unlink_cbk_store(&stub->args_cbk, op_ret, op_errno, preparent,
+                          postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_unlink_stub (call_frame_t *frame, fop_unlink_t fn,
-                 loc_t *loc, int xflag, dict_t *xdata)
+fop_rmdir_stub(call_frame_t *frame, fop_rmdir_t fn, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_UNLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_RMDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.unlink = fn;
-        args_unlink_store (&stub->args, loc, xflag, xdata);
+    stub->fn.rmdir = fn;
+    args_rmdir_store(&stub->args, loc, flags, xdata);
 
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame, fop_unlink_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent,
-		     dict_t *xdata)
+fop_rmdir_cbk_stub(call_frame_t *frame, fop_rmdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_UNLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_RMDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.unlink = fn;
-        args_unlink_cbk_store (&stub->args_cbk, op_ret, op_errno, preparent,
-                               postparent, xdata);
+    stub->fn_cbk.rmdir = fn;
+    args_rmdir_cbk_store(&stub->args_cbk, op_ret, op_errno, preparent,
+                         postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
-
 call_stub_t *
-fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-                loc_t *loc, int flags, dict_t *xdata)
+fop_symlink_stub(call_frame_t *frame, fop_symlink_t fn, const char *linkname,
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 1, GF_FOP_RMDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", linkname, out);
 
-        stub->fn.rmdir = fn;
-        args_rmdir_store (&stub->args, loc, flags, xdata);
+    stub = stub_new(frame, 1, GF_FOP_SYMLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn.symlink = fn;
+    args_symlink_store(&stub->args, linkname, loc, umask, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame, fop_rmdir_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent,
-		    dict_t *xdata)
+fop_symlink_cbk_stub(call_frame_t *frame, fop_symlink_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_RMDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_SYMLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.rmdir = fn;
-        args_rmdir_cbk_store (&stub->args_cbk, op_ret, op_errno, preparent,
-                              postparent, xdata);
+    stub->fn_cbk.symlink = fn;
+    args_symlink_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-                  const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata)
+fop_rename_stub(call_frame_t *frame, fop_rename_t fn, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", linkname, out);
+    GF_VALIDATE_OR_GOTO("call-stub", oldloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", newloc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_SYMLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_RENAME);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.symlink = fn;
-        args_symlink_store (&stub->args, linkname, loc, umask, xdata);
+    stub->fn.rename = fn;
+    args_rename_store(&stub->args, oldloc, newloc, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame, fop_symlink_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-		      dict_t *xdata)
+fop_rename_cbk_stub(call_frame_t *frame, fop_rename_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_RENAME);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_SYMLINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.symlink = fn;
-        args_symlink_cbk_store (&stub->args_cbk, op_ret, op_errno, inode, buf,
-                                preparent, postparent, xdata);
+    stub->fn_cbk.rename = fn;
+    args_rename_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, preoldparent,
+                          postoldparent, prenewparent, postnewparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rename_stub (call_frame_t *frame, fop_rename_t fn,
-                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+fop_link_stub(call_frame_t *frame, fop_link_t fn, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", oldloc, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", newloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", oldloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", newloc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_RENAME);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.rename = fn;
-        args_rename_store (&stub->args, oldloc, newloc, xdata);
+    stub->fn.link = fn;
+    args_link_store(&stub->args, oldloc, newloc, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame, fop_rename_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-		     dict_t *xdata)
+fop_link_cbk_stub(call_frame_t *frame, fop_link_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, inode_t *inode, struct iatt *buf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_RENAME);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_LINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.rename = fn;
-        args_rename_cbk_store (&stub->args_cbk, op_ret, op_errno, buf,
-                               preoldparent, postoldparent, prenewparent,
-                               postnewparent, xdata);
+    stub->fn_cbk.link = fn;
+    args_link_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                        preparent, postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_link_stub (call_frame_t *frame, fop_link_t fn,
-               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+fop_create_stub(call_frame_t *frame, fop_create_t fn, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", oldloc, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", newloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_LINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_CREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.link = fn;
-        args_link_store (&stub->args, oldloc, newloc, xdata);
+    stub->fn.create = fn;
+    args_create_store(&stub->args, loc, flags, mode, umask, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame, fop_link_cbk_t fn,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
-		   dict_t *xdata)
+fop_create_cbk_stub(call_frame_t *frame, fop_create_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_LINK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_CREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.link = fn;
-        args_link_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                             inode, buf, preparent, postparent, xdata);
+    stub->fn_cbk.create = fn;
+    args_create_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, inode, buf,
+                          preparent, postparent, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_create_stub (call_frame_t *frame, fop_create_t fn,
-                 loc_t *loc, int32_t flags, mode_t mode,
-                 mode_t umask, fd_t *fd, dict_t *xdata)
+fop_open_stub(call_frame_t *frame, fop_open_t fn, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_CREATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_OPEN);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.create = fn;
-        args_create_store (&stub->args, loc, flags, mode,
-                           umask, fd, xdata);
+    stub->fn.open = fn;
+    args_open_store(&stub->args, loc, flags, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame, fop_create_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-		     dict_t *xdata)
+fop_open_cbk_stub(call_frame_t *frame, fop_open_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_OPEN);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_CREATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.create = fn;
-        args_create_cbk_store (&stub->args_cbk, op_ret, op_errno, fd, inode,
-                               buf, preparent, postparent, xdata);
+    stub->fn_cbk.open = fn;
+    args_open_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_open_stub (call_frame_t *frame, fop_open_t fn,
-               loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
+fop_readv_stub(call_frame_t *frame, fop_readv_t fn, fd_t *fd, size_t size,
+               off_t off, uint32_t flags, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 1, GF_FOP_READ);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_OPEN);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn.open = fn;
-        args_open_store (&stub->args, loc, flags, fd, xdata);
+    stub->fn.readv = fn;
+    args_readv_store(&stub->args, fd, size, off, flags, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame, fop_open_cbk_t fn,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, dict_t *xdata)
+fop_readv_cbk_stub(call_frame_t *frame, fop_readv_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iovec *vector, int32_t count,
+                   struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_READ);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_OPEN);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.open = fn;
-        args_open_cbk_store (&stub->args_cbk, op_ret, op_errno, fd, xdata);
+    stub->fn_cbk.readv = fn;
+    args_readv_cbk_store(&stub->args_cbk, op_ret, op_errno, vector, count,
+                         stbuf, iobref, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readv_stub (call_frame_t *frame, fop_readv_t fn,
-                fd_t *fd, size_t size, off_t off, uint32_t flags,
-		dict_t *xdata)
+fop_writev_stub(call_frame_t *frame, fop_writev_t fn, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                struct iobref *iobref, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", vector, out);
 
-        stub = stub_new (frame, 1, GF_FOP_READ);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_WRITE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.readv = fn;
-        args_readv_store (&stub->args, fd, size, off, flags, xdata);
+    stub->fn.writev = fn;
+    args_writev_store(&stub->args, fd, vector, count, off, flags, iobref,
+                      xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame, fop_readv_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-		    int32_t count, struct iatt *stbuf,
-                    struct iobref *iobref, dict_t *xdata)
+fop_writev_cbk_stub(call_frame_t *frame, fop_writev_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                    dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_WRITE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_READ);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.readv = fn;
-        args_readv_cbk_store (&stub->args_cbk, op_ret, op_errno, vector,
-                              count, stbuf, iobref, xdata);
+    stub->fn_cbk.writev = fn;
+    args_writev_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_writev_stub (call_frame_t *frame, fop_writev_t fn,
-                 fd_t *fd, struct iovec *vector, int32_t count, off_t off,
-		 uint32_t flags, struct iobref *iobref, dict_t *xdata)
+fop_flush_stub(call_frame_t *frame, fop_flush_t fn, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", vector, out);
+    stub = stub_new(frame, 1, GF_FOP_FLUSH);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_WRITE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn.writev = fn;
-        args_writev_store (&stub->args, fd, vector, count, off, flags,
-                           iobref, xdata);
+    stub->fn.flush = fn;
+    args_flush_store(&stub->args, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame, fop_writev_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+fop_flush_cbk_stub(call_frame_t *frame, fop_flush_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FLUSH);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_WRITE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.writev = fn;
-        args_writev_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                               prebuf, postbuf, xdata);
+    stub->fn_cbk.flush = fn;
+    args_flush_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_flush_stub (call_frame_t *frame, fop_flush_t fn,
-                fd_t *fd, dict_t *xdata)
+fop_fsync_stub(call_frame_t *frame, fop_fsync_t fn, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_FSYNC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FLUSH);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn.flush = fn;
-        args_flush_store (&stub->args, fd, xdata);
+    stub->fn.fsync = fn;
+    args_fsync_store(&stub->args, fd, datasync, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame, fop_flush_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_fsync_cbk_stub(call_frame_t *frame, fop_fsync_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                   dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FSYNC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FLUSH);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.flush = fn;
-        args_flush_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.fsync = fn;
+    args_fsync_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsync_stub (call_frame_t *frame, fop_fsync_t fn,
-                fd_t *fd, int32_t datasync, dict_t *xdata)
+fop_opendir_stub(call_frame_t *frame, fop_opendir_t fn, loc_t *loc, fd_t *fd,
+                 dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FSYNC);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_OPENDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fsync = fn;
-        args_fsync_store (&stub->args, fd, datasync, xdata);
+    stub->fn.opendir = fn;
+    args_opendir_store(&stub->args, loc, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame, fop_fsync_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+fop_opendir_cbk_stub(call_frame_t *frame, fop_opendir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_OPENDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FSYNC);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.fsync = fn;
-        args_fsync_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                              prebuf, postbuf, xdata);
+    stub->fn_cbk.opendir = fn;
+    args_opendir_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_opendir_stub (call_frame_t *frame, fop_opendir_t fn,
-                  loc_t *loc, fd_t *fd, dict_t *xdata)
+fop_fsyncdir_stub(call_frame_t *frame, fop_fsyncdir_t fn, fd_t *fd,
+                  int32_t datasync, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 1, GF_FOP_FSYNCDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_OPENDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn.opendir = fn;
-        args_opendir_store (&stub->args, loc, fd, xdata);
+    stub->fn.fsyncdir = fn;
+    args_fsyncdir_store(&stub->args, fd, datasync, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame, fop_opendir_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, dict_t *xdata)
+fop_fsyncdir_cbk_stub(call_frame_t *frame, fop_fsyncdir_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FSYNCDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_OPENDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.opendir = fn;
-        args_opendir_cbk_store (&stub->args_cbk, op_ret, op_errno, fd, xdata);
+    stub->fn_cbk.fsyncdir = fn;
+    args_fsyncdir_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame, fop_fsyncdir_t fn,
-                   fd_t *fd, int32_t datasync, dict_t *xdata)
+fop_statfs_stub(call_frame_t *frame, fop_statfs_t fn, loc_t *loc, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FSYNCDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_STATFS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fsyncdir = fn;
-        args_fsyncdir_store (&stub->args, fd, datasync, xdata);
+    stub->fn.statfs = fn;
+    args_statfs_store(&stub->args, loc, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame, fop_fsyncdir_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_statfs_cbk_stub(call_frame_t *frame, fop_statfs_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_STATFS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FSYNCDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.fsyncdir = fn;
-        args_fsyncdir_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.statfs = fn;
+    args_statfs_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_statfs_stub (call_frame_t *frame, fop_statfs_t fn,
-                 loc_t *loc, dict_t *xdata)
+fop_setxattr_stub(call_frame_t *frame, fop_setxattr_t fn, loc_t *loc,
+                  dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_STATFS);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_SETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.statfs = fn;
-        args_statfs_store (&stub->args, loc, xdata);
+    stub->fn.setxattr = fn;
+    args_setxattr_store(&stub->args, loc, dict, flags, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame, fop_statfs_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct statvfs *buf, dict_t *xdata)
+fop_setxattr_cbk_stub(call_frame_t *frame, fop_setxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_STATFS);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_SETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.statfs = fn;
-        args_statfs_cbk_store (&stub->args_cbk, op_ret, op_errno, buf, xdata);
+    stub->fn_cbk.setxattr = fn;
+    args_setxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_setxattr_stub (call_frame_t *frame, fop_setxattr_t fn,
-                   loc_t *loc, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+fop_getxattr_stub(call_frame_t *frame, fop_getxattr_t fn, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-        stub = stub_new (frame, 1, GF_FOP_SETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.setxattr = fn;
-        args_setxattr_store (&stub->args, loc, dict, flags, xdata);
+    stub->fn.getxattr = fn;
+    args_getxattr_store(&stub->args, loc, name, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_setxattr_cbk_stub (call_frame_t *frame,
-                       fop_setxattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno, dict_t *xdata)
+fop_getxattr_cbk_stub(call_frame_t *frame, fop_getxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_SETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.setxattr = fn;
-        args_setxattr_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.getxattr = fn;
+    args_getxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, dict, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_getxattr_stub (call_frame_t *frame, fop_getxattr_t fn,
-                   loc_t *loc, const char *name, dict_t *xdata)
+fop_fsetxattr_stub(call_frame_t *frame, fop_fsetxattr_t fn, fd_t *fd,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-        stub = stub_new (frame, 1, GF_FOP_GETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.getxattr = fn;
-        args_getxattr_store (&stub->args, loc, name, xdata);
+    stub->fn.fsetxattr = fn;
+    args_fsetxattr_store(&stub->args, fd, dict, flags, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame, fop_getxattr_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *dict, dict_t *xdata)
+fop_fsetxattr_cbk_stub(call_frame_t *frame, fop_fsetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_GETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FSETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.getxattr = fn;
-        args_getxattr_cbk_store (&stub->args_cbk, op_ret, op_errno, dict,
-                                 xdata);
+    stub->fn_cbk.fsetxattr = fn;
+    args_fsetxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsetxattr_stub (call_frame_t *frame, fop_fsetxattr_t fn,
-                    fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+fop_fgetxattr_stub(call_frame_t *frame, fop_fgetxattr_t fn, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FSETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FGETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fsetxattr = fn;
-        args_fsetxattr_store (&stub->args, fd, dict, flags, xdata);
+    stub->fn.fgetxattr = fn;
+    args_fgetxattr_store(&stub->args, fd, name, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsetxattr_cbk_stub (call_frame_t *frame, fop_fsetxattr_cbk_t fn,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_fgetxattr_cbk_stub(call_frame_t *frame, fop_fgetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_FSETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fsetxattr = fn;
-        args_fsetxattr_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.fgetxattr = fn;
+    args_fgetxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, dict, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fgetxattr_stub (call_frame_t *frame, fop_fgetxattr_t fn,
-                    fd_t *fd, const char *name, dict_t *xdata)
+fop_removexattr_stub(call_frame_t *frame, fop_removexattr_t fn, loc_t *loc,
+                     const char *name, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", name, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FGETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_REMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fgetxattr = fn;
-        args_fgetxattr_store (&stub->args, fd, name, xdata);
+    stub->fn.removexattr = fn;
+    args_removexattr_store(&stub->args, loc, name, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fgetxattr_cbk_stub (call_frame_t *frame, fop_fgetxattr_cbk_t fn,
-                        int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+fop_removexattr_cbk_stub(call_frame_t *frame, fop_removexattr_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_REMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_GETXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.fgetxattr = fn;
-        args_fgetxattr_cbk_store (&stub->args_cbk, op_ret, op_errno, dict,
-                                  xdata);
+    stub->fn_cbk.removexattr = fn;
+    args_removexattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_removexattr_stub (call_frame_t *frame, fop_removexattr_t fn,
-                      loc_t *loc, const char *name, dict_t *xdata)
+fop_fremovexattr_stub(call_frame_t *frame, fop_fremovexattr_t fn, fd_t *fd,
+                      const char *name, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", name, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", name, out);
 
-        stub = stub_new (frame, 1, GF_FOP_REMOVEXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FREMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.removexattr = fn;
-        args_removexattr_store (&stub->args, loc, name, xdata);
+    stub->fn.fremovexattr = fn;
+    args_fremovexattr_store(&stub->args, fd, name, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame, fop_removexattr_cbk_t fn,
+fop_fremovexattr_cbk_stub(call_frame_t *frame, fop_fremovexattr_cbk_t fn,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_REMOVEXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FREMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.removexattr = fn;
-        args_removexattr_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.fremovexattr = fn;
+    args_fremovexattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fremovexattr_stub (call_frame_t *frame, fop_fremovexattr_t fn,
-                       fd_t *fd, const char *name, dict_t *xdata)
+fop_lk_stub(call_frame_t *frame, fop_lk_t fn, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", name, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FREMOVEXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fremovexattr = fn;
-        args_fremovexattr_store (&stub->args, fd, name, xdata);
+    stub->fn.lk = fn;
+    args_lk_store(&stub->args, fd, cmd, lock, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fremovexattr_cbk_stub (call_frame_t *frame, fop_fremovexattr_cbk_t fn,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_lk_cbk_stub(call_frame_t *frame, fop_lk_cbk_t fn, int32_t op_ret,
+                int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_FREMOVEXATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_LK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fremovexattr = fn;
-        args_fremovexattr_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.lk = fn;
+    args_lk_cbk_store(&stub->args_cbk, op_ret, op_errno, lock, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lk_stub (call_frame_t *frame, fop_lk_t fn,
-             fd_t *fd, int32_t cmd,
-             struct gf_flock *lock, dict_t *xdata)
+fop_inodelk_stub(call_frame_t *frame, fop_inodelk_t fn, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", lock, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-        stub = stub_new (frame, 1, GF_FOP_LK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_INODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.lk = fn;
-        args_lk_store (&stub->args, fd, cmd, lock, xdata);
+    stub->fn.inodelk = fn;
+    args_inodelk_store(&stub->args, volume, loc, cmd, lock, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame, fop_lk_cbk_t fn,
-                 int32_t op_ret, int32_t op_errno,
-                 struct gf_flock *lock, dict_t *xdata)
+fop_inodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_INODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_LK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.lk = fn;
-        args_lk_cbk_store (&stub->args_cbk, op_ret, op_errno, lock, xdata);
+    stub->fn_cbk.inodelk = fn;
+    args_inodelk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
-                  const char *volume, loc_t *loc, int32_t cmd,
-                  struct gf_flock *lock, dict_t *xdata)
+fop_finodelk_stub(call_frame_t *frame, fop_finodelk_t fn, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", lock, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-        stub = stub_new (frame, 1, GF_FOP_INODELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FINODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.finodelk = fn;
+    args_finodelk_store(&stub->args, volume, fd, cmd, lock, xdata);
 
-        stub->fn.inodelk = fn;
-        args_inodelk_store (&stub->args, volume, loc, cmd,
-                            lock, xdata);
 out:
-        return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_finodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_FINODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.finodelk = fn;
+    args_finodelk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_entrylk_stub(call_frame_t *frame, fop_entrylk_t fn, const char *volume,
+                 loc_t *loc, const char *name, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_ENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_INODELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.entrylk = fn;
+    args_entrylk_store(&stub->args, volume, loc, name, cmd, type, xdata);
 
-        stub->fn_cbk.inodelk       = fn;
-        args_inodelk_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
-                   const char *volume, fd_t *fd, int32_t cmd,
-                   struct gf_flock *lock, dict_t *xdata)
+fop_entrylk_cbk_stub(call_frame_t *frame, fop_entrylk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", lock, out);
+    stub = stub_new(frame, 0, GF_FOP_ENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FINODELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.entrylk = fn;
+    args_entrylk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
 
-        stub->fn.finodelk = fn;
-        args_finodelk_store (&stub->args, volume, fd, cmd,
-                             lock, xdata);
+call_stub_t *
+fop_fentrylk_stub(call_frame_t *frame, fop_fentrylk_t fn, const char *volume,
+                  fd_t *fd, const char *name, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
+    stub = stub_new(frame, 1, GF_FOP_FENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.fentrylk = fn;
+    args_fentrylk_store(&stub->args, volume, fd, name, cmd, type, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_fentrylk_cbk_stub(call_frame_t *frame, fop_fentrylk_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FINODELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.finodelk       = fn;
-        args_finodelk_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn_cbk.fentrylk = fn;
+    args_fentrylk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
-                  const char *volume, loc_t *loc, const char *name,
-                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+fop_readdirp_cbk_stub(call_frame_t *frame, fop_readdirp_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_READDIRP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_ENTRYLK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.readdirp = fn;
+    args_readdirp_cbk_store(&stub->args_cbk, op_ret, op_errno, entries, xdata);
+out:
+    return stub;
+}
 
-        stub->fn.entrylk = fn;
-        args_entrylk_store (&stub->args, volume, loc, name, cmd, type, xdata);
+call_stub_t *
+fop_readdir_cbk_stub(call_frame_t *frame, fop_readdir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
+    stub = stub_new(frame, 0, GF_FOP_READDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.readdir = fn;
+    args_readdir_cbk_store(&stub->args_cbk, op_ret, op_errno, entries, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_readdir_stub(call_frame_t *frame, fop_readdir_t fn, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_ENTRYLK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_READDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.entrylk       = fn;
-        args_entrylk_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn.readdir = fn;
+    args_readdir_store(&stub->args, fd, size, off, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
-                   const char *volume, fd_t *fd, const char *name,
-                   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+fop_readdirp_stub(call_frame_t *frame, fop_readdirp_t fn, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 1, GF_FOP_FENTRYLK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_READDIRP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fentrylk = fn;
-        args_fentrylk_store (&stub->args, volume, fd, name, cmd, type, xdata);
+    stub->fn.readdirp = fn;
+    args_readdirp_store(&stub->args, fd, size, off, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fentrylk_cbk_stub (call_frame_t *frame, fop_fentrylk_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_rchecksum_stub(call_frame_t *frame, fop_rchecksum_t fn, fd_t *fd,
+                   off_t offset, int32_t len, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FENTRYLK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_RCHECKSUM);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fentrylk       = fn;
-        args_fentrylk_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    stub->fn.rchecksum = fn;
+    args_rchecksum_store(&stub->args, fd, offset, len, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdirp_cbk_stub (call_frame_t *frame, fop_readdirp_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       gf_dirent_t *entries, dict_t *xdata)
+fop_rchecksum_cbk_stub(call_frame_t *frame, fop_rchecksum_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_READDIRP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_RCHECKSUM);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.readdirp = fn;
-        args_readdirp_cbk_store (&stub->args_cbk, op_ret, op_errno, entries,
-                                 xdata);
+    stub->fn_cbk.rchecksum = fn;
+    args_rchecksum_cbk_store(&stub->args_cbk, op_ret, op_errno, weak_checksum,
+                             strong_checksum, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame, fop_readdir_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno,
-                      gf_dirent_t *entries, dict_t *xdata)
+fop_xattrop_cbk_stub(call_frame_t *frame, fop_xattrop_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_READDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_XATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.readdir = fn;
-        args_readdir_cbk_store (&stub->args_cbk, op_ret, op_errno, entries,
-                                xdata);
+    stub->fn_cbk.xattrop = fn;
+    args_xattrop_cbk_store(&stub->args_cbk, op_ret, op_errno, xattr, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdir_stub (call_frame_t *frame, fop_readdir_t fn,
-                  fd_t *fd, size_t size,
-                  off_t off, dict_t *xdata)
+fop_fxattrop_cbk_stub(call_frame_t *frame, fop_fxattrop_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 1, GF_FOP_READDIR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FXATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.readdir = fn;
-        args_readdir_store (&stub->args, fd, size, off, xdata);
+    stub->fn_cbk.fxattrop = fn;
+    args_xattrop_cbk_store(&stub->args_cbk, op_ret, op_errno, xattr, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdirp_stub (call_frame_t *frame, fop_readdirp_t fn,
-                   fd_t *fd, size_t size, off_t off, dict_t *xdata)
+fop_xattrop_stub(call_frame_t *frame, fop_xattrop_t fn, loc_t *loc,
+                 gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", xattr, out);
 
-        stub = stub_new (frame, 1, GF_FOP_READDIRP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_XATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.readdirp = fn;
-        args_readdirp_store (&stub->args, fd, size, off, xdata);
+    stub->fn.xattrop = fn;
+    args_xattrop_store(&stub->args, loc, optype, xattr, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rchecksum_stub (call_frame_t *frame, fop_rchecksum_t fn,
-                    fd_t *fd, off_t offset, int32_t len, dict_t *xdata)
+fop_fxattrop_stub(call_frame_t *frame, fop_fxattrop_t fn, fd_t *fd,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", xattr, out);
 
-        stub = stub_new (frame, 1, GF_FOP_RCHECKSUM);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FXATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.rchecksum = fn;
-        args_rchecksum_store (&stub->args, fd, offset, len, xdata);
+    stub->fn.fxattrop = fn;
+    args_fxattrop_store(&stub->args, fd, optype, xattr, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rchecksum_cbk_stub (call_frame_t *frame, fop_rchecksum_cbk_t fn,
-                        int32_t op_ret, int32_t op_errno,
-                        uint32_t weak_checksum, uint8_t *strong_checksum,
-			dict_t *xdata)
+fop_setattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_SETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_RCHECKSUM);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-        stub->fn_cbk.rchecksum = fn;
-        args_rchecksum_cbk_store (&stub->args_cbk, op_ret, op_errno,
-                                  weak_checksum, strong_checksum, xdata);
+    stub->fn_cbk.setattr = fn;
+    args_setattr_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre, statpost,
+                           xdata);
 out:
-        return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_fsetattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_FSETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.fsetattr = fn;
+    args_fsetattr_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                            statpost, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_xattrop_cbk_stub (call_frame_t *frame, fop_xattrop_cbk_t fn, int32_t op_ret,
-                      int32_t op_errno, dict_t *xattr, dict_t *xdata)
+fop_setattr_stub(call_frame_t *frame, fop_setattr_t fn, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_XATTROP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_SETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.xattrop       = fn;
-        args_xattrop_cbk_store (&stub->args_cbk, op_ret, op_errno, xattr,
-                                xdata);
+    stub->fn.setattr = fn;
+    args_setattr_store(&stub->args, loc, stbuf, valid, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fxattrop_cbk_stub (call_frame_t *frame, fop_fxattrop_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata)
+fop_fsetattr_stub(call_frame_t *frame, fop_fsetattr_t fn, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FXATTROP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fxattrop = fn;
-        args_xattrop_cbk_store (&stub->args_cbk, op_ret, op_errno, xattr,
-                                xdata);
+    stub->fn.fsetattr = fn;
+    args_fsetattr_store(&stub->args, fd, stbuf, valid, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_xattrop_stub (call_frame_t *frame, fop_xattrop_t fn,
-                  loc_t *loc, gf_xattrop_flags_t optype,
-                  dict_t *xattr, dict_t *xdata)
+fop_fallocate_cbk_stub(call_frame_t *frame, fop_fallocate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", xattr, out);
+    stub = stub_new(frame, 0, GF_FOP_FALLOCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_XATTROP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.fallocate = fn;
 
-        stub->fn.xattrop = fn;
-        args_xattrop_store (&stub->args, loc, optype, xattr, xdata);
+    args_fallocate_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                             statpost, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fxattrop_stub (call_frame_t *frame, fop_fxattrop_t fn,
-                   fd_t *fd, gf_xattrop_flags_t optype,
-                   dict_t *xattr, dict_t *xdata)
+fop_fallocate_stub(call_frame_t *frame, fop_fallocate_t fn, fd_t *fd,
+                   int32_t mode, off_t offset, size_t len, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", xattr, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FXATTROP);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FALLOCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fxattrop = fn;
-        args_fxattrop_store (&stub->args, fd, optype, xattr, xdata);
+    stub->fn.fallocate = fn;
+    args_fallocate_store(&stub->args, fd, mode, offset, len, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_setattr_cbk_stub (call_frame_t *frame, fop_setattr_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *statpre, struct iatt *statpost,
-		      dict_t *xdata)
+fop_discard_cbk_stub(call_frame_t *frame, fop_discard_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_DISCARD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_SETATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.discard = fn;
 
-        stub->fn_cbk.setattr = fn;
-        args_setattr_cbk_store (&stub->args_cbk, op_ret, op_errno, statpre,
-                                statpost, xdata);
+    args_discard_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre, statpost,
+                           xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsetattr_cbk_stub (call_frame_t *frame, fop_setattr_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-		       dict_t *xdata)
+fop_discard_stub(call_frame_t *frame, fop_discard_t fn, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_FSETATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_DISCARD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fsetattr = fn;
-        args_fsetattr_cbk_store (&stub->args_cbk, op_ret, op_errno, statpre,
-                                 statpost, xdata);
+    stub->fn.discard = fn;
+    args_discard_store(&stub->args, fd, offset, len, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_setattr_stub (call_frame_t *frame, fop_setattr_t fn,
-                  loc_t *loc, struct iatt *stbuf,
-                  int32_t valid, dict_t *xdata)
+fop_zerofill_cbk_stub(call_frame_t *frame, fop_zerofill_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    stub = stub_new(frame, 0, GF_FOP_ZEROFILL);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_SETATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.zerofill = fn;
 
-        stub->fn.setattr = fn;
-        args_setattr_store (&stub->args, loc, stbuf, valid, xdata);
+    args_zerofill_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                            statpost, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsetattr_stub (call_frame_t *frame, fop_fsetattr_t fn,
-                   fd_t *fd, struct iatt *stbuf,
-                   int32_t valid, dict_t *xdata)
+fop_zerofill_stub(call_frame_t *frame, fop_zerofill_t fn, fd_t *fd,
+                  off_t offset, off_t len, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FSETATTR);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_ZEROFILL);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fsetattr = fn;
-        args_fsetattr_store (&stub->args, fd, stbuf, valid, xdata);
+    stub->fn.zerofill = fn;
+    args_zerofill_store(&stub->args, fd, offset, len, xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_fallocate_cbk_stub(call_frame_t *frame, fop_fallocate_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-		       dict_t *xdata)
+fop_ipc_cbk_stub(call_frame_t *frame, fop_ipc_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_FALLOCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_IPC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.fallocate = fn;
+    stub->fn_cbk.ipc = fn;
 
-        args_fallocate_cbk_store (&stub->args_cbk, op_ret, op_errno, statpre,
-                                  statpost, xdata);
+    args_ipc_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_fallocate_stub(call_frame_t *frame, fop_fallocate_t fn, fd_t *fd,
-		   int32_t mode, off_t offset, size_t len, dict_t *xdata)
+fop_ipc_stub(call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 1, GF_FOP_FALLOCATE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_IPC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.fallocate = fn;
-        args_fallocate_store (&stub->args, fd, mode, offset, len, xdata);
+    stub->fn.ipc = fn;
+    args_ipc_store(&stub->args, op, xdata);
 out:
-        return stub;
-
+    return stub;
 }
 
 call_stub_t *
-fop_discard_cbk_stub(call_frame_t *frame, fop_discard_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-		     dict_t *xdata)
+fop_lease_cbk_stub(call_frame_t *frame, fop_lease_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_LEASE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_DISCARD);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.lease = fn;
+    args_lease_cbk_store(&stub->args_cbk, op_ret, op_errno, lease, xdata);
+out:
+    return stub;
+}
 
-        stub->fn_cbk.discard = fn;
+call_stub_t *
+fop_lease_stub(call_frame_t *frame, fop_lease_t fn, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-        args_discard_cbk_store (&stub->args_cbk, op_ret, op_errno, statpre,
-                                statpost, xdata);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lease, out);
+
+    stub = stub_new(frame, 1, GF_FOP_LEASE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.lease = fn;
+    args_lease_store(&stub->args, loc, lease, xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_discard_stub(call_frame_t *frame, fop_discard_t fn, fd_t *fd,
-		 off_t offset, size_t len, dict_t *xdata)
+fop_seek_cbk_stub(call_frame_t *frame, fop_seek_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    stub = stub_new(frame, 0, GF_FOP_SEEK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_DISCARD);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.seek = fn;
 
-        stub->fn.discard = fn;
-        args_discard_store (&stub->args, fd, offset, len, xdata);
+    args_seek_cbk_store(&stub->args_cbk, op_ret, op_errno, offset, xdata);
 out:
-        return stub;
-
+    return stub;
 }
 
 call_stub_t *
-fop_zerofill_cbk_stub(call_frame_t *frame, fop_zerofill_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata)
+fop_seek_stub(call_frame_t *frame, fop_seek_t fn, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-        stub = stub_new (frame, 0, GF_FOP_ZEROFILL);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub->fn_cbk.zerofill = fn;
+    stub = stub_new(frame, 1, GF_FOP_SEEK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        args_zerofill_cbk_store (&stub->args_cbk, op_ret, op_errno, statpre,
-                                 statpost, xdata);
+    stub->fn.seek = fn;
+    args_seek_store(&stub->args, fd, offset, what, xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_zerofill_stub(call_frame_t *frame, fop_zerofill_t fn, fd_t *fd,
-                 off_t offset, off_t len, dict_t *xdata)
+fop_getactivelk_cbk_stub(call_frame_t *frame, fop_getactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *lmi, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    stub = stub_new(frame, 0, GF_FOP_GETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_ZEROFILL);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.getactivelk = fn;
 
-        stub->fn.zerofill = fn;
-        args_zerofill_store (&stub->args, fd, offset, len, xdata);
+    args_getactivelk_cbk_store(&stub->args_cbk, op_ret, op_errno, lmi, xdata);
 out:
-        return stub;
-
+    return stub;
 }
 
-
 call_stub_t *
-fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_getactivelk_stub(call_frame_t *frame, fop_getactivelk_t fn, loc_t *loc,
+                     dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_GETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_IPC);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.getactivelk = fn;
 
-        stub->fn_cbk.ipc = fn;
+    loc_copy(&stub->args.loc, loc);
 
-        args_ipc_cbk_store (&stub->args_cbk, op_ret, op_errno, xdata);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn,
-              int32_t op, dict_t *xdata)
+fop_setactivelk_cbk_stub(call_frame_t *frame, fop_setactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    stub = stub_new(frame, 0, GF_FOP_SETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 1, GF_FOP_IPC);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.setactivelk = fn;
+    stub->args_cbk.op_ret = op_ret;
+    stub->args_cbk.op_errno = op_errno;
 
-        stub->fn.ipc = fn;
-        args_ipc_store (&stub->args, op, xdata);
-out:
-        return stub;
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_lease_cbk_stub (call_frame_t *frame, fop_lease_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno,
-                    struct gf_lease *lease, dict_t *xdata)
+fop_setactivelk_stub(call_frame_t *frame, fop_setactivelk_t fn, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_LEASE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_SETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.setactivelk = fn;
+
+    args_setactivelk_store(&stub->args, loc, locklist, xdata);
 
-        stub->fn_cbk.lease = fn;
-        args_lease_cbk_store (&stub->args_cbk, op_ret, op_errno, lease, xdata);
 out:
-        return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_lease_stub (call_frame_t *frame, fop_lease_t fn,
-                loc_t *loc, struct gf_lease *lease, dict_t *xdata)
+fop_copy_file_range_stub(call_frame_t *frame, fop_copy_file_range_t fn,
+                         fd_t *fd_in, off64_t off_in, fd_t *fd_out,
+                         off64_t off_out, size_t len, uint32_t flags,
+                         dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", lease, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 1, GF_FOP_LEASE);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_COPY_FILE_RANGE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.lease = fn;
-        args_lease_store (&stub->args, loc, lease, xdata);
-out:
-        return stub;
+    stub->fn.copy_file_range = fn;
+
+    args_copy_file_range_store(&stub->args, fd_in, off_in, fd_out, off_out, len,
+                               flags, xdata);
 
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_seek_cbk_stub (call_frame_t *frame, fop_seek_cbk_t fn,
-                   int32_t op_ret, int32_t op_errno, off_t offset,
-                   dict_t *xdata)
+fop_copy_file_range_cbk_stub(call_frame_t *frame, fop_copy_file_range_cbk_t fn,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_SEEK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_COPY_FILE_RANGE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.seek = fn;
+    stub->fn_cbk.copy_file_range = fn;
+    args_copy_file_range_cbk_store(&stub->args_cbk, op_ret, op_errno, stbuf,
+                                   prebuf_dst, postbuf_dst, xdata);
 
-        args_seek_cbk_store (&stub->args_cbk, op_ret, op_errno, offset, xdata);
 out:
-        return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_seek_stub (call_frame_t *frame, fop_seek_t fn, fd_t *fd,
-               off_t offset, gf_seek_what_t what, dict_t *xdata)
+fop_put_stub(call_frame_t *frame, fop_put_t fn, loc_t *loc, mode_t mode,
+             mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+             off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    GF_VALIDATE_OR_GOTO("call-stub", vector, out);
 
-        stub = stub_new (frame, 1, GF_FOP_SEEK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_PUT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn.seek = fn;
-        args_seek_store (&stub->args, fd, offset, what, xdata);
+    stub->fn.put = fn;
+    args_put_store(&stub->args, loc, mode, umask, flags, vector, count, offset,
+                   iobref, xattr, xdata);
 out:
-        return stub;
+    return stub;
+}
 
+call_stub_t *
+fop_put_cbk_stub(call_frame_t *frame, fop_put_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_PUT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.put = fn;
+    args_put_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_getactivelk_cbk_stub (call_frame_t *frame, fop_getactivelk_cbk_t fn,
-                          int32_t op_ret, int32_t op_errno,
-                          lock_migration_info_t *lmi, dict_t *xdata)
+fop_icreate_stub(call_frame_t *frame, fop_icreate_t fn, loc_t *loc, mode_t mode,
+                 dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_ICREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub = stub_new (frame, 0, GF_FOP_GETACTIVELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.icreate = fn;
 
-        stub->fn_cbk.getactivelk = fn;
+    stub->args.mode = mode;
+    if (loc)
+        loc_copy(&stub->args.loc, loc);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 
-        args_getactivelk_cbk_store (&stub->args_cbk, op_ret, op_errno, lmi,
-                                    xdata);
 out:
-        return stub;
+    return stub;
 }
 
+static void
+args_icreate_store_cbk(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, inode_t *inode, struct iatt *buf,
+                       dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+}
 
 call_stub_t *
-fop_getactivelk_stub (call_frame_t *frame, fop_getactivelk_t fn, loc_t *loc,
-                      dict_t *xdata)
+fop_icreate_cbk_stub(call_frame_t *frame, fop_icreate_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
-
-        stub = stub_new (frame, 1, GF_FOP_GETACTIVELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    call_stub_t *stub = NULL;
 
-        stub->fn.getactivelk = fn;
+    stub = stub_new(frame, 0, GF_FOP_ICREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        loc_copy (&stub->args.loc, loc);
+    stub->fn_cbk.icreate = fn;
+    args_icreate_store_cbk(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                           xdata);
 
-        if (xdata)
-                stub->args.xdata = dict_ref (xdata);
 out:
-        return stub;
-
+    return stub;
 }
 
 call_stub_t *
-fop_setactivelk_cbk_stub (call_frame_t *frame, fop_setactivelk_cbk_t fn,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fop_namelink_stub(call_frame_t *frame, fop_namelink_t fn, loc_t *loc,
+                  dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        stub = stub_new (frame, 0, GF_FOP_SETACTIVELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_NAMELINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->fn_cbk.setactivelk = fn;
-        stub->args_cbk.op_ret = op_ret;
-        stub->args_cbk.op_errno = op_errno;
+    stub->fn.namelink = fn;
 
-        if (xdata)
-                stub->args.xdata = dict_ref (xdata);
+    if (loc)
+        loc_copy(&stub->args.loc, loc);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 
 out:
-        return stub;
+    return stub;
 }
 
-call_stub_t *
-fop_setactivelk_stub (call_frame_t *frame, fop_setactivelk_t fn,
-                        loc_t *loc, lock_migration_info_t *locklist,
-                        dict_t *xdata)
+static void
+args_namelink_store_cbk(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
 
-        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+}
 
-        stub = stub_new (frame, 1, GF_FOP_SETACTIVELK);
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+call_stub_t *
+fop_namelink_cbk_stub(call_frame_t *frame, fop_namelink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-        stub->fn.setactivelk = fn;
+    stub = stub_new(frame, 0, GF_FOP_NAMELINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        args_setactivelk_store (&stub->args, loc, locklist, xdata);
+    stub->fn_cbk.namelink = fn;
+    args_namelink_store_cbk(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
 out:
-        return stub;
-
+    return stub;
 }
 
 void
-call_resume_wind (call_stub_t *stub)
+call_resume_wind(call_stub_t *stub)
 {
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        switch (stub->fop) {
+    switch (stub->fop) {
         case GF_FOP_OPEN:
-                stub->fn.open (stub->frame, stub->frame->this,
-			       &stub->args.loc, stub->args.flags,
-			       stub->args.fd, stub->args.xdata);
-                break;
+            stub->fn.open(stub->frame, stub->frame->this, &stub->args.loc,
+                          stub->args.flags, stub->args.fd, stub->args.xdata);
+            break;
         case GF_FOP_CREATE:
-                stub->fn.create (stub->frame, stub->frame->this,
-				 &stub->args.loc, stub->args.flags,
-				 stub->args.mode, stub->args.umask,
-				 stub->args.fd, stub->args.xdata);
-                break;
+            stub->fn.create(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.flags, stub->args.mode, stub->args.umask,
+                            stub->args.fd, stub->args.xdata);
+            break;
         case GF_FOP_STAT:
-                stub->fn.stat (stub->frame, stub->frame->this,
-			       &stub->args.loc, stub->args.xdata);
-                break;
+            stub->fn.stat(stub->frame, stub->frame->this, &stub->args.loc,
+                          stub->args.xdata);
+            break;
         case GF_FOP_READLINK:
-                stub->fn.readlink (stub->frame, stub->frame->this,
-				   &stub->args.loc, stub->args.size,
-				   stub->args.xdata);
-                break;
+            stub->fn.readlink(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.size, stub->args.xdata);
+            break;
         case GF_FOP_MKNOD:
-                stub->fn.mknod (stub->frame, stub->frame->this,
-				&stub->args.loc, stub->args.mode,
-				stub->args.rdev, stub->args.umask,
-				stub->args.xdata);
-		break;
+            stub->fn.mknod(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.mode, stub->args.rdev, stub->args.umask,
+                           stub->args.xdata);
+            break;
         case GF_FOP_MKDIR:
-                stub->fn.mkdir (stub->frame, stub->frame->this,
-				&stub->args.loc, stub->args.mode,
-				stub->args.umask, stub->args.xdata);
-		break;
+            stub->fn.mkdir(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.mode, stub->args.umask, stub->args.xdata);
+            break;
         case GF_FOP_UNLINK:
-                stub->fn.unlink (stub->frame, stub->frame->this,
-				 &stub->args.loc, stub->args.xflag,
-				 stub->args.xdata);
-		break;
+            stub->fn.unlink(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xflag, stub->args.xdata);
+            break;
         case GF_FOP_RMDIR:
-                stub->fn.rmdir (stub->frame, stub->frame->this,
-				&stub->args.loc, stub->args.flags,
-				stub->args.xdata);
-		break;
+            stub->fn.rmdir(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.flags, stub->args.xdata);
+            break;
         case GF_FOP_SYMLINK:
-                stub->fn.symlink (stub->frame, stub->frame->this,
-				  stub->args.linkname, &stub->args.loc,
-				  stub->args.umask, stub->args.xdata);
-		break;
+            stub->fn.symlink(stub->frame, stub->frame->this,
+                             stub->args.linkname, &stub->args.loc,
+                             stub->args.umask, stub->args.xdata);
+            break;
         case GF_FOP_RENAME:
-                stub->fn.rename (stub->frame, stub->frame->this,
-				 &stub->args.loc, &stub->args.loc2,
-				 stub->args.xdata);
-		break;
+            stub->fn.rename(stub->frame, stub->frame->this, &stub->args.loc,
+                            &stub->args.loc2, stub->args.xdata);
+            break;
         case GF_FOP_LINK:
-                stub->fn.link (stub->frame, stub->frame->this,
-			       &stub->args.loc, &stub->args.loc2,
-			       stub->args.xdata);
-		break;
+            stub->fn.link(stub->frame, stub->frame->this, &stub->args.loc,
+                          &stub->args.loc2, stub->args.xdata);
+            break;
         case GF_FOP_TRUNCATE:
-                stub->fn.truncate (stub->frame, stub->frame->this,
-				   &stub->args.loc, stub->args.offset,
-				   stub->args.xdata);
-                break;
+            stub->fn.truncate(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.offset, stub->args.xdata);
+            break;
         case GF_FOP_READ:
-                stub->fn.readv (stub->frame, stub->frame->this,
-				stub->args.fd, stub->args.size,
-				stub->args.offset, stub->args.flags,
-				stub->args.xdata);
-                break;
+            stub->fn.readv(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.size, stub->args.offset, stub->args.flags,
+                           stub->args.xdata);
+            break;
         case GF_FOP_WRITE:
-                stub->fn.writev (stub->frame, stub->frame->this,
-				 stub->args.fd, stub->args.vector,
-				 stub->args.count, stub->args.offset,
-				 stub->args.flags, stub->args.iobref,
-				 stub->args.xdata);
-                break;
+            stub->fn.writev(stub->frame, stub->frame->this, stub->args.fd,
+                            stub->args.vector, stub->args.count,
+                            stub->args.offset, stub->args.flags,
+                            stub->args.iobref, stub->args.xdata);
+            break;
         case GF_FOP_STATFS:
-                stub->fn.statfs (stub->frame, stub->frame->this,
-				 &stub->args.loc, stub->args.xdata);
-                break;
+            stub->fn.statfs(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xdata);
+            break;
         case GF_FOP_FLUSH:
-                stub->fn.flush (stub->frame, stub->frame->this,
-				stub->args.fd, stub->args.xdata);
-                break;
+            stub->fn.flush(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.xdata);
+            break;
         case GF_FOP_FSYNC:
-                stub->fn.fsync (stub->frame, stub->frame->this,
-				stub->args.fd, stub->args.datasync,
-				stub->args.xdata);
-                break;
+            stub->fn.fsync(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.datasync, stub->args.xdata);
+            break;
         case GF_FOP_SETXATTR:
-                stub->fn.setxattr (stub->frame, stub->frame->this,
-				   &stub->args.loc, stub->args.xattr,
-				   stub->args.flags, stub->args.xdata);
-                break;
+            stub->fn.setxattr(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.xattr, stub->args.flags,
+                              stub->args.xdata);
+            break;
         case GF_FOP_GETXATTR:
-                stub->fn.getxattr (stub->frame, stub->frame->this,
-				   &stub->args.loc, stub->args.name,
-				   stub->args.xdata);
-                break;
+            stub->fn.getxattr(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.name, stub->args.xdata);
+            break;
         case GF_FOP_FSETXATTR:
-                stub->fn.fsetxattr (stub->frame, stub->frame->this,
-				    stub->args.fd, stub->args.xattr,
-				    stub->args.flags, stub->args.xdata);
-                break;
+            stub->fn.fsetxattr(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.xattr, stub->args.flags,
+                               stub->args.xdata);
+            break;
         case GF_FOP_FGETXATTR:
-                stub->fn.fgetxattr (stub->frame, stub->frame->this,
-				    stub->args.fd, stub->args.name,
-				    stub->args.xdata);
-                break;
+            stub->fn.fgetxattr(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.name, stub->args.xdata);
+            break;
         case GF_FOP_REMOVEXATTR:
-                stub->fn.removexattr (stub->frame, stub->frame->this,
-				      &stub->args.loc, stub->args.name,
-				      stub->args.xdata);
-                break;
+            stub->fn.removexattr(stub->frame, stub->frame->this,
+                                 &stub->args.loc, stub->args.name,
+                                 stub->args.xdata);
+            break;
         case GF_FOP_FREMOVEXATTR:
-                stub->fn.fremovexattr (stub->frame, stub->frame->this,
-				       stub->args.fd, stub->args.name,
-				       stub->args.xdata);
-                break;
+            stub->fn.fremovexattr(stub->frame, stub->frame->this, stub->args.fd,
+                                  stub->args.name, stub->args.xdata);
+            break;
         case GF_FOP_OPENDIR:
-                stub->fn.opendir (stub->frame, stub->frame->this,
-				  &stub->args.loc, stub->args.fd,
-				  stub->args.xdata);
-                break;
+            stub->fn.opendir(stub->frame, stub->frame->this, &stub->args.loc,
+                             stub->args.fd, stub->args.xdata);
+            break;
         case GF_FOP_FSYNCDIR:
-                stub->fn.fsyncdir (stub->frame, stub->frame->this,
-				   stub->args.fd, stub->args.datasync,
-				   stub->args.xdata);
-                break;
+            stub->fn.fsyncdir(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.datasync, stub->args.xdata);
+            break;
         case GF_FOP_ACCESS:
-                stub->fn.access (stub->frame, stub->frame->this,
-				 &stub->args.loc, stub->args.mask,
-				 stub->args.xdata);
-                break;
+            stub->fn.access(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.mask, stub->args.xdata);
+            break;
 
         case GF_FOP_FTRUNCATE:
-                stub->fn.ftruncate (stub->frame, stub->frame->this,
-				    stub->args.fd, stub->args.offset,
-				    stub->args.xdata);
-                break;
+            stub->fn.ftruncate(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.offset, stub->args.xdata);
+            break;
         case GF_FOP_FSTAT:
-                stub->fn.fstat (stub->frame, stub->frame->this,
-				stub->args.fd, stub->args.xdata);
-                break;
+            stub->fn.fstat(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.xdata);
+            break;
         case GF_FOP_LK:
-                stub->fn.lk (stub->frame, stub->frame->this,
-			     stub->args.fd, stub->args.cmd,
-			     &stub->args.lock, stub->args.xdata);
-                break;
+            stub->fn.lk(stub->frame, stub->frame->this, stub->args.fd,
+                        stub->args.cmd, &stub->args.lock, stub->args.xdata);
+            break;
         case GF_FOP_INODELK:
-                stub->fn.inodelk (stub->frame, stub->frame->this,
-				  stub->args.volume, &stub->args.loc,
-				  stub->args.cmd, &stub->args.lock,
-				  stub->args.xdata);
-                break;
+            stub->fn.inodelk(stub->frame, stub->frame->this, stub->args.volume,
+                             &stub->args.loc, stub->args.cmd, &stub->args.lock,
+                             stub->args.xdata);
+            break;
         case GF_FOP_FINODELK:
-                stub->fn.finodelk (stub->frame, stub->frame->this,
-				   stub->args.volume, stub->args.fd,
-				   stub->args.cmd, &stub->args.lock,
-				   stub->args.xdata);
-                break;
+            stub->fn.finodelk(stub->frame, stub->frame->this, stub->args.volume,
+                              stub->args.fd, stub->args.cmd, &stub->args.lock,
+                              stub->args.xdata);
+            break;
         case GF_FOP_ENTRYLK:
-                stub->fn.entrylk (stub->frame, stub->frame->this,
-				  stub->args.volume, &stub->args.loc,
-				  stub->args.name, stub->args.entrylkcmd,
-				  stub->args.entrylktype, stub->args.xdata);
-                break;
+            stub->fn.entrylk(stub->frame, stub->frame->this, stub->args.volume,
+                             &stub->args.loc, stub->args.name,
+                             stub->args.entrylkcmd, stub->args.entrylktype,
+                             stub->args.xdata);
+            break;
         case GF_FOP_FENTRYLK:
-                stub->fn.fentrylk (stub->frame, stub->frame->this,
-				   stub->args.volume, stub->args.fd,
-				   stub->args.name, stub->args.entrylkcmd,
-				   stub->args.entrylktype, stub->args.xdata);
-		break;
+            stub->fn.fentrylk(stub->frame, stub->frame->this, stub->args.volume,
+                              stub->args.fd, stub->args.name,
+                              stub->args.entrylkcmd, stub->args.entrylktype,
+                              stub->args.xdata);
+            break;
         case GF_FOP_LOOKUP:
-                stub->fn.lookup (stub->frame, stub->frame->this,
-				 &stub->args.loc, stub->args.xdata);
-                break;
+            stub->fn.lookup(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xdata);
+            break;
         case GF_FOP_RCHECKSUM:
-                stub->fn.rchecksum (stub->frame, stub->frame->this,
-				    stub->args.fd, stub->args.offset,
-				    stub->args.size, stub->args.xdata);
-                break;
+            stub->fn.rchecksum(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.offset, stub->args.size,
+                               stub->args.xdata);
+            break;
         case GF_FOP_READDIR:
-                stub->fn.readdir (stub->frame, stub->frame->this,
-				  stub->args.fd, stub->args.size,
-				  stub->args.offset, stub->args.xdata);
-                break;
+            stub->fn.readdir(stub->frame, stub->frame->this, stub->args.fd,
+                             stub->args.size, stub->args.offset,
+                             stub->args.xdata);
+            break;
         case GF_FOP_READDIRP:
-                stub->fn.readdirp (stub->frame, stub->frame->this,
-				   stub->args.fd, stub->args.size,
-				   stub->args.offset, stub->args.xdata);
-                break;
+            stub->fn.readdirp(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.size, stub->args.offset,
+                              stub->args.xdata);
+            break;
         case GF_FOP_XATTROP:
-                stub->fn.xattrop (stub->frame, stub->frame->this,
-				  &stub->args.loc, stub->args.optype,
-				  stub->args.xattr, stub->args.xdata);
-                break;
+            stub->fn.xattrop(stub->frame, stub->frame->this, &stub->args.loc,
+                             stub->args.optype, stub->args.xattr,
+                             stub->args.xdata);
+            break;
         case GF_FOP_FXATTROP:
-                stub->fn.fxattrop (stub->frame, stub->frame->this,
-				   stub->args.fd, stub->args.optype,
-				   stub->args.xattr, stub->args.xdata);
-                break;
+            stub->fn.fxattrop(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.optype, stub->args.xattr,
+                              stub->args.xdata);
+            break;
         case GF_FOP_SETATTR:
-                stub->fn.setattr (stub->frame, stub->frame->this,
-				  &stub->args.loc, &stub->args.stat,
-				  stub->args.valid, stub->args.xdata);
-                break;
+            stub->fn.setattr(stub->frame, stub->frame->this, &stub->args.loc,
+                             &stub->args.stat, stub->args.valid,
+                             stub->args.xdata);
+            break;
         case GF_FOP_FSETATTR:
-                stub->fn.fsetattr (stub->frame, stub->frame->this,
-				   stub->args.fd, &stub->args.stat,
-				   stub->args.valid, stub->args.xdata);
-                break;
-	case GF_FOP_FALLOCATE:
-		stub->fn.fallocate(stub->frame, stub->frame->this,
-				   stub->args.fd, stub->args.flags,
-				   stub->args.offset, stub->args.size,
-				   stub->args.xdata);
-		break;
-	case GF_FOP_DISCARD:
-		stub->fn.discard(stub->frame, stub->frame->this,
-				 stub->args.fd, stub->args.offset,
-				 stub->args.size, stub->args.xdata);
-		break;
+            stub->fn.fsetattr(stub->frame, stub->frame->this, stub->args.fd,
+                              &stub->args.stat, stub->args.valid,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_FALLOCATE:
+            stub->fn.fallocate(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.flags, stub->args.offset,
+                               stub->args.size, stub->args.xdata);
+            break;
+        case GF_FOP_DISCARD:
+            stub->fn.discard(stub->frame, stub->frame->this, stub->args.fd,
+                             stub->args.offset, stub->args.size,
+                             stub->args.xdata);
+            break;
         case GF_FOP_ZEROFILL:
-                stub->fn.zerofill(stub->frame, stub->frame->this,
-                                 stub->args.fd, stub->args.offset,
-                                 stub->args.size, stub->args.xdata);
-                break;
+            stub->fn.zerofill(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.offset, stub->args.size,
+                              stub->args.xdata);
+            break;
         case GF_FOP_IPC:
-                stub->fn.ipc (stub->frame, stub->frame->this,
-                              stub->args.cmd, stub->args.xdata);
-                break;
+            stub->fn.ipc(stub->frame, stub->frame->this, stub->args.cmd,
+                         stub->args.xdata);
+            break;
         case GF_FOP_SEEK:
-                stub->fn.seek (stub->frame, stub->frame->this,
-                               stub->args.fd, stub->args.offset,
-                               stub->args.what, stub->args.xdata);
-                break;
+            stub->fn.seek(stub->frame, stub->frame->this, stub->args.fd,
+                          stub->args.offset, stub->args.what, stub->args.xdata);
+            break;
         case GF_FOP_LEASE:
-                stub->fn.lease (stub->frame, stub->frame->this,
-                                &stub->args.loc, &stub->args.lease,
-                                stub->args.xdata);
-                break;
+            stub->fn.lease(stub->frame, stub->frame->this, &stub->args.loc,
+                           &stub->args.lease, stub->args.xdata);
+            break;
 
         case GF_FOP_GETACTIVELK:
-                stub->fn.getactivelk (stub->frame, stub->frame->this,
-                                       &stub->args.loc, stub->args.xdata);
+            stub->fn.getactivelk(stub->frame, stub->frame->this,
+                                 &stub->args.loc, stub->args.xdata);
+            break;
 
         case GF_FOP_SETACTIVELK:
-                stub->fn.setactivelk (stub->frame, stub->frame->this,
-                                        &stub->args.loc, &stub->args.locklist,
-                                        stub->args.xdata);
+            stub->fn.setactivelk(stub->frame, stub->frame->this,
+                                 &stub->args.loc, &stub->args.locklist,
+                                 stub->args.xdata);
+            break;
+
+        case GF_FOP_PUT:
+            stub->fn.put(stub->frame, stub->frame->this, &stub->args.loc,
+                         stub->args.mode, stub->args.umask, stub->args.flags,
+                         stub->args.vector, stub->args.count, stub->args.offset,
+                         stub->args.iobref, stub->args.xattr, stub->args.xdata);
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            stub->fn.copy_file_range(
+                stub->frame, stub->frame->this, stub->args.fd,
+                stub->args.off_in, stub->args.fd_dst, stub->args.off_out,
+                stub->args.size, stub->args.flags, stub->args.xdata);
+            break;
 
         default:
-                gf_msg_callingfn ("call-stub", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ENTRY, "Invalid value of FOP"
-                                  " (%d)", stub->fop);
-                break;
-        }
-out:
-        return;
-}
-
-
-#define STUB_UNWIND(stb, fop, args ...) do {				\
-	if (stb->fn_cbk.fop)						\
-		stb->fn_cbk.fop (stb->frame, stb->frame->cookie,	\
-				 stb->frame->this, stb->args_cbk.op_ret, \
-				 stb->args_cbk.op_errno, args);		\
-	else								\
-		STACK_UNWIND_STRICT (fop, stb->frame, stb->args_cbk.op_ret, \
-				     stb->args_cbk.op_errno, args);	\
-	} while (0)
-
+            gf_msg_callingfn("call-stub", GF_LOG_ERROR, EINVAL,
+                             LG_MSG_INVALID_ENTRY,
+                             "Invalid value of FOP"
+                             " (%d)",
+                             stub->fop);
+            break;
+    }
+out:
+    return;
+}
+
+#define STUB_UNWIND(stb, fop, args...)                                         \
+    do {                                                                       \
+        if (stb->fn_cbk.fop)                                                   \
+            stb->fn_cbk.fop(stb->frame, stb->frame->cookie, stb->frame->this,  \
+                            stb->args_cbk.op_ret, stb->args_cbk.op_errno,      \
+                            args);                                             \
+        else                                                                   \
+            STACK_UNWIND_STRICT(fop, stb->frame, stb->args_cbk.op_ret,         \
+                                stb->args_cbk.op_errno, args);                 \
+    } while (0)
 
 static void
-call_resume_unwind (call_stub_t *stub)
+call_resume_unwind(call_stub_t *stub)
 {
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        switch (stub->fop) {
+    switch (stub->fop) {
         case GF_FOP_OPEN:
-		STUB_UNWIND (stub, open, stub->args_cbk.fd,
-			     stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, open, stub->args_cbk.fd, stub->args_cbk.xdata);
+            break;
         case GF_FOP_CREATE:
-		STUB_UNWIND (stub, create, stub->args_cbk.fd,
-			     stub->args_cbk.inode, &stub->args_cbk.stat,
-			     &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent,
-			     stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, create, stub->args_cbk.fd, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
         case GF_FOP_STAT:
-		STUB_UNWIND (stub, stat, &stub->args_cbk.stat,
-			     stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, stat, &stub->args_cbk.stat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_READLINK:
-		STUB_UNWIND (stub, readlink, stub->args_cbk.buf,
-			     &stub->args_cbk.stat, stub->args.xdata);
-		break;
+            STUB_UNWIND(stub, readlink, stub->args_cbk.buf,
+                        &stub->args_cbk.stat, stub->args.xdata);
+            break;
         case GF_FOP_MKNOD:
-		STUB_UNWIND (stub, mknod, stub->args_cbk.inode,
-			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, mknod, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_MKDIR:
-		STUB_UNWIND (stub, mkdir, stub->args_cbk.inode,
-			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, mkdir, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_UNLINK:
-		STUB_UNWIND (stub, unlink, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, unlink, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
         case GF_FOP_RMDIR:
-		STUB_UNWIND (stub, rmdir, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, rmdir, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
         case GF_FOP_SYMLINK:
-		STUB_UNWIND (stub, symlink, stub->args_cbk.inode,
-			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, symlink, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
         case GF_FOP_RENAME:
-		STUB_UNWIND (stub, rename, &stub->args_cbk.stat,
-			     &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent,
-			     &stub->args_cbk.preparent2,
-			     &stub->args_cbk.postparent2,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, rename, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        &stub->args_cbk.preparent2, &stub->args_cbk.postparent2,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_LINK:
-		STUB_UNWIND (stub, link, stub->args_cbk.inode,
-			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
-			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, link, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_TRUNCATE:
-		STUB_UNWIND (stub, truncate, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, truncate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_READ:
-		STUB_UNWIND (stub, readv, stub->args_cbk.vector,
-			     stub->args_cbk.count, &stub->args_cbk.stat,
-			     stub->args_cbk.iobref, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, readv, stub->args_cbk.vector,
+                        stub->args_cbk.count, &stub->args_cbk.stat,
+                        stub->args_cbk.iobref, stub->args_cbk.xdata);
+            break;
         case GF_FOP_WRITE:
-		STUB_UNWIND (stub, writev, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, writev, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_STATFS:
-		STUB_UNWIND (stub, statfs, &stub->args_cbk.statvfs,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, statfs, &stub->args_cbk.statvfs,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_FLUSH:
-		STUB_UNWIND (stub, flush, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, flush, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSYNC:
-		STUB_UNWIND (stub, fsync, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fsync, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_SETXATTR:
-		STUB_UNWIND (stub, setxattr, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, setxattr, stub->args_cbk.xdata);
+            break;
         case GF_FOP_GETXATTR:
-		STUB_UNWIND (stub, getxattr, stub->args_cbk.xattr,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, getxattr, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSETXATTR:
-		STUB_UNWIND (stub, fsetxattr, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fsetxattr, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FGETXATTR:
-		STUB_UNWIND (stub, fgetxattr, stub->args_cbk.xattr,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fgetxattr, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_REMOVEXATTR:
-		STUB_UNWIND (stub, removexattr, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, removexattr, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FREMOVEXATTR:
-		STUB_UNWIND (stub, fremovexattr, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fremovexattr, stub->args_cbk.xdata);
+            break;
         case GF_FOP_OPENDIR:
-		STUB_UNWIND (stub, opendir, stub->args_cbk.fd,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, opendir, stub->args_cbk.fd, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSYNCDIR:
-		STUB_UNWIND (stub, fsyncdir, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, fsyncdir, stub->args_cbk.xdata);
+            break;
         case GF_FOP_ACCESS:
-		STUB_UNWIND (stub, access, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, access, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FTRUNCATE:
-		STUB_UNWIND (stub, ftruncate, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, ftruncate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSTAT:
-		STUB_UNWIND (stub, fstat, &stub->args_cbk.stat,
-			     stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, fstat, &stub->args_cbk.stat,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_LK:
-		STUB_UNWIND (stub, lk, &stub->args_cbk.lock,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, lk, &stub->args_cbk.lock, stub->args_cbk.xdata);
+            break;
         case GF_FOP_INODELK:
-		STUB_UNWIND (stub, inodelk, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, inodelk, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FINODELK:
-		STUB_UNWIND (stub, finodelk, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, finodelk, stub->args_cbk.xdata);
+            break;
         case GF_FOP_ENTRYLK:
-		STUB_UNWIND (stub, entrylk, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, entrylk, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FENTRYLK:
-		STUB_UNWIND (stub, fentrylk, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, fentrylk, stub->args_cbk.xdata);
+            break;
         case GF_FOP_LOOKUP:
-		STUB_UNWIND (stub, lookup, stub->args_cbk.inode,
-			     &stub->args_cbk.stat, stub->args_cbk.xdata,
-			     &stub->args_cbk.postparent);
-                break;
+            STUB_UNWIND(stub, lookup, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, stub->args_cbk.xdata,
+                        &stub->args_cbk.postparent);
+            break;
         case GF_FOP_RCHECKSUM:
-		STUB_UNWIND (stub, rchecksum, stub->args_cbk.weak_checksum,
-			     stub->args_cbk.strong_checksum, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, rchecksum, stub->args_cbk.weak_checksum,
+                        stub->args_cbk.strong_checksum, stub->args_cbk.xdata);
+            break;
         case GF_FOP_READDIR:
-		STUB_UNWIND (stub, readdir, &stub->args_cbk.entries,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, readdir, &stub->args_cbk.entries,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_READDIRP:
-		STUB_UNWIND (stub, readdir, &stub->args_cbk.entries,
-			     stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, readdir, &stub->args_cbk.entries,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_XATTROP:
-		STUB_UNWIND (stub, xattrop, stub->args_cbk.xattr,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, xattrop, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_FXATTROP:
-		STUB_UNWIND (stub, fxattrop, stub->args_cbk.xattr,
-			     stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fxattrop, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_SETATTR:
-		STUB_UNWIND (stub, setattr, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, setattr, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSETATTR:
-		STUB_UNWIND (stub, fsetattr, &stub->args_cbk.prestat,
-			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
-                break;
-	case GF_FOP_FALLOCATE:
-		STUB_UNWIND(stub, fallocate, &stub->args_cbk.prestat,
-			    &stub->args_cbk.poststat, stub->args_cbk.xdata);
-		break;
-	case GF_FOP_DISCARD:
-		STUB_UNWIND(stub, discard, &stub->args_cbk.prestat,
-			    &stub->args_cbk.poststat, stub->args_cbk.xdata);
-		break;
+            STUB_UNWIND(stub, fsetattr, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FALLOCATE:
+            STUB_UNWIND(stub, fallocate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_DISCARD:
+            STUB_UNWIND(stub, discard, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_ZEROFILL:
-                STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat,
-                            &stub->args_cbk.poststat, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_IPC:
-                STUB_UNWIND (stub, ipc, stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, ipc, stub->args_cbk.xdata);
+            break;
         case GF_FOP_SEEK:
-                STUB_UNWIND (stub, seek, stub->args_cbk.offset,
-                             stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, seek, stub->args_cbk.offset,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_LEASE:
-                STUB_UNWIND (stub, lease, &stub->args_cbk.lease,
-                             stub->args_cbk.xdata);
-                break;
+            STUB_UNWIND(stub, lease, &stub->args_cbk.lease,
+                        stub->args_cbk.xdata);
+            break;
 
         case GF_FOP_GETACTIVELK:
-                STUB_UNWIND (stub, getactivelk, &stub->args_cbk.locklist,
-                             stub->args_cbk.xdata);
+            STUB_UNWIND(stub, getactivelk, &stub->args_cbk.locklist,
+                        stub->args_cbk.xdata);
+            break;
 
         case GF_FOP_SETACTIVELK:
-                STUB_UNWIND (stub, setactivelk, stub->args_cbk.xdata);
+            STUB_UNWIND(stub, setactivelk, stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_PUT:
+            STUB_UNWIND(stub, put, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            STUB_UNWIND(stub, copy_file_range, &stub->args_cbk.stat,
+                        &stub->args_cbk.prestat, &stub->args_cbk.poststat,
+                        stub->args_cbk.xdata);
+            break;
 
         default:
-                gf_msg_callingfn ("call-stub", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ENTRY, "Invalid value of FOP"
-                                  " (%d)", stub->fop);
-                break;
-        }
+            gf_msg_callingfn("call-stub", GF_LOG_ERROR, EINVAL,
+                             LG_MSG_INVALID_ENTRY,
+                             "Invalid value of FOP"
+                             " (%d)",
+                             stub->fop);
+            break;
+    }
 out:
-        return;
+    return;
 }
 
-
 static void
-call_stub_wipe_args (call_stub_t *stub)
+call_stub_wipe_args(call_stub_t *stub)
 {
-        args_wipe (&stub->args);
+    args_wipe(&stub->args);
 }
 
 static void
-call_stub_wipe_args_cbk (call_stub_t *stub)
+call_stub_wipe_args_cbk(call_stub_t *stub)
 {
-        args_cbk_wipe (&stub->args_cbk);
+    args_cbk_wipe(&stub->args_cbk);
 }
 
+void
+call_stub_destroy(call_stub_t *stub)
+{
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    if (stub->wind)
+        call_stub_wipe_args(stub);
+    else
+        call_stub_wipe_args_cbk(stub);
+
+    stub->stub_mem_pool = NULL;
+
+    mem_put(stub);
+out:
+    return;
+}
 
 void
-call_stub_destroy (call_stub_t *stub)
+call_resume(call_stub_t *stub)
 {
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    xlator_t *old_THIS = NULL;
+
+    errno = EINVAL;
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    list_del_init(&stub->list);
 
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
         if (stub->wind)
-                call_stub_wipe_args (stub);
+            call_resume_wind(stub);
         else
-                call_stub_wipe_args_cbk (stub);
+            call_resume_unwind(stub);
+    }
+    THIS = old_THIS;
 
-        stub->stub_mem_pool = NULL;
-
-        mem_put (stub);
+    call_stub_destroy(stub);
 out:
-        return;
+    return;
 }
 
-
 void
-call_resume (call_stub_t *stub)
+call_unwind_error(call_stub_t *stub, int op_ret, int op_errno)
 {
-        xlator_t *old_THIS = NULL;
+    xlator_t *old_THIS = NULL;
 
-        errno = EINVAL;
-        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    list_del_init(&stub->list);
 
-        list_del_init (&stub->list);
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        call_resume_unwind(stub);
+    }
+    THIS = old_THIS;
 
-        old_THIS = THIS;
-        THIS = stub->frame->this;
-        {
-                if (stub->wind)
-                        call_resume_wind (stub);
-                else
-                        call_resume_unwind (stub);
-        }
-        THIS = old_THIS;
+    call_stub_destroy(stub);
 
-        call_stub_destroy (stub);
-out:
-        return;
+    return;
 }
 
+void
+call_unwind_error_keep_stub(call_stub_t *stub, int op_ret, int op_errno)
+{
+    xlator_t *old_THIS = NULL;
+
+    list_del_init(&stub->list);
+
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        call_resume_unwind(stub);
+    }
+
+    THIS = old_THIS;
+
+    return;
+}
 
 void
-call_unwind_error (call_stub_t *stub, int op_ret, int op_errno)
+call_resume_keep_stub(call_stub_t *stub)
 {
-        xlator_t *old_THIS = NULL;
+    xlator_t *old_THIS = NULL;
 
-        list_del_init (&stub->list);
+    errno = EINVAL;
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        old_THIS = THIS;
-        THIS = stub->frame->this;
-        {
-		stub->args_cbk.op_ret = op_ret;
-		stub->args_cbk.op_errno = op_errno;
-		call_resume_unwind (stub);
-        }
-        THIS = old_THIS;
+    list_del_init(&stub->list);
 
-        call_stub_destroy (stub);
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        if (stub->wind)
+            call_resume_wind(stub);
+        else
+            call_resume_unwind(stub);
+    }
 
-        return;
+    THIS = old_THIS;
 
+out:
+    return;
 }
diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h
deleted file mode 100644
index 5779ee36e58..00000000000
--- a/libglusterfs/src/call-stub.h
+++ /dev/null
@@ -1,775 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CALL_STUB_H_
-#define _CALL_STUB_H_
-
-#include "xlator.h"
-#include "defaults.h"
-#include "default-args.h"
-#include "stack.h"
-#include "list.h"
-
-typedef struct _call_stub {
-	struct list_head list;
-	char wind;
-	call_frame_t *frame;
-	glusterfs_fop_t fop;
-        struct mem_pool *stub_mem_pool; /* pointer to stub mempool in ctx_t */
-        uint32_t jnl_meta_len;
-        uint32_t jnl_data_len;
-        void (*serialize) (struct _call_stub *, char *, char *);
-
-	union {
-		fop_lookup_t lookup;
-		fop_stat_t stat;
-		fop_fstat_t fstat;
-		fop_truncate_t truncate;
-		fop_ftruncate_t ftruncate;
-		fop_access_t access;
-		fop_readlink_t readlink;
-		fop_mknod_t mknod;
-		fop_mkdir_t mkdir;
-		fop_unlink_t unlink;
-		fop_rmdir_t rmdir;
-		fop_symlink_t symlink;
-		fop_rename_t rename;
-		fop_link_t link;
-		fop_create_t create;
-		fop_open_t open;
-		fop_readv_t readv;
-		fop_writev_t writev;
-		fop_flush_t flush;
-		fop_fsync_t fsync;
-		fop_opendir_t opendir;
-		fop_fsyncdir_t fsyncdir;
-		fop_statfs_t statfs;
-		fop_setxattr_t setxattr;
-		fop_getxattr_t getxattr;
-		fop_fgetxattr_t fgetxattr;
-		fop_fsetxattr_t fsetxattr;
-		fop_removexattr_t removexattr;
-		fop_fremovexattr_t fremovexattr;
-		fop_lk_t lk;
-		fop_inodelk_t inodelk;
-		fop_finodelk_t finodelk;
-		fop_entrylk_t entrylk;
-		fop_fentrylk_t fentrylk;
-		fop_readdir_t readdir;
-		fop_readdirp_t readdirp;
-		fop_rchecksum_t rchecksum;
-		fop_xattrop_t xattrop;
-		fop_fxattrop_t fxattrop;
-		fop_setattr_t setattr;
-		fop_fsetattr_t fsetattr;
-		fop_fallocate_t fallocate;
-		fop_discard_t discard;
-                fop_zerofill_t zerofill;
-                fop_ipc_t ipc;
-                fop_seek_t seek;
-                fop_lease_t lease;
-                fop_getactivelk_t getactivelk;
-	        fop_setactivelk_t setactivelk;
-        } fn;
-
-	union {
-		fop_lookup_cbk_t lookup;
-		fop_stat_cbk_t stat;
-		fop_fstat_cbk_t fstat;
-		fop_truncate_cbk_t truncate;
-		fop_ftruncate_cbk_t ftruncate;
-		fop_access_cbk_t access;
-		fop_readlink_cbk_t readlink;
-		fop_mknod_cbk_t mknod;
-		fop_mkdir_cbk_t mkdir;
-		fop_unlink_cbk_t unlink;
-		fop_rmdir_cbk_t rmdir;
-		fop_symlink_cbk_t symlink;
-		fop_rename_cbk_t rename;
-		fop_link_cbk_t link;
-		fop_create_cbk_t create;
-		fop_open_cbk_t open;
-		fop_readv_cbk_t readv;
-		fop_writev_cbk_t writev;
-		fop_flush_cbk_t flush;
-		fop_fsync_cbk_t fsync;
-		fop_opendir_cbk_t opendir;
-		fop_fsyncdir_cbk_t fsyncdir;
-		fop_statfs_cbk_t statfs;
-		fop_setxattr_cbk_t setxattr;
-		fop_getxattr_cbk_t getxattr;
-		fop_fgetxattr_cbk_t fgetxattr;
-		fop_fsetxattr_cbk_t fsetxattr;
-		fop_removexattr_cbk_t removexattr;
-		fop_fremovexattr_cbk_t fremovexattr;
-		fop_lk_cbk_t lk;
-		fop_inodelk_cbk_t inodelk;
-		fop_finodelk_cbk_t finodelk;
-		fop_entrylk_cbk_t entrylk;
-		fop_fentrylk_cbk_t fentrylk;
-		fop_readdir_cbk_t readdir;
-		fop_readdirp_cbk_t readdirp;
-		fop_rchecksum_cbk_t rchecksum;
-		fop_xattrop_cbk_t xattrop;
-		fop_fxattrop_cbk_t fxattrop;
-		fop_setattr_cbk_t setattr;
-		fop_fsetattr_cbk_t fsetattr;
-		fop_fallocate_cbk_t fallocate;
-		fop_discard_cbk_t discard;
-                fop_zerofill_cbk_t zerofill;
-                fop_ipc_cbk_t ipc;
-                fop_seek_cbk_t seek;
-                fop_lease_cbk_t lease;
-                fop_getactivelk_cbk_t getactivelk;
-                fop_setactivelk_cbk_t setactivelk;
-	} fn_cbk;
-
-        default_args_t args;
-        default_args_cbk_t args_cbk;
-} call_stub_t;
-
-
-call_stub_t *
-fop_lookup_stub (call_frame_t *frame,
-		 fop_lookup_t fn,
-		 loc_t *loc,
-		 dict_t *xdata);
-
-call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame,
-		     fop_lookup_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *xdata,
-                     struct iatt *postparent);
-call_stub_t *
-fop_stat_stub (call_frame_t *frame,
-	       fop_stat_t fn,
-	       loc_t *loc, dict_t *xdata);
-call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame,
-		   fop_stat_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf, dict_t *xdata);
-call_stub_t *
-fop_fstat_stub (call_frame_t *frame,
-		fop_fstat_t fn,
-		fd_t *fd, dict_t *xdata);
-call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame,
-		    fop_fstat_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *buf, dict_t *xdata);
-
-call_stub_t *
-fop_truncate_stub (call_frame_t *frame,
-		   fop_truncate_t fn,
-		   loc_t *loc,
-		   off_t off, dict_t *xdata);
-
-call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame,
-		       fop_truncate_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata);
-
-call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame,
-		    fop_ftruncate_t fn,
-		    fd_t *fd,
-		    off_t off, dict_t *xdata);
-
-call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame,
-			fop_ftruncate_cbk_t fn,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata);
-
-call_stub_t *
-fop_access_stub (call_frame_t *frame,
-		 fop_access_t fn,
-		 loc_t *loc,
-		 int32_t mask, dict_t *xdata);
-
-call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame,
-		     fop_access_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_readlink_stub (call_frame_t *frame,
-		   fop_readlink_t fn,
-		   loc_t *loc,
-		   size_t size, dict_t *xdata);
-
-call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame,
-		       fop_readlink_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       const char *path,
-                       struct iatt *buf, dict_t *xdata);
-
-call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
-                dev_t rdev, mode_t umask, dict_t *xdata);
-
-call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame,
-		    fop_mknod_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn, loc_t *loc, mode_t mode,
-                mode_t umask, dict_t *xdata);
-
-call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame,
-		    fop_mkdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_unlink_stub (call_frame_t *frame, fop_unlink_t fn,
-		 loc_t *loc, int xflag, dict_t *xdata);
-
-call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame,
-		     fop_unlink_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-		loc_t *loc, int flags, dict_t *xdata);
-
-call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame,
-		    fop_rmdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-		  const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata);
-
-call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame,
-		      fop_symlink_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_rename_stub (call_frame_t *frame,
-		 fop_rename_t fn,
-		 loc_t *oldloc,
-		 loc_t *newloc, dict_t *xdata);
-
-call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame,
-		     fop_rename_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent, dict_t *xdata);
-
-call_stub_t *
-fop_link_stub (call_frame_t *frame,
-	       fop_link_t fn,
-	       loc_t *oldloc,
-	       loc_t *newloc, dict_t *xdata);
-
-call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame,
-		   fop_link_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_create_stub (call_frame_t *frame, fop_create_t fn,
-		 loc_t *loc, int32_t flags, mode_t mode,
-                 mode_t umask, fd_t *fd, dict_t *xdata);
-
-call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame,
-		     fop_create_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
-
-call_stub_t *
-fop_open_stub (call_frame_t *frame,
-	       fop_open_t fn,
-	       loc_t *loc,
-	       int32_t flags,
-	       fd_t *fd,
-               dict_t *xdata);
-
-call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame,
-		   fop_open_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd, dict_t *xdata);
-
-call_stub_t *
-fop_readv_stub (call_frame_t *frame,
-		fop_readv_t fn,
-		fd_t *fd,
-		size_t size,
-		off_t off, uint32_t flags, dict_t *xdata);
-
-call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame,
-		    fop_readv_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iovec *vector,
-		    int32_t count,
-		    struct iatt *stbuf,
-                    struct iobref *iobref, dict_t *xdata);
-
-call_stub_t *
-fop_writev_stub (call_frame_t *frame,
-		 fop_writev_t fn,
-		 fd_t *fd,
-		 struct iovec *vector,
-		 int32_t count,
-		 off_t off, uint32_t flags,
-                 struct iobref *iobref, dict_t *xdata);
-
-call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame,
-		     fop_writev_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata);
-
-call_stub_t *
-fop_flush_stub (call_frame_t *frame,
-		fop_flush_t fn,
-		fd_t *fd, dict_t *xdata);
-
-call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame,
-		    fop_flush_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_fsync_stub (call_frame_t *frame,
-		fop_fsync_t fn,
-		fd_t *fd,
-		int32_t datasync, dict_t *xdata);
-
-call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame,
-		    fop_fsync_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata);
-
-call_stub_t *
-fop_opendir_stub (call_frame_t *frame,
-		  fop_opendir_t fn,
-		  loc_t *loc, fd_t *fd, dict_t *xdata);
-
-call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame,
-		      fop_opendir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd, dict_t *xdata);
-
-call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame,
-		   fop_fsyncdir_t fn,
-		   fd_t *fd,
-		   int32_t datasync, dict_t *xdata);
-
-call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame,
-		       fop_fsyncdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_statfs_stub (call_frame_t *frame,
-		 fop_statfs_t fn,
-		 loc_t *loc, dict_t *xdata);
-
-call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame,
-		     fop_statfs_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct statvfs *buf, dict_t *xdata);
-
-call_stub_t *
-fop_setxattr_stub (call_frame_t *frame,
-		   fop_setxattr_t fn,
-		   loc_t *loc,
-		   dict_t *dict,
-		   int32_t flags, dict_t *xdata);
-
-call_stub_t *
-fop_setxattr_cbk_stub (call_frame_t *frame,
-		       fop_setxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_getxattr_stub (call_frame_t *frame,
-		   fop_getxattr_t fn,
-		   loc_t *loc,
-		   const char *name, dict_t *xdata);
-
-call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame,
-		       fop_getxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *value, dict_t *xdata);
-
-call_stub_t *
-fop_fsetxattr_stub (call_frame_t *frame,
-                    fop_fsetxattr_t fn,
-                    fd_t *fd,
-                    dict_t *dict,
-                    int32_t flags, dict_t *xdata);
-
-call_stub_t *
-fop_fsetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fsetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_fgetxattr_stub (call_frame_t *frame,
-                    fop_fgetxattr_t fn,
-                    fd_t *fd,
-                    const char *name, dict_t *xdata);
-
-call_stub_t *
-fop_fgetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fgetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        dict_t *value, dict_t *xdata);
-
-call_stub_t *
-fop_removexattr_stub (call_frame_t *frame,
-		      fop_removexattr_t fn,
-		      loc_t *loc,
-		      const char *name, dict_t *xdata);
-
-call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame,
-			  fop_removexattr_cbk_t fn,
-			  int32_t op_ret,
-			  int32_t op_errno, dict_t *xdata);
-
-
-call_stub_t *
-fop_fremovexattr_stub (call_frame_t *frame,
-                       fop_fremovexattr_t fn,
-                       fd_t *fd,
-                       const char *name, dict_t *xdata);
-
-call_stub_t *
-fop_fremovexattr_cbk_stub (call_frame_t *frame,
-                           fop_fremovexattr_cbk_t fn,
-                           int32_t op_ret,
-                           int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_lk_stub (call_frame_t *frame,
-	     fop_lk_t fn,
-	     fd_t *fd,
-	     int32_t cmd,
-	     struct gf_flock *lock, dict_t *xdata);
-
-call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame,
-		 fop_lk_cbk_t fn,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock, dict_t *xdata);
-
-call_stub_t *
-fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
-		  const char *volume, loc_t *loc, int32_t cmd,
-                  struct gf_flock *lock, dict_t *xdata);
-
-call_stub_t *
-fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
-		   const char *volume, fd_t *fd, int32_t cmd,
-                   struct gf_flock *lock, dict_t *xdata);
-
-call_stub_t *
-fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
-		  const char *volume, loc_t *loc, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-call_stub_t *
-fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
-		   const char *volume, fd_t *fd, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-call_stub_t *
-fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_fentrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_readdir_stub (call_frame_t *frame,
-		  fop_readdir_t fn,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off, dict_t *xdata);
-
-call_stub_t *
-fop_readdirp_stub (call_frame_t *frame,
-		   fop_readdirp_t fn,
-		   fd_t *fd,
-		   size_t size,
-		   off_t off,
-                   dict_t *xdata);
-
-call_stub_t *
-fop_readdirp_cbk_stub (call_frame_t *frame,
-		       fop_readdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       gf_dirent_t *entries, dict_t *xdata);
-
-call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame,
-		      fop_readdir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      gf_dirent_t *entries, dict_t *xdata);
-
-call_stub_t *
-fop_rchecksum_stub (call_frame_t *frame,
-                    fop_rchecksum_t fn,
-                    fd_t *fd, off_t offset,
-                    int32_t len, dict_t *xdata);
-
-call_stub_t *
-fop_rchecksum_cbk_stub (call_frame_t *frame,
-                        fop_rchecksum_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        uint32_t weak_checksum,
-                        uint8_t *strong_checksum, dict_t *xdata);
-
-call_stub_t *
-fop_xattrop_stub (call_frame_t *frame,
-		  fop_xattrop_t fn,
-		  loc_t *loc,
-		  gf_xattrop_flags_t optype,
-		  dict_t *xattr, dict_t *xdata);
-
-call_stub_t *
-fop_xattrop_stub_cbk_stub (call_frame_t *frame,
-			   fop_xattrop_cbk_t fn,
-			   int32_t op_ret,
-			   int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_fxattrop_stub (call_frame_t *frame,
-		   fop_fxattrop_t fn,
-		   fd_t *fd,
-		   gf_xattrop_flags_t optype,
-		   dict_t *xattr, dict_t *xdata);
-
-call_stub_t *
-fop_fxattrop_stub_cbk_stub (call_frame_t *frame,
-			    fop_xattrop_cbk_t fn,
-			    int32_t op_ret,
-			    int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_setattr_stub (call_frame_t *frame,
-                  fop_setattr_t fn,
-                  loc_t *loc,
-                  struct iatt *stbuf,
-                  int32_t valid, dict_t *xdata);
-
-call_stub_t *
-fop_setattr_cbk_stub (call_frame_t *frame,
-                      fop_setattr_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      struct iatt *statpre,
-                      struct iatt *statpost, dict_t *xdata);
-
-call_stub_t *
-fop_fsetattr_stub (call_frame_t *frame,
-                   fop_fsetattr_t fn,
-                   fd_t *fd,
-                   struct iatt *stbuf,
-                   int32_t valid, dict_t *xdata);
-
-call_stub_t *
-fop_fsetattr_cbk_stub (call_frame_t *frame,
-                       fop_setattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *statpre,
-                       struct iatt *statpost, dict_t *xdata);
-
-call_stub_t *
-fop_fallocate_stub(call_frame_t *frame,
-		   fop_fallocate_t fn,
-		   fd_t *fd,
-		   int32_t mode, off_t offset,
-		   size_t len, dict_t *xdata);
-
-call_stub_t *
-fop_fallocate_cbk_stub(call_frame_t *frame,
-		       fop_fallocate_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-                       dict_t *xdata);
-
-call_stub_t *
-fop_discard_stub(call_frame_t *frame,
-		 fop_discard_t fn,
-		 fd_t *fd,
-		 off_t offset,
-		 size_t len, dict_t *xdata);
-
-call_stub_t *
-fop_discard_cbk_stub(call_frame_t *frame,
-		     fop_discard_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata);
-
-call_stub_t *
-fop_zerofill_stub(call_frame_t *frame,
-                 fop_zerofill_t fn,
-                 fd_t *fd,
-                 off_t offset,
-                 off_t len, dict_t *xdata);
-
-call_stub_t *
-fop_zerofill_cbk_stub(call_frame_t *frame,
-                     fop_zerofill_cbk_t fn,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata);
-
-call_stub_t *
-fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata);
-
-call_stub_t *
-fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-call_stub_t *
-fop_seek_stub (call_frame_t *frame, fop_seek_t fn, fd_t *fd, off_t offset,
-               gf_seek_what_t what, dict_t *xdata);
-
-call_stub_t *
-fop_seek_cbk_stub (call_frame_t *frame, fop_seek_cbk_t fn,
-                  int32_t op_ret, int32_t op_errno, off_t offset,
-                  dict_t *xdata);
-
-call_stub_t *
-fop_lease_stub (call_frame_t *frame, fop_lease_t fn, loc_t *loc,
-                struct gf_lease *lease, dict_t *xdata);
-
-call_stub_t *
-fop_lease_cbk_stub (call_frame_t *frame, fop_lease_cbk_t fn,
-                    int32_t op_ret, int32_t op_errno,
-                    struct gf_lease *lease, dict_t *xdata);
-
-call_stub_t *
-fop_getactivelk_stub (call_frame_t *frame, fop_getactivelk_t fn,
-                       loc_t *loc, dict_t *xdata);
-
-call_stub_t *
-fop_getactivelk_cbk_stub (call_frame_t *frame, fop_getactivelk_cbk_t fn,
-                          int32_t op_ret, int32_t op_errno,
-                          lock_migration_info_t *lmi, dict_t *xdata);
-
-call_stub_t *
-fop_setactivelk_stub (call_frame_t *frame, fop_setactivelk_t fn,
-                        loc_t *loc, lock_migration_info_t *locklist,
-                        dict_t *xdata);
-
-call_stub_t *
-fop_setactivelk_cbk_stub (call_frame_t *frame, fop_setactivelk_cbk_t fn,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-void call_resume (call_stub_t *stub);
-void call_stub_destroy (call_stub_t *stub);
-void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno);
-
-/*
- * Sometimes we might want to call just this, perhaps repeatedly, without
- * having (or being able) to destroy and recreate it.
- */
-void call_resume_wind (call_stub_t *stub);
-
-#endif
diff --git a/libglusterfs/src/changelog.h b/libglusterfs/src/changelog.h
index 6f86e5a54cd..a09d9f25287 100644
--- a/libglusterfs/src/changelog.h
+++ b/libglusterfs/src/changelog.h
@@ -16,101 +16,100 @@ struct gf_brick_spec;
 /**
  * Max bit shiter for event selection
  */
-#define CHANGELOG_EV_SELECTION_RANGE  5
-
-#define CHANGELOG_OP_TYPE_JOURNAL    (1<<0)
-#define CHANGELOG_OP_TYPE_OPEN       (1<<1)
-#define CHANGELOG_OP_TYPE_CREATE     (1<<2)
-#define CHANGELOG_OP_TYPE_RELEASE    (1<<3)
-#define CHANGELOG_OP_TYPE_BR_RELEASE (1<<4)  /* logical release (last close()),
-                                                sent by bitrot stub */
-#define CHANGELOG_OP_TYPE_MAX        (1<<CHANGELOG_EV_SELECTION_RANGE)
+#define CHANGELOG_EV_SELECTION_RANGE 5
 
+#define CHANGELOG_OP_TYPE_JOURNAL (1 << 0)
+#define CHANGELOG_OP_TYPE_OPEN (1 << 1)
+#define CHANGELOG_OP_TYPE_CREATE (1 << 2)
+#define CHANGELOG_OP_TYPE_RELEASE (1 << 3)
+#define CHANGELOG_OP_TYPE_BR_RELEASE                                           \
+    (1 << 4) /* logical release (last close()),                                \
+                sent by bitrot stub */
+#define CHANGELOG_OP_TYPE_MAX (1 << CHANGELOG_EV_SELECTION_RANGE)
 
 struct ev_open {
-        unsigned char gfid[16];
-        int32_t flags;
+    unsigned char gfid[16];
+    int32_t flags;
 };
 
 struct ev_creat {
-        unsigned char gfid[16];
-        int32_t flags;
+    unsigned char gfid[16];
+    int32_t flags;
 };
 
 struct ev_release {
-        unsigned char gfid[16];
+    unsigned char gfid[16];
 };
 
 struct ev_release_br {
-        unsigned long version;
-        unsigned char gfid[16];
-        int32_t sign_info;
+    unsigned long version;
+    unsigned char gfid[16];
+    int32_t sign_info;
 };
 
 struct ev_changelog {
-        char path[PATH_MAX];
+    char path[PATH_MAX];
 };
 
 typedef struct changelog_event {
-        unsigned int ev_type;
-
-        union {
-                struct ev_open open;
-                struct ev_creat create;
-                struct ev_release release;
-                struct ev_changelog journal;
-                struct ev_release_br releasebr;
-        } u;
+    unsigned int ev_type;
+
+    union {
+        struct ev_open open;
+        struct ev_creat create;
+        struct ev_release release;
+        struct ev_changelog journal;
+        struct ev_release_br releasebr;
+    } u;
 } changelog_event_t;
 
-#define CHANGELOG_EV_SIZE  (sizeof (changelog_event_t))
+#define CHANGELOG_EV_SIZE (sizeof(changelog_event_t))
 
 /**
  * event callback, connected & disconnection defs
  */
-typedef void (CALLBACK) (void *, char *,
-                        void *, changelog_event_t *);
-typedef void *(INIT) (void *, struct gf_brick_spec *);
-typedef void (FINI) (void *, char *, void *);
-typedef void (CONNECT) (void *, char *, void *);
-typedef void (DISCONNECT) (void *, char *, void *);
+typedef void(CALLBACK)(void *, char *, void *, changelog_event_t *);
+typedef void *(INIT)(void *, struct gf_brick_spec *);
+typedef void(FINI)(void *, char *, void *);
+typedef void(CONNECT)(void *, char *, void *);
+typedef void(DISCONNECT)(void *, char *, void *);
 
 struct gf_brick_spec {
-        char         *brick_path;
-        unsigned int  filter;
+    char *brick_path;
+    unsigned int filter;
 
-        INIT       *init;
-        FINI       *fini;
-        CALLBACK   *callback;
-        CONNECT    *connected;
-        DISCONNECT *disconnected;
+    INIT *init;
+    FINI *fini;
+    CALLBACK *callback;
+    CONNECT *connected;
+    DISCONNECT *disconnected;
 
-        void *ptr;
+    void *ptr;
 };
 
 /* API set */
 
 int
-gf_changelog_register (char *brick_path, char *scratch_dir,
-                       char *log_file, int log_levl, int max_reconnects);
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+                      int log_levl, int max_reconnects);
 ssize_t
-gf_changelog_scan ();
+gf_changelog_scan();
 
 int
-gf_changelog_start_fresh ();
+gf_changelog_start_fresh();
 
 ssize_t
-gf_changelog_next_change (char *bufptr, size_t maxlen);
+gf_changelog_next_change(char *bufptr, size_t maxlen);
 
 int
-gf_changelog_done (char *file);
+gf_changelog_done(char *file);
 
 /* newer flexible API */
 int
-gf_changelog_init (void *xl);
+gf_changelog_init(void *xl);
 
 int
-gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
-                               int ordered, char *logfile, int lvl, void *xl);
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+                              int ordered, char *logfile, int lvl, void *xl);
 
 #endif
diff --git a/libglusterfs/src/checksum.c b/libglusterfs/src/checksum.c
index 5fac1330094..acdaed04ae2 100644
--- a/libglusterfs/src/checksum.c
+++ b/libglusterfs/src/checksum.c
@@ -9,8 +9,10 @@
 */
 
 #include <openssl/md5.h>
+#include <openssl/sha.h>
 #include <zlib.h>
 #include <stdint.h>
+#include <string.h>
 
 /*
  * The "weak" checksum required for the rsync algorithm.
@@ -20,17 +22,23 @@
  * data. Thus int32_t and uint32_t are sufficient
  */
 uint32_t
-gf_rsync_weak_checksum (unsigned char *buf, size_t len)
+gf_rsync_weak_checksum(unsigned char *buf, size_t len)
 {
-        return adler32 (0, buf, len);
+    return adler32(0, buf, len);
 }
 
-
 /*
  * The "strong" checksum required for the rsync algorithm.
  */
 void
-gf_rsync_strong_checksum (unsigned char *data, size_t len, unsigned char *md5)
+gf_rsync_strong_checksum(unsigned char *data, size_t len,
+                         unsigned char *sha256_md)
+{
+    SHA256((const unsigned char *)data, len, sha256_md);
+}
+
+void
+gf_rsync_md5_checksum(unsigned char *data, size_t len, unsigned char *md5)
 {
-        MD5 (data, len, md5);
+    MD5(data, len, md5);
 }
diff --git a/libglusterfs/src/circ-buff.c b/libglusterfs/src/circ-buff.c
index 6259282a917..913115c7be1 100644
--- a/libglusterfs/src/circ-buff.c
+++ b/libglusterfs/src/circ-buff.c
@@ -8,194 +8,186 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "circ-buff.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/circ-buff.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 void
-cb_destroy_data (circular_buffer_t *cb,
-                 void (*destroy_buffer_data) (void *data))
+cb_destroy_data(circular_buffer_t *cb, void (*destroy_buffer_data)(void *data))
 {
-        if (destroy_buffer_data)
-                destroy_buffer_data (cb->data);
-        GF_FREE (cb->data);
-        return;
+    if (destroy_buffer_data)
+        destroy_buffer_data(cb->data);
+    GF_FREE(cb->data);
+    return;
 }
 
-
 /* hold lock while calling this function */
 int
-__cb_add_entry_buffer (buffer_t *buffer, void *item)
+__cb_add_entry_buffer(buffer_t *buffer, void *item)
 {
-        circular_buffer_t   *ptr  = NULL;
-        int    ret   = -1;
-        //DO we really need the assert here?
-        GF_ASSERT (buffer->used_len <= buffer->size_buffer);
-
-        if (buffer->use_once == _gf_true &&
-            buffer->used_len == buffer->size_buffer) {
-                gf_msg  ("circ-buff", GF_LOG_WARNING, 0, LG_MSG_BUFFER_ERROR,
-                         "buffer %p is use once buffer", buffer);
-                return -1;
-        } else {
-                if (buffer->used_len == buffer->size_buffer) {
-                        if (buffer->cb[buffer->w_index]) {
-                                ptr = buffer->cb[buffer->w_index];
-                                if (ptr->data) {
-                                        cb_destroy_data (ptr,
-                                                   buffer->destroy_buffer_data);
-                                        ptr->data = NULL;
-                                        GF_FREE (ptr);
-                                }
-                                buffer->cb[buffer->w_index] = NULL;
-                                ptr = NULL;
-                        }
+    circular_buffer_t *ptr = NULL;
+    int ret = -1;
+    // DO we really need the assert here?
+    GF_ASSERT(buffer->used_len <= buffer->size_buffer);
+
+    if (buffer->use_once == _gf_true &&
+        buffer->used_len == buffer->size_buffer) {
+        gf_msg("circ-buff", GF_LOG_WARNING, 0, LG_MSG_BUFFER_ERROR,
+               "buffer %p is use once buffer", buffer);
+        return -1;
+    } else {
+        if (buffer->used_len == buffer->size_buffer) {
+            if (buffer->cb[buffer->w_index]) {
+                ptr = buffer->cb[buffer->w_index];
+                if (ptr->data) {
+                    cb_destroy_data(ptr, buffer->destroy_buffer_data);
+                    ptr->data = NULL;
+                    GF_FREE(ptr);
                 }
-
-                buffer->cb[buffer->w_index] =
-                        GF_CALLOC (1, sizeof (circular_buffer_t),
-                                   gf_common_mt_circular_buffer_t);
-                if (!buffer->cb[buffer->w_index])
-                        return -1;
-
-                buffer->cb[buffer->w_index]->data = item;
-                ret = gettimeofday (&buffer->cb[buffer->w_index]->tv, NULL);
-                if (ret == -1)
-                        gf_msg_callingfn ("circ-buff", GF_LOG_WARNING, 0,
-                                          LG_MSG_GETTIMEOFDAY_FAILED,
-                                          "getting time of the day failed");
-                buffer->w_index++;
-                buffer->w_index %= buffer->size_buffer;
-                //used_buffer size cannot be greater than the total buffer size
-
-                if (buffer->used_len < buffer->size_buffer)
-                        buffer->used_len++;
-                return buffer->w_index;
+                buffer->cb[buffer->w_index] = NULL;
+                ptr = NULL;
+            }
         }
+
+        buffer->cb[buffer->w_index] = GF_CALLOC(1, sizeof(circular_buffer_t),
+                                                gf_common_mt_circular_buffer_t);
+        if (!buffer->cb[buffer->w_index])
+            return -1;
+
+        buffer->cb[buffer->w_index]->data = item;
+        ret = gettimeofday(&buffer->cb[buffer->w_index]->tv, NULL);
+        if (ret == -1)
+            gf_msg_callingfn("circ-buff", GF_LOG_WARNING, 0,
+                             LG_MSG_GETTIMEOFDAY_FAILED,
+                             "getting time of the day failed");
+        buffer->w_index++;
+        buffer->w_index %= buffer->size_buffer;
+        // used_buffer size cannot be greater than the total buffer size
+
+        if (buffer->used_len < buffer->size_buffer)
+            buffer->used_len++;
+        return buffer->w_index;
+    }
 }
 
 int
-cb_add_entry_buffer (buffer_t *buffer, void *item)
+cb_add_entry_buffer(buffer_t *buffer, void *item)
 {
-        int write_index = -1;
+    int write_index = -1;
 
-        pthread_mutex_lock (&buffer->lock);
-        {
-                write_index = __cb_add_entry_buffer (buffer, item);
-        }
-        pthread_mutex_unlock (&buffer->lock);
+    pthread_mutex_lock(&buffer->lock);
+    {
+        write_index = __cb_add_entry_buffer(buffer, item);
+    }
+    pthread_mutex_unlock(&buffer->lock);
 
-        return write_index;
+    return write_index;
 }
 
 void
-cb_buffer_show (buffer_t *buffer)
+cb_buffer_show(buffer_t *buffer)
 {
-        pthread_mutex_lock (&buffer->lock);
-        {
-                gf_msg_debug ("circ-buff", 0, "w_index: %d, size: %"
-                              GF_PRI_SIZET" used_buffer: %d", buffer->w_index,
-                              buffer->size_buffer, buffer->used_len);
-        }
-        pthread_mutex_unlock (&buffer->lock);
+    pthread_mutex_lock(&buffer->lock);
+    {
+        gf_msg_debug("circ-buff", 0,
+                     "w_index: %d, size: %" GF_PRI_SIZET " used_buffer: %d",
+                     buffer->w_index, buffer->size_buffer, buffer->used_len);
+    }
+    pthread_mutex_unlock(&buffer->lock);
 }
 
 void
-cb_buffer_dump (buffer_t *buffer, void *data,
-                int (fn) (circular_buffer_t *buffer, void *data))
+cb_buffer_dump(buffer_t *buffer, void *data,
+               int(fn)(circular_buffer_t *buffer, void *data))
 {
-        int index = 0;
-        circular_buffer_t *entry = NULL;
-        int  entries = 0;
-        int ul = 0;
-        int w_ind = 0;
-        int size_buff = 0;
-        int i = 0;
-
-        ul = buffer->used_len;
-        w_ind = buffer->w_index;
-        size_buff = buffer->size_buffer;
-
-        pthread_mutex_lock (&buffer->lock);
-        {
-                if (buffer->use_once == _gf_false) {
-                        index = (size_buff + (w_ind - ul))%size_buff;
-                        for (entries = 0; entries < buffer->used_len;
-                             entries++) {
-                                entry = buffer->cb[index];
-                                if (entry)
-                                        fn (entry, data);
-                                else
-                                        gf_msg_callingfn ("circ-buff",
-                                                          GF_LOG_WARNING, 0,
-                                                          LG_MSG_NULL_PTR,
-                                                          "Null entry in "
-                                                          "circular buffer at "
-                                                          "index %d.", index);
-
-                                index++;
-                                index %= buffer->size_buffer;
-                        }
-                } else {
-                        for (i = 0; i < buffer->used_len ; i++) {
-                                entry = buffer->cb[i];
-                                fn (entry, data);
-                        }
-                }
+    int index = 0;
+    circular_buffer_t *entry = NULL;
+    int entries = 0;
+    int ul = 0;
+    int w_ind = 0;
+    int size_buff = 0;
+    int i = 0;
+
+    ul = buffer->used_len;
+    w_ind = buffer->w_index;
+    size_buff = buffer->size_buffer;
+
+    pthread_mutex_lock(&buffer->lock);
+    {
+        if (buffer->use_once == _gf_false) {
+            index = (size_buff + (w_ind - ul)) % size_buff;
+            for (entries = 0; entries < buffer->used_len; entries++) {
+                entry = buffer->cb[index];
+                if (entry)
+                    fn(entry, data);
+                else
+                    gf_msg_callingfn("circ-buff", GF_LOG_WARNING, 0,
+                                     LG_MSG_NULL_PTR,
+                                     "Null entry in "
+                                     "circular buffer at "
+                                     "index %d.",
+                                     index);
+
+                index++;
+                index %= buffer->size_buffer;
+            }
+        } else {
+            for (i = 0; i < buffer->used_len; i++) {
+                entry = buffer->cb[i];
+                fn(entry, data);
+            }
         }
-        pthread_mutex_unlock (&buffer->lock);
+    }
+    pthread_mutex_unlock(&buffer->lock);
 }
 
 buffer_t *
-cb_buffer_new (size_t buffer_size, gf_boolean_t use_once,
-               void (*destroy_buffer_data) (void *data))
+cb_buffer_new(size_t buffer_size, gf_boolean_t use_once,
+              void (*destroy_buffer_data)(void *data))
 {
-        buffer_t    *buffer = NULL;
-
-        buffer = GF_CALLOC (1, sizeof (*buffer), gf_common_mt_buffer_t);
-        if (!buffer) {
-                goto out;
-        }
-
-        buffer->cb = GF_CALLOC (buffer_size,
-                                sizeof (circular_buffer_t *),
-                                gf_common_mt_circular_buffer_t);
-        if (!buffer->cb) {
-                GF_FREE (buffer);
-                buffer = NULL;
-                goto out;
-        }
-
-        buffer->w_index = 0;
-        buffer->size_buffer = buffer_size;
-        buffer->use_once = use_once;
-        buffer->used_len = 0;
-        buffer->destroy_buffer_data = destroy_buffer_data;
-        pthread_mutex_init (&buffer->lock, NULL);
+    buffer_t *buffer = NULL;
+
+    buffer = GF_CALLOC(1, sizeof(*buffer), gf_common_mt_buffer_t);
+    if (!buffer) {
+        goto out;
+    }
+
+    buffer->cb = GF_CALLOC(buffer_size, sizeof(circular_buffer_t *),
+                           gf_common_mt_circular_buffer_t);
+    if (!buffer->cb) {
+        GF_FREE(buffer);
+        buffer = NULL;
+        goto out;
+    }
+
+    buffer->w_index = 0;
+    buffer->size_buffer = buffer_size;
+    buffer->use_once = use_once;
+    buffer->used_len = 0;
+    buffer->destroy_buffer_data = destroy_buffer_data;
+    pthread_mutex_init(&buffer->lock, NULL);
 
 out:
-        return buffer;
+    return buffer;
 }
 
 void
-cb_buffer_destroy (buffer_t *buffer)
+cb_buffer_destroy(buffer_t *buffer)
 {
-        int                     i = 0;
-        circular_buffer_t       *ptr = NULL;
-        if (buffer) {
-                if (buffer->cb) {
-                        for (i = 0; i < buffer->used_len ; i++) {
-                                ptr = buffer->cb[i];
-                                if (ptr->data) {
-                                        cb_destroy_data (ptr,
-                                                   buffer->destroy_buffer_data);
-                                        ptr->data = NULL;
-                                        GF_FREE (ptr);
-                                }
-                        }
-                        GF_FREE (buffer->cb);
+    int i = 0;
+    circular_buffer_t *ptr = NULL;
+    if (buffer) {
+        if (buffer->cb) {
+            for (i = 0; i < buffer->used_len; i++) {
+                ptr = buffer->cb[i];
+                if (ptr->data) {
+                    cb_destroy_data(ptr, buffer->destroy_buffer_data);
+                    ptr->data = NULL;
+                    GF_FREE(ptr);
                 }
-                pthread_mutex_destroy (&buffer->lock);
-                GF_FREE (buffer);
+            }
+            GF_FREE(buffer->cb);
         }
+        pthread_mutex_destroy(&buffer->lock);
+        GF_FREE(buffer);
+    }
 }
-
diff --git a/libglusterfs/src/circ-buff.h b/libglusterfs/src/circ-buff.h
deleted file mode 100644
index e3459f5e3d0..00000000000
--- a/libglusterfs/src/circ-buff.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CB_H
-#define _CB_H
-
-#include "common-utils.h"
-#include "logging.h"
-#include "mem-types.h"
-
-#define BUFFER_SIZE 10
-#define TOTAL_SIZE BUFFER_SIZE + 1
-
-
-struct _circular_buffer {
-        struct timeval tv;
-        void *data;
-};
-
-typedef struct _circular_buffer circular_buffer_t;
-
-struct _buffer {
-        unsigned int w_index;
-        size_t  size_buffer;
-        gf_boolean_t use_once;
-        /* This variable is assigned the proper value at the time of initing */
-        /* the buffer. It indicates, whether the buffer should be used once */
-        /*  it becomes full. */
-
-        int  used_len;
-        /* indicates the amount of circular buffer used. */
-
-        circular_buffer_t **cb;
-        void (*destroy_buffer_data) (void *data);
-        pthread_mutex_t   lock;
-};
-
-typedef struct _buffer buffer_t;
-
-int
-cb_add_entry_buffer (buffer_t *buffer, void *item);
-
-void
-cb_buffer_show (buffer_t *buffer);
-
-buffer_t *
-cb_buffer_new (size_t buffer_size,gf_boolean_t use_buffer_once,
-               void (*destroy_data) (void *data));
-
-void
-cb_buffer_destroy (buffer_t *buffer);
-
-void
-cb_buffer_dump (buffer_t *buffer, void *data,
-                int (fn) (circular_buffer_t *buffer, void *data));
-
-#endif /* _CB_H */
diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c
index b3eb4e4df8c..9d377c3c2e1 100644
--- a/libglusterfs/src/client_t.c
+++ b/libglusterfs/src/client_t.c
@@ -8,623 +8,541 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "dict.h"
-#include "statedump.h"
-#include "client_t.h"
-#include "list.h"
-#include "rpcsvc.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/dict.h"
+#include "glusterfs/statedump.h"
+#include "glusterfs/client_t.h"
+#include "glusterfs/list.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 static int
-gf_client_chain_client_entries (cliententry_t *entries, uint32_t startidx,
-                        uint32_t endcount)
+gf_client_chain_client_entries(cliententry_t *entries, uint32_t startidx,
+                               uint32_t endcount)
 {
-        uint32_t        i = 0;
+    uint32_t i = 0;
 
-        if (!entries) {
-                gf_msg_callingfn ("client_t", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!entries");
-                return -1;
-        }
+    if (!entries) {
+        gf_msg_callingfn("client_t", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!entries");
+        return -1;
+    }
 
-        /* Chain only till the second to last entry because we want to
-         * ensure that the last entry has GF_CLIENTTABLE_END.
-         */
-        for (i = startidx; i < (endcount - 1); i++)
-                entries[i].next_free = i + 1;
+    /* Chain only till the second to last entry because we want to
+     * ensure that the last entry has GF_CLIENTTABLE_END.
+     */
+    for (i = startidx; i < (endcount - 1); i++)
+        entries[i].next_free = i + 1;
 
-        /* i has already been incremented up to the last entry. */
-        entries[i].next_free = GF_CLIENTTABLE_END;
+    /* i has already been incremented up to the last entry. */
+    entries[i].next_free = GF_CLIENTTABLE_END;
 
-        return 0;
+    return 0;
 }
 
-
 static int
-gf_client_clienttable_expand (clienttable_t *clienttable, uint32_t nr)
+gf_client_clienttable_expand(clienttable_t *clienttable, uint32_t nr)
 {
-        cliententry_t   *oldclients     = NULL;
-        uint32_t         oldmax_clients = -1;
-        int              ret            = -1;
-
-        if (clienttable == NULL || nr <= clienttable->max_clients) {
-                gf_msg_callingfn ("client_t", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                ret = EINVAL;
-                goto out;
-        }
-
-        oldclients = clienttable->cliententries;
-        oldmax_clients = clienttable->max_clients;
-
-        clienttable->cliententries = GF_CALLOC (nr, sizeof (cliententry_t),
-                                                gf_common_mt_cliententry_t);
-        if (!clienttable->cliententries) {
-                clienttable->cliententries = oldclients;
-                ret = 0;
-                goto out;
-        }
-        clienttable->max_clients = nr;
-
-        if (oldclients) {
-                uint32_t cpy = oldmax_clients * sizeof (cliententry_t);
-                memcpy (clienttable->cliententries, oldclients, cpy);
-        }
-
-        gf_client_chain_client_entries (clienttable->cliententries,
-                                        oldmax_clients,
-                                        clienttable->max_clients);
-
-        /* Now that expansion is done, we must update the client list
-         * head pointer so that the client allocation functions can continue
-         * using the expanded table.
-         */
-        clienttable->first_free = oldmax_clients;
-        GF_FREE (oldclients);
+    cliententry_t *oldclients = NULL;
+    uint32_t oldmax_clients = -1;
+    int ret = -1;
+
+    if (clienttable == NULL || nr <= clienttable->max_clients) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        ret = EINVAL;
+        goto out;
+    }
+
+    oldclients = clienttable->cliententries;
+    oldmax_clients = clienttable->max_clients;
+
+    clienttable->cliententries = GF_CALLOC(nr, sizeof(cliententry_t),
+                                           gf_common_mt_cliententry_t);
+    if (!clienttable->cliententries) {
+        clienttable->cliententries = oldclients;
         ret = 0;
+        goto out;
+    }
+    clienttable->max_clients = nr;
+
+    if (oldclients) {
+        uint32_t cpy = oldmax_clients * sizeof(cliententry_t);
+        memcpy(clienttable->cliententries, oldclients, cpy);
+    }
+
+    gf_client_chain_client_entries(clienttable->cliententries, oldmax_clients,
+                                   clienttable->max_clients);
+
+    /* Now that expansion is done, we must update the client list
+     * head pointer so that the client allocation functions can continue
+     * using the expanded table.
+     */
+    clienttable->first_free = oldmax_clients;
+    GF_FREE(oldclients);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 clienttable_t *
-gf_clienttable_alloc (void)
-{
-        clienttable_t *clienttable = NULL;
-        int            result = 0;
-
-        clienttable =
-                GF_CALLOC (1, sizeof (clienttable_t), gf_common_mt_clienttable_t);
-        if (!clienttable)
-                return NULL;
-
-        LOCK_INIT (&clienttable->lock);
-
-        result = gf_client_clienttable_expand (clienttable,
-                                               GF_CLIENTTABLE_INITIAL_SIZE);
-        if (result != 0) {
-                gf_msg ("client_t", GF_LOG_ERROR, 0,
-                        LG_MSG_EXPAND_CLIENT_TABLE_FAILED,
-                        "gf_client_clienttable_expand failed");
-                GF_FREE (clienttable);
-                return NULL;
-        }
-
-        return clienttable;
-}
-
-
-void
-gf_client_clienttable_destroy (clienttable_t *clienttable)
+gf_clienttable_alloc(void)
 {
-        client_t         *client        = NULL;
-        cliententry_t    *cliententries = NULL;
-        uint32_t          client_count  = 0;
-        int32_t           i             = 0;
-
-        if (!clienttable) {
-                gf_msg_callingfn ("client_t", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!clienttable");
-                return;
-        }
-
-        LOCK (&clienttable->lock);
-        {
-                client_count = clienttable->max_clients;
-                clienttable->max_clients = 0;
-                cliententries = clienttable->cliententries;
-                clienttable->cliententries = NULL;
-        }
-        UNLOCK (&clienttable->lock);
-
-        if (cliententries != NULL) {
-                for (i = 0; i < client_count; i++) {
-                        client = cliententries[i].client;
-                        if (client != NULL) {
-                                gf_client_unref (client);
-                        }
-                }
-
-                GF_FREE (cliententries);
-                LOCK_DESTROY (&clienttable->lock);
-                GF_FREE (clienttable);
-        }
+    clienttable_t *clienttable = NULL;
+    int result = 0;
+
+    clienttable = GF_CALLOC(1, sizeof(clienttable_t),
+                            gf_common_mt_clienttable_t);
+    if (!clienttable)
+        return NULL;
+
+    LOCK_INIT(&clienttable->lock);
+
+    result = gf_client_clienttable_expand(clienttable,
+                                          GF_CLIENTTABLE_INITIAL_SIZE);
+    if (result != 0) {
+        gf_msg("client_t", GF_LOG_ERROR, 0, LG_MSG_EXPAND_CLIENT_TABLE_FAILED,
+               "gf_client_clienttable_expand failed");
+        GF_FREE(clienttable);
+        return NULL;
+    }
+
+    return clienttable;
 }
 
-
 /*
  * Increments ref.bind if the client is already present or creates a new
  * client with ref.bind = 1,ref.count = 1 it signifies that
  * as long as ref.bind is > 0 client should be alive.
  */
 client_t *
-gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid)
+gf_client_get(xlator_t *this, client_auth_data_t *cred, char *client_uid,
+              char *subdir_mount)
 {
-        client_t      *client      = NULL;
-        cliententry_t *cliententry = NULL;
-        clienttable_t *clienttable = NULL;
-        unsigned int   i           = 0;
-
-        if (this == NULL || client_uid == NULL) {
-                gf_msg_callingfn ("client_t", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        clienttable = this->ctx->clienttable;
-
-        LOCK (&clienttable->lock);
-        {
-                for (; i < clienttable->max_clients; i++) {
-                        client = clienttable->cliententries[i].client;
-                        if (client == NULL)
-                                continue;
-                        /*
-                         * look for matching client_uid, _and_
-                         * if auth was used, matching auth flavour and data
-                         */
-                        if (strcmp (client_uid, client->client_uid) == 0 &&
-                                (cred->flavour != AUTH_NONE &&
-                                        (cred->flavour == client->auth.flavour &&
-                                        (size_t) cred->datalen == client->auth.len &&
-                                        memcmp (cred->authdata,
-                                                client->auth.data,
-                                                client->auth.len) == 0))) {
-                                INCREMENT_ATOMIC (client->ref.lock,
-                                                  client->ref.bind);
-                                goto unlock;
-                        }
-                }
-
-                client = GF_CALLOC (1, sizeof(client_t), gf_common_mt_client_t);
-                if (client == NULL) {
-                        errno = ENOMEM;
-                        goto unlock;
-                }
-
-                client->this = this;
-
-                LOCK_INIT (&client->scratch_ctx.lock);
-                LOCK_INIT (&client->ref.lock);
-
-                client->client_uid = gf_strdup (client_uid);
-                if (client->client_uid == NULL) {
-                        GF_FREE (client);
-                        client = NULL;
-                        errno = ENOMEM;
-                        goto unlock;
-                }
-                client->scratch_ctx.count = GF_CLIENTCTX_INITIAL_SIZE;
-                client->scratch_ctx.ctx =
-                        GF_CALLOC (GF_CLIENTCTX_INITIAL_SIZE,
-                                   sizeof (struct client_ctx),
-                                   gf_common_mt_client_ctx);
-                if (client->scratch_ctx.ctx == NULL) {
-                        GF_FREE (client->client_uid);
-                        GF_FREE (client);
-                        client = NULL;
-                        errno = ENOMEM;
-                        goto unlock;
-                }
-
-                /* no need to do these atomically here */
-                client->ref.bind = client->ref.count = 1;
-
-                client->auth.flavour = cred->flavour;
-                if (cred->flavour != AUTH_NONE) {
-                        client->auth.data =
-                                GF_CALLOC (1, cred->datalen,
-                                           gf_common_mt_client_t);
-                        if (client->auth.data == NULL) {
-                                GF_FREE (client->scratch_ctx.ctx);
-                                GF_FREE (client->client_uid);
-                                GF_FREE (client);
-                                client = NULL;
-                                errno = ENOMEM;
-                                goto unlock;
-                        }
-                        memcpy (client->auth.data, cred->authdata,
-                                cred->datalen);
-                        client->auth.len = cred->datalen;
-                }
-
-                client->tbl_index = clienttable->first_free;
-                cliententry = &clienttable->cliententries[clienttable->first_free];
-                if (cliententry->next_free == GF_CLIENTTABLE_END) {
-                        int result =
-                                gf_client_clienttable_expand (clienttable,
-                                        clienttable->max_clients +
-                                                GF_CLIENTTABLE_INITIAL_SIZE);
-                        if (result != 0) {
-                                GF_FREE (client->scratch_ctx.ctx);
-                                GF_FREE (client->client_uid);
-                                GF_FREE (client);
-                                client = NULL;
-                                errno = result;
-                                goto unlock;
-                        }
-                        cliententry = &clienttable->cliententries[client->tbl_index];
-                        cliententry->next_free = clienttable->first_free;
-                }
-                cliententry->client = client;
-                clienttable->first_free = cliententry->next_free;
-                cliententry->next_free = GF_CLIENTENTRY_ALLOCATED;
-        }
+    client_t *client = NULL;
+    cliententry_t *cliententry = NULL;
+    clienttable_t *clienttable = NULL;
+    unsigned int i = 0;
+
+    if (this == NULL || client_uid == NULL) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
+
+    clienttable = this->ctx->clienttable;
+
+    LOCK(&clienttable->lock);
+    {
+        for (; i < clienttable->max_clients; i++) {
+            client = clienttable->cliententries[i].client;
+            if (client == NULL)
+                continue;
+            /*
+             * look for matching client_uid, _and_
+             * if auth was used, matching auth flavour and data
+             */
+            if (strcmp(client_uid, client->client_uid) == 0 &&
+                (cred->flavour && (cred->flavour == client->auth.flavour &&
+                                   (size_t)cred->datalen == client->auth.len &&
+                                   memcmp(cred->authdata, client->auth.data,
+                                          client->auth.len) == 0))) {
+                GF_ATOMIC_INC(client->bind);
+                goto unlock;
+            }
+        }
+
+        client = GF_CALLOC(1, sizeof(client_t), gf_common_mt_client_t);
+        if (client == NULL) {
+            errno = ENOMEM;
+            goto unlock;
+        }
+
+        client->this = this;
+        if (subdir_mount != NULL)
+            client->subdir_mount = gf_strdup(subdir_mount);
+
+        LOCK_INIT(&client->scratch_ctx.lock);
+
+        client->client_uid = gf_strdup(client_uid);
+        if (client->client_uid == NULL) {
+            GF_FREE(client);
+            client = NULL;
+            errno = ENOMEM;
+            goto unlock;
+        }
+        client->scratch_ctx.count = GF_CLIENTCTX_INITIAL_SIZE;
+        client->scratch_ctx.ctx = GF_CALLOC(GF_CLIENTCTX_INITIAL_SIZE,
+                                            sizeof(struct client_ctx),
+                                            gf_common_mt_client_ctx);
+        if (client->scratch_ctx.ctx == NULL) {
+            GF_FREE(client->client_uid);
+            GF_FREE(client);
+            client = NULL;
+            errno = ENOMEM;
+            goto unlock;
+        }
+
+        GF_ATOMIC_INIT(client->bind, 1);
+        GF_ATOMIC_INIT(client->count, 1);
+        GF_ATOMIC_INIT(client->fd_cnt, 0);
+
+        client->auth.flavour = cred->flavour;
+        if (cred->flavour) {
+            client->auth.data = GF_MALLOC(cred->datalen, gf_common_mt_client_t);
+            if (client->auth.data == NULL) {
+                GF_FREE(client->scratch_ctx.ctx);
+                GF_FREE(client->client_uid);
+                GF_FREE(client);
+                client = NULL;
+                errno = ENOMEM;
+                goto unlock;
+            }
+            memcpy(client->auth.data, cred->authdata, cred->datalen);
+            client->auth.len = cred->datalen;
+        }
+
+        client->tbl_index = clienttable->first_free;
+        cliententry = &clienttable->cliententries[clienttable->first_free];
+        if (cliententry->next_free == GF_CLIENTTABLE_END) {
+            int result = gf_client_clienttable_expand(
+                clienttable,
+                clienttable->max_clients + GF_CLIENTTABLE_INITIAL_SIZE);
+            if (result != 0) {
+                GF_FREE(client->scratch_ctx.ctx);
+                GF_FREE(client->client_uid);
+                GF_FREE(client);
+                client = NULL;
+                errno = result;
+                goto unlock;
+            }
+            cliententry = &clienttable->cliententries[client->tbl_index];
+            cliententry->next_free = clienttable->first_free;
+        }
+        cliententry->client = client;
+        clienttable->first_free = cliententry->next_free;
+        cliententry->next_free = GF_CLIENTENTRY_ALLOCATED;
+    }
 unlock:
-        UNLOCK (&clienttable->lock);
-
-        if (client)
-                gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
-                                  "%s: bind_ref: %d, ref: %d",
-                                  client->client_uid, client->ref.bind,
-                                  client->ref.count);
-        return client;
+    UNLOCK(&clienttable->lock);
+
+    if (client)
+        gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
+                         "%s: bind_ref: %" GF_PRI_ATOMIC
+                         ", ref: "
+                         "%" GF_PRI_ATOMIC,
+                         client->client_uid, GF_ATOMIC_GET(client->bind),
+                         GF_ATOMIC_GET(client->count));
+    return client;
 }
 
 void
-gf_client_put (client_t *client, gf_boolean_t *detached)
+gf_client_put(client_t *client, gf_boolean_t *detached)
 {
-        gf_boolean_t unref = _gf_false;
-        int bind_ref;
-
-        if (client == NULL)
-                goto out;
-
+    gf_boolean_t unref = _gf_false;
+    int bind_ref;
+
+    if (client == NULL)
+        goto out;
+
+    if (detached)
+        *detached = _gf_false;
+
+    bind_ref = GF_ATOMIC_DEC(client->bind);
+    if (bind_ref == 0)
+        unref = _gf_true;
+
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
+                     "%s: "
+                     "bind_ref: %" GF_PRI_ATOMIC ", ref: %" GF_PRI_ATOMIC
+                     ", "
+                     "unref: %d",
+                     client->client_uid, GF_ATOMIC_GET(client->bind),
+                     GF_ATOMIC_GET(client->count), unref);
+    if (unref) {
         if (detached)
-                *detached = _gf_false;
-
-        bind_ref = DECREMENT_ATOMIC (client->ref.lock, client->ref.bind);
-        if (bind_ref == 0)
-                unref = _gf_true;
-
-        gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF, "%s: "
-                          "bind_ref: %d, ref: %d, unref: %d",
-                          client->client_uid, client->ref.bind,
-                          client->ref.count, unref);
-        if (unref) {
-                if (detached)
-                        *detached = _gf_true;
-                gf_client_unref (client);
-        }
+            *detached = _gf_true;
+        gf_client_unref(client);
+    }
 
 out:
-        return;
+    return;
 }
 
 client_t *
-gf_client_ref (client_t *client)
+gf_client_ref(client_t *client)
 {
-        if (!client) {
-                gf_msg_callingfn ("client_t", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "null client");
-                return NULL;
-        }
-
-        INCREMENT_ATOMIC (client->ref.lock, client->ref.count);
-        gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT, "%s: "
-                          "ref-count %d", client->client_uid,
-                          client->ref.count);
-        return client;
+    if (!client) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "null client");
+        return NULL;
+    }
+
+    GF_ATOMIC_INC(client->count);
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT,
+                     "%s: "
+                     "ref-count %" GF_PRI_ATOMIC,
+                     client->client_uid, GF_ATOMIC_GET(client->count));
+    return client;
 }
 
-
 static void
-client_destroy (client_t *client)
+gf_client_destroy_recursive(xlator_t *xl, client_t *client)
 {
-        clienttable_t     *clienttable = NULL;
-        glusterfs_graph_t *gtrav       = NULL;
-        xlator_t          *xtrav       = NULL;
+    xlator_list_t *trav;
 
-        if (client == NULL){
-                gf_msg_callingfn ("xlator", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                goto out;
-        }
+    if (!xl->call_cleanup && xl->cbks->client_destroy) {
+        xl->cbks->client_destroy(xl, client);
+    }
 
-        clienttable = client->this->ctx->clienttable;
-
-        LOCK_DESTROY (&client->scratch_ctx.lock);
-        LOCK_DESTROY (&client->ref.lock);
+    for (trav = xl->children; trav; trav = trav->next) {
+        gf_client_destroy_recursive(trav->xlator, client);
+    }
+}
 
-        LOCK (&clienttable->lock);
-        {
-                clienttable->cliententries[client->tbl_index].client = NULL;
-                clienttable->cliententries[client->tbl_index].next_free =
-                        clienttable->first_free;
-                clienttable->first_free = client->tbl_index;
-        }
-        UNLOCK (&clienttable->lock);
-
-        list_for_each_entry (gtrav, &client->this->ctx->graphs, list) {
-                xtrav = gtrav->top;
-                while (xtrav != NULL) {
-                        if (xtrav->cbks->client_destroy != NULL)
-                                xtrav->cbks->client_destroy (xtrav, client);
-                        xtrav = xtrav->next;
-                }
-        }
-        GF_FREE (client->auth.data);
-        GF_FREE (client->auth.username);
-        GF_FREE (client->auth.passwd);
-        GF_FREE (client->scratch_ctx.ctx);
-        GF_FREE (client->client_uid);
-        GF_FREE (client);
+static void
+client_destroy(client_t *client)
+{
+    clienttable_t *clienttable = NULL;
+    glusterfs_graph_t *gtrav = NULL;
+
+    if (client == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        goto out;
+    }
+
+    clienttable = client->this->ctx->clienttable;
+
+    LOCK(&clienttable->lock);
+    {
+        clienttable->cliententries[client->tbl_index].client = NULL;
+        clienttable->cliententries[client->tbl_index]
+            .next_free = clienttable->first_free;
+        clienttable->first_free = client->tbl_index;
+    }
+    UNLOCK(&clienttable->lock);
+
+    list_for_each_entry(gtrav, &client->this->ctx->graphs, list)
+    {
+        gf_client_destroy_recursive(gtrav->top, client);
+    }
+
+    if (client->subdir_inode)
+        inode_unref(client->subdir_inode);
+
+    LOCK_DESTROY(&client->scratch_ctx.lock);
+
+    GF_FREE(client->auth.data);
+    GF_FREE(client->auth.username);
+    GF_FREE(client->auth.passwd);
+    GF_FREE(client->scratch_ctx.ctx);
+    GF_FREE(client->client_uid);
+    GF_FREE(client->subdir_mount);
+    GF_FREE(client->client_name);
+    GF_FREE(client);
 out:
-        return;
+    return;
 }
 
+static int
+gf_client_disconnect_recursive(xlator_t *xl, client_t *client)
+{
+    int ret = 0;
+    xlator_list_t *trav;
+
+    if (!xl->call_cleanup && xl->cbks->client_disconnect) {
+        ret = xl->cbks->client_disconnect(xl, client);
+    }
+
+    for (trav = xl->children; trav; trav = trav->next) {
+        ret |= gf_client_disconnect_recursive(trav->xlator, client);
+    }
+
+    return ret;
+}
 
 int
-gf_client_disconnect (client_t *client)
+gf_client_disconnect(client_t *client)
 {
-        int                ret   = 0;
-        glusterfs_graph_t *gtrav = NULL;
-        xlator_t          *xtrav = NULL;
-
-        list_for_each_entry (gtrav, &client->this->ctx->graphs, list) {
-                xtrav = gtrav->top;
-                while (xtrav != NULL) {
-                        if (xtrav->cbks->client_disconnect != NULL)
-                                if (xtrav->cbks->client_disconnect (xtrav, client) != 0)
-                                        ret = -1;
-                        xtrav = xtrav->next;
-                }
-        }
+    int ret = 0;
+    glusterfs_graph_t *gtrav = NULL;
 
-        return ret;
-}
+    list_for_each_entry(gtrav, &client->this->ctx->graphs, list)
+    {
+        ret |= gf_client_disconnect_recursive(gtrav->top, client);
+    }
 
+    return ret;
+}
 
 void
-gf_client_unref (client_t *client)
+gf_client_unref(client_t *client)
 {
-        int refcount;
-
-        if (!client) {
-                gf_msg_callingfn ("client_t", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "client is NULL");
-                return;
-        }
+    uint64_t refcount;
 
-        refcount = DECREMENT_ATOMIC (client->ref.lock, client->ref.count);
-        gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT, "%s: "
-                          "ref-count %d", client->client_uid,
-                          (int)client->ref.count);
-        if (refcount == 0) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0, LG_MSG_DISCONNECT_CLIENT,
-                        "Shutting down connection %s", client->client_uid);
-                client_destroy (client);
-        }
+    if (!client) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "client is NULL");
+        return;
+    }
+
+    refcount = GF_ATOMIC_DEC(client->count);
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT,
+                     "%s: "
+                     "ref-count %" GF_PRI_ATOMIC,
+                     client->client_uid, refcount);
+    if (refcount == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, LG_MSG_DISCONNECT_CLIENT,
+               "Shutting down connection %s", client->client_uid);
+        client_destroy(client);
+    }
 }
 
-
 static int
-client_ctx_set_int (client_t *client, void *key, void *value)
+__client_ctx_get_int(client_t *client, void *key, void **value)
 {
-        int index   = 0;
-        int ret     = 0;
-        int set_idx = -1;
-
-        for (index = 0; index < client->scratch_ctx.count; index++) {
-                if (!client->scratch_ctx.ctx[index].ctx_key) {
-                        if (set_idx == -1)
-                                set_idx = index;
-                        /* dont break, to check if key already exists
-                           further on */
-                }
-                if (client->scratch_ctx.ctx[index].ctx_key == key) {
-                        set_idx = index;
-                        break;
-                }
-        }
+    int index = 0;
+    int ret = 0;
 
-        if (set_idx == -1) {
-                ret = -1;
-                goto out;
-        }
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (client->scratch_ctx.ctx[index].ctx_key == key)
+            break;
+    }
+
+    if (index == client->scratch_ctx.count) {
+        ret = -1;
+        goto out;
+    }
 
-        client->scratch_ctx.ctx[set_idx].ctx_key = key;
-        client->scratch_ctx.ctx[set_idx].ctx_value  = value;
+    if (value)
+        *value = client->scratch_ctx.ctx[index].ctx_value;
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-client_ctx_set (client_t *client, void *key, void *value)
+static int
+__client_ctx_set_int(client_t *client, void *key, void *value)
 {
-        int ret = 0;
-
-        if (!client || !key)
-                return -1;
+    int index = 0;
+    int ret = 0;
+    int set_idx = -1;
 
-        LOCK (&client->scratch_ctx.lock);
-        {
-                ret = client_ctx_set_int (client, key, value);
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (!client->scratch_ctx.ctx[index].ctx_key) {
+            if (set_idx == -1)
+                set_idx = index;
+            /* don't break, to check if key already exists
+               further on */
+        }
+        if (client->scratch_ctx.ctx[index].ctx_key == key) {
+            set_idx = index;
+            break;
         }
-        UNLOCK (&client->scratch_ctx.lock);
+    }
 
-        return ret;
-}
+    if (set_idx == -1) {
+        ret = -1;
+        goto out;
+    }
 
+    client->scratch_ctx.ctx[set_idx].ctx_key = key;
+    client->scratch_ctx.ctx[set_idx].ctx_value = value;
 
-static int
-client_ctx_get_int (client_t *client, void *key, void **value)
+out:
+    return ret;
+}
+
+/*will return success with old value if exist*/
+void *
+client_ctx_set(client_t *client, void *key, void *value)
 {
-        int index = 0;
-        int ret   = 0;
+    int ret = 0;
+    void *ret_value = NULL;
 
-        for (index = 0; index < client->scratch_ctx.count; index++) {
-                if (client->scratch_ctx.ctx[index].ctx_key == key)
-                        break;
-        }
+    if (!client || !key || !value)
+        return NULL;
 
-        if (index == client->scratch_ctx.count) {
-                ret = -1;
-                goto out;
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_get_int(client, key, &ret_value);
+        if (!ret && ret_value) {
+            UNLOCK(&client->scratch_ctx.lock);
+            return ret_value;
         }
 
-        if (value)
-                *value = client->scratch_ctx.ctx[index].ctx_value;
+        ret = __client_ctx_set_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
 
-out:
-        return ret;
+    if (ret)
+        return NULL;
+    return value;
 }
 
-
 int
-client_ctx_get (client_t *client, void *key, void **value)
+client_ctx_get(client_t *client, void *key, void **value)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!client || !key)
-                return -1;
+    if (!client || !key)
+        return -1;
 
-        LOCK (&client->scratch_ctx.lock);
-        {
-                ret = client_ctx_get_int (client, key, value);
-        }
-        UNLOCK (&client->scratch_ctx.lock);
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_get_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-client_ctx_del_int (client_t *client, void *key, void **value)
+__client_ctx_del_int(client_t *client, void *key, void **value)
 {
-        int index = 0;
-        int ret   = 0;
+    int index = 0;
+    int ret = 0;
 
-        for (index = 0; index < client->scratch_ctx.count; index++) {
-                if (client->scratch_ctx.ctx[index].ctx_key == key)
-                        break;
-        }
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (client->scratch_ctx.ctx[index].ctx_key == key)
+            break;
+    }
 
-        if (index == client->scratch_ctx.count) {
-                ret = -1;
-                goto out;
-        }
+    if (index == client->scratch_ctx.count) {
+        ret = -1;
+        goto out;
+    }
 
-        if (value)
-                *value = client->scratch_ctx.ctx[index].ctx_value;
+    if (value)
+        *value = client->scratch_ctx.ctx[index].ctx_value;
 
-        client->scratch_ctx.ctx[index].ctx_key   = 0;
-        client->scratch_ctx.ctx[index].ctx_value = 0;
+    client->scratch_ctx.ctx[index].ctx_key = 0;
+    client->scratch_ctx.ctx[index].ctx_value = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-client_ctx_del (client_t *client, void *key, void **value)
+client_ctx_del(client_t *client, void *key, void **value)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!client || !key)
-                return -1;
+    if (!client || !key)
+        return -1;
 
-        LOCK (&client->scratch_ctx.lock);
-        {
-                ret = client_ctx_del_int (client, key, value);
-        }
-        UNLOCK (&client->scratch_ctx.lock);
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_del_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
 
-        return ret;
+    return ret;
 }
 
-
 void
-client_dump (client_t *client, char *prefix)
-{
-        char key[GF_DUMP_MAX_BUF_LEN];
-
-        if (!client)
-                return;
-
-        memset(key, 0, sizeof key);
-        gf_proc_dump_write("refcount", "%d", client->ref.count);
-}
-
-
-void
-cliententry_dump (cliententry_t *cliententry, char *prefix)
-{
-        if (!cliententry)
-                return;
-
-        if (GF_CLIENTENTRY_ALLOCATED != cliententry->next_free)
-                return;
-
-        if (cliententry->client)
-                client_dump(cliententry->client, prefix);
-}
-
-
-void
-clienttable_dump (clienttable_t *clienttable, char *prefix)
-{
-        int     i   = 0;
-        int     ret = -1;
-        char    key[GF_DUMP_MAX_BUF_LEN] = {0};
-
-        if (!clienttable)
-                return;
-
-        ret = TRY_LOCK (&clienttable->lock);
-        {
-                if (ret) {
-                        gf_msg ("client_t", GF_LOG_WARNING, 0,
-                                LG_MSG_LOCK_FAILED,
-                                "Unable to acquire lock");
-                        return;
-                }
-                memset(key, 0, sizeof key);
-                gf_proc_dump_build_key(key, prefix, "maxclients");
-                gf_proc_dump_write(key, "%d", clienttable->max_clients);
-                gf_proc_dump_build_key(key, prefix, "first_free");
-                gf_proc_dump_write(key, "%d", clienttable->first_free);
-                for ( i = 0 ; i < clienttable->max_clients; i++) {
-                        if (GF_CLIENTENTRY_ALLOCATED ==
-                            clienttable->cliententries[i].next_free) {
-                                gf_proc_dump_build_key(key, prefix,
-                                                       "cliententry[%d]", i);
-                                gf_proc_dump_add_section(key);
-                                cliententry_dump(&clienttable->cliententries[i],
-                                                 key);
-                        }
-                }
-        }
-        UNLOCK(&clienttable->lock);
-}
-
-
-void
-client_ctx_dump (client_t *client, char *prefix)
+client_ctx_dump(client_t *client, char *prefix)
 {
 #if 0 /* TBD, FIXME */
         struct client_ctx   *client_ctx = NULL;
@@ -667,243 +585,241 @@ out:
 #endif
 }
 
-
 /*
  * the following functions are here to preserve legacy behavior of the
  * protocol/server xlator dump, but perhaps they should just be folded
  * into the client dump instead?
  */
 int
-gf_client_dump_fdtables_to_dict (xlator_t *this, dict_t *dict)
+gf_client_dump_fdtables_to_dict(xlator_t *this, dict_t *dict)
 {
-        clienttable_t  *clienttable = NULL;
-        int             count       = 0;
-        int             ret         = -1;
+    clienttable_t *clienttable = NULL;
+    int count = 0;
+    int ret = -1;
 #ifdef NOTYET
-        client_t       *client      = NULL;
-        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    client_t *client = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
 #endif
 
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
 
-        clienttable = this->ctx->clienttable;
+    clienttable = this->ctx->clienttable;
 
-        if (!clienttable)
-                return -1;
+    if (!clienttable)
+        return -1;
 
 #ifdef NOTYET
-        ret = TRY_LOCK (&clienttable->lock);
-        {
-                if (ret) {
-                        gf_msg ("client_t", GF_LOG_WARNING, 0,
-                                LG_MSG_LOCK_FAILED,
-                                "Unable to acquire lock");
-                        return -1;
-                }
-                for ( ; count < clienttable->max_clients; count++) {
-                        if (GF_CLIENTENTRY_ALLOCATED !=
-                            clienttable->cliententries[count].next_free)
-                                continue;
-                        client = clienttable->cliententries[count].client;
-                        memset(key, 0, sizeof key);
-                        snprintf (key, sizeof key, "conn%d", count++);
-                        fdtable_dump_to_dict (client->server_ctx.fdtable,
-                                              key, dict);
-                }
-        }
-        UNLOCK(&clienttable->lock);
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->bound_xl &&
+                !strcmp(client->bound_xl->name, this->name)) {
+                snprintf(key, sizeof(key), "conn%d", count++);
+                fdtable_dump_to_dict(client->server_ctx.fdtable, key, dict);
+            }
+        }
+    }
+    UNLOCK(&clienttable->lock);
 #endif
 
-        ret = dict_set_int32 (dict, "conncount", count);
+    ret = dict_set_int32(dict, "conncount", count);
 out:
-        return ret;
+    return ret;
 }
 
 int
-gf_client_dump_fdtables (xlator_t *this)
+gf_client_dump_fdtables(xlator_t *this)
 {
-        client_t       *client = NULL;
-        clienttable_t  *clienttable = NULL;
-        int             count = 1;
-        int             ret = -1;
-        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-
-        clienttable = this->ctx->clienttable;
-
-        if (!clienttable)
-                return -1;
-
-        ret = TRY_LOCK (&clienttable->lock);
-        {
-                if (ret) {
-                        gf_msg ("client_t", GF_LOG_WARNING, 0,
-                                LG_MSG_LOCK_FAILED,
-                                "Unable to acquire lock");
-                        return -1;
-                }
-
-
-                for ( ; count < clienttable->max_clients; count++) {
-                        if (GF_CLIENTENTRY_ALLOCATED !=
-                            clienttable->cliententries[count].next_free)
-                                continue;
-                        client = clienttable->cliententries[count].client;
-                        memset(key, 0, sizeof key);
-                        if (client->client_uid) {
-                                gf_proc_dump_build_key (key, "conn",
-                                                        "%d.id", count);
-                                gf_proc_dump_write (key, "%s",
-                                                    client->client_uid);
-                        }
-
-                        gf_proc_dump_build_key (key, "conn", "%d.ref",
-                                                        count);
-                        gf_proc_dump_write (key, "%d", client->ref.count);
-                        if (client->bound_xl) {
-                                gf_proc_dump_build_key (key, "conn",
-                                                        "%d.bound_xl", count);
-                                gf_proc_dump_write (key, "%s",
-                                                    client->bound_xl->name);
-                        }
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    int count = 1;
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        return -1;
+
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->client_uid) {
+                gf_proc_dump_build_key(key, "conn", "%d.id", count);
+                gf_proc_dump_write(key, "%s", client->client_uid);
+            }
+
+            if (client->subdir_mount) {
+                gf_proc_dump_build_key(key, "conn", "%d.subdir", count);
+                gf_proc_dump_write(key, "%s", client->subdir_mount);
+            }
+            gf_proc_dump_build_key(key, "conn", "%d.ref", count);
+            gf_proc_dump_write(key, "%" GF_PRI_ATOMIC,
+                               GF_ATOMIC_GET(client->count));
+            if (client->bound_xl) {
+                gf_proc_dump_build_key(key, "conn", "%d.bound_xl", count);
+                gf_proc_dump_write(key, "%s", client->bound_xl->name);
+            }
 
 #ifdef NOTYET
-                        gf_proc_dump_build_key (key, "conn","%d.id", count);
-                        fdtable_dump (client->server_ctx.fdtable, key);
+            gf_proc_dump_build_key(key, "conn", "%d.id", count);
+            fdtable_dump(client->server_ctx.fdtable, key);
 #endif
-                }
         }
+    }
 
-        UNLOCK(&clienttable->lock);
+    UNLOCK(&clienttable->lock);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict)
+gf_client_dump_inodes_to_dict(xlator_t *this, dict_t *dict)
 {
-        client_t       *client        = NULL;
-        clienttable_t  *clienttable   = NULL;
-        xlator_t       *prev_bound_xl = NULL;
-        char            key[32]       = {0,};
-        int             count         = 0;
-        int             ret           = -1;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        clienttable = this->ctx->clienttable;
-
-        if (!clienttable)
-                return -1;
-
-        ret = TRY_LOCK (&clienttable->lock);
-        {
-                if (ret) {
-                        gf_msg ("client_t", GF_LOG_WARNING, 0,
-                                LG_MSG_LOCK_FAILED,
-                                "Unable to acquire lock");
-                        return -1;
-                }
-                for ( ; count < clienttable->max_clients; count++) {
-                        if (GF_CLIENTENTRY_ALLOCATED !=
-                            clienttable->cliententries[count].next_free)
-                                continue;
-                        client = clienttable->cliententries[count].client;
-                        memset(key, 0, sizeof key);
-                        if (client->bound_xl && client->bound_xl->itable) {
-                                /* Presently every brick contains only
-                                 * one bound_xl for all connections.
-                                 * This will lead to duplicating of
-                                 *  the inode lists, if listing is
-                                 * done for every connection. This
-                                 * simple check prevents duplication
-                                 * in the present case. If need arises
-                                 * the check can be improved.
-                                 */
-                                if (client->bound_xl == prev_bound_xl)
-                                        continue;
-                                prev_bound_xl = client->bound_xl;
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "conn%d", count);
-                                inode_table_dump_to_dict (client->bound_xl->itable,
-                                                          key, dict);
-                        }
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    xlator_t *prev_bound_xl = NULL;
+    char key[32] = {
+        0,
+    };
+    int count = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        return -1;
+
+    ret = LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (!strcmp(client->bound_xl->name, this->name)) {
+                if (client->bound_xl && client->bound_xl->itable) {
+                    /* Presently every brick contains only
+                     * one bound_xl for all connections.
+                     * This will lead to duplicating of
+                     *  the inode lists, if listing is
+                     * done for every connection. This
+                     * simple check prevents duplication
+                     * in the present case. If need arises
+                     * the check can be improved.
+                     */
+                    if (client->bound_xl == prev_bound_xl)
+                        continue;
+                    prev_bound_xl = client->bound_xl;
+
+                    snprintf(key, sizeof(key), "conn%d", count);
+                    inode_table_dump_to_dict(client->bound_xl->itable, key,
+                                             dict);
                 }
+            }
         }
-        UNLOCK(&clienttable->lock);
+    }
+    UNLOCK(&clienttable->lock);
 
-        ret = dict_set_int32 (dict, "conncount", count);
+    ret = dict_set_int32(dict, "conncount", count);
 
 out:
-        if (prev_bound_xl)
-                prev_bound_xl = NULL;
-        return ret;
+    if (prev_bound_xl)
+        prev_bound_xl = NULL;
+    return ret;
 }
 
 int
-gf_client_dump_inodes (xlator_t *this)
+gf_client_dump_inodes(xlator_t *this)
 {
-        client_t       *client        = NULL;
-        clienttable_t  *clienttable   = NULL;
-        xlator_t       *prev_bound_xl = NULL;
-        int             count         = 0;
-        int             ret           = -1;
-        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-
-        clienttable = this->ctx->clienttable;
-
-        if (!clienttable)
-                goto out;
-
-        ret = TRY_LOCK (&clienttable->lock);
-        {
-                if (ret) {
-                        gf_msg ("client_t", GF_LOG_WARNING, 0,
-                                LG_MSG_LOCK_FAILED,
-                                "Unable to acquire lock");
-                        goto out;
-                }
-
-                for ( ; count < clienttable->max_clients; count++) {
-                        if (GF_CLIENTENTRY_ALLOCATED !=
-                            clienttable->cliententries[count].next_free)
-                                continue;
-                        client = clienttable->cliententries[count].client;
-                        memset(key, 0, sizeof key);
-                        if (client->bound_xl && client->bound_xl->itable) {
-                                /* Presently every brick contains only
-                                 * one bound_xl for all connections.
-                                 * This will lead to duplicating of
-                                 * the inode lists, if listing is
-                                 * done for every connection. This
-                                 * simple check prevents duplication
-                                 * in the present case. If need arises
-                                 * the check can be improved.
-                                 */
-                                if (client->bound_xl == prev_bound_xl)
-                                        continue;
-                                prev_bound_xl = client->bound_xl;
-
-                                gf_proc_dump_build_key(key, "conn",
-                                                       "%d.bound_xl.%s", count,
-                                                       client->bound_xl->name);
-                                inode_table_dump(client->bound_xl->itable,key);
-                        }
-                }
-        }
-        UNLOCK(&clienttable->lock);
-
-        ret = 0;
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    xlator_t *prev_bound_xl = NULL;
+    int count = 0;
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        goto out;
+
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            goto out;
+        }
+
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->bound_xl && client->bound_xl->itable) {
+                /* Presently every brick contains only
+                 * one bound_xl for all connections.
+                 * This will lead to duplicating of
+                 * the inode lists, if listing is
+                 * done for every connection. This
+                 * simple check prevents duplication
+                 * in the present case. If need arises
+                 * the check can be improved.
+                 */
+                if (client->bound_xl == prev_bound_xl)
+                    continue;
+                prev_bound_xl = client->bound_xl;
+
+                gf_proc_dump_build_key(key, "conn", "%d.bound_xl.%s", count,
+                                       client->bound_xl->name);
+                inode_table_dump(client->bound_xl->itable, key);
+            }
+        }
+    }
+    UNLOCK(&clienttable->lock);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
-
diff --git a/libglusterfs/src/client_t.h b/libglusterfs/src/client_t.h
deleted file mode 100644
index 29ea7f29ce8..00000000000
--- a/libglusterfs/src/client_t.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CLIENT_T_H
-#define _CLIENT_T_H
-
-#include "glusterfs.h"
-#include "locking.h"  /* for gf_lock_t, not included by glusterfs.h */
-
-struct client_ctx {
-        void     *ctx_key;
-        void     *ctx_value;
-};
-
-typedef struct _client_t {
-        struct {
-                /* e.g. protocol/server stashes its ctx here */
-                gf_lock_t            lock;
-                unsigned short       count;
-                struct client_ctx   *ctx;
-        }            scratch_ctx;
-        struct {
-                gf_lock_t            lock;
-                volatile int         bind;
-                volatile int         count;
-        }            ref;
-        xlator_t    *bound_xl;
-        xlator_t    *this;
-        int          tbl_index;
-        char        *client_uid;
-        struct {
-                int                  flavour;
-                size_t               len;
-                char                *data;
-                char                *username;
-                char                *passwd;
-        }            auth;
-} client_t;
-
-#define GF_CLIENTCTX_INITIAL_SIZE 8
-
-struct client_table_entry {
-        client_t            *client;
-        int                  next_free;
-};
-typedef struct client_table_entry cliententry_t;
-
-struct clienttable {
-        unsigned int         max_clients;
-        gf_lock_t            lock;
-        cliententry_t       *cliententries;
-        int                  first_free;
-	client_t            *local;
-};
-typedef struct clienttable clienttable_t;
-
-#define GF_CLIENTTABLE_INITIAL_SIZE 128
-
-/* Signifies no more entries in the client table. */
-#define GF_CLIENTTABLE_END  -1
-
-/* This is used to invalidate
- * the next_free value in an cliententry that has been allocated
- */
-#define GF_CLIENTENTRY_ALLOCATED    -2
-
-struct rpcsvc_auth_data;
-
-client_t *
-gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid);
-
-void
-gf_client_put (client_t *client, gf_boolean_t *detached);
-
-clienttable_t *
-gf_clienttable_alloc (void);
-
-void
-gf_client_clienttable_destroy (clienttable_t *clienttable);
-
-client_t *
-gf_client_ref (client_t *client);
-
-void
-gf_client_unref (client_t *client);
-
-int
-gf_client_dump_fdtable_to_dict (xlator_t *this, dict_t *dict);
-
-int
-gf_client_dump_fdtable (xlator_t *this);
-
-int
-gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict);
-
-int
-gf_client_dump_inodes (xlator_t *this);
-
-int
-client_ctx_set (client_t *client, void *key, void *value);
-
-int
-client_ctx_get (client_t *client, void *key, void **value);
-
-int
-client_ctx_del (client_t *client, void *key, void **value);
-
-void
-client_ctx_dump (client_t *client, char *prefix);
-
-int
-gf_client_dump_fdtables_to_dict (xlator_t *this, dict_t *dict);
-
-int
-gf_client_dump_fdtables (xlator_t *this);
-
-int
-gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict);
-
-int
-gf_client_dump_inodes (xlator_t *this);
-
-int
-gf_client_disconnect (client_t *client);
-
-#endif /* _CLIENT_T_H */
diff --git a/libglusterfs/src/cluster-syncop.c b/libglusterfs/src/cluster-syncop.c
index 10993e6088d..6ee89ddfdcf 100644
--- a/libglusterfs/src/cluster-syncop.c
+++ b/libglusterfs/src/cluster-syncop.c
@@ -14,523 +14,499 @@
 /* NOTE: Cluster-syncop, like syncop blocks the executing thread until the
  * responses are gathered if it is not executed as part of synctask. So it
  * shouldn't be invoked in epoll worker thread */
-#include "cluster-syncop.h"
-#include "defaults.h"
-
-#define FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fop, args ...) do {\
-        int __i = 0;                                                    \
-        int __count = 0;					        \
-        cluster_local_t __local = {0,};                                 \
-        void    *__old_local = frame->local;                            \
-                                                                        \
-        __local.replies = replies;                                      \
-        memset (output, 0, numsubvols);                                 \
-        cluster_replies_wipe (replies, numsubvols);                     \
-        for (__i = 0; __i < numsubvols; __i++)                          \
-                INIT_LIST_HEAD (&replies[__i].entries.list);            \
-        if (syncbarrier_init (&__local.barrier))                        \
-                break;                                                  \
-        frame->local = &__local;                                        \
-        for (__i = 0; __i < numsubvols; __i++) {		        \
-                if (!on[__i])                                           \
-                        continue;				        \
-                STACK_WIND_COOKIE (frame, cluster_##fop##_cbk,          \
-                                   (void *)(long) __i, subvols[__i],    \
-                                   subvols[__i]->fops->fop, args);      \
-                __count++;						\
-        }								\
-        syncbarrier_wait (&__local.barrier, __count);			\
-        syncbarrier_destroy (&__local.barrier);                         \
-        frame->local = __old_local;                                     \
-        STACK_RESET (frame->root);                                      \
-        } while (0)
-
-#define FOP_SEQ(subvols, on, numsubvols, replies, output, frame, fop, args ...) do {\
-        int __i = 0;					                \
-                                                                        \
-        cluster_local_t __local = {0,};                                 \
-        void    *__old_local = frame->local;                            \
-        __local.replies = replies;                                      \
-        memset (output, 0, numsubvols);                                 \
-        cluster_replies_wipe (replies, numsubvols);                     \
-        for (__i = 0; __i < numsubvols; __i++)                          \
-                INIT_LIST_HEAD (&replies[__i].entries.list);            \
-        if (syncbarrier_init (&__local.barrier))                        \
-                break;                                                  \
-        frame->local = &__local;                                        \
-        for (__i = 0; __i < numsubvols; __i++) {		        \
-                if (!on[__i])                                           \
-                        continue;				        \
-                STACK_WIND_COOKIE (frame, cluster_##fop##_cbk,          \
-                                   (void *)(long) __i, subvols[__i],    \
-                                   subvols[__i]->fops->fop, args);      \
-                syncbarrier_wait (&__local.barrier, 1);                 \
-        }                                                               \
-        syncbarrier_destroy (&__local.barrier);                         \
-        frame->local = __old_local;                                     \
-        STACK_RESET (frame->root);                                      \
-        } while (0)
-
-#define FOP_CBK(fop, frame, cookie, args ...) do {\
-        cluster_local_t *__local = frame->local;                        \
-        int __i = (long)cookie;                                         \
-        args_##fop##_cbk_store (&__local->replies[__i], args);          \
-        __local->replies[__i].valid = 1;                                \
-        syncbarrier_wake (&__local->barrier);                           \
-        } while (0)
-
-static int
-fop_success_fill (default_args_cbk_t *replies, int numsubvols,
-                  unsigned char *success)
-{
-        int i = 0;
-        int count = 0;
-
-        for (i = 0; i < numsubvols; i++) {
-                if (replies[i].valid && replies[i].op_ret >= 0) {
-                        success[i] = 1;
-                        count++;
-                } else {
-                        success[i] = 0;
-                }
+#include "glusterfs/cluster-syncop.h"
+#include "glusterfs/defaults.h"
+
+#define FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fop,       \
+                   args...)                                                    \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        cluster_local_t __local = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        void *__old_local = frame->local;                                      \
+                                                                               \
+        __local.replies = replies;                                             \
+        memset(output, 0, numsubvols);                                         \
+        cluster_replies_wipe(replies, numsubvols);                             \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+        if (syncbarrier_init(&__local.barrier))                                \
+            break;                                                             \
+        frame->local = &__local;                                               \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (on[__i]) {                                                     \
+                __count++;                                                     \
+            }                                                                  \
+        }                                                                      \
+        __local.barrier.waitfor = __count;                                     \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (!on[__i])                                                      \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, cluster_##fop##_cbk, (void *)(long)__i,   \
+                              subvols[__i], subvols[__i]->fops->fop, args);    \
+        }                                                                      \
+        syncbarrier_wait(&__local.barrier, __count);                           \
+        syncbarrier_destroy(&__local.barrier);                                 \
+        frame->local = __old_local;                                            \
+        STACK_RESET(frame->root);                                              \
+    } while (0)
+
+#define FOP_SEQ(subvols, on, numsubvols, replies, output, frame, fop, args...) \
+    do {                                                                       \
+        int __i = 0;                                                           \
+                                                                               \
+        cluster_local_t __local = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        void *__old_local = frame->local;                                      \
+        __local.replies = replies;                                             \
+        memset(output, 0, numsubvols);                                         \
+        cluster_replies_wipe(replies, numsubvols);                             \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+        if (syncbarrier_init(&__local.barrier))                                \
+            break;                                                             \
+        frame->local = &__local;                                               \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (!on[__i])                                                      \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, cluster_##fop##_cbk, (void *)(long)__i,   \
+                              subvols[__i], subvols[__i]->fops->fop, args);    \
+            syncbarrier_wait(&__local.barrier, 1);                             \
+        }                                                                      \
+        syncbarrier_destroy(&__local.barrier);                                 \
+        frame->local = __old_local;                                            \
+        STACK_RESET(frame->root);                                              \
+    } while (0)
+
+#define FOP_CBK(fop, frame, cookie, args...)                                   \
+    do {                                                                       \
+        cluster_local_t *__local = frame->local;                               \
+        int __i = (long)cookie;                                                \
+        args_##fop##_cbk_store(&__local->replies[__i], args);                  \
+        __local->replies[__i].valid = 1;                                       \
+        syncbarrier_wake(&__local->barrier);                                   \
+    } while (0)
+
+int32_t
+cluster_fop_success_fill(default_args_cbk_t *replies, int numsubvols,
+                         unsigned char *success)
+{
+    int i = 0;
+    int count = 0;
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret >= 0) {
+            success[i] = 1;
+            count++;
+        } else {
+            success[i] = 0;
         }
+    }
 
-        return count;
+    return count;
 }
 
 void
-cluster_replies_wipe (default_args_cbk_t *replies, int numsubvols)
+cluster_replies_wipe(default_args_cbk_t *replies, int numsubvols)
 {
-        int i = 0;
-        for (i = 0; i < numsubvols; i++)
-                args_cbk_wipe (&replies[i]);
-        memset (replies, 0, numsubvols * sizeof (*replies));
+    int i = 0;
+
+    if (!replies)
+        return;
+
+    for (i = 0; i < numsubvols; i++)
+        args_cbk_wipe(&replies[i]);
+    memset(replies, 0, numsubvols * sizeof(*replies));
 }
 
 int32_t
-cluster_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+cluster_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        FOP_CBK (lookup, frame, cookie, op_ret, op_errno, inode, buf,
-                 xdata, postparent);
-        return 0;
+    FOP_CBK(lookup, frame, cookie, op_ret, op_errno, inode, buf, xdata,
+            postparent);
+    return 0;
 }
 
 int32_t
-cluster_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  dict_t *xdata)
+cluster_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
-        FOP_CBK (stat, frame, cookie, op_ret, op_errno, buf, xdata);
-        return 0;
+    FOP_CBK(stat, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
+int32_t
+cluster_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    FOP_CBK(truncate, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
 
 int32_t
-cluster_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+cluster_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf,
-                      dict_t *xdata)
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        FOP_CBK (truncate, frame, cookie, op_ret, op_errno, prebuf,
-                 postbuf, xdata);
-        return 0;
+    FOP_CBK(ftruncate, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-cluster_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf,
-                       dict_t *xdata)
+cluster_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (ftruncate, frame, cookie, op_ret, op_errno, prebuf,
-                 postbuf, xdata);
-        return 0;
+    FOP_CBK(access, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    dict_t *xdata)
+cluster_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
 {
-        FOP_CBK (access, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(readlink, frame, cookie, op_ret, op_errno, path, buf, xdata);
+    return 0;
 }
 
 int32_t
-cluster_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata)
+cluster_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (readlink, frame, cookie, op_ret, op_errno, path, buf,
-                 xdata);
-        return 0;
+    FOP_CBK(mknod, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
 }
 
+int32_t
+cluster_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(mkdir, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
 
 int32_t
-cluster_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
+cluster_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (mknod, frame, cookie, op_ret, op_errno, inode,
-                 buf, preparent, postparent, xdata);
-        return 0;
+    FOP_CBK(unlink, frame, cookie, op_ret, op_errno, preparent, postparent,
+            xdata);
+    return 0;
 }
 
 int32_t
-cluster_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+cluster_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (mkdir, frame, cookie, op_ret, op_errno, inode,
-                 buf, preparent, postparent, xdata);
-        return 0;
+    FOP_CBK(rmdir, frame, cookie, op_ret, op_errno, preparent, postparent,
+            xdata);
+    return 0;
 }
 
 int32_t
-cluster_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+cluster_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
                     struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (unlink, frame, cookie, op_ret, op_errno, preparent,
-                 postparent, xdata);
-        return 0;
+    FOP_CBK(symlink, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
 }
 
 int32_t
-cluster_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent,
+cluster_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
                    dict_t *xdata)
 {
-        FOP_CBK (rmdir, frame, cookie, op_ret, op_errno, preparent,
-                 postparent, xdata);
-        return 0;
+    FOP_CBK(rename, frame, cookie, op_ret, op_errno, buf, preoldparent,
+            postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+cluster_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (symlink, frame, cookie, op_ret, op_errno, inode, buf,
-                 preparent, postparent, xdata);
-        return 0;
+    FOP_CBK(link, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+cluster_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (rename, frame, cookie, op_ret, op_errno, buf, preoldparent,
-                 postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+    FOP_CBK(create, frame, cookie, op_ret, op_errno, fd, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent,
-                  dict_t *xdata)
+cluster_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        FOP_CBK (link, frame, cookie, op_ret, op_errno, inode, buf,
-                 preparent, postparent, xdata);
-        return 0;
+    FOP_CBK(open, frame, cookie, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent,
-                    dict_t *xdata)
+cluster_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-        FOP_CBK (create, frame, cookie, op_ret, op_errno, fd, inode, buf,
-                 preparent, postparent, xdata);
-        return 0;
+    FOP_CBK(readv, frame, cookie, op_ret, op_errno, vector, count, stbuf,
+            iobref, xdata);
+    return 0;
 }
 
 int32_t
-cluster_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd,
-                  dict_t *xdata)
+cluster_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        FOP_CBK (open, frame, cookie, op_ret, op_errno, fd, xdata);
-        return 0;
+    FOP_CBK(writev, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-cluster_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                   dict_t *xdata)
+cluster_put_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        FOP_CBK (readv, frame, cookie, op_ret, op_errno, vector, count,
-                 stbuf, iobref, xdata);
-        return 0;
+    FOP_CBK(put, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+cluster_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (writev, frame, cookie, op_ret, op_errno, prebuf, postbuf,
-                 xdata);
-        return 0;
+    FOP_CBK(flush, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   dict_t *xdata)
+cluster_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        FOP_CBK (flush, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(fsync, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
-
 int32_t
-cluster_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf,
-                   dict_t *xdata)
+cluster_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        FOP_CBK (fsync, frame, cookie, op_ret, op_errno, prebuf, postbuf,
-                 xdata);
-        return 0;
+    FOP_CBK(fstat, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 int32_t
-cluster_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   dict_t *xdata)
+cluster_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        FOP_CBK (fstat, frame, cookie, op_ret, op_errno, buf, xdata);
-        return 0;
+    FOP_CBK(opendir, frame, cookie, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int32_t
-cluster_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd,
-                     dict_t *xdata)
+cluster_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (opendir, frame, cookie, op_ret, op_errno, fd, xdata);
-        return 0;
+    FOP_CBK(fsyncdir, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      dict_t *xdata)
+cluster_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
 {
-        FOP_CBK (fsyncdir, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(statfs, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 int32_t
-cluster_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                    dict_t *xdata)
+cluster_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (statfs, frame, cookie, op_ret, op_errno, buf, xdata);
-        return 0;
+    FOP_CBK(setxattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      dict_t *xdata)
+cluster_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (setxattr, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(fsetxattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+cluster_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        FOP_CBK (fsetxattr, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(fgetxattr, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
-
 int32_t
-cluster_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict,
-                       dict_t *xdata)
+cluster_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        FOP_CBK (fgetxattr, frame, cookie, op_ret, op_errno, dict, xdata);
-        return 0;
+    FOP_CBK(getxattr, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
+cluster_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
-        FOP_CBK (getxattr, frame, cookie, op_ret, op_errno, dict, xdata);
-        return 0;
+    FOP_CBK(xattrop, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-cluster_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+cluster_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, dict_t *dict,
                      dict_t *xdata)
 {
-        FOP_CBK (xattrop, frame, cookie, op_ret, op_errno, dict, xdata);
-        return 0;
+    FOP_CBK(fxattrop, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-cluster_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
+cluster_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (fxattrop, frame, cookie, op_ret, op_errno, dict, xdata);
-        return 0;
+    FOP_CBK(removexattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
+cluster_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (removexattr, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(fremovexattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          dict_t *xdata)
+cluster_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
 {
-        FOP_CBK (fremovexattr, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(lk, frame, cookie, op_ret, op_errno, lock, xdata);
+    return 0;
 }
 
 int32_t
-cluster_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                dict_t *xdata)
+cluster_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (lk, frame, cookie, op_ret, op_errno, lock, xdata);
-        return 0;
+    FOP_CBK(inodelk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     dict_t *xdata)
+cluster_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (inodelk, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(finodelk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      dict_t *xdata)
+cluster_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (finodelk, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(entrylk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     dict_t *xdata)
+cluster_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (entrylk, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(fentrylk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      dict_t *xdata)
+cluster_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                      uint8_t *strong_checksum, dict_t *xdata)
 {
-        FOP_CBK (fentrylk, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(rchecksum, frame, cookie, op_ret, op_errno, weak_checksum,
+            strong_checksum, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum,
-                       dict_t *xdata)
+cluster_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        FOP_CBK (rchecksum, frame, cookie, op_ret, op_errno, weak_checksum,
-                             strong_checksum, xdata);
-        return 0;
+    FOP_CBK(readdir, frame, cookie, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+cluster_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
                      dict_t *xdata)
 {
-        FOP_CBK (readdir, frame, cookie, op_ret, op_errno, entries, xdata);
-        return 0;
+    FOP_CBK(readdirp, frame, cookie, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                      dict_t *xdata)
+cluster_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 {
-        FOP_CBK (readdirp, frame, cookie, op_ret, op_errno, entries, xdata);
-        return 0;
+    FOP_CBK(setattr, frame, cookie, op_ret, op_errno, statpre, statpost, xdata);
+    return 0;
 }
 
 int32_t
-cluster_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+cluster_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost,
-                     dict_t *xdata)
+                     struct iatt *statpost, dict_t *xdata)
 {
-        FOP_CBK (setattr, frame, cookie, op_ret, op_errno, statpre,
-                 statpost, xdata);
-        return 0;
-}
-
-int32_t
-cluster_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost,
-                      dict_t *xdata)
-{
-        FOP_CBK (fsetattr, frame, cookie, op_ret, op_errno, statpre,
-                 statpost, xdata);
-        return 0;
+    FOP_CBK(fsetattr, frame, cookie, op_ret, op_errno, statpre, statpost,
+            xdata);
+    return 0;
 }
 
 int32_t
@@ -538,8 +514,8 @@ cluster_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *pre,
                       struct iatt *post, dict_t *xdata)
 {
-        FOP_CBK (fallocate, frame, cookie, op_ret, op_errno, pre, post, xdata);
-        return 0;
+    FOP_CBK(fallocate, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
 }
 
 int32_t
@@ -547,638 +523,739 @@ cluster_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
                     struct iatt *post, dict_t *xdata)
 {
-        FOP_CBK (discard, frame, cookie, op_ret, op_errno, pre, post, xdata);
-        return 0;
+    FOP_CBK(discard, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
 }
 
 int32_t
 cluster_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                    struct iatt *post, dict_t *xdata)
+                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                     struct iatt *post, dict_t *xdata)
 {
-        FOP_CBK (zerofill, frame, cookie, op_ret, op_errno, pre,
-                 post, xdata);
-        return 0;
+    FOP_CBK(zerofill, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
 }
 
-
 int32_t
-cluster_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+cluster_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        FOP_CBK (ipc, frame, cookie, op_ret, op_errno, xdata);
-        return 0;
+    FOP_CBK(ipc, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-cluster_fgetxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fgetxattr, fd,
-                    name, xdata);
-        return fop_success_fill (replies, numsubvols, output);
-}
-
-int32_t
-cluster_fsetxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+cluster_fgetxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsetxattr, fd,
-                    dict, flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fgetxattr, fd,
+               name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_setxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_fsetxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
                   default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
                   int32_t flags, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, setxattr, loc,
-                    dict, flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsetxattr, fd,
+               dict, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_statfs (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+cluster_setxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, statfs, loc,
-                    xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, setxattr, loc,
+               dict, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fsyncdir (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-                  dict_t *xdata)
+cluster_statfs(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsyncdir, fd,
-                    flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, statfs, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_opendir (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_fsyncdir(xlator_t **subvols, unsigned char *on, int numsubvols,
                  default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
                  dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, opendir, loc,
-                    fd, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsyncdir, fd,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fstat (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+cluster_opendir(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fstat, fd,
-                    xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, opendir, loc,
+               fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fsync (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-               dict_t *xdata)
+cluster_fstat(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsync, fd,
-                    flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fstat, fd,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_flush (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+cluster_fsync(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, flush, fd,
-                    xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsync, fd,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_writev (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+cluster_flush(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, writev, fd,
-                    vector, count, off, flags, iobref, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, flush, fd,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_readv (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_writev(xlator_t **subvols, unsigned char *on, int numsubvols,
                default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+               call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readv, fd, size,
-                    offset, flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, writev, fd,
+               vector, count, off, flags, iobref, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
+int32_t
+cluster_put(xlator_t **subvols, unsigned char *on, int numsubvols,
+            default_args_cbk_t *replies, unsigned char *output,
+            call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+            off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, put, loc, mode,
+               umask, flags, vector, count, offset, iobref, xattr, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
 
 int32_t
-cluster_open (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_readv(xlator_t **subvols, unsigned char *on, int numsubvols,
               default_args_cbk_t *replies, unsigned char *output,
-              call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              fd_t *fd, dict_t *xdata)
+              call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, open, loc,
-                    flags, fd, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readv, fd, size,
+               offset, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_create (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+cluster_open(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, create, loc,
-                    flags, mode, umask, fd, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, open, loc,
+               flags, fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_link (xlator_t **subvols, unsigned char *on, int numsubvols,
-              default_args_cbk_t *replies, unsigned char *output,
-              call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+cluster_create(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, link, oldloc,
-                    newloc, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, create, loc,
+               flags, mode, umask, fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_rename (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
+cluster_link(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rename, oldloc,
-                    newloc, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, link, oldloc,
+               newloc, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
+int32_t
+cluster_rename(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rename, oldloc,
+               newloc, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
 
 int
-cluster_symlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, const char *linkpath,
-                 loc_t *loc, mode_t umask, dict_t *xdata)
+cluster_symlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, symlink,
-                    linkpath, loc, umask, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, symlink,
+               linkpath, loc, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_rmdir (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata)
+cluster_rmdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rmdir, loc,
-                    flags, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rmdir, loc,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_unlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-                dict_t *xdata)
+cluster_unlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, unlink, loc,
-                    xflag, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, unlink, loc,
+               xflag, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int
-cluster_mkdir (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               mode_t umask, dict_t *xdata)
+cluster_mkdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, mkdir, loc,
-                    mode, umask, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, mkdir, loc,
+               mode, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int
-cluster_mknod (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dev_t rdev, mode_t umask, dict_t *xdata)
+cluster_mknod(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, mknod, loc,
-                    mode, rdev, umask, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, mknod, loc,
+               mode, rdev, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_readlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                  dict_t *xdata)
+cluster_readlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readlink, loc,
-                    size, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readlink, loc,
+               size, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int32_t
-cluster_access (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-                dict_t *xdata)
+cluster_access(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+               dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, access, loc,
-                    mask, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, access, loc,
+               mask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_ftruncate (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   dict_t *xdata)
+cluster_ftruncate(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, ftruncate, fd,
-                    offset, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, ftruncate, fd,
+               offset, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_getxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name, dict_t *xdata)
+cluster_getxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, getxattr, loc,
-                    name, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, getxattr, loc,
+               name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
+int32_t
+cluster_xattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, xattrop, loc,
+               flags, dict, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
 
 int32_t
-cluster_xattrop (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_fxattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
                  default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, xattrop, loc,
-                    flags, dict, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fxattrop, fd,
+               flags, dict, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fxattrop (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+cluster_removexattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                    default_args_cbk_t *replies, unsigned char *output,
+                    call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fxattrop, fd,
-                    flags, dict, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, removexattr,
+               loc, name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_removexattr (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_fremovexattr(xlator_t **subvols, unsigned char *on, int numsubvols,
                      default_args_cbk_t *replies, unsigned char *output,
-                     call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     call_frame_t *frame, xlator_t *this, fd_t *fd,
                      const char *name, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, removexattr,
-                    loc, name, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fremovexattr,
+               fd, name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fremovexattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                      default_args_cbk_t *replies, unsigned char *output,
-                      call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
+cluster_lk(xlator_t **subvols, unsigned char *on, int numsubvols,
+           default_args_cbk_t *replies, unsigned char *output,
+           call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *lock, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fremovexattr,
-                    fd, name, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, lk, fd, cmd,
+               lock, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_lk (xlator_t **subvols, unsigned char *on, int numsubvols,
-            default_args_cbk_t *replies, unsigned char *output,
-            call_frame_t *frame, xlator_t *this, fd_t *fd,
-            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+cluster_rchecksum(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  int32_t len, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, lk, fd, cmd,
-                    lock, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rchecksum, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int32_t
-cluster_rchecksum (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   int32_t len, dict_t *xdata)
+cluster_readdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rchecksum, fd,
-                    offset, len, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readdir, fd,
+               size, off, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int32_t
-cluster_readdir (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_readdirp(xlator_t **subvols, unsigned char *on, int numsubvols,
                  default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 size_t size, off_t off, dict_t *xdata)
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readdir, fd,
-                    size, off, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readdirp, fd,
+               size, off, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int32_t
-cluster_readdirp (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  size_t size, off_t off, dict_t *xdata)
+cluster_setattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readdirp, fd,
-                    size, off, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, setattr, loc,
+               stbuf, valid, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_setattr (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_truncate(xlator_t **subvols, unsigned char *on, int numsubvols,
                  default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, setattr, loc,
-                    stbuf, valid, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, truncate, loc,
+               offset, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_truncate (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                  dict_t *xdata)
+cluster_stat(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, truncate, loc,
-                    offset, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, stat, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_stat (xlator_t **subvols, unsigned char *on, int numsubvols,
-              default_args_cbk_t *replies, unsigned char *output,
-              call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+cluster_lookup(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, stat, loc,
-                    xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, lookup, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_lookup (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+cluster_fsetattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, lookup, loc,
-                    xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsetattr, fd,
+               stbuf, valid, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fsetattr (xlator_t **subvols, unsigned char *on, int numsubvols,
+cluster_fallocate(xlator_t **subvols, unsigned char *on, int numsubvols,
                   default_args_cbk_t *replies, unsigned char *output,
                   call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
+                  int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsetattr, fd,
-                    stbuf, valid, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fallocate, fd,
+               keep_size, offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
-cluster_fallocate (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
-{
-        FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fallocate, fd,
-                   keep_size, offset, len, xdata);
-        return fop_success_fill (replies, numsubvols, output);
-}
-
-int32_t
-cluster_discard (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 off_t offset, size_t len, dict_t *xdata)
+cluster_discard(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, discard, fd,
-                   offset, len, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, discard, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int32_t
 cluster_zerofill(xlator_t **subvols, unsigned char *on, int numsubvols,
                  default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 off_t offset, off_t len, dict_t *xdata)
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, zerofill, fd,
-                   offset, len, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, zerofill, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
-
 int32_t
-cluster_ipc (xlator_t **subvols, unsigned char *on, int numsubvols,
-             default_args_cbk_t *replies, unsigned char *output,
-             call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+cluster_ipc(xlator_t **subvols, unsigned char *on, int numsubvols,
+            default_args_cbk_t *replies, unsigned char *output,
+            call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, ipc, op, xdata);
-        return fop_success_fill (replies, numsubvols, output);
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, ipc, op, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int
-cluster_uninodelk (xlator_t **subvols, unsigned char *locked_on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, char *dom,
-                   inode_t *inode, off_t off, size_t size)
+cluster_uninodelk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, off_t off, size_t size)
 {
-        loc_t loc = {0,};
-        struct gf_flock flock = {0, };
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-        flock.l_type = F_UNLCK;
-        flock.l_start = off;
-        flock.l_len = size;
+    FOP_ONLIST(subvols, locked_on, numsubvols, replies, output, frame, inodelk,
+               dom, &loc, F_SETLK, &flock, NULL);
 
-        FOP_ONLIST (subvols, locked_on, numsubvols, replies, output, frame, inodelk,
-                    dom, &loc, F_SETLK, &flock, NULL);
+    loc_wipe(&loc);
 
-        loc_wipe (&loc);
-
-        return fop_success_fill (replies, numsubvols, output);
+    return cluster_fop_success_fill(replies, numsubvols, output);
 }
 
 int
-cluster_tryinodelk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                    default_args_cbk_t *replies, unsigned char *locked_on,
-                    call_frame_t *frame, xlator_t *this, char *dom,
-                    inode_t *inode, off_t off, size_t size)
+cluster_tryinodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
+                   call_frame_t *frame, xlator_t *this, char *dom,
+                   inode_t *inode, off_t off, size_t size)
 {
-        struct gf_flock flock = {0, };
-        loc_t loc = {0};
+    struct gf_flock flock = {
+        0,
+    };
+    loc_t loc = {0};
 
-        flock.l_type = F_WRLCK;
-        flock.l_start = off;
-        flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        FOP_ONLIST (subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
-                    &loc, F_SETLK, &flock, NULL);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
 
-        loc_wipe (&loc);
-        return fop_success_fill (replies, numsubvols, locked_on);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
 }
 
 int
-cluster_inodelk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *locked_on,
-                 call_frame_t *frame, xlator_t *this, char *dom,
-                 inode_t *inode, off_t off, size_t size)
-{
-        struct gf_flock flock = {0, };
-        int i = 0;
-        loc_t loc = {0};
-        unsigned char *output = NULL;
-
-        flock.l_type = F_WRLCK;
-        flock.l_start = off;
-        flock.l_len = size;
-
-        output = alloca(numsubvols);
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        FOP_ONLIST (subvols, on, numsubvols, replies, locked_on, frame,
-                    inodelk, dom, &loc, F_SETLK, &flock, NULL);
-
-        for (i = 0; i < numsubvols; i++) {
-                if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
-                        fop_success_fill (replies, numsubvols, locked_on);
-                        cluster_uninodelk (subvols, locked_on, numsubvols,
-                                           replies, output, frame, this, dom, inode, off, size);
-
-                        FOP_SEQ (subvols, on, numsubvols, replies, locked_on,
-                                 frame, inodelk, dom, &loc, F_SETLKW, &flock,
-                                 NULL);
-                        break;
-                }
+cluster_inodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                off_t off, size_t size)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_uninodelk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, off, size);
+
+            FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame, inodelk,
+                    dom, &loc, F_SETLKW, &flock, NULL);
+            break;
         }
+    }
 
-        loc_wipe (&loc);
-        return fop_success_fill (replies, numsubvols, locked_on);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
 }
 
+int
+cluster_unentrylk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, const char *name)
+{
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    FOP_ONLIST(subvols, locked_on, numsubvols, replies, output, frame, entrylk,
+               dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+
+    loc_wipe(&loc);
+
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
 
 int
-cluster_unentrylk (xlator_t **subvols, unsigned char *locked_on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
+cluster_tryentrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
                    call_frame_t *frame, xlator_t *this, char *dom,
                    inode_t *inode, const char *name)
 {
-        loc_t loc = {0,};
-
+    loc_t loc = {0};
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-        FOP_ONLIST (subvols, locked_on, numsubvols, replies, output, frame,
-                    entrylk, dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK,
-                    NULL);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
 
-        loc_wipe (&loc);
+int
+cluster_entrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                const char *name)
+{
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_unentrylk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, name);
+            FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame, entrylk,
+                    dom, &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
 
-        return fop_success_fill (replies, numsubvols, output);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
 }
 
 int
-cluster_tryentrylk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                    default_args_cbk_t *replies, unsigned char *locked_on,
-                    call_frame_t *frame, xlator_t *this, char *dom,
-                    inode_t *inode, const char *name)
-{
-        loc_t loc = {0};
+cluster_tiebreaker_inodelk(xlator_t **subvols, unsigned char *on,
+                           int numsubvols, default_args_cbk_t *replies,
+                           unsigned char *locked_on, call_frame_t *frame,
+                           xlator_t *this, char *dom, inode_t *inode, off_t off,
+                           size_t size)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+    int num_success = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            num_success++;
+            continue;
+        }
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        FOP_ONLIST (subvols, on, numsubvols, replies, locked_on, frame,
-                    entrylk, dom, &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                    NULL);
+        /* TODO: If earlier subvols fail with an error other
+         * than EAGAIN, we could still have 2 clients competing
+         * for the lock*/
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_uninodelk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, off, size);
+
+            if (num_success) {
+                FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame,
+                        inodelk, dom, &loc, F_SETLKW, &flock, NULL);
+            } else {
+                loc_wipe(&loc);
+                memset(locked_on, 0, numsubvols);
+                return 0;
+            }
+            break;
+        }
+    }
 
-        loc_wipe (&loc);
-        return fop_success_fill (replies, numsubvols, locked_on);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
 }
 
 int
-cluster_entrylk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *locked_on,
-                 call_frame_t *frame, xlator_t *this, char *dom,
-                 inode_t *inode, const char *name)
-{
-        int i = 0;
-        loc_t loc = {0};
-        unsigned char *output = NULL;
-
-        output = alloca(numsubvols);
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        FOP_ONLIST (subvols, on, numsubvols, replies, locked_on, frame,
-                    entrylk, dom, &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                    NULL);
-
-        for (i = 0; i < numsubvols; i++) {
-                if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
-                        fop_success_fill (replies, numsubvols, locked_on);
-                        cluster_unentrylk (subvols, locked_on, numsubvols,
-                                           replies, output, frame, this, dom,
-                                           inode, name);
-                        FOP_SEQ (subvols, on, numsubvols, replies,
-                                 locked_on, frame, entrylk, dom, &loc, name,
-                                 ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                        break;
-                }
+cluster_tiebreaker_entrylk(xlator_t **subvols, unsigned char *on,
+                           int numsubvols, default_args_cbk_t *replies,
+                           unsigned char *locked_on, call_frame_t *frame,
+                           xlator_t *this, char *dom, inode_t *inode,
+                           const char *name)
+{
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+    int num_success = 0;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            num_success++;
+            continue;
+        }
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_unentrylk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, name);
+            if (num_success) {
+                FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame,
+                        entrylk, dom, &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK,
+                        NULL);
+            } else {
+                loc_wipe(&loc);
+                memset(locked_on, 0, numsubvols);
+                return 0;
+            }
+            break;
         }
+    }
 
-        loc_wipe (&loc);
-        return fop_success_fill (replies, numsubvols, locked_on);
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
 }
diff --git a/libglusterfs/src/cluster-syncop.h b/libglusterfs/src/cluster-syncop.h
deleted file mode 100644
index 7e17faae0e0..00000000000
--- a/libglusterfs/src/cluster-syncop.h
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CLUSTER_SYNCOP_H
-#define _CLUSTER_SYNCOP_H
-
-#include "xlator.h"
-#include <sys/time.h>
-#include <pthread.h>
-#include <ucontext.h>
-#include "defaults.h"
-#include "default-args.h"
-#include "syncop.h"
-
-typedef struct cluster_local_ {
-        default_args_cbk_t *replies;
-        syncbarrier_t barrier;
-} cluster_local_t;
-
-int32_t
-cluster_lookup (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
-int32_t
-cluster_setattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
-int32_t
-cluster_getxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name, dict_t *xdata);
-int32_t
-cluster_setxattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags, dict_t *xdata);
-
-int
-cluster_inodelk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *locked_on,
-                 call_frame_t *frame, xlator_t *this, char *dom,
-                 inode_t *inode, off_t off, size_t size);
-
-int
-cluster_uninodelk (xlator_t **subvols, unsigned char *locked_on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, char *dom,
-                   inode_t *inode, off_t off, size_t size);
-
-int
-cluster_entrylk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *locked_on,
-                 call_frame_t *frame, xlator_t *this, char *dom,
-                 inode_t *inode, const char *name);
-
-int32_t
-cluster_rmdir (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata);
-
-int32_t
-cluster_unlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                default_args_cbk_t *replies, unsigned char *output,
-                call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-                dict_t *xdata);
-
-int
-cluster_mkdir (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               mode_t umask, dict_t *xdata);
-
-int32_t
-cluster_readlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                  dict_t *xdata);
-
-int
-cluster_symlink (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, const char *linkpath,
-                 loc_t *loc, mode_t umask, dict_t *xdata);
-
-int32_t
-cluster_link (xlator_t **subvols, unsigned char *on, int numsubvols,
-              default_args_cbk_t *replies, unsigned char *output,
-              call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata);
-
-int
-cluster_mknod (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dev_t rdev, mode_t umask, dict_t *xdata);
-
-int
-cluster_unentrylk (xlator_t **subvols, unsigned char *locked_on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, char *dom,
-                   inode_t *inode, const char *name);
-
-int
-cluster_tryentrylk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                    default_args_cbk_t *replies, unsigned char *locked_on,
-                    call_frame_t *frame, xlator_t *this, char *dom,
-                    inode_t *inode, const char *name);
-
-int32_t
-cluster_fxattrop (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
-
-int32_t
-cluster_xattrop (xlator_t **subvols, unsigned char *on, int numsubvols,
-                 default_args_cbk_t *replies, unsigned char *output,
-                 call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
-
-int32_t
-cluster_fstat (xlator_t **subvols, unsigned char *on, int numsubvols,
-               default_args_cbk_t *replies, unsigned char *output,
-               call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
-
-int32_t
-cluster_ftruncate (xlator_t **subvols, unsigned char *on, int numsubvols,
-                   default_args_cbk_t *replies, unsigned char *output,
-                   call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   dict_t *xdata);
-
-int32_t
-cluster_open (xlator_t **subvols, unsigned char *on, int numsubvols,
-              default_args_cbk_t *replies, unsigned char *output,
-              call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              fd_t *fd, dict_t *xdata);
-
-int
-cluster_tryinodelk (xlator_t **subvols, unsigned char *on, int numsubvols,
-                    default_args_cbk_t *replies, unsigned char *locked_on,
-                    call_frame_t *frame, xlator_t *this, char *dom,
-                    inode_t *inode, off_t off, size_t size);
-
-int32_t
-cluster_fsetattr (xlator_t **subvols, unsigned char *on, int numsubvols,
-                  default_args_cbk_t *replies, unsigned char *output,
-                  call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iatt *stbuf, int32_t valid, dict_t *xdata);
-
-void
-cluster_replies_wipe (default_args_cbk_t *replies, int num_subvols);
-#endif /* !_CLUSTER_SYNCOP_H */
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 14a344958fd..682cbf28055 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -24,162 +24,377 @@
 #include <time.h>
 #include <locale.h>
 #include <sys/socket.h>
-#include <sys/wait.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <signal.h>
 #include <assert.h>
 #include <libgen.h> /* for dirname() */
+#include <grp.h>
 
 #if defined(GF_BSD_HOST_OS) || defined(GF_DARWIN_HOST_OS)
 #include <sys/sysctl.h>
 #endif
-#include <libgen.h>
-
-#include "glusterfs-acl.h"
-#include "compat-errno.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "revision.h"
-#include "glusterfs.h"
-#include "stack.h"
-#include "globals.h"
-#include "lkowner.h"
-#include "syscall.h"
-#include "cli1-xdr.h"
+#ifndef GF_LINUX_HOST_OS
+#include <sys/resource.h>
+#endif
+#ifdef HAVE_SYNCFS_SYS
+#include <sys/syscall.h>
+#endif
+
+#include "glusterfs/compat-errno.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/revision.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/stack.h"
+#include "glusterfs/lkowner.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/globals.h"
+#define XXH_INLINE_ALL
+#include "xxhash.h"
 #include <ifaddrs.h>
-#include "libglusterfs-messages.h"
+#include "glusterfs/libglusterfs-messages.h"
+#include "glusterfs/glusterfs-acl.h"
+#ifdef __FreeBSD__
+#include <pthread_np.h>
+#undef BIT_SET
+#endif
 
 #ifndef AI_ADDRCONFIG
 #define AI_ADDRCONFIG 0
 #endif /* AI_ADDRCONFIG */
 
-char *vol_type_str[] = {"Distribute",
-                        "Stripe",
-                        "Replicate",
-                        "Striped-Replicate",
-                        "Disperse",
-                        "Tier",
-                        "Distributed-Stripe",
-                        "Distributed-Replicate",
-                        "Distributed-Striped-Replicate",
-                        "Distributed-Disperse",
-                       };
+char *vol_type_str[] = {
+    "Distribute",
+    "Stripe [NOT SUPPORTED from v6.0]",
+    "Replicate",
+    "Striped-Replicate [NOT SUPPORTED from v6.0]",
+    "Disperse",
+    "Tier [NOT SUPPORTED from v6.0]",
+    "Distributed-Stripe [NOT SUPPORTED from v6.0]",
+    "Distributed-Replicate",
+    "Distributed-Striped-Replicate [NOT SUPPORTED from v6.0]",
+    "Distributed-Disperse",
+};
 
 typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size);
 typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size);
 
+char *xattrs_to_heal[] = {"user.",
+                          POSIX_ACL_ACCESS_XATTR,
+                          POSIX_ACL_DEFAULT_XATTR,
+                          QUOTA_LIMIT_KEY,
+                          QUOTA_LIMIT_OBJECTS_KEY,
+                          GF_SELINUX_XATTR_KEY,
+                          GF_XATTR_MDATA_KEY,
+                          NULL};
+
 void
-md5_wrapper(const unsigned char *data, size_t len, char *md5)
+gf_xxh64_wrapper(const unsigned char *data, size_t const len,
+                 unsigned long long const seed, char *xxh64)
 {
-        unsigned short i = 0;
-        unsigned short lim = MD5_DIGEST_LENGTH*2+1;
-        unsigned char scratch[MD5_DIGEST_LENGTH] = {0,};
-        MD5(data, len, scratch);
-        for (; i < MD5_DIGEST_LENGTH; i++)
-                snprintf(md5 + i * 2, lim-i*2, "%02x", scratch[i]);
+    unsigned short i = 0;
+    const unsigned short lim = GF_XXH64_DIGEST_LENGTH * 2 + 1;
+    XXH64_hash_t hash = 0;
+    XXH64_canonical_t c_hash = {
+        {
+            0,
+        },
+    };
+    const uint8_t *p = (const uint8_t *)&c_hash;
+
+    hash = XXH64(data, len, seed);
+    XXH64_canonicalFromHash(&c_hash, hash);
+
+    for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++)
+        snprintf(xxh64 + i * 2, lim - i * 2, "%02x", p[i]);
+}
+
+/**
+ * This function takes following arguments
+ * @this: xlator
+ * @gfid: The gfid which has to be filled
+ * @hash: the 8 byte hash which has to be filled inside the gfid
+ * @index: the array element of the uuid_t structure (which is
+ *         a array of unsigned char) from where the 8 bytes of
+ *         the hash has to be filled. Since uuid_t contains 16
+ *        char elements in the array, each byte of the hash has
+ *        to be filled in one array element.
+ *
+ * This function is called twice for 2 hashes (of 8 byte each) to
+ * be filled in the gfid.
+ *
+ * The for loop in this function actually is doing these 2 things
+ * for each hash
+ *
+ * 1) One of the hashes
+ *      tmp[0] = (hash_2 >> 56) & 0xff;
+ *      tmp[1] = (hash_2 >> 48) & 0xff;
+ *      tmp[2] = (hash_2 >> 40) & 0xff;
+ *      tmp[3] = (hash_2 >> 32) & 0xff;
+ *      tmp[4] = (hash_2 >> 24) & 0xff;
+ *      tmp[5] = (hash_2 >> 16) & 0xff;
+ *      tmp[6] = (hash_2 >> 8) & 0xff;
+ *      tmp[7] = (hash_2) & 0xff;
+ *
+ * 2) The other hash:
+ *      tmp[8] = (hash_1 >> 56) & 0xff;
+ *      tmp[9] = (hash_1 >> 48) & 0xff;
+ *      tmp[10] = (hash_1 >> 40) & 0xff;
+ *      tmp[11] = (hash_1 >> 32) & 0xff;
+ *      tmp[12] = (hash_1 >> 24) & 0xff;
+ *      tmp[13] = (hash_1 >> 16) & 0xff;
+ *      tmp[14] = (hash_1 >> 8) & 0xff;
+ *      tmp[15] = (hash_1) & 0xff;
+ **/
+static int
+gf_gfid_from_xxh64(xlator_t *this, uuid_t gfid, XXH64_hash_t hash,
+                   unsigned short index)
+{
+    int ret = -1;
+    int i = -1;
+
+    if ((index != 0) && (index != 8)) {
+        gf_msg_callingfn("gfid-from-xxh64", GF_LOG_WARNING, 0,
+                         LG_MSG_INDEX_NOT_FOUND,
+                         "index can only be either 0 or 8, as this"
+                         "function's purpose is to encode a 8 byte "
+                         "hash inside the gfid (index: %d)",
+                         index);
+        goto out;
+    }
+
+    for (i = 0; i < sizeof(hash); i++) {
+        /*
+         * As of now the below statement is equivalent of this.
+         * gfid[index+i] = (hash >> (64 - (8 * (i+1)))) & 0xff;
+         */
+        gfid[index + i] = (hash >> ((sizeof(hash) * 8) - (8 * (i + 1)))) &
+                          (0xff);
+    }
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+/**
+ * This function does the same thing as gf_xxh64_wrapper. But gf_xxh64_wrapper
+ * does not return anything and in this xlator there is a need for both the
+ * actual hash and the canonicalized form of the hash.
+ *
+ * To summarize:
+ * - XXH64_hash_t is needed as return because, those bytes which contain the
+ *   hash can be used for different purposes as needed. One example is
+ *   to have those bytes copied into the uuid_t structure to be used as gfid
+ * - xxh64 string is needed because, it can be used as the key for generating
+ *   the next hash (and any other purpose which might require canonical form
+ *   of the hash).
+ **/
+XXH64_hash_t
+gf_xxh64_hash_wrapper(const unsigned char *data, size_t const len,
+                      unsigned long long const seed, char *xxh64)
+{
+    unsigned short i = 0;
+    const unsigned short lim = GF_XXH64_DIGEST_LENGTH * 2 + 1;
+    XXH64_hash_t hash = 0;
+    XXH64_canonical_t c_hash = {
+        {
+            0,
+        },
+    };
+    const uint8_t *p = (const uint8_t *)&c_hash;
+
+    hash = XXH64(data, len, seed);
+    XXH64_canonicalFromHash(&c_hash, hash);
+
+    for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++)
+        snprintf(xxh64 + i * 2, lim - i * 2, "%02x", p[i]);
+
+    return hash;
+}
+
+/**
+ * This is the algorithm followed for generating new gfid
+ * 1) generate xxh64 hash using snapname and original gfid of the object
+ * 2) Using the canonicalized form of above hash as the key, generate
+ *    another hash
+ * 3) Combine both of the  8 byte hashes to generate a 16 byte uuid_t type
+ * 4) Use the above uuid as the gfid
+ *
+ * Each byte of the hash is stored separately in different elements of the
+ * character array represented by uuid_t
+ * Ex: tmp[0] = (hash_2 >> 56) & 0xFF
+ *     This saves the most significant byte of hash_2 in tmp[0]
+ *     tmp[1] = (hash_2 >> 48) & 0xFF
+ *     This saves next most significant byte of hash_2 in tmp[1]
+ *     .
+ *     .
+ *     So on.
+ *     tmp[0] - tmp[7] holds the contents of hash_2
+ *     tmp[8] - tmp[15] hold the conents of hash_1
+ *
+ * The hash generated (i.e. of type XXH64_hash_t) is 8 bytes long. And for
+ * gfid 16 byte uuid is needed. Hecne the 2 hashes are combined to form
+ * one 16 byte entity.
+ **/
+int
+gf_gfid_generate_from_xxh64(uuid_t gfid, char *key)
+{
+    char xxh64_1[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {
+        0,
+    };
+    char xxh64_2[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {
+        0,
+    };
+    XXH64_hash_t hash_1 = 0;
+    XXH64_hash_t hash_2 = 0;
+    int ret = -1;
+    xlator_t *this = THIS;
+
+    hash_1 = gf_xxh64_hash_wrapper((unsigned char *)key, strlen(key),
+                                   GF_XXHSUM64_DEFAULT_SEED, xxh64_1);
+
+    hash_2 = gf_xxh64_hash_wrapper((unsigned char *)xxh64_1, strlen(xxh64_1),
+                                   GF_XXHSUM64_DEFAULT_SEED, xxh64_2);
+
+    /* hash_2 is saved in 1st 8 elements of uuid_t char array */
+    if (gf_gfid_from_xxh64(this, gfid, hash_2, 0)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         LG_MSG_XXH64_TO_GFID_FAILED,
+                         "failed to encode the hash %llx into the 1st"
+                         "half of gfid",
+                         hash_2);
+        goto out;
+    }
+
+    /* hash_1 is saved in the remaining 8 elements of uuid_t */
+    if (gf_gfid_from_xxh64(this, gfid, hash_1, 8)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         LG_MSG_XXH64_TO_GFID_FAILED,
+                         "failed to encode the hash %llx into the 2nd"
+                         "half of gfid",
+                         hash_1);
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "gfid generated is %s (hash1: %llx) "
+                 "hash2: %llx, xxh64_1: %s xxh64_2: %s",
+                 uuid_utoa(gfid), hash_1, hash_2, xxh64_1, xxh64_2);
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
 /* works similar to mkdir(1) -p.
  */
 int
-mkdir_p (char *path, mode_t mode, gf_boolean_t allow_symlinks)
-{
-        int             i               = 0;
-        int             ret             = -1;
-        char            dir[PATH_MAX]   = {0,};
-        struct stat     stbuf           = {0,};
-
-        strncpy (dir, path, (PATH_MAX - 1));
-        dir[PATH_MAX - 1] = '\0';
-
-        i = (dir[0] == '/')? 1: 0;
-        do {
-                if (path[i] != '/' && path[i] != '\0')
-                        continue;
-
-                dir[i] = '\0';
-                ret = sys_mkdir (dir, mode);
-                if (ret && errno != EEXIST) {
-                        gf_msg ("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED,
-                                "Failed due to reason");
-                        goto out;
-                }
-
-                if (ret && errno == EEXIST && !allow_symlinks) {
-                        ret = sys_lstat (dir, &stbuf);
-                        if (ret)
-                                goto out;
-
-                        if (S_ISLNK (stbuf.st_mode)) {
-                                ret = -1;
-                                gf_msg ("", GF_LOG_ERROR, 0,
-                                        LG_MSG_DIR_IS_SYMLINK, "%s is a "
-                                        "symlink", dir);
-                                goto out;
-                        }
-                }
-                dir[i] = '/';
-
-        } while (path[i++] != '\0');
-
-        ret = sys_stat (dir, &stbuf);
-        if (ret || !S_ISDIR (stbuf.st_mode)) {
-                if (ret == 0)
-                        errno = 0;
+mkdir_p(char *path, mode_t mode, gf_boolean_t allow_symlinks)
+{
+    int i = 0;
+    int ret = -1;
+    char dir[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+
+    const int path_len = min(strlen(path), PATH_MAX - 1);
+
+    snprintf(dir, path_len + 1, "%s", path);
+
+    i = (dir[0] == '/') ? 1 : 0;
+    do {
+        if (path[i] != '/' && path[i] != '\0')
+            continue;
+
+        dir[i] = '\0';
+        ret = sys_mkdir(dir, mode);
+        if (ret && errno != EEXIST) {
+            gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED, NULL);
+            goto out;
+        }
+
+        if (ret && errno == EEXIST && !allow_symlinks) {
+            ret = sys_lstat(dir, &stbuf);
+            if (ret)
+                goto out;
+
+            if (S_ISLNK(stbuf.st_mode)) {
                 ret = -1;
-                gf_msg ("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED, "Failed"
-                        " to create directory, possibly some of the components"
-                        " were not directories");
+                gf_smsg("", GF_LOG_ERROR, 0, LG_MSG_DIR_IS_SYMLINK, "dir=%s",
+                        dir, NULL);
                 goto out;
+            }
         }
+        dir[i] = '/';
 
-        ret = 0;
+    } while (path[i++] != '\0');
+
+    ret = sys_stat(dir, &stbuf);
+    if (ret || !S_ISDIR(stbuf.st_mode)) {
+        if (ret == 0)
+            errno = 0;
+        ret = -1;
+        gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED,
+                "possibly some of the components"
+                " were not directories",
+                NULL);
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 int
-gf_lstat_dir (const char *path, struct stat *stbuf_in)
+gf_lstat_dir(const char *path, struct stat *stbuf_in)
 {
-        int ret           = -1;
-        struct stat stbuf = {0,};
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
 
-        if (path == NULL) {
-                errno = EINVAL;
-                goto out;
-        }
+    if (path == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
 
-        ret = sys_lstat (path, &stbuf);
-        if (ret)
-                goto out;
+    ret = sys_lstat(path, &stbuf);
+    if (ret)
+        goto out;
 
-        if (!S_ISDIR (stbuf.st_mode)) {
-                errno = ENOTDIR;
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    if (!S_ISDIR(stbuf.st_mode)) {
+        errno = ENOTDIR;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
 out:
-        if (!ret && stbuf_in)
-                *stbuf_in = stbuf;
+    if (!ret && stbuf_in)
+        *stbuf_in = stbuf;
 
-        return ret;
+    return ret;
 }
 
 int
-log_base2 (unsigned long x)
+log_base2(unsigned long x)
 {
-        int val = 0;
+    int val = 0;
 
-        while (x > 1) {
-                x /= 2;
-                val++;
-        }
+    while (x > 1) {
+        x /= 2;
+        val++;
+    }
 
-        return val;
+    return val;
 }
 
 /**
@@ -191,32 +406,21 @@ log_base2 (unsigned long x)
  *          failure: NULL
  */
 char *
-gf_rev_dns_lookup (const char *ip)
+gf_rev_dns_lookup(const char *ip)
 {
-        char               *fqdn = NULL;
-        int                ret  = 0;
-        struct sockaddr_in sa   = {0};
-        char               host_addr[256] = {0, };
+    char *fqdn = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO ("resolver", ip, out);
-
-        sa.sin_family = AF_INET;
-        inet_pton (AF_INET, ip, &sa.sin_addr);
-        ret = getnameinfo ((struct sockaddr *)&sa, sizeof (sa), host_addr,
-                          sizeof (host_addr), NULL, 0, 0);
-
-        if (ret != 0) {
-                gf_msg ("resolver", GF_LOG_INFO, errno,
-                        LG_MSG_RESOLVE_HOSTNAME_FAILED, "could not resolve "
-                        "hostname for %s", ip);
-                goto out;
-        }
-
-        /* Get the FQDN */
-        fqdn = gf_strdup (host_addr);
+    GF_VALIDATE_OR_GOTO("resolver", ip, out);
 
+    /* Get the FQDN */
+    ret = gf_get_hostname_from_ip((char *)ip, &fqdn);
+    if (ret != 0) {
+        gf_smsg("resolver", GF_LOG_INFO, errno, LG_MSG_RESOLVE_HOSTNAME_FAILED,
+                "hostname=%s", ip, NULL);
+    }
 out:
-       return fqdn;
+    return fqdn;
 }
 
 /**
@@ -226,144 +430,139 @@ out:
  * @return: The parent path if found, NULL otherwise
  */
 char *
-gf_resolve_path_parent (const char *path)
+gf_resolve_path_parent(const char *path)
 {
-        char    *parent = NULL;
-        char    *tmp    = NULL;
-        char    *pathc  = NULL;
+    char *parent = NULL;
+    char *tmp = NULL;
+    char *pathc = NULL;
 
-        GF_VALIDATE_OR_GOTO (THIS->name, path, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, path, out);
 
-        if (strlen (path) <= 0) {
-                gf_msg_callingfn (THIS->name, GF_LOG_DEBUG, 0,
-                                  LG_MSG_INVALID_STRING,
-                                  "invalid string for 'path'");
-                goto out;
-        }
+    if (0 == strlen(path)) {
+        gf_msg_callingfn(THIS->name, GF_LOG_DEBUG, 0, LG_MSG_INVALID_STRING,
+                         "invalid string for 'path'");
+        goto out;
+    }
 
-        /* dup the parameter, we don't want to modify it */
-        pathc = strdupa (path);
-        if (!pathc) {
-                goto out;
-        }
+    /* dup the parameter, we don't want to modify it */
+    pathc = strdupa(path);
+    if (!pathc) {
+        goto out;
+    }
 
-        /* Get the parent directory */
-        tmp = dirname (pathc);
-        if (strcmp (tmp, "/") == 0)
-                goto out;
+    /* Get the parent directory */
+    tmp = dirname(pathc);
+    if (strcmp(tmp, "/") == 0)
+        goto out;
 
-        parent = gf_strdup (tmp);
+    parent = gf_strdup(tmp);
 out:
-        return parent;
+    return parent;
 }
 
 int32_t
-gf_resolve_ip6 (const char *hostname,
-                uint16_t port,
-                int family,
-                void **dnscache,
-                struct addrinfo **addr_info)
-{
-        int32_t ret = 0;
-        struct addrinfo hints;
-        struct dnscache6 *cache = NULL;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-
-        if (!hostname) {
-                gf_msg_callingfn ("resolver", GF_LOG_WARNING, 0,
-                                  LG_MSG_HOSTNAME_NULL, "hostname is NULL");
-                return -1;
-        }
+gf_resolve_ip6(const char *hostname, uint16_t port, int family, void **dnscache,
+               struct addrinfo **addr_info)
+{
+    int32_t ret = 0;
+    struct addrinfo hints;
+    struct dnscache6 *cache = NULL;
+    char service[NI_MAXSERV], host[NI_MAXHOST];
 
-        if (!*dnscache) {
-                *dnscache = GF_CALLOC (1, sizeof (struct dnscache6),
-                                       gf_common_mt_dnscache6);
-                if (!*dnscache)
-                        return -1;
+    if (!hostname) {
+        gf_msg_callingfn("resolver", GF_LOG_WARNING, 0, LG_MSG_HOSTNAME_NULL,
+                         "hostname is NULL");
+        return -1;
+    }
+
+    if (!*dnscache) {
+        *dnscache = GF_CALLOC(1, sizeof(struct dnscache6),
+                              gf_common_mt_dnscache6);
+        if (!*dnscache)
+            return -1;
+    }
+
+    cache = *dnscache;
+    if (cache->first && !cache->next) {
+        freeaddrinfo(cache->first);
+        cache->first = cache->next = NULL;
+        gf_msg_trace("resolver", 0, "flushing DNS cache");
+    }
+
+    if (!cache->first) {
+        char *port_str = NULL;
+        gf_msg_trace("resolver", 0,
+                     "DNS cache not present, freshly "
+                     "probing hostname: %s",
+                     hostname);
+
+        memset(&hints, 0, sizeof(hints));
+        hints.ai_family = family;
+        hints.ai_socktype = SOCK_STREAM;
+
+        ret = gf_asprintf(&port_str, "%d", port);
+        if (-1 == ret) {
+            return -1;
+        }
+        if ((ret = getaddrinfo(hostname, port_str, &hints, &cache->first)) !=
+            0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
+                    "family=%d", family, "ret=%s", gai_strerror(ret), NULL);
+
+            GF_FREE(*dnscache);
+            *dnscache = NULL;
+            GF_FREE(port_str);
+            return -1;
+        }
+        GF_FREE(port_str);
+
+        cache->next = cache->first;
+    }
+
+    if (cache->next) {
+        ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
+                          cache->next->ai_addrlen, host, sizeof(host), service,
+                          sizeof(service), NI_NUMERICHOST);
+        if (ret != 0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto err;
         }
 
-        cache = *dnscache;
-        if (cache->first && !cache->next) {
-                freeaddrinfo(cache->first);
-                cache->first = cache->next = NULL;
-                gf_msg_trace ("resolver", 0, "flushing DNS cache");
-        }
+        gf_msg_debug("resolver", 0,
+                     "returning ip-%s (port-%s) for "
+                     "hostname: %s and port: %d",
+                     host, service, hostname, port);
 
-        if (!cache->first) {
-                char *port_str = NULL;
-                gf_msg_trace ("resolver", 0, "DNS cache not present, freshly "
-                              "probing hostname: %s", hostname);
-
-                memset(&hints, 0, sizeof(hints));
-                hints.ai_family   = family;
-                hints.ai_socktype = SOCK_STREAM;
-
-                ret = gf_asprintf (&port_str, "%d", port);
-                if (-1 == ret) {
-                        return -1;
-                }
-                if ((ret = getaddrinfo(hostname, port_str, &hints, &cache->first)) != 0) {
-                        gf_msg ("resolver", GF_LOG_ERROR, 0,
-                                LG_MSG_GETADDRINFO_FAILED, "getaddrinfo failed"
-                                " (%s)", gai_strerror (ret));
-
-                        GF_FREE (*dnscache);
-                        *dnscache = NULL;
-                        GF_FREE (port_str);
-                        return -1;
-                }
-                GF_FREE (port_str);
-
-                cache->next = cache->first;
-        }
+        *addr_info = cache->next;
+    }
 
-        if (cache->next) {
-                ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
-                                  cache->next->ai_addrlen,
-                                  host, sizeof (host),
-                                  service, sizeof (service),
-                                  NI_NUMERICHOST);
-                if (ret != 0) {
-                        gf_msg ("resolver", GF_LOG_ERROR, 0,
-                                LG_MSG_GETNAMEINFO_FAILED, "getnameinfo failed"
-                                " (%s)", gai_strerror (ret));
-                        goto err;
-                }
-
-                gf_msg_debug ("resolver", 0, "returning ip-%s (port-%s) for "
-                              "hostname: %s and port: %d", host, service,
-                              hostname, port);
-
-                *addr_info = cache->next;
+    if (cache->next)
+        cache->next = cache->next->ai_next;
+    if (cache->next) {
+        ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
+                          cache->next->ai_addrlen, host, sizeof(host), service,
+                          sizeof(service), NI_NUMERICHOST);
+        if (ret != 0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto err;
         }
 
-        if (cache->next)
-                cache->next = cache->next->ai_next;
-        if (cache->next) {
-                ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
-                                  cache->next->ai_addrlen,
-                                  host, sizeof (host),
-                                  service, sizeof (service),
-                                  NI_NUMERICHOST);
-                if (ret != 0) {
-                        gf_msg ("resolver", GF_LOG_ERROR, 0,
-                                LG_MSG_GETNAMEINFO_FAILED, "getnameinfo failed"
-                                " (%s)", gai_strerror (ret));
-                        goto err;
-                }
-
-                gf_msg_debug ("resolver", 0, "next DNS query will return: "
-                              "ip-%s port-%s", host, service);
-        }
+        gf_msg_debug("resolver", 0,
+                     "next DNS query will return: "
+                     "ip-%s port-%s",
+                     host, service);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        freeaddrinfo (cache->first);
-        cache->first = cache->next = NULL;
-        GF_FREE (cache);
-        *dnscache = NULL;
-        return -1;
+    freeaddrinfo(cache->first);
+    cache->first = cache->next = NULL;
+    GF_FREE(cache);
+    *dnscache = NULL;
+    return -1;
 }
 
 /**
@@ -375,13 +574,35 @@ err:
  *          FAILURE: NULL
  */
 struct dnscache *
-gf_dnscache_init (time_t ttl)
+gf_dnscache_init(time_t ttl)
 {
-        struct dnscache *cache = GF_MALLOC (sizeof (*cache),
-                                            gf_common_mt_dnscache);
-        cache->cache_dict = NULL;
+    struct dnscache *cache = GF_MALLOC(sizeof(*cache), gf_common_mt_dnscache);
+    if (!cache)
+        return NULL;
+
+    cache->cache_dict = dict_new();
+    if (!cache->cache_dict) {
+        GF_FREE(cache);
+        cache = NULL;
+    } else {
         cache->ttl = ttl;
-        return cache;
+    }
+
+    return cache;
+}
+
+/**
+ * gf_dnscache_deinit -- cleanup resources used by struct dnscache
+ */
+void
+gf_dnscache_deinit(struct dnscache *cache)
+{
+    if (!cache) {
+        gf_msg_plain(GF_LOG_WARNING, "dnscache is NULL");
+        return;
+    }
+    dict_unref(cache->cache_dict);
+    GF_FREE(cache);
 }
 
 /**
@@ -391,11 +612,11 @@ gf_dnscache_init (time_t ttl)
  *          FAILURE: NULL
  */
 struct dnscache_entry *
-gf_dnscache_entry_init ()
+gf_dnscache_entry_init()
 {
-        struct dnscache_entry *entry = GF_CALLOC (1, sizeof (*entry),
-                                                 gf_common_mt_dnscache_entry);
-        return entry;
+    struct dnscache_entry *entry = GF_CALLOC(1, sizeof(*entry),
+                                             gf_common_mt_dnscache_entry);
+    return entry;
 }
 
 /**
@@ -404,11 +625,11 @@ gf_dnscache_entry_init ()
  * @entry: Pointer to deallocate
  */
 void
-gf_dnscache_entry_deinit (struct dnscache_entry *entry)
+gf_dnscache_entry_deinit(struct dnscache_entry *entry)
 {
-        GF_FREE (entry->ip);
-        GF_FREE (entry->fqdn);
-        GF_FREE (entry);
+    GF_FREE(entry->ip);
+    GF_FREE(entry->fqdn);
+    GF_FREE(entry);
 }
 
 /**
@@ -420,1537 +641,1490 @@ gf_dnscache_entry_deinit (struct dnscache_entry *entry)
  *          failure: NULL
  */
 char *
-gf_rev_dns_lookup_cached (const char *ip, struct dnscache *dnscache)
-{
-        char               *fqdn = NULL;
-        int                ret  = 0;
-        dict_t             *cache = NULL;
-        data_t             *entrydata = NULL;
-        struct dnscache_entry *dnsentry = NULL;
-        gf_boolean_t        from_cache = _gf_false;
-
-        if (!dnscache)
-                goto out;
-
-        if (!dnscache->cache_dict) {
-                dnscache->cache_dict = dict_new ();
-                if (!dnscache->cache_dict) {
-                        goto out;
-                }
-        }
-        cache = dnscache->cache_dict;
-
-        /* Quick cache lookup to see if we already hold it */
-        entrydata = dict_get (cache, (char *)ip);
-        if (entrydata) {
-                dnsentry = (struct dnscache_entry *)entrydata->data;
-                /* First check the TTL & timestamp */
-                if (time (NULL) - dnsentry->timestamp > dnscache->ttl) {
-                        gf_dnscache_entry_deinit (dnsentry);
-                        entrydata->data = NULL; /* Mark this as 'null' so
-                                                 * dict_del () doesn't try free
-                                                 * this after we've already
-                                                 * freed it.
-                                                 */
-
-                        dict_del (cache, (char *)ip); /* Remove this entry */
-                } else {
-                        /* Cache entry is valid, get the FQDN and return */
-                        fqdn = dnsentry->fqdn;
-                        from_cache = _gf_true; /* Mark this as from cache */
-                        goto out;
-                }
+gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache)
+{
+    char *fqdn = NULL;
+    int ret = 0;
+    dict_t *cache = NULL;
+    data_t *entrydata = NULL;
+    struct dnscache_entry *dnsentry = NULL;
+    gf_boolean_t from_cache = _gf_false;
+
+    if (!dnscache)
+        goto out;
+
+    cache = dnscache->cache_dict;
+
+    /* Quick cache lookup to see if we already hold it */
+    entrydata = dict_get(cache, (char *)ip);
+    if (entrydata) {
+        dnsentry = (struct dnscache_entry *)entrydata->data;
+        /* First check the TTL & timestamp */
+        if (gf_time() - dnsentry->timestamp > dnscache->ttl) {
+            gf_dnscache_entry_deinit(dnsentry);
+            entrydata->data = NULL; /* Mark this as 'null' so
+                                     * dict_del () doesn't try free
+                                     * this after we've already
+                                     * freed it.
+                                     */
+
+            dict_del(cache, (char *)ip); /* Remove this entry */
+        } else {
+            /* Cache entry is valid, get the FQDN and return */
+            fqdn = dnsentry->fqdn;
+            from_cache = _gf_true; /* Mark this as from cache */
+            goto out;
         }
+    }
 
-        /* Get the FQDN */
-        ret =  gf_get_hostname_from_ip ((char *)ip, &fqdn);
-        if (ret != 0)
-                goto out;
+    /* Get the FQDN */
+    ret = gf_get_hostname_from_ip((char *)ip, &fqdn);
+    if (ret != 0)
+        goto out;
 
-        if (!fqdn) {
-                gf_log_callingfn ("resolver", GF_LOG_CRITICAL,
-                                  "Allocation failed for the host address");
-                goto out;
-        }
+    if (!fqdn) {
+        gf_log_callingfn("resolver", GF_LOG_CRITICAL,
+                         "Allocation failed for the host address");
+        goto out;
+    }
 
-        from_cache = _gf_false;
+    from_cache = _gf_false;
 out:
-        /* Insert into the cache */
-        if (fqdn && !from_cache) {
-                struct dnscache_entry *entry = gf_dnscache_entry_init ();
-
-                if (!entry) {
-                        goto out;
-                }
-                entry->fqdn = fqdn;
-                entry->ip = gf_strdup (ip);
-                if (!ip) {
-                        gf_dnscache_entry_deinit (entry);
-                        goto out;
-                }
-                entry->timestamp = time (NULL);
-
-                entrydata = bin_to_data (entry, sizeof (*entry));
-                dict_set (cache, (char *)ip, entrydata);
+    /* Insert into the cache */
+    if (fqdn && !from_cache && ip) {
+        struct dnscache_entry *entry = gf_dnscache_entry_init();
+
+        if (entry) {
+            entry->fqdn = fqdn;
+            entry->ip = gf_strdup(ip);
+            entry->timestamp = gf_time();
+            entrydata = bin_to_data(entry, sizeof(*entry));
+            dict_set(cache, (char *)ip, entrydata);
         }
-        return fqdn;
+    }
+    return fqdn;
 }
 
 struct xldump {
-	int lineno;
+    int lineno;
 };
 
 /* to catch any format discrepencies that may arise in code */
-static int nprintf (struct xldump *dump, const char *fmt, ...)
-                    __attribute__ ((__format__ (__printf__, 2, 3)));
 static int
-nprintf (struct xldump *dump, const char *fmt, ...)
+nprintf(struct xldump *dump, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+static int
+nprintf(struct xldump *dump, const char *fmt, ...)
 {
-        va_list  ap;
-        char    *msg = NULL;
-        char     header[32];
-        int      ret = 0;
+    va_list ap;
+    char *msg = NULL;
+    char header[32];
+    int ret = 0;
 
-        ret = snprintf (header, 32, "%3d:", ++dump->lineno);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(header, 32, "%3d:", ++dump->lineno);
+    if (ret < 0)
+        goto out;
 
-        va_start (ap, fmt);
-        ret = vasprintf (&msg, fmt, ap);
-        va_end (ap);
-        if (-1 == ret)
-                goto out;
+    va_start(ap, fmt);
+    ret = vasprintf(&msg, fmt, ap);
+    va_end(ap);
+    if (-1 == ret)
+        goto out;
 
-        /* NOTE: No ret value from gf_msg_plain, so unable to compute printed
-         * characters. The return value from nprintf is not used, so for now
-         * living with it */
-        gf_msg_plain (GF_LOG_WARNING, "%s %s", header, msg);
+    /* NOTE: No ret value from gf_msg_plain, so unable to compute printed
+     * characters. The return value from nprintf is not used, so for now
+     * living with it */
+    gf_msg_plain(GF_LOG_WARNING, "%s %s", header, msg);
 
 out:
-        FREE (msg);
-        return 0;
+    FREE(msg);
+    return 0;
 }
 
-
 static int
-xldump_options (dict_t *this, char *key, data_t *value,	void *d)
+xldump_options(dict_t *this, char *key, data_t *value, void *d)
 {
-	nprintf (d, "    option %s %s", key, value->data);
-	return 0;
+    nprintf(d, "    option %s %s", key, value->data);
+    return 0;
 }
 
-
 static void
-xldump_subvolumes (xlator_t *this, void *d)
+xldump_subvolumes(xlator_t *this, void *d)
 {
-	xlator_list_t *subv = NULL;
-	int len = 0;
-	char *subvstr = NULL;
+    xlator_list_t *subv = NULL;
+    int len = 0;
+    char *subvstr = NULL;
 
-	subv = this->children;
-	if (!this->children)
-		return;
+    if (!this->children)
+        return;
 
-	for (subv = this->children; subv; subv = subv->next)
-		len += (strlen (subv->xlator->name) + 1);
+    for (subv = this->children; subv; subv = subv->next)
+        len += (strlen(subv->xlator->name) + 1);
 
-	subvstr = GF_CALLOC (1, len, gf_common_mt_strdup);
+    subvstr = GF_MALLOC(len, gf_common_mt_strdup);
 
-	len = 0;
-	for (subv = this->children; subv; subv= subv->next)
-		len += sprintf (subvstr + len, "%s%s", subv->xlator->name,
-				subv->next ? " " : "");
+    len = 0;
+    for (subv = this->children; subv; subv = subv->next)
+        len += sprintf(subvstr + len, "%s%s", subv->xlator->name,
+                       subv->next ? " " : "");
 
-	nprintf (d, "    subvolumes %s", subvstr);
+    nprintf(d, "    subvolumes %s", subvstr);
 
-	GF_FREE (subvstr);
+    GF_FREE(subvstr);
 }
 
-
 static void
-xldump (xlator_t *each, void *d)
+xldump(xlator_t *each, void *d)
 {
-	nprintf (d, "volume %s", each->name);
-	nprintf (d, "    type %s", each->type);
-	dict_foreach (each->options, xldump_options, d);
+    nprintf(d, "volume %s", each->name);
+    nprintf(d, "    type %s", each->type);
+    dict_foreach(each->options, xldump_options, d);
 
-	xldump_subvolumes (each, d);
+    xldump_subvolumes(each, d);
 
-	nprintf (d, "end-volume");
-	nprintf (d, " ");
+    nprintf(d, "end-volume");
+    nprintf(d, " ");
 }
 
-
 void
-gf_log_dump_graph (FILE *specfp, glusterfs_graph_t *graph)
+gf_log_dump_graph(FILE *specfp, glusterfs_graph_t *graph)
 {
-	struct xldump xld = {0, };
+    struct xldump xld = {
+        0,
+    };
 
-        gf_msg_plain (GF_LOG_WARNING, "Final graph:");
-        gf_msg_plain (GF_LOG_WARNING,
-                      "+---------------------------------------"
-                      "---------------------------------------+");
+    gf_msg_plain(GF_LOG_WARNING, "Final graph:");
+    gf_msg_plain(GF_LOG_WARNING,
+                 "+---------------------------------------"
+                 "---------------------------------------+");
 
-	xlator_foreach_depth_first (graph->top, xldump, &xld);
+    xlator_foreach_depth_first(graph->top, xldump, &xld);
 
-        gf_msg_plain (GF_LOG_WARNING,
-                      "+---------------------------------------"
-                      "---------------------------------------+");
+    gf_msg_plain(GF_LOG_WARNING,
+                 "+---------------------------------------"
+                 "---------------------------------------+");
 }
 
 static void
-gf_dump_config_flags ()
+gf_dump_config_flags()
 {
-        gf_msg_plain_nomem (GF_LOG_ALERT, "configuration details:");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "configuration details:");
 
 /* have argp */
 #ifdef HAVE_ARGP
-        gf_msg_plain_nomem (GF_LOG_ALERT, "argp 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "argp 1");
 #endif
 
 /* ifdef if found backtrace */
 #ifdef HAVE_BACKTRACE
-        gf_msg_plain_nomem (GF_LOG_ALERT, "backtrace 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "backtrace 1");
 #endif
 
 /* Berkeley-DB version has cursor->get() */
 #ifdef HAVE_BDB_CURSOR_GET
-        gf_msg_plain_nomem (GF_LOG_ALERT, "bdb->cursor->get 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "bdb->cursor->get 1");
 #endif
 
 /* Define to 1 if you have the <db.h> header file. */
 #ifdef HAVE_DB_H
-        gf_msg_plain_nomem (GF_LOG_ALERT, "db.h 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "db.h 1");
 #endif
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #ifdef HAVE_DLFCN_H
-        gf_msg_plain_nomem (GF_LOG_ALERT, "dlfcn 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "dlfcn 1");
 #endif
 
 /* define if fdatasync exists */
 #ifdef HAVE_FDATASYNC
-        gf_msg_plain_nomem (GF_LOG_ALERT, "fdatasync 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "fdatasync 1");
 #endif
 
 /* Define to 1 if you have the `pthread' library (-lpthread). */
 #ifdef HAVE_LIBPTHREAD
-        gf_msg_plain_nomem (GF_LOG_ALERT, "libpthread 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "libpthread 1");
 #endif
 
 /* define if llistxattr exists */
 #ifdef HAVE_LLISTXATTR
-        gf_msg_plain_nomem (GF_LOG_ALERT, "llistxattr 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "llistxattr 1");
 #endif
 
 /* define if found setfsuid setfsgid */
 #ifdef HAVE_SET_FSID
-        gf_msg_plain_nomem (GF_LOG_ALERT, "setfsid 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "setfsid 1");
 #endif
 
 /* define if found spinlock */
 #ifdef HAVE_SPINLOCK
-        gf_msg_plain_nomem (GF_LOG_ALERT, "spinlock 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "spinlock 1");
 #endif
 
 /* Define to 1 if you have the <sys/epoll.h> header file. */
 #ifdef HAVE_SYS_EPOLL_H
-        gf_msg_plain_nomem (GF_LOG_ALERT, "epoll.h 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "epoll.h 1");
 #endif
 
 /* Define to 1 if you have the <sys/extattr.h> header file. */
 #ifdef HAVE_SYS_EXTATTR_H
-        gf_msg_plain_nomem (GF_LOG_ALERT, "extattr.h 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "extattr.h 1");
 #endif
 
 /* Define to 1 if you have the <sys/xattr.h> header file. */
 #ifdef HAVE_SYS_XATTR_H
-        gf_msg_plain_nomem (GF_LOG_ALERT, "xattr.h 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "xattr.h 1");
 #endif
 
 /* define if found st_atim.tv_nsec */
 #ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-        gf_msg_plain_nomem (GF_LOG_ALERT, "st_atim.tv_nsec 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "st_atim.tv_nsec 1");
 #endif
 
 /* define if found st_atimespec.tv_nsec */
 #ifdef HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC
-        gf_msg_plain_nomem (GF_LOG_ALERT, "st_atimespec.tv_nsec 1");
+    gf_msg_plain_nomem(GF_LOG_ALERT, "st_atimespec.tv_nsec 1");
 #endif
 
 /* Define to the full name and version of this package. */
 #ifdef PACKAGE_STRING
-        {
-                char *msg = NULL;
-                int   ret = -1;
-
-                ret = gf_asprintf (&msg, "package-string: %s", PACKAGE_STRING);
-                if (ret >= 0) {
-                        gf_msg_plain_nomem (GF_LOG_ALERT, msg);
-                        GF_FREE (msg);
-                }
+    {
+        char *msg = NULL;
+        int ret = -1;
+
+        ret = gf_asprintf(&msg, "package-string: %s", PACKAGE_STRING);
+        if (ret >= 0) {
+            gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+            GF_FREE(msg);
         }
+    }
 #endif
 
-        return;
+    return;
 }
 
 /* Obtain a backtrace and print it to the log */
 void
-gf_print_trace (int32_t signum, glusterfs_ctx_t *ctx)
-{
-        char         msg[1024] = {0,};
-        char         timestr[64] = {0,};
-        call_stack_t *stack = NULL;
-
-        /* Now every gf_log call will just write to a buffer and when the
-         * buffer becomes full, its written to the log-file. Suppose the process
-         * crashes and prints the backtrace in the log-file, then the previous
-         * log information will still be in the buffer itself. So flush the
-         * contents of the buffer to the log file before printing the backtrace
-         * which helps in debugging.
-         */
-        gf_log_flush();
-
-        gf_log_disable_suppression_before_exit (ctx);
-
-        /* Pending frames, (if any), list them in order */
-        gf_msg_plain_nomem (GF_LOG_ALERT, "pending frames:");
+gf_print_trace(int32_t signum, glusterfs_ctx_t *ctx)
+{
+    char msg[1024] = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    call_stack_t *stack = NULL;
+
+    /* Now every gf_log call will just write to a buffer and when the
+     * buffer becomes full, its written to the log-file. Suppose the process
+     * crashes and prints the backtrace in the log-file, then the previous
+     * log information will still be in the buffer itself. So flush the
+     * contents of the buffer to the log file before printing the backtrace
+     * which helps in debugging.
+     */
+    gf_log_flush();
+
+    gf_log_disable_suppression_before_exit(ctx);
+
+    /* Pending frames, (if any), list them in order */
+    gf_msg_plain_nomem(GF_LOG_ALERT, "pending frames:");
+    {
+        /* FIXME: traversing stacks outside pool->lock */
+        list_for_each_entry(stack, &ctx->pool->all_frames, all_frames)
         {
-                /* FIXME: traversing stacks outside pool->lock */
-                list_for_each_entry (stack, &ctx->pool->all_frames,
-                                     all_frames) {
-                        if (stack->type == GF_OP_TYPE_FOP)
-                                sprintf (msg,"frame : type(%d) op(%s)",
-                                         stack->type,
-                                         gf_fop_list[stack->op]);
-                        else
-                                sprintf (msg,"frame : type(%d) op(%d)",
-                                         stack->type,
-                                         stack->op);
-
-                        gf_msg_plain_nomem (GF_LOG_ALERT, msg);
-                }
+            if (stack->type == GF_OP_TYPE_FOP)
+                sprintf(msg, "frame : type(%d) op(%s)", stack->type,
+                        gf_fop_list[stack->op]);
+            else
+                sprintf(msg, "frame : type(%d) op(%d)", stack->type, stack->op);
+
+            gf_msg_plain_nomem(GF_LOG_ALERT, msg);
         }
+    }
 
-        sprintf (msg, "patchset: %s", GLUSTERFS_REPOSITORY_REVISION);
-        gf_msg_plain_nomem (GF_LOG_ALERT, msg);
+    sprintf(msg, "patchset: %s", GLUSTERFS_REPOSITORY_REVISION);
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
 
-        sprintf (msg, "signal received: %d", signum);
-        gf_msg_plain_nomem (GF_LOG_ALERT, msg);
-        {
-                /* Dump the timestamp of the crash too, so the previous logs
-                   can be related */
-                gf_time_fmt (timestr, sizeof timestr, time (NULL),
-                             gf_timefmt_FT);
-                gf_msg_plain_nomem (GF_LOG_ALERT, "time of crash: ");
-                gf_msg_plain_nomem (GF_LOG_ALERT, timestr);
-        }
+    sprintf(msg, "signal received: %d", signum);
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+    {
+        /* Dump the timestamp of the crash too, so the previous logs
+           can be related */
+        gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT);
+        gf_msg_plain_nomem(GF_LOG_ALERT, "time of crash: ");
+        gf_msg_plain_nomem(GF_LOG_ALERT, timestr);
+    }
 
-        gf_dump_config_flags ();
-        gf_msg_backtrace_nomem (GF_LOG_ALERT, 200);
-        sprintf (msg, "---------");
-        gf_msg_plain_nomem (GF_LOG_ALERT, msg);
+    gf_dump_config_flags();
+    gf_msg_backtrace_nomem(GF_LOG_ALERT, 200);
+    sprintf(msg, "---------");
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
 
-        /* Send a signal to terminate the process */
-        signal (signum, SIG_DFL);
-        raise (signum);
+    /* Send a signal to terminate the process */
+    signal(signum, SIG_DFL);
+    raise(signum);
 }
 
 void
-trap (void)
+trap(void)
 {
-
 }
 
 char *
-gf_trim (char *string)
+gf_trim(char *string)
 {
-        register char *s, *t;
+    register char *s, *t;
 
-        if (string == NULL) {
-                return NULL;
-        }
-
-        for (s = string; isspace (*s); s++)
-                ;
-
-        if (*s == 0)
-                return s;
+    if (string == NULL) {
+        return NULL;
+    }
 
-        t = s + strlen (s) - 1;
-        while (t > s && isspace (*t))
-                t--;
-        *++t = '\0';
+    for (s = string; isspace(*s); s++)
+        ;
 
+    if (*s == 0)
         return s;
-}
-
-int
-gf_strsplit (const char *str, const char *delim,
-             char ***tokens, int *token_count)
-{
-        char *_running = NULL;
-        char *running = NULL;
-        char *token = NULL;
-        char **token_list = NULL;
-        int count = 0;
-        int i = 0;
-        int j = 0;
-
-        if (str == NULL || delim == NULL || tokens == NULL || token_count == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                return -1;
-        }
-
-        _running = gf_strdup (str);
-        if (_running == NULL)
-                return -1;
 
-        running = _running;
+    t = s + strlen(s) - 1;
+    while (t > s && isspace(*t))
+        t--;
+    *++t = '\0';
 
-        while ((token = strsep (&running, delim)) != NULL) {
-                if (token[0] != '\0')
-                        count++;
-        }
-        GF_FREE (_running);
-
-        _running = gf_strdup (str);
-        if (_running == NULL)
-                return -1;
-
-        running = _running;
-
-        if ((token_list = GF_CALLOC (count, sizeof (char *),
-                                     gf_common_mt_char)) == NULL) {
-                GF_FREE (_running);
-                return -1;
-        }
-
-        while ((token = strsep (&running, delim)) != NULL) {
-                if (token[0] == '\0')
-                        continue;
-
-                token_list[i] = gf_strdup (token);
-                if (token_list[i] == NULL)
-                        goto free_exit;
-                i++;
-        }
-
-        GF_FREE (_running);
-
-        *tokens = token_list;
-        *token_count = count;
-        return 0;
-
-free_exit:
-        GF_FREE (_running);
-        for (j = 0; j < i; j++)
-                GF_FREE (token_list[j]);
-
-        GF_FREE (token_list);
-        return -1;
+    return s;
 }
 
 int
-gf_strstr (const char *str, const char *delim, const char *match)
+gf_strstr(const char *str, const char *delim, const char *match)
 {
-        char *tmp      = NULL;
-        char *save_ptr = NULL;
-        char *tmp_str  = NULL;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *tmp_str = NULL;
 
-        int  ret       = 0;
+    int ret = 0;
 
-        tmp_str = strdup (str);
-
-        if (str == NULL || delim == NULL || match == NULL || tmp_str == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                ret = -1;
-                goto out;
-        }
+    tmp_str = strdup(str);
 
+    if (str == NULL || delim == NULL || match == NULL || tmp_str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        ret = -1;
+        goto out;
+    }
 
-        tmp = strtok_r (tmp_str, delim, &save_ptr);
+    tmp = strtok_r(tmp_str, delim, &save_ptr);
 
-        while (tmp) {
-                ret = strcmp (tmp, match);
+    while (tmp) {
+        ret = strcmp(tmp, match);
 
-                if (ret == 0)
-                        break;
+        if (ret == 0)
+            break;
 
-                tmp = strtok_r (NULL, delim, &save_ptr);
-        }
+        tmp = strtok_r(NULL, delim, &save_ptr);
+    }
 
 out:
-        free (tmp_str);
-
-        return ret;
+    free(tmp_str);
 
+    return ret;
 }
 
 int
-gf_volume_name_validate (const char *volume_name)
+gf_volume_name_validate(const char *volume_name)
 {
-        const char *vname = NULL;
+    const char *vname = NULL;
 
-        if (volume_name == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                return -1;
-        }
+    if (volume_name == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
 
-        if (!isalpha (volume_name[0]))
-                return 1;
+    if (!isalpha(volume_name[0]))
+        return 1;
 
-        for (vname = &volume_name[1]; *vname != '\0'; vname++) {
-                if (!(isalnum (*vname) || *vname == '_'))
-                        return 1;
-        }
+    for (vname = &volume_name[1]; *vname != '\0'; vname++) {
+        if (!(isalnum(*vname) || *vname == '_'))
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-gf_string2time (const char *str, uint32_t *n)
-{
-        unsigned long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtol (str, &tail, 0);
-        if (str == tail)
-                errno = EINVAL;
-
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
-
-        if (errno == 0)
-                errno = old_errno;
-
-        if (((tail[0] == '\0') ||
-              ((tail[0] == 's') && (tail[1] == '\0')) ||
-              ((tail[0] == 's') && (tail[1] == 'e') &&
-	       (tail[2] == 'c') && (tail[3] == '\0'))))
-               goto out;
-
-        else if (((tail[0] == 'm') && (tail[1] == '\0')) ||
-                 ((tail[0] == 'm') && (tail[1] == 'i') &&
-                  (tail[2] == 'n') && (tail[3] == '\0'))) {
-                value = value * GF_MINUTE_IN_SECONDS;
-                goto out;
-        }
-
-        else if (((tail[0] == 'h') && (tail[1] == '\0')) ||
-                 ((tail[0] == 'h') && (tail[1] == 'r') &&
-	         (tail[2] == '\0'))) {
-                value = value * GF_HOUR_IN_SECONDS;
-                goto out;
-        }
+gf_string2time(const char *str, uint32_t *n)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
 
-        else if (((tail[0] == 'd') && (tail[1] == '\0')) ||
-                 ((tail[0] == 'd') && (tail[1] == 'a') &&
-	         (tail[2] == 'y') && (tail[3] == 's') &&
-                 (tail[4] == '\0'))) {
-                value = value * GF_DAY_IN_SECONDS;
-                goto out;
-        }
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtol(str, &tail, 0);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        else if (((tail[0] == 'w') && (tail[1] == '\0')) ||
-                 ((tail[0] == 'w') && (tail[1] == 'k') &&
-	         (tail[2] == '\0'))) {
-                value = value * GF_WEEK_IN_SECONDS;
-                goto out;
-        } else {
-                return -1;
-        }
+    if (errno == 0)
+        errno = old_errno;
+
+    if (((tail[0] == '\0') || ((tail[0] == 's') && (tail[1] == '\0')) ||
+         ((tail[0] == 's') && (tail[1] == 'e') && (tail[2] == 'c') &&
+          (tail[3] == '\0'))))
+        goto out;
+
+    else if (((tail[0] == 'm') && (tail[1] == '\0')) ||
+             ((tail[0] == 'm') && (tail[1] == 'i') && (tail[2] == 'n') &&
+              (tail[3] == '\0'))) {
+        value = value * GF_MINUTE_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'h') && (tail[1] == '\0')) ||
+             ((tail[0] == 'h') && (tail[1] == 'r') && (tail[2] == '\0'))) {
+        value = value * GF_HOUR_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'd') && (tail[1] == '\0')) ||
+             ((tail[0] == 'd') && (tail[1] == 'a') && (tail[2] == 'y') &&
+              (tail[3] == 's') && (tail[4] == '\0'))) {
+        value = value * GF_DAY_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'w') && (tail[1] == '\0')) ||
+             ((tail[0] == 'w') && (tail[1] == 'k') && (tail[2] == '\0'))) {
+        value = value * GF_WEEK_IN_SECONDS;
+        goto out;
+    } else {
+        return -1;
+    }
 
 out:
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 int
-gf_string2percent (const char *str, double *n)
-{
-        double value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtod (str, &tail);
-        if (str == tail)
-                errno = EINVAL;
+gf_string2percent(const char *str, double *n)
+{
+    double value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (!((tail[0] == '\0') ||
-              ((tail[0] == '%') && (tail[1] == '\0'))))
-                return -1;
+    if (!((tail[0] == '\0') || ((tail[0] == '%') && (tail[1] == '\0'))))
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
-
 static int
-_gf_string2long (const char *str, long *n, int base)
+_gf_string2long(const char *str, long *n, int base)
 {
-        long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
+    long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        old_errno = errno;
-        errno = 0;
-        value = strtol (str, &tail, base);
-        if (str == tail)
-                errno = EINVAL;
+    old_errno = errno;
+    errno = 0;
+    value = strtol(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_string2ulong (const char *str, unsigned long *n, int base)
-{
-        unsigned long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtoul (str, &tail, base);
-        if (str == tail)
-                errno = EINVAL;
+_gf_string2ulong(const char *str, unsigned long *n, int base)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoul(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_string2uint (const char *str, unsigned int *n, int base)
-{
-        unsigned long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtoul (str, &tail, base);
-        if (str == tail)
-                errno = EINVAL;
+_gf_string2uint(const char *str, unsigned int *n, int base)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoul(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = (unsigned int)value;
+    *n = (unsigned int)value;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_string2double (const char *str, double *n)
+_gf_string2double(const char *str, double *n)
 {
-        double value     = 0.0;
-        char   *tail     = NULL;
-        int    old_errno = 0;
+    double value = 0.0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        old_errno = errno;
-        errno = 0;
-        value = strtod (str, &tail);
-        if (str == tail)
-                errno = EINVAL;
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_string2longlong (const char *str, long long *n, int base)
+_gf_string2longlong(const char *str, long long *n, int base)
 {
-        long long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
+    long long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        old_errno = errno;
-        errno = 0;
-        value = strtoll (str, &tail, base);
-        if (str == tail)
-                errno = EINVAL;
+    old_errno = errno;
+    errno = 0;
+    value = strtoll(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_string2ulonglong (const char *str, unsigned long long *n, int base)
-{
-        unsigned long long value = 0;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
-
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
-
-        old_errno = errno;
-        errno = 0;
-        value = strtoull (str, &tail, base);
-        if (str == tail)
-                errno = EINVAL;
+_gf_string2ulonglong(const char *str, unsigned long long *n, int base)
+{
+    unsigned long long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoull(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-                return -1;
+    if (tail[0] != '\0')
+        return -1;
 
-        *n = value;
+    *n = value;
 
-        return 0;
+    return 0;
 }
 
 int
-gf_string2long (const char *str, long *n)
+gf_string2long(const char *str, long *n)
 {
-        return _gf_string2long (str, n, 0);
+    return _gf_string2long(str, n, 0);
 }
 
 int
-gf_string2ulong (const char *str, unsigned long *n)
+gf_string2ulong(const char *str, unsigned long *n)
 {
-        return _gf_string2ulong (str, n, 0);
+    return _gf_string2ulong(str, n, 0);
 }
 
 int
-gf_string2int (const char *str, int *n)
+gf_string2int(const char *str, int *n)
 {
-        long l = 0;
-        int  ret = 0;
+    long l = 0;
+    int ret = 0;
 
-        ret = _gf_string2long (str, &l, 0);
+    ret = _gf_string2long(str, &l, 0);
 
-        *n = l;
-        return ret;
+    *n = l;
+    return ret;
 }
 
 int
-gf_string2uint (const char *str, unsigned int *n)
+gf_string2uint(const char *str, unsigned int *n)
 {
-        return _gf_string2uint (str, n, 0);
+    return _gf_string2uint(str, n, 0);
 }
 
 int
-gf_string2double (const char *str, double *n)
+gf_string2double(const char *str, double *n)
 {
-        return _gf_string2double (str, n);
+    return _gf_string2double(str, n);
 }
 
 int
-gf_string2longlong (const char *str, long long *n)
+gf_string2longlong(const char *str, long long *n)
 {
-        return _gf_string2longlong (str, n, 0);
+    return _gf_string2longlong(str, n, 0);
 }
 
 int
-gf_string2ulonglong (const char *str, unsigned long long *n)
+gf_string2ulonglong(const char *str, unsigned long long *n)
 {
-        return _gf_string2ulonglong (str, n, 0);
+    return _gf_string2ulonglong(str, n, 0);
 }
 
 int
-gf_string2int8 (const char *str, int8_t *n)
+gf_string2int8(const char *str, int8_t *n)
 {
-        long l = 0L;
-        int rv = 0;
+    long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2long (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if ((l >= INT8_MIN) && (l <= INT8_MAX)) {
-                *n = (int8_t) l;
-                return 0;
-        }
+    if ((l >= INT8_MIN) && (l <= INT8_MAX)) {
+        *n = (int8_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2int16 (const char *str, int16_t *n)
+gf_string2int16(const char *str, int16_t *n)
 {
-        long l = 0L;
-        int rv = 0;
+    long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2long (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if ((l >= INT16_MIN) && (l <= INT16_MAX)) {
-                *n = (int16_t) l;
-                return 0;
-        }
+    if ((l >= INT16_MIN) && (l <= INT16_MAX)) {
+        *n = (int16_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2int32 (const char *str, int32_t *n)
+gf_string2int32(const char *str, int32_t *n)
 {
-        long l = 0L;
-        int rv = 0;
+    long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2long (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if ((l >= INT32_MIN) && (l <= INT32_MAX)) {
-                *n = (int32_t) l;
-                return 0;
-        }
+    if ((l >= INT32_MIN) && (l <= INT32_MAX)) {
+        *n = (int32_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2int64 (const char *str, int64_t *n)
+gf_string2int64(const char *str, int64_t *n)
 {
-        long long l = 0LL;
-        int rv = 0;
+    long long l = 0LL;
+    int rv = 0;
 
-        rv = _gf_string2longlong (str, &l, 0);
-        if (rv != 0)
-                return rv;
-
-        if (l <= INT64_MAX) {
-                *n = (int64_t) l;
-                return 0;
-        }
+    rv = _gf_string2longlong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        errno = ERANGE;
-        return -1;
+    *n = (int64_t)l;
+    return 0;
 }
 
 int
-gf_string2uint8 (const char *str, uint8_t *n)
+gf_string2uint8(const char *str, uint8_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT8_MAX) {
-                *n = (uint8_t) l;
-                return 0;
-        }
+    if (l <= UINT8_MAX) {
+        *n = (uint8_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint16 (const char *str, uint16_t *n)
+gf_string2uint16(const char *str, uint16_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT16_MAX) {
-                *n = (uint16_t) l;
-                return 0;
-        }
+    if (l <= UINT16_MAX) {
+        *n = (uint16_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint32 (const char *str, uint32_t *n)
+gf_string2uint32(const char *str, uint32_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	if (l <= UINT32_MAX) {
-                *n = (uint32_t) l;
-                return 0;
-        }
+    if (l <= UINT32_MAX) {
+        *n = (uint32_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint64 (const char *str, uint64_t *n)
+gf_string2uint64(const char *str, uint64_t *n)
 {
-        unsigned long long l = 0ULL;
-        int rv = 0;
+    unsigned long long l = 0ULL;
+    int rv = 0;
 
-        rv = _gf_string2ulonglong (str, &l, 0);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulonglong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT64_MAX) {
-                *n = (uint64_t) l;
-                return 0;
-        }
+    if (l <= UINT64_MAX) {
+        *n = (uint64_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2ulong_base10 (const char *str, unsigned long *n)
+gf_string2ulong_base10(const char *str, unsigned long *n)
 {
-        return _gf_string2ulong (str, n, 10);
+    return _gf_string2ulong(str, n, 10);
 }
 
 int
-gf_string2uint_base10 (const char *str, unsigned int *n)
+gf_string2uint_base10(const char *str, unsigned int *n)
 {
-        return _gf_string2uint (str,  n, 10);
+    return _gf_string2uint(str, n, 10);
 }
 
 int
-gf_string2uint8_base10 (const char *str, uint8_t *n)
+gf_string2uint8_base10(const char *str, uint8_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 10);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT8_MAX) {
-                *n = (uint8_t) l;
-                return 0;
-        }
+    if (l <= UINT8_MAX) {
+        *n = (uint8_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint16_base10 (const char *str, uint16_t *n)
+gf_string2uint16_base10(const char *str, uint16_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 10);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT16_MAX) {
-                *n = (uint16_t) l;
-                return 0;
-        }
+    if (l <= UINT16_MAX) {
+        *n = (uint16_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint32_base10 (const char *str, uint32_t *n)
+gf_string2uint32_base10(const char *str, uint32_t *n)
 {
-        unsigned long l = 0L;
-        int rv = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-        rv = _gf_string2ulong (str, &l, 10);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT32_MAX) {
-                *n = (uint32_t) l;
-                return 0;
-        }
+    if (l <= UINT32_MAX) {
+        *n = (uint32_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 int
-gf_string2uint64_base10 (const char *str, uint64_t *n)
+gf_string2uint64_base10(const char *str, uint64_t *n)
 {
-        unsigned long long l = 0ULL;
-        int rv = 0;
+    unsigned long long l = 0ULL;
+    int rv = 0;
 
-        rv = _gf_string2ulonglong (str, &l, 10);
-        if (rv != 0)
-                return rv;
+    rv = _gf_string2ulonglong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-        if (l <= UINT64_MAX) {
-                *n = (uint64_t) l;
-                return 0;
-        }
+    if (l <= UINT64_MAX) {
+        *n = (uint64_t)l;
+        return 0;
+    }
 
-        errno = ERANGE;
-        return -1;
+    errno = ERANGE;
+    return -1;
 }
 
 char *
-gf_uint64_2human_readable (uint64_t n)
-{
-        int   ret = 0;
-        char *str = NULL;
-
-        if (n >= GF_UNIT_PB) {
-                ret = gf_asprintf (&str, "%.1lfPB", ((double) n)/GF_UNIT_PB);
-                if (ret < 0)
-                        goto err;
-        } else if (n >= GF_UNIT_TB) {
-                ret = gf_asprintf (&str, "%.1lfTB", ((double) n)/GF_UNIT_TB);
-                if (ret < 0)
-                        goto err;
-        } else if (n >= GF_UNIT_GB) {
-                ret = gf_asprintf (&str, "%.1lfGB", ((double) n)/GF_UNIT_GB);
-                if (ret < 0)
-                        goto err;
-        } else if (n >= GF_UNIT_MB) {
-                ret = gf_asprintf (&str, "%.1lfMB", ((double) n)/GF_UNIT_MB);
-                if (ret < 0)
-                        goto err;
-        } else if (n >= GF_UNIT_KB) {
-                ret = gf_asprintf (&str, "%.1lfKB", ((double) n)/GF_UNIT_KB);
-                if (ret < 0)
-                        goto err;
-        } else {
-                ret = gf_asprintf (&str, "%luBytes", n);
-                if (ret < 0)
-                        goto err;
-        }
-        return str;
+gf_uint64_2human_readable(uint64_t n)
+{
+    int ret = 0;
+    char *str = NULL;
+
+    if (n >= GF_UNIT_PB) {
+        ret = gf_asprintf(&str, "%.1lfPB", ((double)n) / GF_UNIT_PB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_TB) {
+        ret = gf_asprintf(&str, "%.1lfTB", ((double)n) / GF_UNIT_TB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_GB) {
+        ret = gf_asprintf(&str, "%.1lfGB", ((double)n) / GF_UNIT_GB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_MB) {
+        ret = gf_asprintf(&str, "%.1lfMB", ((double)n) / GF_UNIT_MB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_KB) {
+        ret = gf_asprintf(&str, "%.1lfKB", ((double)n) / GF_UNIT_KB);
+        if (ret < 0)
+            goto err;
+    } else {
+        ret = gf_asprintf(&str, "%" PRIu64 "Bytes", n);
+        if (ret < 0)
+            goto err;
+    }
+    return str;
 err:
-        return NULL;
+    return NULL;
 }
 
 int
-gf_string2bytesize_range (const char *str, uint64_t *n, uint64_t umax)
-{
-        double        value      = 0.0;
-        int64_t       int_value  = 0;
-        uint64_t      unit       = 0;
-        int64_t       max        = 0;
-        char         *tail       = NULL;
-        int           old_errno  = 0;
-        const char   *s          = NULL;
-        gf_boolean_t  fraction   = _gf_false;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
+gf_string2bytesize_range(const char *str, uint64_t *n, uint64_t umax)
+{
+    double value = 0.0;
+    int64_t int_value = 0;
+    uint64_t unit = 0;
+    int64_t max = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+    gf_boolean_t fraction = _gf_false;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        max = umax & 0x7fffffffffffffffLL;
+    max = umax & 0x7fffffffffffffffLL;
 
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
 
-        if (strrchr (str, '.'))
-                fraction = _gf_true;
+    if (strrchr(str, '.'))
+        fraction = _gf_true;
 
-        old_errno = errno;
-        errno = 0;
-        if (fraction)
-                value = strtod (str, &tail);
-        else
-                int_value = strtoll (str, &tail, 10);
+    old_errno = errno;
+    errno = 0;
+    if (fraction)
+        value = strtod(str, &tail);
+    else
+        int_value = strtoll(str, &tail, 10);
 
-        if (str == tail)
-                errno = EINVAL;
+    if (str == tail)
+        errno = EINVAL;
 
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        if (errno == 0)
-                errno = old_errno;
+    if (errno == 0)
+        errno = old_errno;
 
-        if (tail[0] != '\0')
-        {
-                if (strcasecmp (tail, GF_UNIT_KB_STRING) == 0)
-                        unit = GF_UNIT_KB;
-                else if (strcasecmp (tail, GF_UNIT_MB_STRING) == 0)
-                        unit = GF_UNIT_MB;
-                else if (strcasecmp (tail, GF_UNIT_GB_STRING) == 0)
-                        unit = GF_UNIT_GB;
-                else if (strcasecmp (tail, GF_UNIT_TB_STRING) == 0)
-                        unit = GF_UNIT_TB;
-                else if (strcasecmp (tail, GF_UNIT_PB_STRING) == 0)
-                        unit = GF_UNIT_PB;
-                else if (strcasecmp (tail, GF_UNIT_B_STRING) != 0)
-                        return -1;
-
-                if (unit > 0) {
-                        if (fraction)
-                                value *= unit;
-                        else
-                                int_value *= unit;
-                }
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, GF_UNIT_KB_STRING) == 0)
+            unit = GF_UNIT_KB;
+        else if (strcasecmp(tail, GF_UNIT_MB_STRING) == 0)
+            unit = GF_UNIT_MB;
+        else if (strcasecmp(tail, GF_UNIT_GB_STRING) == 0)
+            unit = GF_UNIT_GB;
+        else if (strcasecmp(tail, GF_UNIT_TB_STRING) == 0)
+            unit = GF_UNIT_TB;
+        else if (strcasecmp(tail, GF_UNIT_PB_STRING) == 0)
+            unit = GF_UNIT_PB;
+        else if (strcasecmp(tail, GF_UNIT_B_STRING) != 0)
+            return -1;
+
+        if (unit > 0) {
+            if (fraction)
+                value *= unit;
+            else
+                int_value *= unit;
         }
+    }
 
-        if (fraction) {
-                if ((max - value) < 0) {
-                        errno = ERANGE;
-                        return -1;
-                }
-                *n = (uint64_t) value;
-        } else {
-                if ((max - int_value) < 0) {
-                        errno = ERANGE;
-                        return -1;
-                }
-                *n = int_value;
+    if (fraction) {
+        if ((max - value) < 0) {
+            errno = ERANGE;
+            return -1;
         }
+        *n = (uint64_t)value;
+    } else {
+        if ((max - int_value) < 0) {
+            errno = ERANGE;
+            return -1;
+        }
+        *n = int_value;
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-gf_string2bytesize_size (const char *str, size_t *n)
+gf_string2bytesize_uint64(const char *str, uint64_t *n)
 {
-        uint64_t u64;
-        size_t max = (size_t) - 1;
-        int val = gf_string2bytesize_range (str, &u64, max);
-        *n = (size_t) u64;
-        return val;
+    return gf_string2bytesize_range(str, n, UINT64_MAX);
 }
 
 int
-gf_string2bytesize (const char *str, uint64_t *n)
+gf_string2bytesize_int64(const char *str, int64_t *n)
 {
-        return gf_string2bytesize_range(str, n, UINT64_MAX);
-}
+    uint64_t u64 = 0;
+    int ret = 0;
 
-int
-gf_string2bytesize_uint64 (const char *str, uint64_t *n)
-{
-        return gf_string2bytesize_range(str, n, UINT64_MAX);
+    ret = gf_string2bytesize_range(str, &u64, INT64_MAX);
+    *n = (int64_t)u64;
+    return ret;
 }
 
 int
-gf_string2bytesize_int64 (const char *str, int64_t *n)
-{
-        uint64_t u64 = 0;
-        int      ret = 0;
+gf_string2percent_or_bytesize(const char *str, double *n,
+                              gf_boolean_t *is_percent)
+{
+    double value = 0ULL;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-        ret = gf_string2bytesize_range(str, &u64, INT64_MAX);
-        *n = (int64_t) u64;
-        return ret;
-}
+    if (errno == 0)
+        errno = old_errno;
+
+    /*Maximum accepted value for 64 bit OS will be (2^14 -1)PB*/
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, GF_UNIT_KB_STRING) == 0)
+            value *= GF_UNIT_KB;
+        else if (strcasecmp(tail, GF_UNIT_MB_STRING) == 0)
+            value *= GF_UNIT_MB;
+        else if (strcasecmp(tail, GF_UNIT_GB_STRING) == 0)
+            value *= GF_UNIT_GB;
+        else if (strcasecmp(tail, GF_UNIT_TB_STRING) == 0)
+            value *= GF_UNIT_TB;
+        else if (strcasecmp(tail, GF_UNIT_PB_STRING) == 0)
+            value *= GF_UNIT_PB;
+        else if (strcasecmp(tail, GF_UNIT_PERCENT_STRING) == 0)
+            *is_percent = _gf_true;
+        else
+            return -1;
+    }
 
-int
-gf_string2percent_or_bytesize (const char *str, double *n,
-			       gf_boolean_t *is_percent)
-{
-        double value = 0ULL;
-        char *tail = NULL;
-        int old_errno = 0;
-        const char *s = NULL;
-
-        if (str == NULL || n == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                errno = EINVAL;
-                return -1;
-        }
+    /* Error out if we cannot store the value in uint64 */
+    if ((UINT64_MAX - value) < 0) {
+        errno = ERANGE;
+        return -1;
+    }
 
-        for (s = str; *s != '\0'; s++) {
-                if (isspace (*s))
-                        continue;
-                if (*s == '-')
-                        return -1;
-                break;
-        }
+    *n = value;
 
-        old_errno = errno;
-        errno = 0;
-        value = strtod (str, &tail);
-        if (str == tail)
-                errno = EINVAL;
-
-        if (errno == ERANGE || errno == EINVAL)
-                return -1;
-
-        if (errno == 0)
-                errno = old_errno;
-
-        /*Maximum accepted value for 64 bit OS will be (2^14 -1)PB*/
-        if (tail[0] != '\0') {
-                if (strcasecmp (tail, GF_UNIT_KB_STRING) == 0)
-                        value *= GF_UNIT_KB;
-                else if (strcasecmp (tail, GF_UNIT_MB_STRING) == 0)
-                        value *= GF_UNIT_MB;
-                else if (strcasecmp (tail, GF_UNIT_GB_STRING) == 0)
-                        value *= GF_UNIT_GB;
-                else if (strcasecmp (tail, GF_UNIT_TB_STRING) == 0)
-                        value *= GF_UNIT_TB;
-                else if (strcasecmp (tail, GF_UNIT_PB_STRING) == 0)
-                        value *= GF_UNIT_PB;
-		else if (strcasecmp (tail, GF_UNIT_PERCENT_STRING) == 0)
-			*is_percent = _gf_true;
-                else
-                        return -1;
-        }
+    return 0;
+}
 
-        /* Error out if we cannot store the value in uint64 */
-        if ((UINT64_MAX - value) < 0) {
-                errno = ERANGE;
-                return -1;
-        }
+int64_t
+gf_str_to_long_long(const char *number)
+{
+    int64_t unit = 1;
+    int64_t ret = 0;
+    char *endptr = NULL;
+    if (!number)
+        return 0;
 
-        *n = value;
+    ret = strtoll(number, &endptr, 0);
 
-        return 0;
+    if (endptr) {
+        switch (*endptr) {
+            case 'G':
+            case 'g':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024 * 1024 * 1024;
+                break;
+            case 'M':
+            case 'm':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024 * 1024;
+                break;
+            case 'K':
+            case 'k':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024;
+                break;
+            case '%':
+                unit = 1;
+                break;
+            default:
+                unit = 1;
+                break;
+        }
+    }
+    return ret * unit;
 }
 
-int64_t
-gf_str_to_long_long (const char *number)
+int
+gf_string2boolean(const char *str, gf_boolean_t *b)
 {
-        int64_t unit = 1;
-        int64_t ret = 0;
-        char *endptr = NULL ;
-        if (!number)
-                return 0;
+    if (str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
 
-        ret = strtoll (number, &endptr, 0);
-
-        if (endptr) {
-                switch (*endptr) {
-                case 'G':
-                case 'g':
-                        if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-                                unit = 1024 * 1024 * 1024;
-                        break;
-                case 'M':
-                case 'm':
-                        if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-                                unit = 1024 * 1024;
-                        break;
-                case 'K':
-                case 'k':
-                        if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-                                unit = 1024;
-                        break;
-                case '%':
-                        unit = 1;
-                        break;
-                default:
-                        unit = 1;
-                        break;
-                }
-        }
-        return ret * unit;
+    if ((strcasecmp(str, "1") == 0) || (strcasecmp(str, "on") == 0) ||
+        (strcasecmp(str, "yes") == 0) || (strcasecmp(str, "true") == 0) ||
+        (strcasecmp(str, "enable") == 0)) {
+        *b = _gf_true;
+        return 0;
+    }
+
+    if ((strcasecmp(str, "0") == 0) || (strcasecmp(str, "off") == 0) ||
+        (strcasecmp(str, "no") == 0) || (strcasecmp(str, "false") == 0) ||
+        (strcasecmp(str, "disable") == 0)) {
+        *b = _gf_false;
+        return 0;
+    }
+
+    return -1;
 }
 
 int
-gf_string2boolean (const char *str, gf_boolean_t *b)
+gf_strn2boolean(const char *str, const int len, gf_boolean_t *b)
 {
-        if (str == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                return -1;
-        }
+    if (str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
 
-        if ((strcasecmp (str, "1") == 0) ||
-            (strcasecmp (str, "on") == 0) ||
-            (strcasecmp (str, "yes") == 0) ||
-            (strcasecmp (str, "true") == 0) ||
-            (strcasecmp (str, "enable") == 0)) {
+    switch (len) {
+        case 1:
+            if (strcasecmp(str, "1") == 0) {
                 *b = _gf_true;
                 return 0;
-        }
-
-        if ((strcasecmp (str, "0") == 0) ||
-            (strcasecmp (str, "off") == 0) ||
-            (strcasecmp (str, "no") == 0) ||
-            (strcasecmp (str, "false") == 0) ||
-            (strcasecmp (str, "disable") == 0)) {
+            } else if (strcasecmp(str, "0") == 0) {
                 *b = _gf_false;
                 return 0;
-        }
-
-        return -1;
+            }
+            break;
+        case 2:
+            if (strcasecmp(str, "on") == 0) {
+                *b = _gf_true;
+                return 0;
+            } else if (strcasecmp(str, "no") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 3:
+            if (strcasecmp(str, "yes") == 0) {
+                *b = _gf_true;
+                return 0;
+            } else if (strcasecmp(str, "off") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 4:
+            if (strcasecmp(str, "true") == 0) {
+                *b = _gf_true;
+                return 0;
+            }
+            break;
+        case 5:
+            if (strcasecmp(str, "false") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 6:
+            if (strcasecmp(str, "enable") == 0) {
+                *b = _gf_true;
+                return 0;
+            }
+            break;
+        case 7:
+            if (strcasecmp(str, "disable") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        default:
+            return -1;
+            break;
+    }
+    return -1;
 }
 
-
 int
-gf_lockfd (int fd)
+gf_lockfd(int fd)
 {
-        struct gf_flock fl;
+    struct gf_flock fl;
 
-        fl.l_type = F_WRLCK;
-        fl.l_whence = SEEK_SET;
-        fl.l_start = 0;
-        fl.l_len = 0;
+    fl.l_type = F_WRLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 0;
+    fl.l_len = 0;
 
-        return fcntl (fd, F_SETLK, &fl);
+    return fcntl(fd, F_SETLK, &fl);
 }
 
-
 int
-gf_unlockfd (int fd)
+gf_unlockfd(int fd)
 {
-        struct gf_flock fl;
+    struct gf_flock fl;
 
-        fl.l_type = F_UNLCK;
-        fl.l_whence = SEEK_SET;
-        fl.l_start = 0;
-        fl.l_len = 0;
+    fl.l_type = F_UNLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 0;
+    fl.l_len = 0;
 
-        return fcntl (fd, F_SETLK, &fl);
+    return fcntl(fd, F_SETLK, &fl);
 }
 
 static void
-compute_checksum (char *buf, size_t size, uint32_t *checksum)
+compute_checksum(char *buf, const ssize_t size, uint32_t *checksum)
 {
-        int  ret = -1;
-        char *checksum_buf = NULL;
+    int ret = -1;
+    char *checksum_buf = NULL;
 
-        checksum_buf = (char *)(checksum);
+    checksum_buf = (char *)(checksum);
 
-        if (!(*checksum)) {
-                checksum_buf [0] = 0xba;
-                checksum_buf [1] = 0xbe;
-                checksum_buf [2] = 0xb0;
-                checksum_buf [3] = 0x0b;
-        }
+    if (!(*checksum)) {
+        checksum_buf[0] = 0xba;
+        checksum_buf[1] = 0xbe;
+        checksum_buf[2] = 0xb0;
+        checksum_buf[3] = 0x0b;
+    }
 
-        for (ret = 0; ret < (size - 4); ret += 4) {
-                checksum_buf[0] ^= (buf[ret]);
-                checksum_buf[1] ^= (buf[ret + 1] << 1) ;
-                checksum_buf[2] ^= (buf[ret + 2] << 2);
-                checksum_buf[3] ^= (buf[ret + 3] << 3);
-        }
+    for (ret = 0; ret < (size - 4); ret += 4) {
+        checksum_buf[0] ^= (buf[ret]);
+        checksum_buf[1] ^= (buf[ret + 1] << 1);
+        checksum_buf[2] ^= (buf[ret + 2] << 2);
+        checksum_buf[3] ^= (buf[ret + 3] << 3);
+    }
 
-        for (ret = 0; ret <= (size % 4); ret++) {
-                checksum_buf[ret] ^= (buf[(size - 4) + ret] << ret);
-        }
+    for (ret = 0; ret <= (size % 4); ret++) {
+        checksum_buf[ret] ^= (buf[(size - 4) + ret] << ret);
+    }
 
-        return;
+    return;
 }
 
 #define GF_CHECKSUM_BUF_SIZE 1024
 
 int
-get_checksum_for_file (int fd, uint32_t *checksum)
+get_checksum_for_file(int fd, uint32_t *checksum, int op_version)
 {
-        int ret = -1;
-        char buf[GF_CHECKSUM_BUF_SIZE] = {0,};
+    int ret = -1;
+    char buf[GF_CHECKSUM_BUF_SIZE] = {
+        0,
+    };
 
-        /* goto first place */
-        sys_lseek (fd, 0L, SEEK_SET);
-        do {
-                ret = sys_read (fd, &buf, GF_CHECKSUM_BUF_SIZE);
-                if (ret > 0)
-                        compute_checksum (buf, GF_CHECKSUM_BUF_SIZE,
-                                          checksum);
-        } while (ret > 0);
+    /* goto first place */
+    sys_lseek(fd, 0L, SEEK_SET);
+    do {
+        ret = sys_read(fd, &buf, GF_CHECKSUM_BUF_SIZE);
+        if (ret > 0) {
+            if (op_version < GD_OP_VERSION_5_4)
+                compute_checksum(buf, GF_CHECKSUM_BUF_SIZE, checksum);
+            else
+                compute_checksum(buf, ret, checksum);
+        }
+    } while (ret > 0);
 
-        /* set it back */
-        sys_lseek (fd, 0L, SEEK_SET);
+    /* set it back */
+    sys_lseek(fd, 0L, SEEK_SET);
 
-        return ret;
+    return ret;
 }
 
-
 int
-get_checksum_for_path (char *path, uint32_t *checksum)
+get_checksum_for_path(char *path, uint32_t *checksum, int op_version)
 {
-        int     ret = -1;
-        int     fd = -1;
+    int ret = -1;
+    int fd = -1;
 
-        GF_ASSERT (path);
-        GF_ASSERT (checksum);
+    GF_ASSERT(path);
+    GF_ASSERT(checksum);
 
-        fd = open (path, O_RDWR);
+    fd = open(path, O_RDWR);
 
-        if (fd == -1) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno, LG_MSG_PATH_ERROR,
-                        "Unable to open %s", path);
-                goto out;
-        }
+    if (fd == -1) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, LG_MSG_PATH_OPEN_FAILED,
+                "path=%s", path, NULL);
+        goto out;
+    }
 
-        ret = get_checksum_for_file (fd, checksum);
+    ret = get_checksum_for_file(fd, checksum, op_version);
 
 out:
-        if (fd != -1)
-                sys_close (fd);
+    if (fd != -1)
+        sys_close(fd);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -1963,26 +2137,25 @@ out:
  *           errors : Errors returned by the stat () call
  */
 int
-get_file_mtime (const char *path, time_t *stamp)
+get_file_mtime(const char *path, time_t *stamp)
 {
-        struct stat     f_stat  = {0};
-        int             ret     = -EINVAL;
+    struct stat f_stat = {0};
+    int ret = -EINVAL;
 
-        GF_VALIDATE_OR_GOTO (THIS->name, path, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, stamp, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, path, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, stamp, out);
 
-        ret = sys_stat (path, &f_stat);
-        if (ret < 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        LG_MSG_FILE_STAT_FAILED, "failed to stat %s",
-                        path);
-                goto out;
-        }
+    ret = sys_stat(path, &f_stat);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, LG_MSG_FILE_STAT_FAILED,
+                "path=%s", path, NULL);
+        goto out;
+    }
 
-        /* Set the mtime */
-        *stamp = f_stat.st_mtime;
+    /* Set the mtime */
+    *stamp = f_stat.st_mtime;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1994,268 +2167,402 @@ out:
  * @ip_str : The IP to check
  * @network: The network to check the IP against.
  *
- * @return: success: 0
+ * @return: success: _gf_true
  *          failure: -EINVAL for bad args, retval of inet_pton otherwise
  */
 gf_boolean_t
-gf_is_ip_in_net (const char *network, const char *ip_str)
-{
-        unsigned long ip_buf       = 0;
-        unsigned long net_ip_buf   = 0;
-        unsigned long subnet_mask  = 0;
-        int           ret          = -EINVAL;
-        char          *slash       = NULL;
-        char          *net_ip      = NULL;
-        char          *subnet      = NULL;
-        char          *net_str     = NULL;
-        int           family       = AF_INET;
-        gf_boolean_t  result       = _gf_false;
-
-        GF_ASSERT (network);
-        GF_ASSERT (ip_str);
-
-        if (strchr (network, ':'))
-                family = AF_INET6;
-        else if (strchr (network, '.'))
-                family = AF_INET;
-        else {
-                family = -1;
-                goto out;
-        }
-
-        net_str = strdupa (network);
-        slash = strchr (net_str, '/');
-        if (!slash)
-                goto out;
-        *slash = '\0';
-
-        subnet = slash + 1;
-        net_ip = net_str;
-
-        /* Convert IP address to a long */
-        ret = inet_pton (family, ip_str, &ip_buf);
-        if (ret < 0)
-                gf_msg ("common-utils", GF_LOG_ERROR, errno,
-                        LG_MSG_INET_PTON_FAILED, "inet_pton() failed");
-
-        /* Convert network IP address to a long */
-        ret = inet_pton (family, net_ip, &net_ip_buf);
-        if (ret < 0) {
-                gf_msg ("common-utils", GF_LOG_ERROR, errno,
-                        LG_MSG_INET_PTON_FAILED, "inet_pton() failed");
-                goto out;
-        }
-
-        /* Converts /x into a mask */
-        subnet_mask = (1 << atoi (subnet)) - 1;
-
-        result = ((ip_buf & subnet_mask) == (net_ip_buf & subnet_mask));
+gf_is_ip_in_net(const char *network, const char *ip_str)
+{
+    unsigned long ip_buf = 0;
+    unsigned long net_ip_buf = 0;
+    unsigned long subnet_mask = 0;
+    int ret = -EINVAL;
+    char *slash = NULL;
+    char *net_ip = NULL;
+    char *subnet = NULL;
+    char *net_str = NULL;
+    int family = AF_INET;
+    gf_boolean_t result = _gf_false;
+
+    GF_ASSERT(network);
+    GF_ASSERT(ip_str);
+
+    if (strchr(network, ':'))
+        family = AF_INET6;
+    else if (strchr(network, '.'))
+        family = AF_INET;
+    else {
+        goto out;
+    }
+
+    net_str = strdupa(network);
+    slash = strchr(net_str, '/');
+    if (!slash)
+        goto out;
+    *slash = '\0';
+
+    subnet = slash + 1;
+    net_ip = net_str;
+
+    /* Convert IP address to a long */
+    ret = inet_pton(family, ip_str, &ip_buf);
+    if (ret < 0)
+        gf_smsg("common-utils", GF_LOG_ERROR, errno, LG_MSG_INET_PTON_FAILED,
+                NULL);
+
+    /* Convert network IP address to a long */
+    ret = inet_pton(family, net_ip, &net_ip_buf);
+    if (ret < 0) {
+        gf_smsg("common-utils", GF_LOG_ERROR, errno, LG_MSG_INET_PTON_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* Converts /x into a mask */
+    subnet_mask = (1 << atoi(subnet)) - 1;
+
+    result = ((ip_buf & subnet_mask) == (net_ip_buf & subnet_mask));
 out:
-        return result;
+    return result;
 }
 
 char *
-strtail (char *str, const char *pattern)
+strtail(char *str, const char *pattern)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; str[i] == pattern[i] && str[i]; i++);
+    for (i = 0; str[i] == pattern[i] && str[i]; i++)
+        ;
 
-        if (pattern[i] == '\0')
-                return str + i;
+    if (pattern[i] == '\0')
+        return str + i;
 
-        return NULL;
+    return NULL;
 }
 
 void
-skipwhite (char **s)
+skipwhite(char **s)
 {
-        while (isspace (**s))
-                (*s)++;
+    while (isspace(**s))
+        (*s)++;
 }
 
-char *
-nwstrtail (char *str, char *pattern)
+void
+gf_strTrim(char **s)
 {
-        for (;;) {
-                skipwhite (&str);
-                skipwhite (&pattern);
+    char *end = NULL;
 
-                if (*str != *pattern || !*str)
-                        break;
+    end = *s + strlen(*s) - 1;
+    while (end > *s && isspace((unsigned char)*end))
+        end--;
 
-                str++;
-                pattern++;
-        }
+    *(end + 1) = '\0';
+
+    while (isspace(**s))
+        (*s)++;
 
-        return *pattern ? NULL : str;
+    return;
 }
 
-void
-skipword (char **s)
+char *
+nwstrtail(char *str, char *pattern)
 {
-        if (!*s)
-                return;
+    for (;;) {
+        skipwhite(&str);
+        skipwhite(&pattern);
 
-        skipwhite (s);
+        if (*str != *pattern || !*str)
+            break;
 
-        while (!isspace(**s))
-                (*s)++;
+        str++;
+        pattern++;
+    }
+
+    return *pattern ? NULL : str;
 }
 
+/**
+ * token_iter_init -- initialize tokenization
+ *
+ * @str: string to be tokenized
+ * @sep: token separator character
+ * @tit: pointer to iteration state
+ *
+ * @return: token string
+ *
+ * The returned token string and tit are
+ * not to be used directly, but through
+ * next_token().
+ */
 char *
-get_nth_word (const char *str, int n)
+token_iter_init(char *str, char sep, token_iter_t *tit)
 {
-        char           buf[4096] = {0};
-        char          *start     = NULL;
-        char          *word      = NULL;
-        int            i         = 0;
-        int            word_len  = 0;
-        const char    *end       = NULL;
+    tit->end = str + strlen(str);
+    tit->sep = sep;
 
-        if (!str)
-                goto out;
+    return str;
+}
+
+/**
+ * next_token -- fetch next token in tokenization
+ * inited by token_iter_init().
+ *
+ * @tokenp: pointer to token
+ * @tit:    pointer to iteration state
+ *
+ * @return: true if iteration ends, else false
+ *
+ * The token pointed by @tokenp can be used
+ * after a call to next_token(). When next_token()
+ * returns true the iteration is to be stopped
+ * and the string with which the tokenization
+ * was inited (see token_iter_init() is restored,
+ * apart from dropped tokens (see drop_token()).
+ */
+gf_boolean_t
+next_token(char **tokenp, token_iter_t *tit)
+{
+    char *cursor = NULL;
+    gf_boolean_t is_last = _gf_false;
 
-        snprintf (buf, sizeof (buf), "%s", str);
-        start = buf;
+    for (cursor = *tokenp; *cursor; cursor++)
+        ;
+    if (cursor < tit->end) {
+        /*
+         * We detect that in between current token and end a zero
+         * marker has already been inserted. This means that the
+         * token has already been returned. We restore the
+         * separator and move ahead.
+         */
+        *cursor = tit->sep;
+        *tokenp = cursor + 1;
+    }
 
-        for (i = 0; i < n-1; i++)
-                skipword (&start);
+    for (cursor = *tokenp; *cursor && *cursor != tit->sep; cursor++)
+        ;
+    /* If the cursor ended up on a zero byte, then it's the last token. */
+    is_last = !*cursor;
+    /* Zero-terminate the token. */
+    *cursor = 0;
 
-        skipwhite (&start);
-        end = strpbrk ((const char *)start, " \t\n\0");
+    return is_last;
+}
 
-        if (!end)
-                goto out;
+/*
+ * drop_token -- drop a token during iterated calls of next_token().
+ *
+ * Sample program that uses these functions to tokenize
+ * a comma-separated first argument while dropping the
+ * rest of the arguments if they occur as token:
+ *
+ * #include <stdio.h>
+ * #include <stdlib.h>
+ * #include <string.h>
+ * #include "glusterfs/common-utils.h"
+ *
+ * int
+ * main (int argc, char **argv)
+ * {
+ *         char *buf;
+ *         char *token;
+ *         token_iter_t tit;
+ *         int i;
+ *         gf_boolean_t iter_end;
+ *
+ *         if (argc <= 1)
+ *                 abort();
+ *
+ *         buf = strdup (argv[1]);
+ *         if (!buf)
+ *                 abort();
+ *
+ *         for (token = token_iter_init (buf, ',', &tit) ;;) {
+ *                 iter_end = next_token (&token, &tit);
+ *                 printf("found token: '%s'\n", token);
+ *                 for (i = 2; i < argc; i++) {
+ *                         if (strcmp (argv[i], token) == 0) {
+ *                                 printf ("%s\n", "dropping token!");
+ *                                 drop_token (token, &tit);
+ *                                 break;
+ *                          }
+ *                 }
+ *                 if (iter_end)
+ *                         break;
+ *         }
+ *
+ *         printf ("finally: '%s'\n", buf);
+ *
+ *         return 0;
+ * }
+ */
+void
+drop_token(char *token, token_iter_t *tit)
+{
+    char *cursor = NULL;
 
-        word_len = labs (end - start);
+    for (cursor = token; *cursor; cursor++)
+        ;
+    if (cursor < tit->end) {
+        /*
+         * We detect a zero inserted by next_token().
+         * Step the cursor and copy what comes after
+         * to token.
+         */
+        for (cursor++; cursor < tit->end; *token++ = *cursor++)
+            ;
+    }
 
-        word = GF_CALLOC (1, word_len + 1, gf_common_mt_strdup);
-        if (!word)
-                goto out;
+    /*
+     * Zero out the remainder of the buffer.
+     * It would be enough to insert just a single zero,
+     * but we continue 'till the end to have cleaner
+     * memory content.
+     */
+    for (cursor = token; cursor < tit->end; *cursor++ = 0)
+        ;
 
-        strncpy (word, start, word_len);
-        *(word + word_len) = '\0';
- out:
-        return word;
+    /* Adjust the end to point to the new terminating zero. */
+    tit->end = token;
 }
 
 /* Syntax formed according to RFC 1912 (RFC 1123 & 952 are more restrictive)  *
    <hname> ::= <gen-name>*["."<gen-name>]                                     *
    <gen-name> ::= <let-or-digit> <[*[<let-or-digit-or-hyphen>]<let-or-digit>] */
 char
-valid_host_name (char *address, int length)
+valid_host_name(char *address, int length)
 {
-        int             i = 0;
-        int             str_len = 0;
-        char            ret = 1;
-        char            *dup_addr = NULL;
-        char            *temp_str = NULL;
-        char            *save_ptr = NULL;
+    int i = 0;
+    int str_len = 0;
+    char ret = 1;
+    char *dup_addr = NULL;
+    char *temp_str = NULL;
+    char *save_ptr = NULL;
 
-        if ((length > _POSIX_HOST_NAME_MAX) || (length < 1)) {
-                ret = 0;
-                goto out;
-        }
+    if ((length > _POSIX_HOST_NAME_MAX) || (length < 1)) {
+        ret = 0;
+        goto out;
+    }
 
-        dup_addr = gf_strdup (address);
-        if (!dup_addr) {
-                ret = 0;
-                goto out;
-        }
+    dup_addr = gf_strdup(address);
+    if (!dup_addr) {
+        ret = 0;
+        goto out;
+    }
+
+    if (!isalnum(dup_addr[length - 1]) && (dup_addr[length - 1] != '*')) {
+        ret = 0;
+        goto out;
+    }
+
+    /* Check for consecutive dots, which is invalid in a hostname and is
+     * ignored by strtok()
+     */
+    if (strstr(dup_addr, "..")) {
+        ret = 0;
+        goto out;
+    }
+
+    /* gen-name */
+    temp_str = strtok_r(dup_addr, ".", &save_ptr);
+    do {
+        str_len = strlen(temp_str);
 
-        if (!isalnum (dup_addr[length - 1]) && (dup_addr[length - 1] != '*')) {
+        if (!isalnum(temp_str[0]) || !isalnum(temp_str[str_len - 1])) {
+            ret = 0;
+            goto out;
+        }
+        for (i = 1; i < str_len; i++) {
+            if (!isalnum(temp_str[i]) && (temp_str[i] != '-')) {
                 ret = 0;
                 goto out;
+            }
         }
+    } while ((temp_str = strtok_r(NULL, ".", &save_ptr)));
 
-        /* Check for consecutive dots, which is invalid in a hostname and is
-         * ignored by strtok()
-         */
-        if (strstr (dup_addr, "..")) {
+out:
+    GF_FREE(dup_addr);
+    return ret;
+}
+
+/*  Matches all ipv4 address, if wildcard_acc is true  '*' wildcard pattern for*
+  subnets is considered as valid strings as well */
+char
+valid_ipv4_address(char *address, int length, gf_boolean_t wildcard_acc)
+{
+    int octets = 0;
+    int value = 0;
+    char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
+    char ret = 1;
+    int is_wildcard = 0;
+
+    tmp = gf_strdup(address);
+
+    /*
+     * To prevent cases where last character is '.' and which have
+     * consecutive dots like ".." as strtok ignore consecutive
+     * delimiters.
+     */
+    if (length <= 0 || (strstr(address, "..")) ||
+        (!isdigit(tmp[length - 1]) && (tmp[length - 1] != '*'))) {
+        ret = 0;
+        goto out;
+    }
+
+    prev = strtok_r(tmp, ".", &ptr);
+
+    while (prev != NULL) {
+        octets++;
+        if (wildcard_acc && !strcmp(prev, "*")) {
+            is_wildcard = 1;
+        } else {
+            value = strtol(prev, &endptr, 10);
+            if ((value > 255) || (value < 0) ||
+                (endptr != NULL && *endptr != '\0')) {
                 ret = 0;
                 goto out;
+            }
         }
+        prev = strtok_r(NULL, ".", &ptr);
+    }
 
-        /* gen-name */
-        temp_str = strtok_r (dup_addr, ".", &save_ptr);
-        do {
-                str_len = strlen (temp_str);
-
-                if (!isalnum (temp_str[0]) ||
-                    !isalnum (temp_str[str_len-1])) {
-                        ret = 0;
-                        goto out;
-                }
-                for (i = 1; i < str_len; i++) {
-                        if (!isalnum (temp_str[i]) && (temp_str[i] != '-')) {
-                                ret = 0;
-                                goto out;
-                        }
-                }
-        } while ((temp_str = strtok_r (NULL, ".", &save_ptr)));
+    if ((octets > 4) || (octets < 4 && !is_wildcard)) {
+        ret = 0;
+    }
 
 out:
-        GF_FREE (dup_addr);
-        return ret;
+    GF_FREE(tmp);
+    return ret;
 }
 
-/*  Matches all ipv4 address, if wildcard_acc is true  '*' wildcard pattern for*
-  subnets is considered as valid strings as well                               */
 char
-valid_ipv4_address (char *address, int length, gf_boolean_t wildcard_acc)
+valid_cidr_address(char *cidr_address, gf_boolean_t wildcard_acc)
 {
-        int octets = 0;
-        int value = 0;
-        char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
-        char ret = 1;
-        int is_wildcard = 0;
+    unsigned int net_mask = 0, len = 0;
+    char *temp = NULL, *cidr_str = NULL, ret = 1;
 
-        tmp = gf_strdup (address);
+    cidr_str = strdupa(cidr_address);
+    temp = strstr(cidr_str, "/");
+    if (temp == NULL)
+        return 0; /* Since Invalid cidr ip address we return 0 */
 
-        /*
-         * To prevent cases where last character is '.' and which have
-         * consecutive dots like ".." as strtok ignore consecutive
-         * delimeters.
-         */
-        if (length <= 0 ||
-            (strstr (address, "..")) ||
-            (!isdigit (tmp[length - 1]) && (tmp[length - 1] != '*'))) {
-                ret = 0;
-                goto out;
-        }
+    *temp = '\0';
+    temp++;
+    net_mask = (unsigned int)atoi(temp);
 
-        prev = tmp;
-        prev = strtok_r (tmp, ".", &ptr);
-
-        while (prev != NULL) {
-                octets++;
-                if (wildcard_acc && !strcmp (prev, "*")) {
-                        is_wildcard = 1;
-                } else {
-                        value = strtol (prev, &endptr, 10);
-                        if ((value > 255) || (value < 0) ||
-                            (endptr != NULL && *endptr != '\0')) {
-                                ret = 0;
-                                goto out;
-                        }
-                }
-                prev = strtok_r (NULL, ".", &ptr);
-        }
+    if (net_mask > 32 || net_mask < 1)
+        return 0; /* Since Invalid cidr ip address we return 0*/
 
-        if ((octets > 4) || (octets < 4 && !is_wildcard)) {
-                ret = 0;
-        }
+    len = strlen(cidr_str);
 
-out:
-        GF_FREE (tmp);
-        return ret;
+    ret = valid_ipv4_address(cidr_str, len, wildcard_acc);
+
+    return ret;
 }
 
 /**
  * valid_ipv4_subnetwork() takes the pattern and checks if it contains
  * a valid ipv4 subnetwork pattern i.e. xx.xx.xx.xx/n. IPv4 address
- * part (xx.xx.xx.xx) and mask bits lengh part (n). The mask bits lengh
+ * part (xx.xx.xx.xx) and mask bits length part (n). The mask bits length
  * must be in 0-32 range (ipv4 addr is 32 bit). The pattern must be
  * in this format.
  *
@@ -2263,149 +2570,158 @@ out:
  *         _gf_false otherwise.
  */
 gf_boolean_t
-valid_ipv4_subnetwork (const char *address)
+valid_ipv4_subnetwork(const char *address)
 {
-        char         *slash     = NULL;
-        char         *paddr     = NULL;
-        char         *endptr    = NULL;
-        long         prefixlen  = -1;
-        gf_boolean_t retv       = _gf_true;
-
-        if (address == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                return _gf_false;
-        }
-
-        paddr = gf_strdup (address);
-        if (paddr == NULL) /* ENOMEM */
-                return _gf_false;
-
-        /*
-         * INVALID: If '/' is not present OR
-         *          Nothing specified after '/'
-         */
-        slash = strchr(paddr, '/');
-        if ((slash == NULL) || (slash[1] == '\0')) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, 0,
-                                  LG_MSG_INVALID_IPV4_FORMAT, "Invalid IPv4 "
-                                  "subnetwork format");
-                retv = _gf_false;
-                goto out;
-        }
+    char *slash = NULL;
+    char *paddr = NULL;
+    char *endptr = NULL;
+    long prefixlen = -1;
+    gf_boolean_t retv = _gf_true;
 
-        *slash = '\0';
-        retv = valid_ipv4_address (paddr, strlen(paddr), _gf_false);
-        if (retv == _gf_false) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, 0,
-                                  LG_MSG_INVALID_IPV4_FORMAT,
-                                  "Invalid IPv4 subnetwork address");
-                goto out;
-        }
+    if (address == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return _gf_false;
+    }
 
-        prefixlen = strtol (slash + 1, &endptr, 10);
-        if ((errno != 0) || (*endptr != '\0') ||
-            (prefixlen < 0) || (prefixlen > IPv4_ADDR_SIZE)) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, 0,
-                                  LG_MSG_INVALID_IPV4_FORMAT,
-                                  "Invalid IPv4 subnetwork mask");
-                retv = _gf_false;
-                goto out;
-        }
+    paddr = gf_strdup(address);
+    if (paddr == NULL) /* ENOMEM */
+        return _gf_false;
 
-        retv = _gf_true;
+    /*
+     * INVALID: If '/' is not present OR
+     *          Nothing specified after '/'
+     */
+    slash = strchr(paddr, '/');
+    if ((slash == NULL) || (slash[1] == '\0')) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 "
+                         "subnetwork format");
+        retv = _gf_false;
+        goto out;
+    }
+
+    *slash = '\0';
+    retv = valid_ipv4_address(paddr, strlen(paddr), _gf_false);
+    if (retv == _gf_false) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 subnetwork address");
+        goto out;
+    }
+    /*
+     * Reset errno before checking it
+     */
+    errno = 0;
+    prefixlen = strtol(slash + 1, &endptr, 10);
+    if ((errno != 0) || (*endptr != '\0') || (prefixlen < 0) ||
+        (prefixlen > IPv4_ADDR_SIZE)) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 subnetwork mask");
+        retv = _gf_false;
+        goto out;
+    }
+
+    retv = _gf_true;
 out:
-        GF_FREE (paddr);
-        return retv;
+    GF_FREE(paddr);
+    return retv;
 }
 
 char
-valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc)
-{
-        int hex_numbers = 0;
-        int value = 0;
-        int i = 0;
-        char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
-        char ret = 1;
-        int is_wildcard = 0;
-        int is_compressed = 0;
-
-        tmp = gf_strdup (address);
-
-        /* Check for '%' for link local addresses */
-        endptr = strchr(tmp, '%');
-        if (endptr) {
-                *endptr = '\0';
-                length = strlen(tmp);
-                endptr = NULL;
-        }
-
-        /* Check for compressed form */
-        if (length <= 0 || tmp[length - 1] == ':') {
+valid_ipv6_address(char *address, int length, gf_boolean_t wildcard_acc)
+{
+    int hex_numbers = 0;
+    int value = 0;
+    int i = 0;
+    char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
+    char ret = 1;
+    int is_wildcard = 0;
+    int is_compressed = 0;
+
+    tmp = gf_strdup(address);
+
+    /* Check for '%' for link local addresses */
+    endptr = strchr(tmp, '%');
+    if (endptr) {
+        *endptr = '\0';
+        length = strlen(tmp);
+        endptr = NULL;
+    }
+
+    /* Check for compressed form */
+    if (length <= 0 || tmp[length - 1] == ':') {
+        ret = 0;
+        goto out;
+    }
+    for (i = 0; i < (length - 1); i++) {
+        if (tmp[i] == ':' && tmp[i + 1] == ':') {
+            if (is_compressed == 0)
+                is_compressed = 1;
+            else {
                 ret = 0;
                 goto out;
+            }
         }
-        for (i = 0; i < (length - 1) ; i++) {
-                if (tmp[i] == ':' && tmp[i + 1] == ':') {
-                        if (is_compressed == 0)
-                                is_compressed = 1;
-                        else {
-                                ret = 0;
-                                goto out;
-                        }
-                }
-        }
+    }
 
-        prev = strtok_r (tmp, ":", &ptr);
-
-        while (prev != NULL) {
-                hex_numbers++;
-                if (wildcard_acc && !strcmp (prev, "*")) {
-                        is_wildcard = 1;
-                } else {
-                        value = strtol (prev, &endptr, 16);
-                        if ((value > 0xffff) || (value < 0)
-                        || (endptr != NULL && *endptr != '\0')) {
-                                ret = 0;
-                                goto out;
-                        }
-                }
-                prev = strtok_r (NULL, ":", &ptr);
-        }
+    prev = strtok_r(tmp, ":", &ptr);
 
-        if ((hex_numbers > 8) || (hex_numbers < 8 && !is_wildcard
-            && !is_compressed)) {
+    while (prev != NULL) {
+        hex_numbers++;
+        if (wildcard_acc && !strcmp(prev, "*")) {
+            is_wildcard = 1;
+        } else {
+            value = strtol(prev, &endptr, 16);
+            if ((value > 0xffff) || (value < 0) ||
+                (endptr != NULL && *endptr != '\0')) {
                 ret = 0;
+                goto out;
+            }
         }
+        prev = strtok_r(NULL, ":", &ptr);
+    }
+
+    if ((hex_numbers > 8) ||
+        (hex_numbers < 8 && !is_wildcard && !is_compressed)) {
+        ret = 0;
+    }
 
 out:
-        GF_FREE (tmp);
-        return ret;
+    GF_FREE(tmp);
+    return ret;
 }
 
 char
-valid_internet_address (char *address, gf_boolean_t wildcard_acc)
+valid_internet_address(char *address, gf_boolean_t wildcard_acc,
+                       gf_boolean_t cidr)
 {
-        char ret = 0;
-        int length = 0;
+    char ret = 0;
+    int length = 0;
 
-        if (address == NULL) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                goto out;
-        }
+    if (address == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        goto out;
+    }
 
-        length = strlen (address);
-        if (length == 0)
-                goto out;
+    length = strlen(address);
+    if (length == 0)
+        goto out;
+
+    if (cidr && valid_cidr_address(address, wildcard_acc)) {
+        ret = 1;
+    }
 
-        if (valid_ipv4_address (address, length, wildcard_acc)
-            || valid_ipv6_address (address, length, wildcard_acc)
-            || valid_host_name (address, length))
-                ret = 1;
+    if (valid_ipv4_address(address, length, wildcard_acc) ||
+        valid_ipv6_address(address, length, wildcard_acc) ||
+        valid_host_name(address, length))
+        ret = 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -2415,56 +2731,56 @@ out:
  *
  * @return _gf_true if "address" is "*" (anonymous) 'OR'
  *                  if "address" is valid FQDN or valid IPv4/6 address 'OR'
- *                  if "address" contains wildcard chars e.g. "'*' or '?' or '['"
- *                  if "address" is valid ipv4 subnet pattern (xx.xx.xx.xx/n)
- *         _gf_false otherwise
+ *                  if "address" contains wildcard chars e.g. "'*' or '?' or
+ * '['" if "address" is valid ipv4 subnet pattern (xx.xx.xx.xx/n) _gf_false
+ * otherwise
  *
  *
  * NB: If the user/admin set for wildcard pattern, then it does not have
  *     to be validated. Make it similar to the way exportfs (kNFS) works.
  */
 gf_boolean_t
-valid_mount_auth_address (char *address)
+valid_mount_auth_address(char *address)
 {
-        int    length = 0;
-        char   *cp    = NULL;
+    int length = 0;
+    char *cp = NULL;
 
-        /* 1. Check for "NULL and empty string */
-        if ((address == NULL) || (address[0] == '\0')){
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "argument invalid");
-                return _gf_false;
-        }
+    /* 1. Check for "NULL and empty string */
+    if ((address == NULL) || (address[0] == '\0')) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return _gf_false;
+    }
 
-        /* 2. Check for Anonymous */
-        if (strcmp(address, "*") == 0)
-                return _gf_true;
+    /* 2. Check for Anonymous */
+    if (strcmp(address, "*") == 0)
+        return _gf_true;
 
-        for (cp = address; *cp; cp++) {
-                /* 3. Check for wildcard pattern */
-                if (*cp == '*' || *cp == '?' || *cp == '[') {
-                        return _gf_true;
-                }
-
-                /*
-                 * 4. check for IPv4 subnetwork i.e. xx.xx.xx.xx/n
-                 * TODO: check for IPv6 subnetwork
-                 * NB: Wildcard must not be mixed with subnetwork.
-                 */
-                if (*cp == '/') {
-                        return valid_ipv4_subnetwork (address);
-                }
+    for (cp = address; *cp; cp++) {
+        /* 3. Check for wildcard pattern */
+        if (*cp == '*' || *cp == '?' || *cp == '[') {
+            return _gf_true;
         }
 
-        /* 5. Check for v4/v6 IP addr and FQDN/hostname */
-        length = strlen (address);
-        if ((valid_ipv4_address (address, length, _gf_false)) ||
-            (valid_ipv6_address (address, length, _gf_false)) ||
-            (valid_host_name (address, length))) {
-                return _gf_true;
+        /*
+         * 4. check for IPv4 subnetwork i.e. xx.xx.xx.xx/n
+         * TODO: check for IPv6 subnetwork
+         * NB: Wildcard must not be mixed with subnetwork.
+         */
+        if (*cp == '/') {
+            return valid_ipv4_subnetwork(address);
         }
+    }
 
-        return _gf_false;
+    /* 5. Check for v4/v6 IP addr and FQDN/hostname */
+    length = strlen(address);
+    if ((valid_ipv4_address(address, length, _gf_false)) ||
+        (valid_ipv6_address(address, length, _gf_false)) ||
+        (valid_host_name(address, length))) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 /**
@@ -2475,40 +2791,39 @@ valid_mount_auth_address (char *address)
  * @return _gf_true if a and b have same ipv{4,6} addr, _gf_false otherwise
  */
 gf_boolean_t
-gf_sock_union_equal_addr (union gf_sock_union *a,
-                          union gf_sock_union *b)
+gf_sock_union_equal_addr(union gf_sock_union *a, union gf_sock_union *b)
 {
-        if (!a || !b) {
-                gf_msg ("common-utils", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
-                        "Invalid arguments to gf_sock_union_equal_addr");
-                return _gf_false;
-        }
+    if (!a || !b) {
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
+                "gf_sock_union_equal_addr", NULL);
+        return _gf_false;
+    }
 
-        if (a->storage.ss_family != b->storage.ss_family)
-                return _gf_false;
+    if (a->storage.ss_family != b->storage.ss_family)
+        return _gf_false;
 
-        switch (a->storage.ss_family) {
+    switch (a->storage.ss_family) {
         case AF_INET:
-                if (a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr)
-                        return _gf_true;
-                else
-                        return _gf_false;
+            if (a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr)
+                return _gf_true;
+            else
+                return _gf_false;
 
         case AF_INET6:
-                if (memcmp ((void *)(&a->sin6.sin6_addr),
-                            (void *)(&b->sin6.sin6_addr),
-                            sizeof (a->sin6.sin6_addr)))
-                        return _gf_false;
-                else
-                        return _gf_true;
+            if (memcmp((void *)(&a->sin6.sin6_addr),
+                       (void *)(&b->sin6.sin6_addr), sizeof(a->sin6.sin6_addr)))
+                return _gf_false;
+            else
+                return _gf_true;
 
         default:
-                gf_msg_debug ("common-utils", 0, "Unsupported/invalid address "
-                              "family");
-                break;
-        }
+            gf_msg_debug("common-utils", 0,
+                         "Unsupported/invalid address "
+                         "family");
+            break;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 /*
@@ -2524,62 +2839,61 @@ gf_sock_union_equal_addr (union gf_sock_union *a,
 gf_boolean_t
 mask_match(const uint32_t a, const uint32_t b, const uint32_t m)
 {
-        return (((a ^ b) & m) == 0);
+    return (((a ^ b) & m) == 0);
 }
 
-
 /*Thread safe conversion function*/
 char *
-uuid_utoa (uuid_t uuid)
+uuid_utoa(uuid_t uuid)
 {
-        char *uuid_buffer = glusterfs_uuid_buf_get ();
-        gf_uuid_unparse (uuid, uuid_buffer);
-        return uuid_buffer;
+    char *uuid_buffer = glusterfs_uuid_buf_get();
+    gf_uuid_unparse(uuid, uuid_buffer);
+    return uuid_buffer;
 }
 
 /*Re-entrant conversion function*/
 char *
-uuid_utoa_r (uuid_t uuid, char *dst)
+uuid_utoa_r(uuid_t uuid, char *dst)
 {
-        if(!dst)
-                return NULL;
-        gf_uuid_unparse (uuid, dst);
-        return dst;
+    if (!dst)
+        return NULL;
+    gf_uuid_unparse(uuid, dst);
+    return dst;
 }
 
 /*Thread safe conversion function*/
 char *
-lkowner_utoa (gf_lkowner_t *lkowner)
+lkowner_utoa(gf_lkowner_t *lkowner)
 {
-        char *lkowner_buffer = glusterfs_lkowner_buf_get ();
-        lkowner_unparse (lkowner, lkowner_buffer, GF_LKOWNER_BUF_SIZE);
-        return lkowner_buffer;
+    char *lkowner_buffer = glusterfs_lkowner_buf_get();
+    lkowner_unparse(lkowner, lkowner_buffer, GF_LKOWNER_BUF_SIZE);
+    return lkowner_buffer;
 }
 
 /*Re-entrant conversion function*/
 char *
-lkowner_utoa_r (gf_lkowner_t *lkowner, char *dst, int len)
+lkowner_utoa_r(gf_lkowner_t *lkowner, char *dst, int len)
 {
-        if(!dst)
-                return NULL;
-        lkowner_unparse (lkowner, dst, len);
-        return dst;
+    if (!dst)
+        return NULL;
+    lkowner_unparse(lkowner, dst, len);
+    return dst;
 }
 
 gf_boolean_t
-is_valid_lease_id (const char *lease_id)
+is_valid_lease_id(const char *lease_id)
 {
-        int i = 0;
-        gf_boolean_t valid = _gf_false;
+    int i = 0;
+    gf_boolean_t valid = _gf_false;
 
-        for (i = 0; i < LEASE_ID_SIZE; i++) {
-                if (lease_id[i] != 0) {
-                        valid = _gf_true;
-                        goto out;
-                }
+    for (i = 0; i < LEASE_ID_SIZE; i++) {
+        if (lease_id[i] != 0) {
+            valid = _gf_true;
+            goto out;
         }
+    }
 out:
-        return valid;
+    return valid;
 }
 
 /* Lease_id can be a either in printable or non printable binary
@@ -2596,96 +2910,107 @@ out:
  * subsequent call to this function will override the same buffer.
  */
 char *
-leaseid_utoa (const char *lease_id)
-{
-        char *buf = NULL;
-        int   i   = 0;
-        int   j   = 0;
+leaseid_utoa(const char *lease_id)
+{
+    char *buf = NULL;
+    int i = 0;
+    int j = 0;
+
+    buf = glusterfs_leaseid_buf_get();
+    if (!buf)
+        goto out;
+
+    for (i = 0; i < LEASE_ID_SIZE; i++) {
+        if (i && !(i % 2)) {
+            buf[j] = '-';
+            j++;
+        }
+        sprintf(&buf[j], "%02hhx", lease_id[i]);
+        j += 2;
+        if (j == GF_LEASE_ID_BUF_SIZE)
+            break;
+    }
+    buf[GF_LEASE_ID_BUF_SIZE - 1] = '\0';
+out:
+    return buf;
+}
 
-        buf = glusterfs_leaseid_buf_get ();
-        if (!buf)
-                goto out;
+char *
+gf_leaseid_get()
+{
+    return glusterfs_leaseid_buf_get();
+}
 
-        for (i = 0; i < LEASE_ID_SIZE; i++) {
-                if (i && !(i % 2)) {
-                        buf[j] = '-';
-                        j++;
-                }
-                sprintf (&buf[j], "%02hhx", lease_id[i]);
-                j += 2;
-                if (j == GF_LEASE_ID_BUF_SIZE)
-                        break;
-        }
-        buf[GF_LEASE_ID_BUF_SIZE - 1] = '\0';
-out:
-        return buf;
+char *
+gf_existing_leaseid()
+{
+    return glusterfs_leaseid_exist();
 }
 
-void* gf_array_elem (void *a, int index, size_t elem_size)
+void *
+gf_array_elem(void *a, int index, size_t elem_size)
 {
-        uint8_t* ptr = a;
-        return (void*)(ptr + index * elem_size);
+    uint8_t *ptr = a;
+    return (void *)(ptr + index * elem_size);
 }
 
 void
-gf_elem_swap (void *x, void *y, size_t l) {
-        uint8_t *a = x, *b = y, c;
-        while(l--) {
-                c = *a;
-                *a++ = *b;
-                *b++ = c;
-        }
+gf_elem_swap(void *x, void *y, size_t l)
+{
+    uint8_t *a = x, *b = y, c;
+    while (l--) {
+        c = *a;
+        *a++ = *b;
+        *b++ = c;
+    }
 }
 
 void
-gf_array_insertionsort (void *A, int l, int r, size_t elem_size,
-                        gf_cmp cmp)
-{
-        int  i = l;
-        int  N = r+1;
-        void *Temp = NULL;
-        int  j = 0;
-
-        for(i = l; i < N; i++) {
-                Temp = gf_array_elem (A, i, elem_size);
-                j = i - 1;
-                while (j >= 0 && (cmp (Temp, gf_array_elem (A, j, elem_size))
-                      < 0)) {
-                        gf_elem_swap (Temp, gf_array_elem (A, j, elem_size),
-                                      elem_size);
-                        Temp = gf_array_elem (A, j, elem_size);
-                        j = j-1;
-                }
+gf_array_insertionsort(void *A, int l, int r, size_t elem_size, gf_cmp cmp)
+{
+    int i = l;
+    int N = r + 1;
+    void *Temp = NULL;
+    int j = 0;
+
+    for (i = l; i < N; i++) {
+        Temp = gf_array_elem(A, i, elem_size);
+        j = i - 1;
+        while (j >= 0 && (cmp(Temp, gf_array_elem(A, j, elem_size)) < 0)) {
+            gf_elem_swap(Temp, gf_array_elem(A, j, elem_size), elem_size);
+            Temp = gf_array_elem(A, j, elem_size);
+            j = j - 1;
         }
+    }
 }
 
 int
-gf_is_str_int (const char *value)
+gf_is_str_int(const char *value)
 {
-        int     flag = 0;
-        char   *str  = NULL;
-        char   *fptr = NULL;
+    int flag = 0;
+    char *str = NULL;
+    char *fptr = NULL;
 
-        GF_VALIDATE_OR_GOTO (THIS->name, value, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, value, out);
 
-        str = gf_strdup (value);
-        if (!str)
-                goto out;
+    str = gf_strdup(value);
+    if (!str)
+        goto out;
 
-        fptr = str;
+    fptr = str;
 
-        while (*str) {
-                if (!isdigit(*str)) {
-                        flag = 1;
-                        goto out;
-                }
-                str++;
+    while (*str) {
+        if (!isdigit(*str)) {
+            flag = 1;
+            goto out;
         }
+        str++;
+    }
 
 out:
-        GF_FREE (fptr);
+    GF_FREE(fptr);
 
-        return flag;
+    return flag;
 }
 /*
  * rounds up nr to power of two. If nr is already a power of two, just returns
@@ -2693,22 +3018,22 @@ out:
  */
 
 int32_t
-gf_roundup_power_of_two (int32_t nr)
+gf_roundup_power_of_two(int32_t nr)
 {
-        int32_t result = 1;
+    int32_t result = 1;
 
-        if (nr < 0) {
-                gf_msg ("common-utils", GF_LOG_WARNING, 0,
-                        LG_MSG_NEGATIVE_NUM_PASSED, "negative number passed");
-                result = -1;
-                goto out;
-        }
+    if (nr < 0) {
+        gf_smsg("common-utils", GF_LOG_WARNING, 0, LG_MSG_NEGATIVE_NUM_PASSED,
+                NULL);
+        result = -1;
+        goto out;
+    }
 
-        while (result < nr)
-                result *= 2;
+    while (result < nr)
+        result *= 2;
 
 out:
-        return result;
+    return result;
 }
 
 /*
@@ -2717,403 +3042,365 @@ out:
  */
 
 int32_t
-gf_roundup_next_power_of_two (int32_t nr)
+gf_roundup_next_power_of_two(int32_t nr)
 {
-        int32_t result = 1;
+    int32_t result = 1;
 
-        if (nr < 0) {
-                gf_msg ("common-utils", GF_LOG_WARNING, 0,
-                        LG_MSG_NEGATIVE_NUM_PASSED, "negative number passed");
-                result = -1;
-                goto out;
-        }
+    if (nr < 0) {
+        gf_smsg("common-utils", GF_LOG_WARNING, 0, LG_MSG_NEGATIVE_NUM_PASSED,
+                NULL);
+        result = -1;
+        goto out;
+    }
 
-        while (result <= nr)
-                result *= 2;
+    while (result <= nr)
+        result *= 2;
 
 out:
-        return result;
+    return result;
 }
 
 int
-get_vol_type (int type, int dist_count, int brick_count)
+validate_brick_name(char *brick)
 {
-        if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
-             (dist_count < brick_count))
-              type = type + GF_CLUSTER_TYPE_MAX - 1;
+    char *delimiter = NULL;
+    int ret = 0;
+    delimiter = strrchr(brick, ':');
+    if (!delimiter || delimiter == brick || *(delimiter + 1) != '/')
+        ret = -1;
 
-        return type;
-}
-
-int
-validate_brick_name (char *brick)
-{
-        char *delimiter = NULL;
-        int  ret = 0;
-        delimiter = strrchr (brick, ':');
-        if (!delimiter || delimiter == brick
-            || *(delimiter+1) != '/')
-                ret = -1;
-
-        return ret;
+    return ret;
 }
 
 char *
-get_host_name (char *word, char **host)
+get_host_name(char *word, char **host)
 {
-        char *delimiter = NULL;
-        delimiter = strrchr (word, ':');
-        if (delimiter)
-                *delimiter = '\0';
-        else
-                return NULL;
-        *host = word;
-        return *host;
+    char *delimiter = NULL;
+    delimiter = strrchr(word, ':');
+    if (delimiter)
+        *delimiter = '\0';
+    else
+        return NULL;
+    *host = word;
+    return *host;
 }
 
-
 char *
-get_path_name (char *word, char **path)
+get_path_name(char *word, char **path)
 {
-        char *delimiter = NULL;
-        delimiter = strchr (word, '/');
-        if (!delimiter)
-                return NULL;
-        *path = delimiter;
-        return *path;
+    char *delimiter = NULL;
+    delimiter = strchr(word, '/');
+    if (!delimiter)
+        return NULL;
+    *path = delimiter;
+    return *path;
 }
 
 void
-gf_path_strip_trailing_slashes (char *path)
+gf_path_strip_trailing_slashes(char *path)
 {
-        int i = 0;
-        int len = 0;
+    int i = 0;
+    int len = 0;
 
-        if (!path)
-                return;
+    if (!path)
+        return;
 
-        len = strlen (path);
-        for (i = len - 1; i > 0; i--) {
-                if (path[i] != '/')
-                        break;
-	}
+    len = strlen(path);
+    for (i = len - 1; i > 0; i--) {
+        if (path[i] != '/')
+            break;
+    }
 
-        if (i < (len -1))
-                path [i+1] = '\0';
+    if (i < (len - 1))
+        path[i + 1] = '\0';
 
-        return;
+    return;
 }
 
 uint64_t
-get_mem_size ()
+get_mem_size()
 {
-        uint64_t memsize = -1;
+    uint64_t memsize = -1;
 
 #if defined GF_LINUX_HOST_OS || defined GF_SOLARIS_HOST_OS
 
-	uint64_t page_size = 0;
-	uint64_t num_pages = 0;
+    uint64_t page_size = 0;
+    uint64_t num_pages = 0;
+
+    page_size = sysconf(_SC_PAGESIZE);
+    num_pages = sysconf(_SC_PHYS_PAGES);
+
+    memsize = page_size * num_pages;
+#endif
+
+#if defined GF_DARWIN_HOST_OS || defined __FreeBSD__
 
-	page_size = sysconf (_SC_PAGESIZE);
-	num_pages = sysconf (_SC_PHYS_PAGES);
+    size_t len = sizeof(memsize);
+    int name[] = {CTL_HW, HW_PHYSMEM};
 
-	memsize = page_size * num_pages;
+    sysctl(name, 2, &memsize, &len, NULL, 0);
 #endif
 
-#if defined GF_BSD_HOST_OS || defined GF_DARWIN_HOST_OS
+#if defined __NetBSD__
 
-	size_t len = sizeof(memsize);
-	int name [] = { CTL_HW, HW_PHYSMEM };
+    size_t len = sizeof(memsize);
+    int name64[] = {CTL_HW, HW_PHYSMEM64};
 
-	sysctl (name, 2, &memsize, &len, NULL, 0);
+    sysctl(name64, 2, &memsize, &len, NULL, 0);
+    if (memsize == -1)
+        sysctl(name64, 2, &memsize, &len, NULL, 0);
 #endif
-	return memsize;
+    return memsize;
 }
 
 /* Strips all whitespace characters in a string and returns length of new string
  * on success
  */
 int
-gf_strip_whitespace (char *str, int len)
+gf_strip_whitespace(char *str, int len)
 {
-        int     i = 0;
-        int     new_len = 0;
-        char    *new_str = NULL;
+    int i = 0;
+    int new_len = 0;
+    char *new_str = NULL;
 
-        GF_ASSERT (str);
+    GF_ASSERT(str);
 
-        new_str = GF_CALLOC (1, len + 1, gf_common_mt_char);
-        if (new_str == NULL)
-                return -1;
+    new_str = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (new_str == NULL)
+        return -1;
 
-        for (i = 0; i < len; i++) {
-                if (!isspace (str[i]))
-                        new_str[new_len++] = str[i];
-        }
-        new_str[new_len] = '\0';
+    for (i = 0; i < len; i++) {
+        if (!isspace(str[i]))
+            new_str[new_len++] = str[i];
+    }
+    new_str[new_len] = '\0';
 
-        if (new_len != len) {
-                memset (str, 0, len);
-                strncpy (str, new_str, new_len);
-        }
+    if (new_len != len) {
+        snprintf(str, new_len + 1, "%s", new_str);
+    }
 
-        GF_FREE (new_str);
-        return new_len;
+    GF_FREE(new_str);
+    return new_len;
 }
 
 int
-gf_canonicalize_path (char *path)
+gf_canonicalize_path(char *path)
 {
-        int             ret                  = -1;
-        int             path_len             = 0;
-        int             dir_path_len         = 0;
-        char           *tmppath              = NULL;
-        char           *dir                  = NULL;
-        char           *tmpstr               = NULL;
+    int ret = -1;
+    int path_len = 0;
+    int dir_path_len = 0;
+    char *tmppath = NULL;
+    char *dir = NULL;
+    char *tmpstr = NULL;
 
-        if (!path || *path != '/')
-                goto out;
+    if (!path || *path != '/')
+        goto out;
 
-        if (!strcmp (path, "/"))
-                return 0;
+    if (!strcmp(path, "/"))
+        return 0;
 
-        tmppath = gf_strdup (path);
-        if (!tmppath)
-                goto out;
+    tmppath = gf_strdup(path);
+    if (!tmppath)
+        goto out;
+
+    /* Strip the extra slashes and return */
+    bzero(path, strlen(path));
+    path[0] = '/';
+    dir = strtok_r(tmppath, "/", &tmpstr);
 
-        /* Strip the extra slashes and return */
-        bzero (path, strlen(path));
-        path[0] = '/';
-        dir = strtok_r(tmppath, "/", &tmpstr);
-
-        while (dir) {
-                dir_path_len = strlen(dir);
-                strncpy ((path + path_len + 1), dir, dir_path_len);
-                path_len += dir_path_len + 1;
-                dir = strtok_r (NULL, "/", &tmpstr);
-                if (dir)
-                        strncpy ((path + path_len), "/", 1);
+    while (dir) {
+        dir_path_len = strlen(dir);
+        memcpy((path + path_len + 1), dir, dir_path_len);
+        path_len += dir_path_len + 1;
+        dir = strtok_r(NULL, "/", &tmpstr);
+        if (dir) {
+            path[path_len] = '/';
         }
-        path[path_len] = '\0';
-        ret = 0;
+    }
+    path[path_len] = '\0';
+    ret = 0;
 
- out:
-        if (ret)
-                gf_msg ("common-utils", GF_LOG_ERROR, 0, LG_MSG_PATH_ERROR,
-                        "Path manipulation failed");
+out:
+    if (ret)
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_PATH_ERROR, NULL);
 
-        GF_FREE(tmppath);
+    GF_FREE(tmppath);
 
-        return ret;
+    return ret;
 }
 
 static const char *__gf_timefmts[] = {
-        "%F %T",
-        "%Y/%m/%d-%T",
-        "%b %d %T",
-        "%F %H%M%S",
-	"%Y-%m-%d-%T",
-        "%s",
+    "%F %T", "%Y/%m/%d-%T", "%b %d %T", "%F %H%M%S", "%Y-%m-%d-%T", "%s",
 };
 
 static const char *__gf_zerotimes[] = {
-        "0000-00-00 00:00:00",
-        "0000/00/00-00:00:00",
-        "xxx 00 00:00:00",
-        "0000-00-00 000000",
-	"0000-00-00-00:00:00",
-        "0",
+    "0000-00-00 00:00:00", "0000/00/00-00:00:00", "xxx 00 00:00:00",
+    "0000-00-00 000000",   "0000-00-00-00:00:00", "0",
 };
 
 void
-_gf_timestuff (gf_timefmts *fmt, const char ***fmts, const char ***zeros)
+_gf_timestuff(const char ***fmts, const char ***zeros)
 {
-        *fmt = gf_timefmt_last;
-        *fmts = __gf_timefmts;
-        *zeros = __gf_zerotimes;
+    *fmts = __gf_timefmts;
+    *zeros = __gf_zerotimes;
 }
 
-
 char *
-generate_glusterfs_ctx_id (void)
+generate_glusterfs_ctx_id(void)
 {
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        char           now_str[32];
+    uuid_t ctxid;
+    char *tmp = NULL;
 
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno,
-                        LG_MSG_GETTIMEOFDAY_FAILED, "gettimeofday: "
-                        "failed");
-        }
-
-        if (gethostname (hostname, 256) == -1) {
-                gf_msg ("glusterfsd", GF_LOG_ERROR, errno,
-                        LG_MSG_GETHOSTNAME_FAILED, "gethostname: failed");
-        }
+    gf_uuid_generate(ctxid);
+    tmp = uuid_utoa(ctxid);
 
-        gf_time_fmt (now_str, sizeof now_str, tv.tv_sec, gf_timefmt_Ymd_T);
-        snprintf (tmp_str, sizeof tmp_str, "%s-%d-%s:%"
-#ifdef GF_DARWIN_HOST_OS
-                  PRId32,
-#else
-                  "ld",
-#endif
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
+    return gf_strdup(tmp);
 }
 
 char *
-gf_get_reserved_ports ()
+gf_get_reserved_ports()
 {
-        char    *ports_info  = NULL;
+    char *ports_info = NULL;
 #if defined GF_LINUX_HOST_OS
-        int     proc_fd      = -1;
-        char    *proc_file   = "/proc/sys/net/ipv4/ip_local_reserved_ports";
-        char    buffer[4096] = {0,};
-        int32_t ret          = -1;
-
-        proc_fd = open (proc_file, O_RDONLY);
-        if (proc_fd == -1) {
-                /* What should be done in this case? error out from here
-                 * and thus stop the glusterfs process from starting or
-                 * continue with older method of using any of the available
-                 * port? For now 2nd option is considered.
-                 */
-                gf_msg ("glusterfs", GF_LOG_WARNING, errno,
-                        LG_MSG_FILE_OP_FAILED, "could not open the file "
-                        "/proc/sys/net/ipv4/ip_local_reserved_ports for "
-                        "getting reserved ports info");
-                goto out;
-        }
+    int proc_fd = -1;
+    char *proc_file = "/proc/sys/net/ipv4/ip_local_reserved_ports";
+    char buffer[4096] = {
+        0,
+    };
+    int32_t ret = -1;
+
+    proc_fd = open(proc_file, O_RDONLY);
+    if (proc_fd == -1) {
+        /* What should be done in this case? error out from here
+         * and thus stop the glusterfs process from starting or
+         * continue with older method of using any of the available
+         * port? For now 2nd option is considered.
+         */
+        gf_smsg("glusterfs", GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                " /proc/sys/net/ipv4/ip_local_reserved_ports", NULL);
+        goto out;
+    }
 
-        ret = sys_read (proc_fd, buffer, sizeof (buffer));
-        if (ret < 0) {
-                gf_msg ("glusterfs", GF_LOG_WARNING, errno,
-                        LG_MSG_FILE_OP_FAILED, "could not read the file %s for"
-                        " getting reserved ports info", proc_file);
-                goto out;
-        }
-        ports_info = gf_strdup (buffer);
+    ret = sys_read(proc_fd, buffer, sizeof(buffer) - 1);
+    if (ret < 0) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                "file=%s", proc_file, NULL);
+        goto out;
+    }
+
+    buffer[ret] = '\0';
+    ports_info = gf_strdup(buffer);
 
 out:
-        if (proc_fd != -1)
-                sys_close (proc_fd);
+    if (proc_fd != -1)
+        sys_close(proc_fd);
 #endif /* GF_LINUX_HOST_OS */
-        return ports_info;
+    return ports_info;
 }
 
 int
-gf_process_reserved_ports (gf_boolean_t *ports, uint32_t ceiling)
+gf_process_reserved_ports(unsigned char *ports, uint32_t ceiling)
 {
-        int      ret         = -1;
-        int      i           = 0;
+    int ret = -1;
 
-        for (i = 0; i < GF_PORT_MAX; i++) {
-                *(ports + i) = _gf_false;
-        }
+    memset(ports, 0, GF_PORT_ARRAY_SIZE);
 
 #if defined GF_LINUX_HOST_OS
-        char    *ports_info  = NULL;
-        char    *tmp         = NULL;
-        char    *blocked_port = NULL;
-
-        ports_info = gf_get_reserved_ports ();
-        if (!ports_info) {
-                gf_msg ("glusterfs", GF_LOG_WARNING, 0,
-                        LG_MSG_RESERVED_PORTS_ERROR, "Not able to get reserved"
-                        " ports, hence there is a possibility that glusterfs "
-                        "may consume reserved port");
-                goto out;
-        }
+    char *ports_info = NULL;
+    char *tmp = NULL;
+    char *blocked_port = NULL;
 
-        blocked_port = strtok_r (ports_info, ",\n",&tmp);
+    ports_info = gf_get_reserved_ports();
+    if (!ports_info) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, 0, LG_MSG_RESERVED_PORTS_ERROR,
+                NULL);
+        goto out;
+    }
 
-        while (blocked_port) {
-                gf_ports_reserved (blocked_port, ports, ceiling);
-                blocked_port = strtok_r (NULL, ",\n", &tmp);
-        }
+    blocked_port = strtok_r(ports_info, ",\n", &tmp);
 
-        ret = 0;
+    while (blocked_port) {
+        gf_ports_reserved(blocked_port, ports, ceiling);
+        blocked_port = strtok_r(NULL, ",\n", &tmp);
+    }
+
+    ret = 0;
 
 out:
-        GF_FREE (ports_info);
+    GF_FREE(ports_info);
 
 #else  /* FIXME: Non Linux Host */
-        ret = 0;
+    ret = 0;
 #endif /* GF_LINUX_HOST_OS */
 
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling)
-{
-        gf_boolean_t    result      = _gf_false;
-        char            *range_port = NULL;
-        int16_t         tmp_port1   = -1;
-        int16_t         tmp_port2   = -1;
-
-        if (strstr (blocked_port, "-") == NULL) {
-                /* get rid of the new line character*/
-                if (blocked_port[strlen(blocked_port) -1] == '\n')
-                        blocked_port[strlen(blocked_port) -1] = '\0';
-                if (gf_string2int16 (blocked_port, &tmp_port1) == 0) {
-                        if (tmp_port1 > ceiling
-                            || tmp_port1 < 0) {
-                                gf_msg ("glusterfs-socket", GF_LOG_WARNING, 0,
-                                        LG_MSG_INVALID_PORT, "invalid port %d",
-                                        tmp_port1);
-                                result = _gf_true;
-                                goto out;
-                        } else {
-                                gf_msg_debug ("glusterfs", 0, "blocking port "
-                                              "%d", tmp_port1);
-                                ports[tmp_port1] = _gf_true;
-                        }
-                } else {
-                        gf_msg ("glusterfs-socket", GF_LOG_WARNING, 0,
-                                LG_MSG_INVALID_PORT, "%s is not a valid port "
-                                "identifier", blocked_port);
-                        result = _gf_true;
-                        goto out;
-                }
+gf_ports_reserved(char *blocked_port, unsigned char *ports, uint32_t ceiling)
+{
+    gf_boolean_t result = _gf_false;
+    char *range_port = NULL;
+    int32_t tmp_port1 = -1;
+    int32_t tmp_port2 = -1;
+
+    if (strstr(blocked_port, "-") == NULL) {
+        /* get rid of the new line character*/
+        if (blocked_port[strlen(blocked_port) - 1] == '\n')
+            blocked_port[strlen(blocked_port) - 1] = '\0';
+        if (gf_string2int32(blocked_port, &tmp_port1) == 0) {
+            if (tmp_port1 > GF_PORT_MAX || tmp_port1 < 0) {
+                gf_smsg("glusterfs-socket", GF_LOG_WARNING, 0,
+                        LG_MSG_INVALID_PORT, "port=%d", tmp_port1, NULL);
+                result = _gf_true;
+                goto out;
+            } else {
+                gf_msg_debug("glusterfs", 0,
+                             "blocking port "
+                             "%d",
+                             tmp_port1);
+                BIT_SET(ports, tmp_port1);
+            }
         } else {
-                range_port = strtok (blocked_port, "-");
-                if (!range_port){
-                        result = _gf_true;
-                        goto out;
-                }
-                if (gf_string2int16 (range_port, &tmp_port1) == 0) {
-                        if (tmp_port1 > ceiling)
-                                tmp_port1 = ceiling;
-                        if (tmp_port1 < 0)
-                                tmp_port1 = 0;
-                }
-                range_port = strtok (NULL, "-");
-                if (!range_port) {
-                        result = _gf_true;
-                        goto out;
-                }
-                /* get rid of the new line character*/
-                if (range_port[strlen(range_port) -1] == '\n')
-                        range_port[strlen(range_port) - 1] = '\0';
-                if (gf_string2int16 (range_port, &tmp_port2) == 0) {
-                        if (tmp_port2 > ceiling)
-                                tmp_port2 = ceiling;
-                        if (tmp_port2 < 0)
-                                tmp_port2 = 0;
-                }
-                gf_msg_debug ("glusterfs", 0, "lower: %d, higher: %d",
-                              tmp_port1, tmp_port2);
-                for (; tmp_port1 <= tmp_port2; tmp_port1++)
-                        ports[tmp_port1] = _gf_true;
-        }
+            gf_smsg("glusterfs-socket", GF_LOG_WARNING, 0, LG_MSG_INVALID_PORT,
+                    "port=%s", blocked_port, NULL);
+            result = _gf_true;
+            goto out;
+        }
+    } else {
+        range_port = strtok(blocked_port, "-");
+        if (!range_port) {
+            result = _gf_true;
+            goto out;
+        }
+        if (gf_string2int32(range_port, &tmp_port1) == 0) {
+            if (tmp_port1 > ceiling)
+                tmp_port1 = ceiling;
+            if (tmp_port1 < 0)
+                tmp_port1 = 0;
+        }
+        range_port = strtok(NULL, "-");
+        if (!range_port) {
+            result = _gf_true;
+            goto out;
+        }
+        /* get rid of the new line character*/
+        if (range_port[strlen(range_port) - 1] == '\n')
+            range_port[strlen(range_port) - 1] = '\0';
+        if (gf_string2int32(range_port, &tmp_port2) == 0) {
+            if (tmp_port2 > ceiling)
+                tmp_port2 = ceiling;
+            if (tmp_port2 < 0)
+                tmp_port2 = 0;
+        }
+        gf_msg_debug("glusterfs", 0, "lower: %d, higher: %d", tmp_port1,
+                     tmp_port2);
+        for (; tmp_port1 <= tmp_port2; tmp_port1++)
+            BIT_SET(ports, tmp_port1);
+    }
 
 out:
-        return result;
+    return result;
 }
 
 /* Takes in client ip{v4,v6} and returns associated hostname, if any
@@ -3121,619 +3408,890 @@ out:
  * Returns: 0 for success, -1 for failure
  */
 int
-gf_get_hostname_from_ip (char *client_ip, char **hostname)
-{
-        int                      ret                          = -1;
-        struct sockaddr         *client_sockaddr              = NULL;
-        struct sockaddr_in       client_sock_in               = {0};
-        struct sockaddr_in6      client_sock_in6              = {0};
-        char                     client_hostname[NI_MAXHOST]  = {0};
-        char                    *client_ip_copy               = NULL;
-        char                    *tmp                          = NULL;
-        char                    *ip                           = NULL;
-
-        /* if ipv4, reverse lookup the hostname to
-         * allow FQDN based rpc authentication
-         */
-        if (valid_ipv4_address (client_ip, strlen (client_ip), 0) == _gf_false) {
-                /* most times, we get a.b.c.d:port form, so check that */
-                client_ip_copy = gf_strdup (client_ip);
-                if (!client_ip_copy)
-                        goto out;
+gf_get_hostname_from_ip(char *client_ip, char **hostname)
+{
+    int ret = -1;
+    struct sockaddr *client_sockaddr = NULL;
+    struct sockaddr_in client_sock_in = {0};
+    struct sockaddr_in6 client_sock_in6 = {0};
+    char client_hostname[NI_MAXHOST] = {0};
+    char *client_ip_copy = NULL;
+    char *tmp = NULL;
+    char *ip = NULL;
+    size_t addr_sz = 0;
+
+    /* if ipv4, reverse lookup the hostname to
+     * allow FQDN based rpc authentication
+     */
+    if (!valid_ipv6_address(client_ip, strlen(client_ip), 0) &&
+        !valid_ipv4_address(client_ip, strlen(client_ip), 0)) {
+        /* most times, we get a.b.c.d:port form, so check that */
+        client_ip_copy = gf_strdup(client_ip);
+        if (!client_ip_copy)
+            goto out;
+
+        ip = strtok_r(client_ip_copy, ":", &tmp);
+    } else {
+        ip = client_ip;
+    }
+
+    if (valid_ipv4_address(ip, strlen(ip), 0) == _gf_true) {
+        client_sockaddr = (struct sockaddr *)&client_sock_in;
+        addr_sz = sizeof(client_sock_in);
+        client_sock_in.sin_family = AF_INET;
+        ret = inet_pton(AF_INET, ip, (void *)&client_sock_in.sin_addr.s_addr);
+
+    } else if (valid_ipv6_address(ip, strlen(ip), 0) == _gf_true) {
+        client_sockaddr = (struct sockaddr *)&client_sock_in6;
+        addr_sz = sizeof(client_sock_in6);
+
+        client_sock_in6.sin6_family = AF_INET6;
+        ret = inet_pton(AF_INET6, ip, (void *)&client_sock_in6.sin6_addr);
+    } else {
+        goto out;
+    }
+
+    if (ret != 1) {
+        ret = -1;
+        goto out;
+    }
+
+    /* You cannot just use sizeof (*client_sockaddr), as per the man page
+     * the (getnameinfo) size must be the size of the underlying sockaddr
+     * struct e.g. sockaddr_in6 or sockaddr_in.  Failure to do so will
+     * break IPv6 hostname resolution (IPv4 will work only because
+     * the sockaddr_in struct happens to be of the correct size).
+     */
+    ret = getnameinfo(client_sockaddr, addr_sz, client_hostname,
+                      sizeof(client_hostname), NULL, 0, 0);
+    if (ret) {
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                "ip=%s", client_ip, "ret=%s", gai_strerror(ret), NULL);
+        ret = -1;
+        goto out;
+    }
+
+    *hostname = gf_strdup((char *)client_hostname);
+out:
+    if (client_ip_copy)
+        GF_FREE(client_ip_copy);
 
-                ip = strtok_r (client_ip_copy, ":", &tmp);
-        } else {
-                ip = client_ip;
-        }
+    return ret;
+}
 
-        if (valid_ipv4_address (ip, strlen (ip), 0) == _gf_true) {
-                client_sockaddr = (struct sockaddr *)&client_sock_in;
-                client_sock_in.sin_family = AF_INET;
-                ret = inet_pton (AF_INET, ip,
-                                 (void *)&client_sock_in.sin_addr.s_addr);
+gf_boolean_t
+gf_interface_search(char *ip)
+{
+    int32_t ret = -1;
+    gf_boolean_t found = _gf_false;
+    struct ifaddrs *ifaddr, *ifa;
+    int family;
+    char host[NI_MAXHOST];
+    xlator_t *this = NULL;
+    char *pct = NULL;
 
-        } else if (valid_ipv6_address (ip, strlen (ip), 0) == _gf_true) {
-                client_sockaddr = (struct sockaddr *) &client_sock_in6;
+    this = THIS;
 
-                client_sock_in6.sin6_family = AF_INET6;
-                ret = inet_pton (AF_INET6, ip,
-                                 (void *)&client_sock_in6.sin6_addr);
-        } else {
-                goto out;
-        }
+    ret = getifaddrs(&ifaddr);
 
-        if (ret != 1) {
-                ret = -1;
-                goto out;
-        }
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETIFADDRS_FAILED, "ret=%s",
+                gai_strerror(ret), NULL);
+        goto out;
+    }
 
-        ret = getnameinfo (client_sockaddr,
-                           sizeof (*client_sockaddr),
-                           client_hostname, sizeof (client_hostname),
-                           NULL, 0, 0);
-        if (ret) {
-                gf_msg ("common-utils", GF_LOG_ERROR, 0,
-                        LG_MSG_GETNAMEINFO_FAILED, "Could not lookup hostname "
-                        "of %s : %s", client_ip, gai_strerror (ret));
-                ret = -1;
-                goto out;
+    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+        if (!ifa->ifa_addr) {
+            /*
+             * This seemingly happens if an interface hasn't
+             * been bound to a particular protocol (seen with
+             * TUN devices).
+             */
+            continue;
         }
+        family = ifa->ifa_addr->sa_family;
 
-        *hostname = gf_strdup ((char *)client_hostname);
- out:
-        if (client_ip_copy)
-                GF_FREE (client_ip_copy);
+        if (family != AF_INET && family != AF_INET6)
+            continue;
 
-        return ret;
-}
-
-gf_boolean_t
-gf_interface_search (char *ip)
-{
-        int32_t         ret = -1;
-        gf_boolean_t    found = _gf_false;
-        struct          ifaddrs *ifaddr, *ifa;
-        int             family;
-        char            host[NI_MAXHOST];
-        xlator_t        *this = NULL;
-        char            *pct = NULL;
-
-        this = THIS;
-
-        ret = getifaddrs (&ifaddr);
+        ret = getnameinfo(ifa->ifa_addr,
+                          (family == AF_INET) ? sizeof(struct sockaddr_in)
+                                              : sizeof(struct sockaddr_in6),
+                          host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
 
         if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_GETIFADDRS_FAILED,
-                        "getifaddrs() failed: %s\n", gai_strerror(ret));
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto out;
+        }
+
+        /*
+         * Sometimes the address comes back as addr%eth0 or
+         * similar.  Since % is an invalid character, we can
+         * strip it out with confidence that doing so won't
+         * harm anything.
+         */
+        pct = index(host, '%');
+        if (pct) {
+            *pct = '\0';
         }
 
-        for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-                if (!ifa->ifa_addr) {
-                        /*
-                         * This seemingly happens if an interface hasn't
-                         * been bound to a particular protocol (seen with
-                         * TUN devices).
-                         */
-                        continue;
-                }
-                family = ifa->ifa_addr->sa_family;
-
-                if (family != AF_INET && family != AF_INET6)
-                        continue;
-
-                ret = getnameinfo (ifa->ifa_addr,
-                        (family == AF_INET) ? sizeof(struct sockaddr_in) :
-                                              sizeof(struct sockaddr_in6),
-                        host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                LG_MSG_GETNAMEINFO_FAILED, "getnameinfo() "
-                                "failed: %s\n", gai_strerror(ret));
-                        goto out;
-                }
-
-                /*
-                 * Sometimes the address comes back as addr%eth0 or
-                 * similar.  Since % is an invalid character, we can
-                 * strip it out with confidence that doing so won't
-                 * harm anything.
-                 */
-                pct = index(host,'%');
-                if (pct) {
-                        *pct = '\0';
-                }
-
-                if (strncmp (ip, host, NI_MAXHOST) == 0) {
-                        gf_msg_debug (this->name, 0, "%s is local address at "
-                                      "interface %s", ip, ifa->ifa_name);
-                        found = _gf_true;
-                        goto out;
-                }
+        if (strncmp(ip, host, NI_MAXHOST) == 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s is local address at "
+                         "interface %s",
+                         ip, ifa->ifa_name);
+            found = _gf_true;
+            goto out;
         }
+    }
 out:
-        if(ifaddr)
-                freeifaddrs (ifaddr);
-        return found;
+    if (ifaddr)
+        freeifaddrs(ifaddr);
+    return found;
 }
 
 char *
-get_ip_from_addrinfo (struct addrinfo *addr, char **ip)
+get_ip_from_addrinfo(struct addrinfo *addr, char **ip)
 {
-        char buf[64];
-        void *in_addr = NULL;
-        struct sockaddr_in *s4 = NULL;
-        struct sockaddr_in6 *s6 = NULL;
+    char buf[64];
+    void *in_addr = NULL;
+    struct sockaddr_in *s4 = NULL;
+    struct sockaddr_in6 *s6 = NULL;
 
-        switch (addr->ai_family)
-        {
-                case AF_INET:
-                        s4 = (struct sockaddr_in *)addr->ai_addr;
-                        in_addr = &s4->sin_addr;
-                        break;
-
-                case AF_INET6:
-                        s6 = (struct sockaddr_in6 *)addr->ai_addr;
-                        in_addr = &s6->sin6_addr;
-                        break;
-
-                default:
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                LG_MSG_INVALID_FAMILY, "Invalid family");
-                        return NULL;
-        }
+    switch (addr->ai_family) {
+        case AF_INET:
+            s4 = (struct sockaddr_in *)addr->ai_addr;
+            in_addr = &s4->sin_addr;
+            break;
 
-        if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0, LG_MSG_CONVERSION_FAILED,
-                        "String conversion failed");
-                return NULL;
-        }
+        case AF_INET6:
+            s6 = (struct sockaddr_in6 *)addr->ai_addr;
+            in_addr = &s6->sin6_addr;
+            break;
 
-        *ip = gf_strdup (buf);
-        return *ip;
-}
+        default:
+            gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_INVALID_FAMILY, NULL);
+            return NULL;
+    }
 
-gf_boolean_t
-gf_is_loopback_localhost (const struct sockaddr *sa, char *hostname)
-{
-        GF_ASSERT (sa);
-
-        gf_boolean_t is_local = _gf_false;
-        const struct in_addr *addr4 = NULL;
-        const struct in6_addr *addr6 = NULL;
-        uint8_t      *ap   = NULL;
-        struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
-
-        switch (sa->sa_family) {
-                case AF_INET:
-                        addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
-                        ap = (uint8_t*)&addr4->s_addr;
-                        if (ap[0] == 127)
-                                is_local = _gf_true;
-                        break;
-
-                case AF_INET6:
-                        addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
-                        if (memcmp (addr6, &loopbackaddr6,
-                                    sizeof (loopbackaddr6)) == 0)
-                                is_local = _gf_true;
-                        break;
-
-                default:
-                        if (hostname)
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        LG_MSG_INVALID_FAMILY, "unknown "
-                                        "address family %d for %s",
-                                        sa->sa_family, hostname);
-                        break;
-        }
+    if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_CONVERSION_FAILED, NULL);
+        return NULL;
+    }
 
-        return is_local;
+    *ip = gf_strdup(buf);
+    return *ip;
 }
 
 gf_boolean_t
-gf_is_local_addr (char *hostname)
+gf_is_loopback_localhost(const struct sockaddr *sa, char *hostname)
 {
-        int32_t         ret = -1;
-        struct          addrinfo *result = NULL;
-        struct          addrinfo *res = NULL;
-        gf_boolean_t    found = _gf_false;
-        char            *ip = NULL;
-        xlator_t        *this = NULL;
-        struct          addrinfo hints;
+    GF_ASSERT(sa);
 
-        this = THIS;
+    gf_boolean_t is_local = _gf_false;
+    const struct in_addr *addr4 = NULL;
+    const struct in6_addr *addr6 = NULL;
+    uint8_t *ap = NULL;
+    struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
 
-        memset (&hints, 0, sizeof (hints));
-        /*
-         * Removing AI_ADDRCONFIG from default_hints
-         * for being able to use link local ipv6 addresses
-         */
-        hints.ai_family = AF_UNSPEC;
+    switch (sa->sa_family) {
+        case AF_INET:
+            addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
+            ap = (uint8_t *)&addr4->s_addr;
+            if (ap[0] == 127)
+                is_local = _gf_true;
+            break;
 
-        ret = getaddrinfo (hostname, NULL, &hints, &result);
+        case AF_INET6:
+            addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
+            if (memcmp(addr6, &loopbackaddr6, sizeof(loopbackaddr6)) == 0)
+                is_local = _gf_true;
+            break;
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
-                        "error in getaddrinfo: %s\n", gai_strerror(ret));
-                goto out;
-        }
+        default:
+            if (hostname)
+                gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_INVALID_FAMILY,
+                        "family=%d", sa->sa_family, "hostname=%s", hostname,
+                        NULL);
+            break;
+    }
 
-        for (res = result; res != NULL; res = res->ai_next) {
-                gf_msg_debug (this->name, 0, "%s ",
-                              get_ip_from_addrinfo (res, &ip));
-
-                found = gf_is_loopback_localhost (res->ai_addr, hostname)
-                        || gf_interface_search (ip);
-                if (found) {
-                        GF_FREE (ip);
-                        goto out;
-                }
-                GF_FREE (ip);
-        }
+    return is_local;
+}
+
+gf_boolean_t
+gf_is_local_addr(char *hostname)
+{
+    int32_t ret = -1;
+    struct addrinfo *result = NULL;
+    struct addrinfo *res = NULL;
+    gf_boolean_t found = _gf_false;
+    char *ip = NULL;
+    xlator_t *this = NULL;
+    struct addrinfo hints;
+
+    this = THIS;
+
+    memset(&hints, 0, sizeof(hints));
+    /*
+     * Removing AI_ADDRCONFIG from default_hints
+     * for being able to use link local ipv6 addresses
+     */
+    hints.ai_family = AF_UNSPEC;
+
+    ret = getaddrinfo(hostname, NULL, &hints, &result);
+
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
+                "ret=%s", gai_strerror(ret), NULL);
+        goto out;
+    }
+
+    for (res = result; res != NULL; res = res->ai_next) {
+        get_ip_from_addrinfo(res, &ip);
+        gf_msg_debug(this->name, 0, "%s ", ip);
+
+        if (ip) {
+            found = (gf_is_loopback_localhost(res->ai_addr, hostname) ||
+                     gf_interface_search(ip));
+        }
+        if (found) {
+            GF_FREE(ip);
+            goto out;
+        }
+        GF_FREE(ip);
+        /* the above free will not set ip to NULL, and hence, there is
+           double free possible as the loop continues. set ip to NULL. */
+        ip = NULL;
+    }
 
 out:
-        if (result)
-                freeaddrinfo (result);
+    if (result)
+        freeaddrinfo(result);
 
-        if (!found)
-                gf_msg_debug (this->name, 0, "%s is not local", hostname);
+    if (!found)
+        gf_msg_debug(this->name, 0, "%s is not local", hostname);
 
-        return found;
+    return found;
 }
 
 gf_boolean_t
-gf_is_same_address (char *name1, char *name2)
-{
-        struct addrinfo         *addr1 = NULL;
-        struct addrinfo         *addr2 = NULL;
-        struct addrinfo         *p = NULL;
-        struct addrinfo         *q = NULL;
-        gf_boolean_t            ret = _gf_false;
-        int                     gai_err = 0;
-        struct                  addrinfo hints;
-
-        memset (&hints, 0, sizeof (hints));
-        hints.ai_family = AF_UNSPEC;
-
-        gai_err = getaddrinfo(name1, NULL, &hints, &addr1);
-        if (gai_err != 0) {
-                gf_msg (name1, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
-                        "error in getaddrinfo: %s\n", gai_strerror(gai_err));
-                goto out;
-        }
+gf_is_same_address(char *name1, char *name2)
+{
+    struct addrinfo *addr1 = NULL;
+    struct addrinfo *addr2 = NULL;
+    struct addrinfo *p = NULL;
+    struct addrinfo *q = NULL;
+    gf_boolean_t ret = _gf_false;
+    int gai_err = 0;
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
+
+    gai_err = getaddrinfo(name1, NULL, &hints, &addr1);
+    if (gai_err != 0) {
+        gf_smsg(name1, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED, "error=%s",
+                gai_strerror(gai_err), NULL);
+        goto out;
+    }
+
+    gai_err = getaddrinfo(name2, NULL, &hints, &addr2);
+    if (gai_err != 0) {
+        gf_smsg(name2, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED, "error=%s",
+                gai_strerror(gai_err), NULL);
+        goto out;
+    }
+
+    for (p = addr1; p; p = p->ai_next) {
+        for (q = addr2; q; q = q->ai_next) {
+            if (p->ai_addrlen != q->ai_addrlen) {
+                continue;
+            }
+            if (memcmp(p->ai_addr, q->ai_addr, p->ai_addrlen)) {
+                continue;
+            }
+            ret = _gf_true;
+            goto out;
+        }
+    }
 
-        gai_err = getaddrinfo(name2, NULL, &hints, &addr2);
-        if (gai_err != 0) {
-                gf_msg (name2, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED,
-                        "error in getaddrinfo: %s\n", gai_strerror(gai_err));
-                goto out;
-        }
+out:
+    if (addr1) {
+        freeaddrinfo(addr1);
+    }
+    if (addr2) {
+        freeaddrinfo(addr2);
+    }
+    return ret;
+}
 
-        for (p = addr1; p; p = p->ai_next) {
-                for (q = addr2; q; q = q->ai_next) {
-                        if (p->ai_addrlen != q->ai_addrlen) {
-                                continue;
-                        }
-                        if (memcmp(p->ai_addr,q->ai_addr,p->ai_addrlen)) {
-                                continue;
-                        }
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
+/*
+ * Processes list of volfile servers.
+ * Format: <host1>:<port1> <host2>:<port2>...
+ */
+int
+gf_process_getspec_servers_list(cmd_args_t *cmd_args, const char *servers_list)
+{
+    char *tmp = NULL;
+    char *address = NULL;
+    char *host = NULL;
+    char *last_colon = NULL;
+    char *save_ptr = NULL;
+    int port = 0;
+    int ret = -1;
+
+    tmp = gf_strdup(servers_list);
+    if (!tmp) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    address = strtok_r(tmp, " ", &save_ptr);
+    if (!address) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    while (1) {
+        last_colon = strrchr(address, ':');
+        if (!last_colon) {
+            errno = EINVAL;
+            ret = -1;
+            break;
+        }
+        *last_colon = '\0';
+        host = address;
+        port = atoi(last_colon + 1);
+        if (port <= 0) {
+            errno = EINVAL;
+            ret = -1;
+            break;
+        }
+        ret = gf_set_volfile_server_common(cmd_args, host,
+                                           GF_DEFAULT_VOLFILE_TRANSPORT, port);
+        if (ret && errno != EEXIST) {
+            break;
+        }
+        address = strtok_r(NULL, " ", &save_ptr);
+        if (!address) {
+            errno = 0;
+            ret = 0;
+            break;
+        }
+    }
 
 out:
-        if (addr1) {
-                freeaddrinfo(addr1);
-        }
-        if (addr2) {
-                freeaddrinfo(addr2);
+    if (tmp) {
+        GF_FREE(tmp);
+    }
+
+    return ret;
+}
+
+int
+gf_set_volfile_server_common(cmd_args_t *cmd_args, const char *host,
+                             const char *transport, int port)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, host, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, transport, out);
+
+    server = GF_CALLOC(1, sizeof(server_cmdline_t),
+                       gf_common_mt_server_cmdline_t);
+    if (!server) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&server->list);
+
+    server->volfile_server = gf_strdup(host);
+    if (!server->volfile_server) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    server->transport = gf_strdup(transport);
+    if (!server->transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    server->port = port;
+
+    if (!cmd_args->volfile_server) {
+        cmd_args->volfile_server = server->volfile_server;
+        cmd_args->volfile_server_transport = server->transport;
+        cmd_args->volfile_server_port = server->port;
+        cmd_args->curr_server = server;
+    }
+
+    list_for_each_entry(tmp, &cmd_args->volfile_servers, list)
+    {
+        if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
+             !strcmp(tmp->transport, server->transport) &&
+             (tmp->port == server->port))) {
+            /* Duplicate option given, log and ignore */
+            gf_smsg("gluster", GF_LOG_INFO, EEXIST, LG_MSG_DUPLICATE_ENTRY,
+                    NULL);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    list_add_tail(&server->list, &cmd_args->volfile_servers);
+
+    ret = 0;
+out:
+    if (-1 == ret) {
+        if (server) {
+            GF_FREE(server->volfile_server);
+            GF_FREE(server->transport);
+            GF_FREE(server);
         }
-        return ret;
+    }
 
+    return ret;
 }
 
 /* Sets log file path from user provided arguments */
 int
-gf_set_log_file_path (cmd_args_t *cmd_args, glusterfs_ctx_t *ctx)
-{
-        int   i = 0;
-        int   j = 0;
-        int   ret = 0;
-        char  tmp_str[1024] = {0,};
-
-        if (!cmd_args)
-                goto done;
-
-        if (cmd_args->mount_point) {
-                j = 0;
-                i = 0;
-                if (cmd_args->mount_point[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->mount_point); i++,j++) {
-                        tmp_str[j] = cmd_args->mount_point[i];
-                        if (cmd_args->mount_point[i] == '/')
-                                tmp_str[j] = '-';
-                }
-
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   tmp_str);
-                if (ret > 0)
-                        ret = 0;
-                goto done;
-        }
+gf_set_log_file_path(cmd_args_t *cmd_args, glusterfs_ctx_t *ctx)
+{
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int tmp_len = 0;
+    char tmp_str[1024] = {
+        0,
+    };
 
-        if (ctx && GF_GLUSTERD_PROCESS == ctx->process_mode) {
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   GLUSTERD_NAME);
-                if (ret > 0)
-                        ret = 0;
+    if (!cmd_args)
+        goto done;
 
-                goto done;
+    if (cmd_args->mount_point) {
+        j = 0;
+        i = 0;
+        if (cmd_args->mount_point[0] == '/')
+            i = 1;
+        for (; i < strlen(cmd_args->mount_point); i++, j++) {
+            tmp_str[j] = cmd_args->mount_point[i];
+            if (cmd_args->mount_point[i] == '/')
+                tmp_str[j] = '-';
         }
 
-        if (cmd_args->volfile) {
-                j = 0;
-                i = 0;
-                if (cmd_args->volfile[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->volfile); i++,j++) {
-                        tmp_str[j] = cmd_args->volfile[i];
-                        if (cmd_args->volfile[i] == '/')
-                                tmp_str[j] = '-';
-                }
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   tmp_str);
-                if (ret > 0)
-                        ret = 0;
-                goto done;
-        }
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", tmp_str);
+        if (ret > 0)
+            ret = 0;
+        goto done;
+    }
+
+    if (ctx && GF_GLUSTERD_PROCESS == ctx->process_mode) {
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", GLUSTERD_NAME);
+        if (ret > 0)
+            ret = 0;
 
-        if (cmd_args->volfile_server) {
+        goto done;
+    }
 
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s-%s-%d.log",
-                                   cmd_args->volfile_server,
-                                   cmd_args->volfile_id, getpid());
-                if (ret > 0)
-                        ret = 0;
+    if (cmd_args->volfile) {
+        j = 0;
+        i = 0;
+        if (cmd_args->volfile[0] == '/')
+            i = 1;
+        for (; i < strlen(cmd_args->volfile); i++, j++) {
+            tmp_str[j] = cmd_args->volfile[i];
+            if (cmd_args->volfile[i] == '/')
+                tmp_str[j] = '-';
+        }
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", tmp_str);
+        if (ret > 0)
+            ret = 0;
+        goto done;
+    }
+
+    if (cmd_args->volfile_server) {
+        if (strncmp(cmd_args->volfile_server_transport, "unix", 4) == 0) {
+            if (cmd_args->volfile_server[0] == '/')
+                i = 1;
+            tmp_len = strlen(cmd_args->volfile_server);
+            for (j = 0; i < tmp_len; i++, j++) {
+                tmp_str[j] = cmd_args->volfile_server[i];
+                if (cmd_args->volfile_server[i] == '/')
+                    tmp_str[j] = '-';
+            }
+            ret = gf_asprintf(&cmd_args->log_file, "%s/%s-%s-%d.log",
+                              DEFAULT_LOG_FILE_DIRECTORY, tmp_str,
+                              cmd_args->volfile_id, getpid());
+        } else {
+            ret = gf_asprintf(&cmd_args->log_file, "%s/%s-%s-%d.log",
+                              DEFAULT_LOG_FILE_DIRECTORY,
+                              cmd_args->volfile_server, cmd_args->volfile_id,
+                              getpid());
         }
+        if (ret > 0)
+            ret = 0;
+    }
 done:
-        return ret;
+    return ret;
 }
 
 int
-gf_set_log_ident (cmd_args_t *cmd_args)
+gf_set_log_ident(cmd_args_t *cmd_args)
 {
-        int              ret = 0;
-        char            *ptr = NULL;
-
-        if (cmd_args->log_file == NULL) {
-                /* no ident source */
-                return 0;
-        }
+    int ret = 0;
+    char *ptr = NULL;
 
-        /* TODO: Some idents would look like, etc-glusterfs-glusterd.vol, which
-         * seems ugly and can be bettered? */
-        /* just get the filename as the ident */
-        if (NULL != (ptr = strrchr (cmd_args->log_file, '/'))) {
-                ret = gf_asprintf (&cmd_args->log_ident, "%s", ptr + 1);
-        } else {
-                ret = gf_asprintf (&cmd_args->log_ident, "%s",
-                                   cmd_args->log_file);
-        }
-
-        if (ret > 0)
-                ret = 0;
-        else
-                return ret;
+    if (cmd_args->log_file == NULL) {
+        /* no ident source */
+        return 0;
+    }
+
+    /* TODO: Some idents would look like, etc-glusterfs-glusterd.vol, which
+     * seems ugly and can be bettered? */
+    /* just get the filename as the ident */
+    if (NULL != (ptr = strrchr(cmd_args->log_file, '/'))) {
+        ret = gf_asprintf(&cmd_args->log_ident, "%s", ptr + 1);
+    } else {
+        ret = gf_asprintf(&cmd_args->log_ident, "%s", cmd_args->log_file);
+    }
+
+    if (ret > 0)
+        ret = 0;
+    else
+        return ret;
 
-        /* remove .log suffix */
-        if (NULL != (ptr = strrchr (cmd_args->log_ident, '.'))) {
-                if (strcmp (ptr, ".log") == 0) {
-                        ptr[0] = '\0';
-                }
+    /* remove .log suffix */
+    if (NULL != (ptr = strrchr(cmd_args->log_ident, '.'))) {
+        if (strcmp(ptr, ".log") == 0) {
+            ptr[0] = '\0';
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-gf_thread_cleanup_xint (pthread_t thread)
+gf_thread_cleanup_xint(pthread_t thread)
 {
-        int ret = 0;
-        void *res = NULL;
+    int ret = 0;
+    void *res = NULL;
 
-        ret = pthread_cancel (thread);
-        if (ret != 0)
-                goto error_return;
+    ret = pthread_cancel(thread);
+    if (ret != 0)
+        goto error_return;
 
-        ret = pthread_join (thread, &res);
-        if (ret != 0)
-                goto error_return;
+    ret = pthread_join(thread, &res);
+    if (ret != 0)
+        goto error_return;
 
-        if (res != PTHREAD_CANCELED)
-                goto error_return;
+    if (res != PTHREAD_CANCELED)
+        goto error_return;
 
-        ret = 0;
+    ret = 0;
 
- error_return:
-        return ret;
+error_return:
+    return ret;
+}
+
+void
+gf_thread_set_vname(pthread_t thread, const char *name, va_list args)
+{
+    char thread_name[GF_THREAD_NAME_LIMIT];
+    int ret;
+
+    /* Initialize the thread name with the prefix (not NULL terminated). */
+    memcpy(thread_name, GF_THREAD_NAME_PREFIX,
+           sizeof(GF_THREAD_NAME_PREFIX) - 1);
+
+    ret = vsnprintf(thread_name + sizeof(GF_THREAD_NAME_PREFIX) - 1,
+                    sizeof(thread_name) - sizeof(GF_THREAD_NAME_PREFIX) + 1,
+                    name, args);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_PTHREAD_NAMING_FAILED,
+                "name=%s", name, NULL);
+        return;
+    }
+
+    if (ret >= sizeof(thread_name)) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_THREAD_NAME_TOO_LONG,
+                "name=%s", thread_name, NULL);
+    }
+
+#ifdef GF_LINUX_HOST_OS
+    ret = pthread_setname_np(thread, thread_name);
+#elif defined(__NetBSD__)
+    ret = pthread_setname_np(thread, thread_name, NULL);
+#elif defined(__FreeBSD__)
+    pthread_set_name_np(thread, thread_name);
+    ret = 0;
+#else
+    ret = ENOSYS;
+#endif
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, ret, LG_MSG_SET_THREAD_FAILED,
+                "name=%s", thread_name, NULL);
+    }
+}
+
+void
+gf_thread_set_name(pthread_t thread, const char *name, ...)
+{
+    va_list args;
+
+    va_start(args, name);
+    gf_thread_set_vname(thread, name, args);
+    va_end(args);
 }
 
 int
-gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
-                  void *(*start_routine)(void *), void *arg)
+gf_thread_vcreate(pthread_t *thread, const pthread_attr_t *attr,
+                  void *(*start_routine)(void *), void *arg, const char *name,
+                  va_list args)
 {
-        sigset_t set, old;
-        int ret;
+    sigset_t set, old;
+    int ret;
+
+    sigemptyset(&old);
+    sigfillset(&set);
+    sigdelset(&set, SIGSEGV);
+    sigdelset(&set, SIGBUS);
+    sigdelset(&set, SIGILL);
+    sigdelset(&set, SIGSYS);
+    sigdelset(&set, SIGFPE);
+    sigdelset(&set, SIGABRT);
 
-        sigemptyset (&set);
+    pthread_sigmask(SIG_BLOCK, &set, &old);
 
-        sigfillset (&set);
-        sigdelset (&set, SIGSEGV);
-        sigdelset (&set, SIGBUS);
-        sigdelset (&set, SIGILL);
-        sigdelset (&set, SIGSYS);
-        sigdelset (&set, SIGFPE);
-        sigdelset (&set, SIGABRT);
+    ret = pthread_create(thread, attr, start_routine, arg);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ret, LG_MSG_THREAD_CREATE_FAILED,
+                NULL);
+        ret = -1;
+    } else if (name != NULL) {
+        gf_thread_set_vname(*thread, name, args);
+    }
 
-        pthread_sigmask (SIG_BLOCK, &set, &old);
+    pthread_sigmask(SIG_SETMASK, &old, NULL);
 
-        ret = pthread_create (thread, attr, start_routine, arg);
+    return ret;
+}
 
-        pthread_sigmask (SIG_SETMASK, &old, NULL);
+int
+gf_thread_create(pthread_t *thread, const pthread_attr_t *attr,
+                 void *(*start_routine)(void *), void *arg, const char *name,
+                 ...)
+{
+    va_list args;
+    int ret;
 
-        return ret;
+    va_start(args, name);
+    ret = gf_thread_vcreate(thread, attr, start_routine, arg, name, args);
+    va_end(args);
+
+    return ret;
 }
 
 int
-gf_thread_create_detached (pthread_t *thread,
-                         void *(*start_routine)(void *), void *arg)
+gf_thread_create_detached(pthread_t *thread, void *(*start_routine)(void *),
+                          void *arg, const char *name, ...)
 {
-        pthread_attr_t attr;
-        int ret = -1;
+    pthread_attr_t attr;
+    va_list args;
+    int ret = -1;
 
-        ret = pthread_attr_init (&attr);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ret,
-                        LG_MSG_PTHREAD_ATTR_INIT_FAILED,
-                        "Thread attribute initialization failed");
-                return -1;
-        }
+    ret = pthread_attr_init(&attr);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ret, LG_MSG_PTHREAD_ATTR_INIT_FAILED,
+                NULL);
+        return -1;
+    }
 
-        pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-        ret = gf_thread_create (thread, &attr, start_routine, arg);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ret,
-                        LG_MSG_PTHREAD_FAILED,
-                        "Thread creation failed");
-                ret = -1;
-        }
+    va_start(args, name);
+    ret = gf_thread_vcreate(thread, &attr, start_routine, arg, name, args);
+    va_end(args);
 
-        pthread_attr_destroy (&attr);
+    pthread_attr_destroy(&attr);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_skip_header_section (int fd, int header_len)
+gf_skip_header_section(int fd, int header_len)
 {
-        int  ret           = -1;
+    int ret = -1;
 
-        ret = sys_lseek (fd, header_len, SEEK_SET);
-        if (ret == (off_t) -1) {
-                gf_msg ("", GF_LOG_ERROR, 0, LG_MSG_SKIP_HEADER_FAILED,
-                        "Failed to skip header section");
-        } else {
-                ret = 0;
-        }
+    ret = sys_lseek(fd, header_len, SEEK_SET);
+    if (ret == (off_t)-1) {
+        gf_smsg("", GF_LOG_ERROR, 0, LG_MSG_SKIP_HEADER_FAILED, NULL);
+    } else {
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
+/* Below function is use to check at runtime if pid is running */
+
 gf_boolean_t
-gf_is_service_running (char *pidfile, int *pid)
+gf_is_pid_running(int pid)
 {
-        FILE            *file = NULL;
-        gf_boolean_t    running = _gf_false;
-        int             ret = 0;
-        int             fno = 0;
+#ifdef __FreeBSD__
+    int ret = -1;
 
-        file = fopen (pidfile, "r+");
-        if (!file)
-                goto out;
+    ret = sys_kill(pid, 0);
+    if (ret < 0) {
+        return _gf_false;
+    }
+#else
+    char fname[32] = {
+        0,
+    };
+    int fd = -1;
 
-        fno = fileno (file);
-        ret = lockf (fno, F_TEST, 0);
-        if (ret == -1)
-                running = _gf_true;
-        if (!pid)
-                goto out;
+    snprintf(fname, sizeof(fname), "/proc/%d/cmdline", pid);
 
-        ret = fscanf (file, "%d", pid);
-        if (ret <= 0) {
-                gf_msg ("", GF_LOG_ERROR, errno, LG_MSG_FILE_OP_FAILED,
-                        "Unable to read pidfile: %s", pidfile);
-                *pid = -1;
-        }
+    fd = sys_open(fname, O_RDONLY, 0);
+    if (fd < 0) {
+        return _gf_false;
+    }
 
+    sys_close(fd);
+#endif
+    return _gf_true;
+}
+
+gf_boolean_t
+gf_is_service_running(char *pidfile, int *pid)
+{
+    FILE *file = NULL;
+    gf_boolean_t running = _gf_false;
+    int ret = 0;
+    int fno = 0;
+
+    file = fopen(pidfile, "r+");
+    if (!file) {
+        goto out;
+    }
+
+    fno = fileno(file);
+    ret = lockf(fno, F_TEST, 0);
+    if (ret == -1) {
+        running = _gf_true;
+    }
+
+    ret = fscanf(file, "%d", pid);
+    if (ret <= 0) {
+        gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_FILE_OP_FAILED, "pidfile=%s",
+                pidfile, NULL);
+        *pid = -1;
+        running = _gf_false;
+        goto out;
+    }
+
+    running = gf_is_pid_running(*pid);
 out:
-        if (file)
-                fclose (file);
-        return running;
+    if (file)
+        fclose(file);
+    return running;
+}
+
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid(const char *pid, int length)
+{
+    gf_boolean_t ret = _gf_true;
+    pid_t value = 0;
+    char *end_ptr = NULL;
+
+    if (length <= 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    value = strtol(pid, &end_ptr, 10);
+    if (value <= 0) {
+        ret = _gf_false;
+    }
+out:
+    return ret;
 }
 
 static int
-dht_is_linkfile_key (dict_t *this, char *key, data_t *value, void *data)
+dht_is_linkfile_key(dict_t *this, char *key, data_t *value, void *data)
 {
-        gf_boolean_t *linkfile_key_found = NULL;
+    gf_boolean_t *linkfile_key_found = NULL;
 
-        if (!data)
-                goto out;
+    if (!data)
+        goto out;
 
-        linkfile_key_found = data;
+    linkfile_key_found = data;
 
-        *linkfile_key_found = _gf_true;
+    *linkfile_key_found = _gf_true;
 out:
-        return 0;
+    return 0;
 }
 
-
 gf_boolean_t
-dht_is_linkfile (struct iatt *buf, dict_t *dict)
+dht_is_linkfile(struct iatt *buf, dict_t *dict)
 {
-        gf_boolean_t linkfile_key_found = _gf_false;
+    gf_boolean_t linkfile_key_found = _gf_false;
 
-        if (!IS_DHT_LINKFILE_MODE (buf))
-                return _gf_false;
+    if (!IS_DHT_LINKFILE_MODE(buf))
+        return _gf_false;
 
-        dict_foreach_fnmatch (dict, "*."DHT_LINKFILE_STR, dht_is_linkfile_key,
-                              &linkfile_key_found);
+    dict_foreach_fnmatch(dict, "*." DHT_LINKFILE_STR, dht_is_linkfile_key,
+                         &linkfile_key_found);
 
-        return linkfile_key_found;
+    return linkfile_key_found;
 }
 
 int
-gf_check_log_format (const char *value)
+gf_check_log_format(const char *value)
 {
-        int log_format = -1;
+    int log_format = -1;
 
-        if (!strcasecmp (value, GF_LOG_FORMAT_NO_MSG_ID))
-                log_format = gf_logformat_traditional;
-        else if (!strcasecmp (value, GF_LOG_FORMAT_WITH_MSG_ID))
-                log_format = gf_logformat_withmsgid;
+    if (!strcasecmp(value, GF_LOG_FORMAT_NO_MSG_ID))
+        log_format = gf_logformat_traditional;
+    else if (!strcasecmp(value, GF_LOG_FORMAT_WITH_MSG_ID))
+        log_format = gf_logformat_withmsgid;
 
-        if (log_format == -1)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
-                        "Invalid log-format. possible values are "
-                        GF_LOG_FORMAT_NO_MSG_ID "|" GF_LOG_FORMAT_WITH_MSG_ID);
+    if (log_format == -1)
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
+                "possible_values=" GF_LOG_FORMAT_NO_MSG_ID
+                "|" GF_LOG_FORMAT_WITH_MSG_ID,
+                NULL);
 
-        return log_format;
+    return log_format;
 }
 
 int
-gf_check_logger (const char *value)
+gf_check_logger(const char *value)
 {
-        int logger = -1;
+    int logger = -1;
 
-        if (!strcasecmp (value, GF_LOGGER_GLUSTER_LOG))
-                logger = gf_logger_glusterlog;
-        else if (!strcasecmp (value, GF_LOGGER_SYSLOG))
-                logger = gf_logger_syslog;
+    if (!strcasecmp(value, GF_LOGGER_GLUSTER_LOG))
+        logger = gf_logger_glusterlog;
+    else if (!strcasecmp(value, GF_LOGGER_SYSLOG))
+        logger = gf_logger_syslog;
 
-        if (logger == -1)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
-                        "Invalid logger. possible values are "
-                        GF_LOGGER_GLUSTER_LOG "|" GF_LOGGER_SYSLOG);
+    if (logger == -1)
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
+                "possible_values=" GF_LOGGER_GLUSTER_LOG "|" GF_LOGGER_SYSLOG,
+                NULL);
 
-        return logger;
+    return logger;
 }
 
 /* gf_compare_sockaddr compares the given addresses @addr1 and @addr2 for
@@ -3743,30 +4301,28 @@ gf_check_logger (const char *value)
  * https://www.opensource.apple.com/source/postfix/postfix-197/postfix/src/util/sock_addr.c
  */
 gf_boolean_t
-gf_compare_sockaddr (const struct sockaddr *addr1,
-                     const struct sockaddr *addr2)
+gf_compare_sockaddr(const struct sockaddr *addr1, const struct sockaddr *addr2)
 {
-        GF_ASSERT (addr1 != NULL);
-        GF_ASSERT (addr2 != NULL);
-
-        /* Obviously, the addresses don't match if their families are different
-         */
-        if (addr1->sa_family != addr2->sa_family)
-                return _gf_false;
+    GF_ASSERT(addr1 != NULL);
+    GF_ASSERT(addr2 != NULL);
 
+    /* Obviously, the addresses don't match if their families are different
+     */
+    if (addr1->sa_family != addr2->sa_family)
+        return _gf_false;
 
-        if (AF_INET == addr1->sa_family) {
-                if (((struct sockaddr_in *)addr1)->sin_addr.s_addr ==
-                       ((struct sockaddr_in *)addr2)->sin_addr.s_addr)
-                        return _gf_true;
+    if (AF_INET == addr1->sa_family) {
+        if (((struct sockaddr_in *)addr1)->sin_addr.s_addr ==
+            ((struct sockaddr_in *)addr2)->sin_addr.s_addr)
+            return _gf_true;
 
-        } else if (AF_INET6 == addr1->sa_family) {
-                if (memcmp ((char *)&((struct sockaddr_in6 *)addr1)->sin6_addr,
-                            (char *)&((struct sockaddr_in6 *)addr2)->sin6_addr,
-                            sizeof (struct in6_addr)) == 0)
-                        return _gf_true;
-        }
-        return _gf_false;
+    } else if (AF_INET6 == addr1->sa_family) {
+        if (memcmp((char *)&((struct sockaddr_in6 *)addr1)->sin6_addr,
+                   (char *)&((struct sockaddr_in6 *)addr2)->sin6_addr,
+                   sizeof(struct in6_addr)) == 0)
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 /*
@@ -3775,129 +4331,167 @@ gf_compare_sockaddr (const struct sockaddr *addr1,
  */
 
 int
-gf_set_timestamp  (const char *src, const char* dest)
-{
-        struct stat    sb             = {0, };
-        struct timeval new_time[2]    = {{0, },{0,}};
-        int            ret            = 0;
-        xlator_t       *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (src);
-        GF_ASSERT (dest);
-
-        ret = sys_stat (src, &sb);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        LG_MSG_FILE_STAT_FAILED, "stat on %s", src);
-                goto out;
-        }
-        new_time[0].tv_sec = sb.st_atime;
-        new_time[0].tv_usec = ST_ATIM_NSEC (&sb)/1000;
+gf_set_timestamp(const char *src, const char *dest)
+{
+    struct stat sb = {
+        0,
+    };
+#if defined(HAVE_UTIMENSAT)
+    struct timespec new_time[2] = {{
+                                       0,
+                                   },
+                                   {
+                                       0,
+                                   }};
+#else
+    struct timeval new_time[2] = {{
+                                      0,
+                                  },
+                                  {
+                                      0,
+                                  }};
+#endif
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(src);
+    GF_ASSERT(dest);
+
+    ret = sys_stat(src, &sb);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_FILE_STAT_FAILED,
+                "stat=%s", src, NULL);
+        goto out;
+    }
+    /* The granularity is nano seconds if `utimensat()` is available,
+     * and micro seconds otherwise.
+     */
+#if defined(HAVE_UTIMENSAT)
+    new_time[0].tv_sec = sb.st_atime;
+    new_time[0].tv_nsec = ST_ATIM_NSEC(&sb);
+
+    new_time[1].tv_sec = sb.st_mtime;
+    new_time[1].tv_nsec = ST_MTIM_NSEC(&sb);
+
+    /* dirfd = 0 is ignored because `dest` is an absolute path. */
+    ret = sys_utimensat(AT_FDCWD, dest, new_time, AT_SYMLINK_NOFOLLOW);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_UTIMENSAT_FAILED,
+                "dest=%s", dest, NULL);
+    }
+#else
+    new_time[0].tv_sec = sb.st_atime;
+    new_time[0].tv_usec = ST_ATIM_NSEC(&sb) / 1000;
 
-        new_time[1].tv_sec = sb.st_mtime;
-        new_time[1].tv_usec = ST_MTIM_NSEC (&sb)/1000;
+    new_time[1].tv_sec = sb.st_mtime;
+    new_time[1].tv_usec = ST_MTIM_NSEC(&sb) / 1000;
 
-        /* The granularity is micro seconds as per the current
-         * requiremnt. Hence using 'utimes'. This can be updated
-         * to 'utimensat' if we need timestamp in nanoseconds.
-         */
-        ret = sys_utimes (dest, new_time);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno, LG_MSG_UTIMES_FAILED,
-                        "utimes on %s", dest);
-        }
+    ret = sys_utimes(dest, new_time);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_UTIMES_FAILED,
+                "dest=%s", dest, NULL);
+    }
+#endif
 out:
-        return ret;
+    return ret;
 }
 
 static void
-gf_backtrace_end (char *buf, size_t frames)
+gf_backtrace_end(char *buf, size_t frames)
 {
-        size_t pos = 0;
+    size_t pos = 0;
 
-        if (!buf)
-                return;
+    if (!buf)
+        return;
 
-        pos = strlen (buf);
+    pos = strlen(buf);
 
-        frames = min(frames, GF_BACKTRACE_LEN - pos -1);
+    frames = min(frames, GF_BACKTRACE_LEN - pos - 1);
 
-        if (frames <= 0)
-                return;
+    if (0 == frames)
+        return;
 
-        memset (buf+pos, ')', frames);
-        buf[pos+frames] = '\0';
+    memset(buf + pos, ')', frames);
+    buf[pos + frames] = '\0';
 }
 
 /*Returns bytes written*/
 static int
-gf_backtrace_append (char *buf, size_t pos, char *framestr)
+gf_backtrace_append(char *buf, size_t pos, char *framestr)
 {
-        if (pos >= GF_BACKTRACE_LEN)
-                return -1;
-        return snprintf (buf+pos, GF_BACKTRACE_LEN-pos, "(--> %s ", framestr);
+    if (pos >= GF_BACKTRACE_LEN)
+        return -1;
+    return snprintf(buf + pos, GF_BACKTRACE_LEN - pos, "(--> %s ", framestr);
 }
 
 static int
-gf_backtrace_fillframes (char *buf)
-{
-        void    *array[GF_BACKTRACE_FRAME_COUNT];
-        size_t  frames                  = 0;
-        FILE    *fp                     = NULL;
-        char    callingfn[GF_BACKTRACE_FRAME_COUNT-2][1024] = {{0},};
-        int     ret                     = -1;
-        int     fd                      = -1;
-        size_t  idx                     = 0;
-        size_t  pos                     = 0;
-        size_t  inc                     = 0;
-        char    tmpl[32]                = "/tmp/btXXXXXX";
-
-        frames = backtrace (array, GF_BACKTRACE_FRAME_COUNT);
-        if (!frames)
-                return -1;
-
-        fd = gf_mkostemp (tmpl, 0, O_RDWR);
-        if (fd == -1)
-                return -1;
-
-        /*The most recent two frames are the calling function and
-         * gf_backtrace_save, which we can infer.*/
-
-        backtrace_symbols_fd (&array[2], frames-2, fd);
-
-        fp = fdopen (fd, "r");
-        if (!fp) {
-                sys_close (fd);
-                ret = -1;
-                goto out;
-        }
+gf_backtrace_fillframes(char *buf)
+{
+    void *array[GF_BACKTRACE_FRAME_COUNT];
+    size_t frames = 0;
+    FILE *fp = NULL;
+    char callingfn[GF_BACKTRACE_FRAME_COUNT - 2][1024] = {
+        {0},
+    };
+    int ret = -1;
+    int fd = -1;
+    size_t idx = 0;
+    size_t pos = 0;
+    size_t inc = 0;
+    char tmpl[] = "/tmp/glfs-bt-XXXXXX";
+
+    frames = backtrace(array, GF_BACKTRACE_FRAME_COUNT);
+    if (!frames)
+        return -1;
 
-        ret = fseek (fp, 0L, SEEK_SET);
-        if (ret)
-                goto out;
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    fd = mkstemp(tmpl);
+    if (fd == -1)
+        return -1;
 
-        pos = 0;
-        for (idx = 0; idx < frames - 2; idx++) {
-                ret = fscanf (fp, "%s", callingfn[idx]);
-                if (ret == EOF)
-                        break;
-                inc = gf_backtrace_append (buf, pos, callingfn[idx]);
-                if (inc == -1)
-                        break;
-                pos += inc;
-        }
-        gf_backtrace_end (buf, idx);
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(tmpl);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_INFO, 0, LG_MSG_FILE_DELETE_FAILED,
+                "temporary_file=%s", tmpl, NULL);
+    }
+
+    /*The most recent two frames are the calling function and
+     * gf_backtrace_save, which we can infer.*/
+
+    backtrace_symbols_fd(&array[2], frames - 2, fd);
+
+    fp = fdopen(fd, "r");
+    if (!fp) {
+        sys_close(fd);
+        goto out;
+    }
+
+    ret = fseek(fp, 0L, SEEK_SET);
+    if (ret)
+        goto out;
+
+    pos = 0;
+    for (idx = 0; idx < frames - 2; idx++) {
+        ret = fscanf(fp, "%1023s", callingfn[idx]);
+        if (ret == EOF)
+            break;
+        inc = gf_backtrace_append(buf, pos, callingfn[idx]);
+        if (inc == -1)
+            break;
+        pos += inc;
+    }
+    gf_backtrace_end(buf, idx);
 
 out:
-        if (fp)
-                fclose (fp);
-
-        sys_unlink (tmpl);
-
-        return (idx > 0)? 0: -1;
+    if (fp)
+        fclose(fp);
 
+    return (idx > 0) ? 0 : -1;
 }
 
 /* Optionally takes @buf to save backtrace.  If @buf is NULL, uses the
@@ -3908,66 +4502,69 @@ out:
  * when there is a real-use for that.*/
 
 char *
-gf_backtrace_save (char *buf)
+gf_backtrace_save(char *buf)
 {
-        char *bt = NULL;
+    char *bt = NULL;
 
-        if (!buf) {
-                bt = THIS->ctx->btbuf;
-                GF_ASSERT (bt);
+    if (!buf) {
+        bt = THIS->ctx->btbuf;
+        GF_ASSERT(bt);
 
-        } else {
-                bt = buf;
-
-        }
+    } else {
+        bt = buf;
+    }
 
-        if ((0 == gf_backtrace_fillframes (bt)))
-                return bt;
+    if ((0 == gf_backtrace_fillframes(bt)))
+        return bt;
 
-        gf_msg (THIS->name, GF_LOG_WARNING, 0, LG_MSG_BACKTRACE_SAVE_FAILED,
-                "Failed to save the backtrace.");
-        return NULL;
+    gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_BACKTRACE_SAVE_FAILED, NULL);
+    return NULL;
 }
 
 gf_loglevel_t
-fop_log_level (glusterfs_fop_t fop, int op_errno)
-{
-        /* if gfid doesn't exist ESTALE comes */
-        if (op_errno == ENOENT || op_errno == ESTALE)
-                return GF_LOG_DEBUG;
-
-        if ((fop == GF_FOP_ENTRYLK) ||
-            (fop == GF_FOP_FENTRYLK) ||
-            (fop == GF_FOP_FINODELK) ||
-            (fop == GF_FOP_INODELK) ||
-            (fop == GF_FOP_LK)) {
-                /*
-                 * if non-blocking lock fails EAGAIN comes
-                 * if locks xlator is not loaded ENOSYS comes
-                 */
-                if (op_errno == EAGAIN || op_errno == ENOSYS)
-                        return GF_LOG_DEBUG;
-        }
-
-        if ((fop == GF_FOP_GETXATTR) ||
-            (fop == GF_FOP_FGETXATTR)) {
-                if (op_errno == ENOTSUP || op_errno == ENODATA)
-                        return GF_LOG_DEBUG;
-        }
+fop_log_level(glusterfs_fop_t fop, int op_errno)
+{
+    /* if gfid doesn't exist ESTALE comes */
+    if (op_errno == ENOENT || op_errno == ESTALE)
+        return GF_LOG_DEBUG;
 
-        if ((fop == GF_FOP_SETXATTR) ||
-            (fop == GF_FOP_FSETXATTR) ||
-            (fop == GF_FOP_REMOVEXATTR) ||
-            (fop == GF_FOP_FREMOVEXATTR)) {
-                if (op_errno == ENOTSUP)
-                        return GF_LOG_DEBUG;
+    if ((fop == GF_FOP_ENTRYLK) || (fop == GF_FOP_FENTRYLK) ||
+        (fop == GF_FOP_FINODELK) || (fop == GF_FOP_INODELK) ||
+        (fop == GF_FOP_LK)) {
+        /*
+         * if non-blocking lock fails EAGAIN comes
+         * if locks xlator is not loaded ENOSYS comes
+         */
+        if (op_errno == EAGAIN || op_errno == ENOSYS)
+            return GF_LOG_DEBUG;
+    }
+
+    if ((fop == GF_FOP_GETXATTR) || (fop == GF_FOP_FGETXATTR)) {
+        if (op_errno == ENOTSUP || op_errno == ENODATA)
+            return GF_LOG_DEBUG;
+    }
+
+    if ((fop == GF_FOP_SETXATTR) || (fop == GF_FOP_FSETXATTR) ||
+        (fop == GF_FOP_REMOVEXATTR) || (fop == GF_FOP_FREMOVEXATTR)) {
+        if (op_errno == ENOTSUP)
+            return GF_LOG_DEBUG;
+    }
+
+    if (fop == GF_FOP_MKNOD || fop == GF_FOP_MKDIR)
+        if (op_errno == EEXIST)
+            return GF_LOG_DEBUG;
+
+    if (fop == GF_FOP_SEEK) {
+#ifdef HAVE_SEEK_HOLE
+        if (op_errno == ENXIO) {
+            return GF_LOG_DEBUG;
         }
+#else
+        return GF_LOG_DEBUG;
+#endif
+    }
 
-        if (fop == GF_FOP_MKNOD || fop == GF_FOP_MKDIR)
-                if (op_errno == EEXIST)
-                        return GF_LOG_DEBUG;
-
-        return GF_LOG_ERROR;
+    return GF_LOG_ERROR;
 }
 
 /* This function will build absolute path of file/directory from the
@@ -3985,116 +4582,115 @@ fop_log_level (glusterfs_fop_t fop, int op_errno)
  */
 
 int32_t
-gf_build_absolute_path (char *current_path, char *relative_path, char **path)
-{
-        char                    *absolute_path          = NULL;
-        char                    *token                  = NULL;
-        char                    *component              = NULL;
-        char                    *saveptr                = NULL;
-        char                    *end                    = NULL;
-        int                     ret                     = 0;
-        size_t                  relativepath_len        = 0;
-        size_t                  currentpath_len         = 0;
-        size_t                  max_absolutepath_len    = 0;
-
-        GF_ASSERT (current_path);
-        GF_ASSERT (relative_path);
-        GF_ASSERT (path);
-
-        if (!path || !current_path || !relative_path) {
-                ret = -EFAULT;
-                goto err;
-        }
-        /* Check for current and relative path
-         * current path should be absolute one and  start from '/'
-         * relative path should not start from '/'
-         */
-        currentpath_len = strlen (current_path);
-        if (current_path[0] != '/' || (currentpath_len > PATH_MAX)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
-                        "Wrong value for current path %s", current_path);
-                ret = -EINVAL;
+gf_build_absolute_path(char *current_path, char *relative_path, char **path)
+{
+    char *absolute_path = NULL;
+    char *token = NULL;
+    char *component = NULL;
+    char *saveptr = NULL;
+    char *end = NULL;
+    int ret = 0;
+    size_t relativepath_len = 0;
+    size_t currentpath_len = 0;
+    size_t max_absolutepath_len = 0;
+
+    GF_ASSERT(current_path);
+    GF_ASSERT(relative_path);
+    GF_ASSERT(path);
+
+    if (!path || !current_path || !relative_path) {
+        ret = -EFAULT;
+        goto err;
+    }
+    /* Check for current and relative path
+     * current path should be absolute one and  start from '/'
+     * relative path should not start from '/'
+     */
+    currentpath_len = strlen(current_path);
+    if (current_path[0] != '/' || (currentpath_len > PATH_MAX)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_WRONG_VALUE,
+                "current-path=%s", current_path, NULL);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    relativepath_len = strlen(relative_path);
+    if (relative_path[0] == '/' || (relativepath_len > PATH_MAX)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_WRONG_VALUE,
+                "relative-path=%s", relative_path, NULL);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    /* It is maximum possible value for absolute path */
+    max_absolutepath_len = currentpath_len + relativepath_len + 2;
+
+    absolute_path = GF_CALLOC(1, max_absolutepath_len, gf_common_mt_char);
+    if (!absolute_path) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    absolute_path[0] = '\0';
+
+    /* If current path is root i.e contains only "/", we do not
+     * need to copy it
+     */
+    if (strcmp(current_path, "/") != 0) {
+        strcpy(absolute_path, current_path);
+
+        /* We trim '/' at the end for easier string manipulation */
+        gf_path_strip_trailing_slashes(absolute_path);
+    }
+
+    /* Used to spilt relative path based on '/' */
+    component = gf_strdup(relative_path);
+    if (!component) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    /* In the relative path, we want to consider ".." and "."
+     * if token is ".." , we just need to reduce one level hierarchy
+     * if token is "." , we just ignore it
+     * if token is NULL , end of relative path
+     * if absolute path becomes '\0' and still "..", then it is a bad
+     * relative path,  it points to out of boundary area and stop
+     * building the absolute path
+     * All other cases we just concatenate token to the absolute path
+     */
+    for (token = strtok_r(component, "/", &saveptr),
+        end = strchr(absolute_path, '\0');
+         token; token = strtok_r(NULL, "/", &saveptr)) {
+        if (strcmp(token, ".") == 0)
+            continue;
+
+        else if (strcmp(token, "..") == 0) {
+            if (absolute_path[0] == '\0') {
+                ret = -EACCES;
                 goto err;
-        }
-
-        relativepath_len = strlen (relative_path);
-        if (relative_path[0] == '/' || (relativepath_len > PATH_MAX)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
-                        "Wrong value for relative path %s", relative_path);
-                ret = -EINVAL;
-                goto err;
-        }
-
-        /* It is maximum possible value for absolute path */
-        max_absolutepath_len = currentpath_len + relativepath_len + 2;
-
-        absolute_path = GF_CALLOC (1, max_absolutepath_len, gf_common_mt_char);
-        if (!absolute_path) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        absolute_path[0] = '\0';
-
-        /* If current path is root i.e contains only "/", we do not
-         * need to copy it
-         */
-        if (strcmp (current_path, "/") != 0) {
-                strcpy (absolute_path, current_path);
-
-                /* We trim '/' at the end for easier string manipulation */
-                gf_path_strip_trailing_slashes (absolute_path);
-        }
-
-        /* Used to spilt relative path based on '/' */
-        component = gf_strdup (relative_path);
-        if (!component) {
-                ret = -ENOMEM;
-                goto err;
-        }
+            }
 
-        /* In the relative path, we want to consider ".." and "."
-         * if token is ".." , we just need to reduce one level hierarchy
-         * if token is "." , we just ignore it
-         * if token is NULL , end of relative path
-         * if absolute path becomes '\0' and still "..", then it is a bad
-         * relative path,  it points to out of boundary area and stop
-         * building the absolute path
-         * All other cases we just concatenate token to the absolute path
-         */
-        for (token = strtok_r (component,  "/", &saveptr),
-             end = strchr (absolute_path, '\0'); token;
-             token = strtok_r (NULL, "/", &saveptr)) {
-                if (strcmp (token, ".") == 0)
-                        continue;
-
-                else if (strcmp (token, "..") == 0) {
-
-                        if (absolute_path[0] == '\0') {
-                                ret = -EACCES;
-                                goto err;
-                         }
-
-                         end = strrchr (absolute_path, '/');
-                         *end = '\0';
-                } else {
-                        ret = snprintf (end, max_absolutepath_len -
-                                        strlen (absolute_path), "/%s", token);
-                        end = strchr (absolute_path , '\0');
-                }
+            end = strrchr(absolute_path, '/');
+            *end = '\0';
+        } else {
+            ret = snprintf(end, max_absolutepath_len - strlen(absolute_path),
+                           "/%s", token);
+            end = strchr(absolute_path, '\0');
         }
+    }
 
-        if (strlen (absolute_path) > PATH_MAX) {
-                ret = -EINVAL;
-                goto err;
-        }
-        *path = gf_strdup (absolute_path);
+    if (strlen(absolute_path) > PATH_MAX) {
+        ret = -EINVAL;
+        goto err;
+    }
+    *path = gf_strdup(absolute_path);
 
 err:
-        if (component)
-                GF_FREE (component);
-        if (absolute_path)
-                GF_FREE (absolute_path);
-        return ret;
+    if (component)
+        GF_FREE(component);
+    if (absolute_path)
+        GF_FREE(absolute_path);
+    return ret;
 }
 
 /* This is an utility function which will recursively delete
@@ -4105,68 +4701,84 @@ err:
  * @return 0 on success and -1 on failure.
  */
 int
-recursive_rmdir (const char *delete_path)
-{
-        int             ret             = -1;
-        char            path[PATH_MAX]  = {0,};
-        struct stat     st              = {0,};
-        DIR            *dir             = NULL;
-        struct dirent  *entry           = NULL;
-        struct dirent   scratch[2]      = {{0,},};
-        xlator_t       *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, delete_path, out);
-
-        dir = sys_opendir (delete_path);
-        if (!dir) {
-                gf_msg_debug (this->name, 0, "Failed to open directory %s. "
-                              "Reason : %s", delete_path, strerror (errno));
-                ret = 0;
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir, scratch);
-        while (entry) {
-                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
-                ret = sys_lstat (path, &st);
-                if (ret == -1) {
-                        gf_msg_debug (this->name, 0, "Failed to stat entry %s :"
-                                      " %s", path, strerror (errno));
-                        goto out;
-                }
-
-                if (S_ISDIR (st.st_mode))
-                        ret = recursive_rmdir (path);
-                else
-                        ret = sys_unlink (path);
-
-                if (ret) {
-                        gf_msg_debug (this->name, 0, " Failed to remove %s. "
-                                      "Reason : %s", path, strerror (errno));
-                }
-
-                gf_msg_debug (this->name, 0, "%s %s", ret ?
-                              "Failed to remove" : "Removed", entry->d_name);
-
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir, scratch);
-        }
+recursive_rmdir(const char *delete_path)
+{
+    int ret = -1;
+    char path[PATH_MAX] = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, delete_path, out);
+
+    dir = sys_opendir(delete_path);
+    if (!dir) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to open directory %s. "
+                     "Reason : %s",
+                     delete_path, strerror(errno));
+        ret = 0;
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+        ret = sys_lstat(path, &st);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to stat entry %s :"
+                         " %s",
+                         path, strerror(errno));
+            (void)sys_closedir(dir);
+            goto out;
+        }
+
+        if (S_ISDIR(st.st_mode))
+            ret = recursive_rmdir(path);
+        else
+            ret = sys_unlink(path);
 
-        ret = sys_closedir (dir);
         if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to close dir %s. Reason :"
-                              " %s", delete_path, strerror (errno));
-        }
-
-        ret = sys_rmdir (delete_path);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to rmdir: %s,err: %s",
-                              delete_path, strerror (errno));
-        }
+            gf_msg_debug(this->name, 0,
+                         " Failed to remove %s. "
+                         "Reason : %s",
+                         path, strerror(errno));
+        }
+
+        gf_msg_debug(this->name, 0, "%s %s",
+                     ret ? "Failed to remove" : "Removed", entry->d_name);
+    }
+
+    ret = sys_closedir(dir);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to close dir %s. Reason :"
+                     " %s",
+                     delete_path, strerror(errno));
+    }
+
+    ret = sys_rmdir(delete_path);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s,err: %s", delete_path,
+                     strerror(errno));
+    }
 
 out:
-        return ret;
+    return ret;
 }
 /*
  * Input: Array of strings 'array' terminating in NULL
@@ -4175,197 +4787,198 @@ out:
  * Output: Index of the element in the array if found, '-1' otherwise
  */
 int
-gf_get_index_by_elem (char **array, char *elem)
+gf_get_index_by_elem(char **array, char *elem)
 {
-        int     i = 0;
+    int i = 0;
 
-        for (i = 0; array[i]; i++) {
-                if (strcmp (elem, array[i]) == 0)
-                        return i;
-        }
+    for (i = 0; array[i]; i++) {
+        if (strcmp(elem, array[i]) == 0)
+            return i;
+    }
 
-        return -1;
+    return -1;
 }
 
 static int
-get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
+get_pathinfo_host(char *pathinfo, char *hostname, size_t size)
 {
-        char    *start = NULL;
-        char    *end = NULL;
-        int     ret  = -1;
-        int     i    = 0;
+    char *start = NULL;
+    char *end = NULL;
+    int ret = -1;
+    int i = 0;
 
-        if (!pathinfo)
-                goto out;
+    if (!pathinfo)
+        goto out;
 
-        start = strchr (pathinfo, ':');
-        if (!start)
-                goto out;
+    start = strchr(pathinfo, ':');
+    if (!start)
+        goto out;
 
-        end = strrchr (pathinfo, ':');
-        if (start == end)
-                goto out;
+    end = strrchr(pathinfo, ':');
+    if (start == end)
+        goto out;
 
-        memset (hostname, 0, size);
-        i = 0;
-        while (++start != end)
-                hostname[i++] = *start;
-        ret = 0;
+    memset(hostname, 0, size);
+    i = 0;
+    while (++start != end)
+        hostname[i++] = *start;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*Note: 'pathinfo' should be gathered only from one brick*/
 int
-glusterfs_is_local_pathinfo (char *pathinfo, gf_boolean_t *is_local)
+glusterfs_is_local_pathinfo(char *pathinfo, gf_boolean_t *is_local)
 {
-        int             ret   = 0;
-        char            pathinfohost[1024] = {0};
-        char            localhost[1024] = {0};
+    int ret = 0;
+    char pathinfohost[1024] = {0};
+    char localhost[1024] = {0};
 
-        *is_local = _gf_false;
-        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
-        if (ret)
-                goto out;
+    *is_local = _gf_false;
+    ret = get_pathinfo_host(pathinfo, pathinfohost, sizeof(pathinfohost));
+    if (ret)
+        goto out;
 
-        ret = gethostname (localhost, sizeof (localhost));
-        if (ret)
-                goto out;
+    ret = gethostname(localhost, sizeof(localhost));
+    if (ret)
+        goto out;
 
-        if (!strcmp (localhost, pathinfohost))
-                *is_local = _gf_true;
+    if (!strcmp(localhost, pathinfohost))
+        *is_local = _gf_true;
 out:
-        return ret;
+    return ret;
 }
 
 ssize_t
-gf_nread (int fd, void *buf, size_t count)
-{
-        ssize_t  ret           = 0;
-        ssize_t  read_bytes    = 0;
-
-        for (read_bytes = 0; read_bytes < count; read_bytes += ret) {
-                ret = sys_read (fd, buf + read_bytes, count - read_bytes);
-                if (ret == 0) {
-                        break;
-                } else if (ret < 0) {
-                        if (errno == EINTR)
-                                ret = 0;
-                        else
-                                goto out;
-                }
+gf_nread(int fd, void *buf, size_t count)
+{
+    ssize_t ret = 0;
+    ssize_t read_bytes = 0;
+
+    for (read_bytes = 0; read_bytes < count; read_bytes += ret) {
+        ret = sys_read(fd, buf + read_bytes, count - read_bytes);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            if (errno == EINTR)
+                ret = 0;
+            else
+                goto out;
         }
+    }
 
-        ret = read_bytes;
+    ret = read_bytes;
 out:
-        return ret;
+    return ret;
 }
 
 ssize_t
-gf_nwrite (int fd, const void *buf, size_t count)
-{
-        ssize_t  ret        = 0;
-        ssize_t  written    = 0;
-
-        for (written = 0; written != count; written += ret) {
-                ret = sys_write (fd, buf + written, count - written);
-                if (ret < 0) {
-                        if (errno == EINTR)
-                                ret = 0;
-                        else
-                                goto out;
-                }
+gf_nwrite(int fd, const void *buf, size_t count)
+{
+    ssize_t ret = 0;
+    ssize_t written = 0;
+
+    for (written = 0; written != count; written += ret) {
+        ret = sys_write(fd, buf + written, count - written);
+        if (ret < 0) {
+            if (errno == EINTR)
+                ret = 0;
+            else
+                goto out;
         }
+    }
 
-        ret = written;
+    ret = written;
 out:
-        return ret;
+    return ret;
 }
 
 void
-gf_free_mig_locks (lock_migration_info_t *locks)
+gf_free_mig_locks(lock_migration_info_t *locks)
 {
-        lock_migration_info_t    *current       = NULL;
-        lock_migration_info_t    *temp          = NULL;
+    lock_migration_info_t *current = NULL;
+    lock_migration_info_t *temp = NULL;
 
-        if (!locks)
-                return;
+    if (!locks)
+        return;
 
-        if (list_empty (&locks->list))
-                return;
+    if (list_empty(&locks->list))
+        return;
 
-        list_for_each_entry_safe (current, temp, &locks->list, list) {
-                list_del_init (&current->list);
-                GF_FREE (current->client_uid);
-                GF_FREE (current);
-        }
+    list_for_each_entry_safe(current, temp, &locks->list, list)
+    {
+        list_del_init(&current->list);
+        GF_FREE(current->client_uid);
+        GF_FREE(current);
+    }
 }
 
 void
-_mask_cancellation (void)
+_mask_cancellation(void)
 {
-        (void) pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, NULL);
+    (void)pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
 }
 
 void
-_unmask_cancellation (void)
+_unmask_cancellation(void)
 {
-        (void) pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, NULL);
+    (void)pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
 }
 
 /* This is a wrapper function to add a pointer to a list,
  * which doesn't contain list member
  */
-struct list_node*
-_list_node_add (void *ptr, struct list_head *list,
+struct list_node *
+_list_node_add(void *ptr, struct list_head *list,
                int (*compare)(struct list_head *, struct list_head *))
 {
-        struct list_node  *node = NULL;
+    struct list_node *node = NULL;
 
-        if (ptr == NULL || list == NULL)
-                goto out;
+    if (ptr == NULL || list == NULL)
+        goto out;
 
-        node = GF_CALLOC (1, sizeof (struct list_node), gf_common_list_node);
+    node = GF_CALLOC(1, sizeof(struct list_node), gf_common_list_node);
 
-        if (node == NULL)
-                goto out;
+    if (node == NULL)
+        goto out;
 
-        node->ptr = ptr;
-        if (compare)
-                list_add_order (&node->list, list, compare);
-        else
-                list_add_tail (&node->list, list);
+    node->ptr = ptr;
+    if (compare)
+        list_add_order(&node->list, list, compare);
+    else
+        list_add_tail(&node->list, list);
 out:
-        return node;
+    return node;
 }
 
-struct list_node*
-list_node_add (void *ptr, struct list_head *list)
+struct list_node *
+list_node_add(void *ptr, struct list_head *list)
 {
-        return _list_node_add (ptr, list, NULL);
+    return _list_node_add(ptr, list, NULL);
 }
 
-struct list_node*
-list_node_add_order (void *ptr, struct list_head *list,
-                     int (*compare)(struct list_head *, struct list_head *))
+struct list_node *
+list_node_add_order(void *ptr, struct list_head *list,
+                    int (*compare)(struct list_head *, struct list_head *))
 {
-        return _list_node_add (ptr, list, compare);
+    return _list_node_add(ptr, list, compare);
 }
 
 void
-list_node_del (struct list_node *node)
+list_node_del(struct list_node *node)
 {
-        if (node == NULL)
-                return;
+    if (node == NULL)
+        return;
 
-        list_del_init (&node->list);
-        GF_FREE (node);
+    list_del_init(&node->list);
+    GF_FREE(node);
 }
 
 const char *
-fop_enum_to_pri_string (glusterfs_fop_t fop)
+fop_enum_to_pri_string(glusterfs_fop_t fop)
 {
-        switch (fop) {
+    switch (fop) {
         case GF_FOP_OPEN:
         case GF_FOP_STAT:
         case GF_FOP_FSTAT:
@@ -4378,7 +4991,9 @@ fop_enum_to_pri_string (glusterfs_fop_t fop)
         case GF_FOP_READDIRP:
         case GF_FOP_GETACTIVELK:
         case GF_FOP_SETACTIVELK:
-                return "HIGH";
+        case GF_FOP_ICREATE:
+        case GF_FOP_NAMELINK:
+            return "HIGH";
 
         case GF_FOP_CREATE:
         case GF_FOP_FLUSH:
@@ -4404,7 +5019,7 @@ fop_enum_to_pri_string (glusterfs_fop_t fop)
         case GF_FOP_FREMOVEXATTR:
         case GF_FOP_IPC:
         case GF_FOP_LEASE:
-                return "NORMAL";
+            return "NORMAL";
 
         case GF_FOP_READ:
         case GF_FOP_WRITE:
@@ -4418,7 +5033,7 @@ fop_enum_to_pri_string (glusterfs_fop_t fop)
         case GF_FOP_ZEROFILL:
         case GF_FOP_FALLOCATE:
         case GF_FOP_SEEK:
-                return "LOW";
+            return "LOW";
 
         case GF_FOP_NULL:
         case GF_FOP_FORGET:
@@ -4427,164 +5042,424 @@ fop_enum_to_pri_string (glusterfs_fop_t fop)
         case GF_FOP_GETSPEC:
         case GF_FOP_MAXVALUE:
         case GF_FOP_DISCARD:
-                return "LEAST";
+            return "LEAST";
         default:
-                return "UNKNOWN";
-        }
-}
-
-const char *
-fop_enum_to_string (glusterfs_fop_t fop)
-{
-        static const char *const str_map[] = {
-                "NULL",
-                "STAT",
-                "READLINK",
-                "MKNOD",
-                "MKDIR",
-                "UNLINK",
-                "RMDIR",
-                "SYMLINK",
-                "RENAME",
-                "LINK",
-                "TRUNCATE",
-                "OPEN",
-                "READ",
-                "WRITE",
-                "STATFS",
-                "FLUSH",
-                "FSYNC",
-                "SETXATTR",
-                "GETXATTR",
-                "REMOVEXATTR",
-                "OPENDIR",
-                "FSYNCDIR",
-                "ACCESS",
-                "CREATE",
-                "FTRUNCATE",
-                "FSTAT",
-                "LK",
-                "LOOKUP",
-                "READDIR",
-                "INODELK",
-                "FINODELK",
-                "ENTRYLK",
-                "FENTRYLK",
-                "XATTROP",
-                "FXATTROP",
-                "FGETXATTR",
-                "FSETXATTR",
-                "RCHECKSUM",
-                "SETATTR",
-                "FSETATTR",
-                "READDIRP",
-                "FORGET",
-                "RELEASE",
-                "RELEASEDIR",
-                "GETSPEC",
-                "FREMOVEXATTR",
-                "FALLOCATE",
-                "DISCARD",
-                "ZEROFILL",
-                "IPC",
-                "SEEK",
-                "COMPOUND",
-                "MAXVALUE"};
-        if (fop <= GF_FOP_MAXVALUE)
-                return str_map[fop];
-
-        return "UNKNOWNFOP";
+            return "UNKNOWN";
+    }
 }
 
 const char *
-gf_inode_type_to_str (ia_type_t type)
+gf_inode_type_to_str(ia_type_t type)
 {
-        static const char *const str_ia_type[] = {
-                "UNKNOWN",
-                "REGULAR FILE",
-                "DIRECTORY",
-                "LINK",
-                "BLOCK DEVICE",
-                "CHARACTER DEVICE",
-                "PIPE",
-                "SOCKET"};
-        return str_ia_type[type];
+    static const char *const str_ia_type[] = {
+        "UNKNOWN",      "REGULAR FILE",     "DIRECTORY", "LINK",
+        "BLOCK DEVICE", "CHARACTER DEVICE", "PIPE",      "SOCKET"};
+    return str_ia_type[type];
 }
 
 gf_boolean_t
-gf_is_zero_filled_stat (struct iatt *buf)
+gf_is_zero_filled_stat(struct iatt *buf)
 {
-        if (!buf)
-                return 1;
+    if (!buf)
+        return 1;
 
-        /* Do not use st_dev because it is transformed to store the xlator id
-         * in place of the device number. Do not use st_ino because by this time
-         * we've already mapped the root ino to 1 so it is not guaranteed to be
-         * 0.
-         */
-        if ((buf->ia_nlink == 0) && (buf->ia_ctime == 0))
-                return 1;
+    /* Do not use st_dev because it is transformed to store the xlator id
+     * in place of the device number. Do not use st_ino because by this time
+     * we've already mapped the root ino to 1 so it is not guaranteed to be
+     * 0.
+     */
+    if ((buf->ia_nlink == 0) && (buf->ia_ctime == 0))
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 void
-gf_zero_fill_stat (struct iatt *buf)
+gf_zero_fill_stat(struct iatt *buf)
 {
-        buf->ia_nlink = 0;
-        buf->ia_ctime = 0;
+    buf->ia_nlink = 0;
+    buf->ia_ctime = 0;
 }
 
 gf_boolean_t
-gf_is_valid_xattr_namespace (char *key)
+gf_is_valid_xattr_namespace(char *key)
 {
-        static char *xattr_namespaces[] = {"trusted.", "security.", "system.",
-                                           "user.", NULL };
-        int i = 0;
+    static char *xattr_namespaces[] = {"trusted.", "system.", "user.",
+                                       "security.", NULL};
+    int i = 0;
 
-        for (i = 0; xattr_namespaces[i]; i++) {
-                if (strncmp (key, xattr_namespaces[i],
-                             strlen (xattr_namespaces[i])) == 0)
-                        return _gf_true;
-        }
+    for (i = 0; xattr_namespaces[i]; i++) {
+        if (strncmp(key, xattr_namespaces[i], strlen(xattr_namespaces[i])) == 0)
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 ino_t
-gfid_to_ino (uuid_t gfid)
+gfid_to_ino(uuid_t gfid)
 {
-        ino_t ino = 0;
-        int32_t i;
+    ino_t ino = 0;
+    int32_t i;
 
-        for (i = 8; i < 16; i++) {
-                ino <<= 8;
-                ino += (uint8_t)gfid[i];
-        }
+    for (i = 8; i < 16; i++) {
+        ino <<= 8;
+        ino += (uint8_t)gfid[i];
+    }
 
-        return ino;
+    return ino;
 }
 
 int
-gf_bits_count (uint64_t n)
+gf_bits_count(uint64_t n)
 {
-        int val = 0;
-#ifdef _GNU_SOURCE
-        val = __builtin_popcountll (n);
+    int val = 0;
+#if defined(__GNUC__) || defined(__clang__)
+    val = __builtin_popcountll(n);
 #else
-        n -= (n >> 1) & 0x5555555555555555ULL;
-        n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL);
-        n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
-        n += n >> 8;
-        n += n >> 16;
-        n += n >> 32;
-        val = n & 0xFF;
+    n -= (n >> 1) & 0x5555555555555555ULL;
+    n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL);
+    n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+    n += n >> 8;
+    n += n >> 16;
+    n += n >> 32;
+    val = n & 0xFF;
 #endif
-        return val;
+    return val;
 }
 
 int
-gf_bits_index (uint64_t n)
+gf_bits_index(uint64_t n)
 {
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_ffsll(n) - 1;
+#else
     return ffsll(n) - 1;
+#endif
+}
+
+const char *
+gf_fop_string(glusterfs_fop_t fop)
+{
+    if ((fop > GF_FOP_NULL) && (fop < GF_FOP_MAXVALUE))
+        return gf_fop_list[fop];
+    return "INVALID";
+}
+
+int
+gf_fop_int(char *fop)
+{
+    int i = 0;
+
+    for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++) {
+        if (strcasecmp(fop, gf_fop_list[i]) == 0)
+            return i;
+    }
+    return -1;
+}
+
+int
+close_fds_except(int *fdv, size_t count)
+{
+    int i = 0;
+    size_t j = 0;
+    gf_boolean_t should_close = _gf_true;
+#ifdef GF_LINUX_HOST_OS
+    DIR *d = NULL;
+    struct dirent *de = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char *e = NULL;
+
+    d = sys_opendir("/proc/self/fd");
+    if (!d)
+        return -1;
+
+    for (;;) {
+        should_close = _gf_true;
+
+        errno = 0;
+        de = sys_readdir(d, scratch);
+        if (!de || errno != 0)
+            break;
+        i = strtoul(de->d_name, &e, 10);
+        if (*e != '\0' || i == dirfd(d))
+            continue;
+
+        for (j = 0; j < count; j++) {
+            if (i == fdv[j]) {
+                should_close = _gf_false;
+                break;
+            }
+        }
+        if (should_close)
+            sys_close(i);
+    }
+    sys_closedir(d);
+#else  /* !GF_LINUX_HOST_OS */
+    struct rlimit rl;
+    int ret = -1;
+
+    ret = getrlimit(RLIMIT_NOFILE, &rl);
+    if (ret)
+        return ret;
+
+    for (i = 0; i < rl.rlim_cur; i++) {
+        should_close = _gf_true;
+        for (j = 0; j < count; j++) {
+            if (i == fdv[j]) {
+                should_close = _gf_false;
+                break;
+            }
+        }
+        if (should_close)
+            sys_close(i);
+    }
+#endif /* !GF_LINUX_HOST_OS */
+    return 0;
+}
+
+/**
+ * gf_getgrouplist - get list of groups to which a user belongs
+ *
+ * A convenience wrapper for getgrouplist(3).
+ *
+ * @param user - same as in getgrouplist(3)
+ * @param group - same as in getgrouplist(3)
+ * @param groups - pointer to a gid_t pointer
+ *
+ * gf_getgrouplist allocates a gid_t buffer which is big enough to
+ * hold the list of auxiliary group ids for user, up to the GF_MAX_AUX_GROUPS
+ * threshold. Upon successful invocation groups will be pointed to that buffer.
+ *
+ * @return success: the number of auxiliary group ids retrieved
+ *         failure: -1
+ */
+int
+gf_getgrouplist(const char *user, gid_t group, gid_t **groups)
+{
+    int ret = -1;
+    int ngroups = SMALL_GROUP_COUNT;
+
+    *groups = GF_CALLOC(sizeof(gid_t), ngroups, gf_common_mt_groups_t);
+    if (!*groups)
+        return -1;
+
+    /*
+     * We are running getgrouplist() in a loop until we succeed (or hit
+     * certain exit conditions, see the comments below). This is because
+     * the indicated number of auxiliary groups that we obtain in case of
+     * the failure of the first invocation is not guaranteed to keep its
+     * validity upon the next invocation with a gid buffer of that size.
+     */
+    for (;;) {
+        int ngroups_old = ngroups;
+        ret = getgrouplist(user, group, *groups, &ngroups);
+        if (ret != -1)
+            break;
+
+        if (ngroups >= GF_MAX_AUX_GROUPS) {
+            /*
+             * This should not happen as GF_MAX_AUX_GROUPS is set
+             * to the max value of number of supported auxiliary
+             * groups across all platforms supported by GlusterFS.
+             * However, if it still happened some way, we wouldn't
+             * care about the incompleteness of the result, we'd
+             * just go on with what we got.
+             */
+            return GF_MAX_AUX_GROUPS;
+        } else if (ngroups <= ngroups_old) {
+            /*
+             * There is an edge case that getgrouplist() fails but
+             * ngroups remains the same. This is actually not
+             * specified in getgrouplist(3), but implementations
+             * can do this upon internal failure[1]. To avoid
+             * falling into an infinite loop when this happens, we
+             * break the loop if the getgrouplist call failed
+             * without an increase in the indicated group number.
+             *
+             * [1]
+             * https://sourceware.org/git/?p=glibc.git;a=blob;f=grp/initgroups.c;hb=refs/heads/release/2.25/master#l168
+             */
+            GF_FREE(*groups);
+            return -1;
+        }
+
+        *groups = GF_REALLOC(*groups, ngroups * sizeof(gid_t));
+        if (!*groups)
+            return -1;
+    }
+    return ret;
+}
+
+int
+glusterfs_compute_sha256(const unsigned char *content, size_t size,
+                         char *sha256_hash)
+{
+    SHA256_CTX sha256;
+
+    SHA256_Init(&sha256);
+    SHA256_Update(&sha256, (const unsigned char *)(content), size);
+    SHA256_Final((unsigned char *)sha256_hash, &sha256);
+
+    return 0;
+}
+
+/* * Safe wrapper function for strncpy.
+ * This wrapper makes sure that when there is no null byte among the first n in
+ * source srting for strncpy function call, the string placed in dest will be
+ * null-terminated.
+ */
+
+char *
+gf_strncpy(char *dest, const char *src, const size_t dest_size)
+{
+    strncpy(dest, src, dest_size - 1);
+    dest[dest_size - 1] = '\0';
+    return dest;
+}
+
+int
+gf_replace_old_iatt_in_dict(dict_t *xdata)
+{
+    int ret;
+    struct old_iatt *o_iatt; /* old iatt structure */
+    struct iatt *c_iatt;     /* current iatt */
+
+    if (!xdata) {
+        return 0;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&c_iatt);
+    if (ret < 0) {
+        return 0;
+    }
+
+    o_iatt = GF_CALLOC(1, sizeof(struct old_iatt), gf_common_mt_char);
+    if (!o_iatt) {
+        return -1;
+    }
+
+    oldiatt_from_iatt(o_iatt, c_iatt);
+
+    ret = dict_set_bin(xdata, DHT_IATT_IN_XDATA_KEY, o_iatt,
+                       sizeof(struct old_iatt));
+    if (ret) {
+        GF_FREE(o_iatt);
+    }
+
+    return ret;
+}
+
+int
+gf_replace_new_iatt_in_dict(dict_t *xdata)
+{
+    int ret;
+    struct old_iatt *o_iatt; /* old iatt structure */
+    struct iatt *c_iatt;     /* new iatt */
+
+    if (!xdata) {
+        return 0;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&o_iatt);
+    if (ret < 0) {
+        return 0;
+    }
+
+    c_iatt = GF_CALLOC(1, sizeof(struct iatt), gf_common_mt_char);
+    if (!c_iatt) {
+        return -1;
+    }
+
+    iatt_from_oldiatt(c_iatt, o_iatt);
+
+    ret = dict_set_bin(xdata, DHT_IATT_IN_XDATA_KEY, c_iatt,
+                       sizeof(struct iatt));
+    if (ret) {
+        GF_FREE(c_iatt);
+    }
+
+    return ret;
+}
+
+xlator_cmdline_option_t *
+find_xlator_option_in_cmd_args_t(const char *option_name, cmd_args_t *args)
+{
+    xlator_cmdline_option_t *pos = NULL;
+    xlator_cmdline_option_t *tmp = NULL;
+
+    list_for_each_entry_safe(pos, tmp, &args->xlator_options, cmd_args)
+    {
+        if (strcmp(pos->key, option_name) == 0)
+            return pos;
+    }
+    return NULL;
+}
+
+int
+gf_d_type_from_ia_type(ia_type_t type)
+{
+    switch (type) {
+        case IA_IFDIR:
+            return DT_DIR;
+        case IA_IFCHR:
+            return DT_CHR;
+        case IA_IFBLK:
+            return DT_BLK;
+        case IA_IFIFO:
+            return DT_FIFO;
+        case IA_IFLNK:
+            return DT_LNK;
+        case IA_IFREG:
+            return DT_REG;
+        case IA_IFSOCK:
+            return DT_SOCK;
+        default:
+            return DT_UNKNOWN;
+    }
+}
+
+int
+gf_nanosleep(uint64_t nsec)
+{
+    struct timespec req;
+    struct timespec rem;
+    int ret = -1;
+
+    req.tv_sec = nsec / GF_SEC_IN_NS;
+    req.tv_nsec = nsec % GF_SEC_IN_NS;
+
+    do {
+        ret = nanosleep(&req, &rem);
+        req = rem;
+    } while (ret == -1 && errno == EINTR);
+
+    return ret;
+}
+
+int
+gf_syncfs(int fd)
+{
+    int ret = 0;
+#if defined(HAVE_SYNCFS)
+    /* Linux with glibc recent enough. */
+    ret = syncfs(fd);
+#elif defined(HAVE_SYNCFS_SYS)
+    /* Linux with no library function. */
+    ret = syscall(SYS_syncfs, fd);
+#else
+    /* Fallback to generic UNIX stuff. */
+    sync();
+#endif
+    return ret;
+}
+
+char **
+get_xattrs_to_heal()
+{
+    return xattrs_to_heal;
 }
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
deleted file mode 100644
index 72613acba3b..00000000000
--- a/libglusterfs/src/common-utils.h
+++ /dev/null
@@ -1,867 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _COMMON_UTILS_H
-#define _COMMON_UTILS_H
-
-#include <stdint.h>
-#include <sys/uio.h>
-#include <netdb.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <pthread.h>
-#include <openssl/md5.h>
-#ifndef GF_BSD_HOST_OS
-#include <alloca.h>
-#endif
-#include <limits.h>
-#include <fnmatch.h>
-
-#ifndef ffsll
-#define ffsll(x) __builtin_ffsll(x)
-#endif
-
-void trap (void);
-
-#define GF_UNIVERSAL_ANSWER 42    /* :O */
-
-/* To solve type punned error */
-#define VOID(ptr) ((void **) ((void *) ptr))
-
-#include "logging.h"
-#include "glusterfs.h"
-#include "locking.h"
-#include "mem-pool.h"
-#include "compat-uuid.h"
-#include "iatt.h"
-#include "uuid.h"
-#include "libglusterfs-messages.h"
-
-#define STRINGIFY(val) #val
-#define TOSTRING(val) STRINGIFY(val)
-
-#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr; })
-
-#define min(a,b) ((a)<(b)?(a):(b))
-#define max(a,b) ((a)>(b)?(a):(b))
-#define roof(a,b) ((((a)+(b)-1)/((b)?(b):1))*(b))
-#define floor(a,b) (((a)/((b)?(b):1))*(b))
-
-#define IPv4_ADDR_SIZE 32
-
-
-#define GF_UNIT_KB    1024ULL
-#define GF_UNIT_MB    1048576ULL
-#define GF_UNIT_GB    1073741824ULL
-#define GF_UNIT_TB    1099511627776ULL
-#define GF_UNIT_PB    1125899906842624ULL
-
-#define GF_UNIT_B_STRING     "B"
-#define GF_UNIT_KB_STRING    "KB"
-#define GF_UNIT_MB_STRING    "MB"
-#define GF_UNIT_GB_STRING    "GB"
-#define GF_UNIT_TB_STRING    "TB"
-#define GF_UNIT_PB_STRING    "PB"
-
-#define GF_UNIT_PERCENT_STRING      "%"
-
-#define GEOREP "geo-replication"
-#define GHADOOP "glusterfs-hadoop"
-#define GLUSTERD_NAME "glusterd"
-
-#define GF_SELINUX_XATTR_KEY "security.selinux"
-
-#define WIPE(statp) do { typeof(*statp) z = {0,}; if (statp) *statp = z; } while (0)
-
-#define IS_EXT_FS(fs_name)          \
-        (!strcmp (fs_name, "ext2") || \
-         !strcmp (fs_name, "ext3") || \
-         !strcmp (fs_name, "ext4"))
-
-/* process mode definitions */
-#define GF_SERVER_PROCESS   0
-#define GF_CLIENT_PROCESS   1
-#define GF_GLUSTERD_PROCESS 2
-
-/* Defining this here as it is needed by glusterd for setting
- * nfs port in volume status.
- */
-#define GF_NFS3_PORT    2049
-
-#define GF_CLIENT_PORT_CEILING 1024
-#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
-#define GF_CLNT_INSECURE_PORT_CEILING (GF_IANA_PRIV_PORTS_START - 1)
-#define GF_PORT_MAX 65535
-
-#define GF_MINUTE_IN_SECONDS 60
-#define GF_HOUR_IN_SECONDS (60*60)
-#define GF_DAY_IN_SECONDS (24*60*60)
-#define GF_WEEK_IN_SECONDS (7*24*60*60)
-
-/* Default timeout for both barrier and changelog translator */
-#define BARRIER_TIMEOUT "120"
-
-/* Default value of signing waiting time to sign a file for bitrot */
-#define SIGNING_TIMEOUT "120"
-
-/* Shard */
-#define GF_XATTR_SHARD_FILE_SIZE  "trusted.glusterfs.shard.file-size"
-#define SHARD_ROOT_GFID "be318638-e8a0-4c6d-977d-7a937aa84806"
-
-/* Lease: buffer length for stringified lease id
- * Format: 4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum
- * Eg:6c69-6431-2d63-6c6e-7431-0000-0000-0000
- */
-#define GF_LEASE_ID_BUF_SIZE  ((LEASE_ID_SIZE * 2) +     \
-                               (LEASE_ID_SIZE / 2))
-
-enum _gf_boolean
-{
-	_gf_false = 0,
-	_gf_true = 1
-};
-
-/*
- * we could have initialized these as +ve values and treated
- * them as negative while comparing etc.. (which would have
- * saved us with the pain of assigning values), but since we
- * only have a few clients that use this feature, it's okay.
- */
-enum _gf_special_pid
-{
-        GF_CLIENT_PID_MAX               =  0,
-        GF_CLIENT_PID_GSYNCD            = -1,
-        GF_CLIENT_PID_HADOOP            = -2,
-        GF_CLIENT_PID_DEFRAG            = -3,
-        GF_CLIENT_PID_NO_ROOT_SQUASH    = -4,
-        GF_CLIENT_PID_QUOTA_MOUNT       = -5,
-        GF_CLIENT_PID_SELF_HEALD        = -6,
-        GF_CLIENT_PID_GLFS_HEAL         = -7,
-        GF_CLIENT_PID_BITD              = -8,
-        GF_CLIENT_PID_SCRUB             = -9,
-        GF_CLIENT_PID_TIER_DEFRAG       = -10,
-        GF_SERVER_PID_TRASH             = -11
-};
-
-enum _gf_xlator_ipc_targets {
-        GF_IPC_TARGET_CHANGELOG = 0,
-        GF_IPC_TARGET_CTR = 1,
-        GF_IPC_TARGET_UPCALL = 2
-};
-
-typedef enum _gf_boolean gf_boolean_t;
-typedef enum _gf_special_pid gf_special_pid_t;
-typedef enum _gf_xlator_ipc_targets _gf_xlator_ipc_targets_t;
-
-/* The DHT file rename operation is not a straightforward rename.
- * It involves creating linkto and linkfiles, and can unlink or rename the
- * source file depending on the hashed and cached subvols for the source
- * and target files. this makes it difficult for geo-rep to figure out that
- * a rename operation has taken place.
- *
- * We now send a special key and the values of the source and target pargfids
- * and basenames to indicate to changelog that the operation in question
- * should be treated as a rename. We are explicitly filling and sending this
- * as a binary value in the dictionary as the unlink op will not have the
- * source file information. The lengths of the src and target basenames
- * are used to calculate where to start reading the names in the structure.
- * XFS allows a max of 255 chars for filenames but other file systems might
- * not have such restrictions
- */
-typedef struct dht_changelog_rename_info {
-         uuid_t  old_pargfid;
-         uuid_t  new_pargfid;
-         int32_t oldname_len;
-         int32_t newname_len;
-         char    buffer[1];
- } dht_changelog_rename_info_t;
-
-
-typedef int (*gf_cmp) (void *, void *);
-
-struct _dict;
-
-struct dnscache {
-        struct _dict *cache_dict;
-        time_t ttl;
-};
-
-struct dnscache_entry {
-        char *ip;
-        char *fqdn;
-        time_t timestamp;
-};
-
-struct dnscache6 {
-        struct addrinfo *first;
-        struct addrinfo *next;
-};
-
-struct list_node {
-        void *ptr;
-        struct list_head list;
-};
-
-extern char *vol_type_str[];
-
-struct list_node *list_node_add (void *ptr, struct list_head *list);
-struct list_node *list_node_add_order (void *ptr, struct list_head *list,
-                                       int (*compare)(struct list_head *,
-                                            struct list_head *));
-void list_node_del (struct list_node *node);
-
-struct dnscache *gf_dnscache_init (time_t ttl);
-struct dnscache_entry *gf_dnscache_entry_init (void);
-void gf_dnscache_entry_deinit (struct dnscache_entry *entry);
-char *gf_rev_dns_lookup_cached (const char *ip, struct dnscache *dnscache);
-
-char *gf_resolve_path_parent (const char *path);
-
-void gf_global_variable_init(void);
-
-int32_t gf_resolve_ip6 (const char *hostname, uint16_t port, int family,
-                        void **dnscache, struct addrinfo **addr_info);
-
-void gf_log_dump_graph (FILE *specfp, glusterfs_graph_t *graph);
-void gf_print_trace (int32_t signal, glusterfs_ctx_t *ctx);
-int  gf_set_log_file_path (cmd_args_t *cmd_args, glusterfs_ctx_t *ctx);
-int  gf_set_log_ident (cmd_args_t *cmd_args);
-
-#define VECTORSIZE(count) (count * (sizeof (struct iovec)))
-
-#define STRLEN_0(str) (strlen(str) + 1)
-
-#define VALIDATE_OR_GOTO(arg,label)   do {				\
-		if (!arg) {						\
-			errno = EINVAL;					\
-			gf_msg_callingfn ((this ? (this->name) :        \
-                                           "(Govinda! Govinda!)"),      \
-                                          GF_LOG_WARNING, EINVAL,       \
-                                          LG_MSG_INVALID_ARG,           \
-                                          "invalid argument: " #arg);   \
-			goto label;					\
-		}							\
-	} while (0)
-
-#define GF_VALIDATE_OR_GOTO(name,arg,label)   do {                      \
-		if (!arg) {                                             \
-			errno = EINVAL;                                 \
-			gf_msg_callingfn (name, GF_LOG_ERROR, errno,    \
-                                          LG_MSG_INVALID_ARG,           \
-                                          "invalid argument: " #arg);	\
-			goto label;                                     \
-		}                                                       \
-	} while (0)
-
-#define GF_VALIDATE_OR_GOTO_WITH_ERROR(name, arg, label, errno, error) do { \
-                if (!arg) {                                                 \
-                        errno = error;                                  \
-                        gf_msg_callingfn (name, GF_LOG_ERROR, EINVAL,   \
-                                          LG_MSG_INVALID_ARG,         \
-                                          "invalid argument: " #arg);   \
-                        goto label;                                     \
-                }                                                       \
-        }while (0)
-
-#define GF_CHECK_ALLOC(arg, retval, label)   do {                       \
-                if (!(arg)) {                                           \
-                        retval = -ENOMEM;                               \
-                        goto label;                                     \
-                }                                                       \
-        } while (0)                                                     \
-
-#define GF_CHECK_ALLOC_AND_LOG(name, item, retval, msg, errlabel) do {  \
-                if (!(item)) {                                          \
-                        (retval) = -ENOMEM;                             \
-                        gf_msg (name, GF_LOG_CRITICAL, ENOMEM,          \
-                                LG_MSG_NO_MEMORY, (msg));               \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)
-
-#define GF_ASSERT_AND_GOTO_WITH_ERROR(name, arg, label, errno, error) do { \
-                if (!arg) {                                             \
-                        GF_ASSERT (0);                                  \
-                        errno = error;                                  \
-                        goto label;                                     \
-                }                                                       \
-        }while (0)
-
-#define GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO(name,arg,label)               \
-        do {                                                            \
-                GF_VALIDATE_OR_GOTO (name, arg, label);                 \
-                if ((arg[0]) != '/') {                                  \
-                        errno = EINVAL;                                 \
-			gf_msg_callingfn (name, GF_LOG_ERROR, EINVAL,   \
-                                          LG_MSG_INVALID_ARG,           \
-                                          "invalid argument: " #arg);	\
-                        goto label;                                     \
-                }                                                       \
-	} while (0)
-
-#define GF_REMOVE_SLASH_FROM_PATH(path, string)                         \
-        do {                                                            \
-                int i = 0;                                              \
-                for (i = 1; i < strlen (path); i++) {                   \
-                        string[i-1] = path[i];                          \
-                        if (string[i-1] == '/')                         \
-                                string[i-1] = '-';                      \
-                }                                                       \
-        } while (0)
-
-#define GF_REMOVE_INTERNAL_XATTR(pattern, dict)                         \
-        do {                                                            \
-                if (!dict) {                                            \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,            \
-                                LG_MSG_DICT_NULL, "dict is null");      \
-                        break;                                          \
-                }                                                       \
-                dict_foreach_fnmatch (dict, pattern,                    \
-                                      dict_remove_foreach_fn,           \
-                                      NULL);                            \
-        } while (0)
-
-#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, op_errno, label)       \
-        do {                                                            \
-                if (!dict) {                                            \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,            \
-                                LG_MSG_DICT_NULL,                        \
-                                "setxattr dict is null");               \
-                        goto label;                                     \
-                }                                                       \
-                if (dict_foreach_fnmatch (dict, pattern,                \
-                                          dict_null_foreach_fn,         \
-                                          NULL) > 0) {                  \
-                        op_errno = EPERM;                               \
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,     \
-                                LG_MSG_NO_PERM,                         \
-                                "attempt to set internal"               \
-                                " xattr: %s", pattern);                 \
-                        goto label;                                     \
-                }                                                       \
-        } while (0)
-
-#define GF_IF_NATIVE_XATTR_GOTO(pattern, key, op_errno, label)          \
-        do {                                                            \
-                if (!key) {                                             \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,            \
-                                LG_MSG_NO_KEY,                          \
-                                "no key for removexattr");              \
-                        goto label;                                     \
-                }                                                       \
-                if (!fnmatch (pattern, key, 0)) {                       \
-                        op_errno = EPERM;                               \
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,     \
-                                LG_MSG_NO_PERM,                         \
-                                "attempt to remove internal "           \
-                                "xattr: %s", key);                      \
-                        goto label;                                     \
-                }                                                       \
-        } while (0)
-
-
-#define GF_FILE_CONTENT_REQUESTED(_xattr_req,_content_limit) \
-	(dict_get_uint64 (_xattr_req, "glusterfs.content", _content_limit) == 0)
-
-#ifdef DEBUG
-#define GF_ASSERT(x) assert (x);
-#else
-#define GF_ASSERT(x)                                                    \
-        do {                                                            \
-                if (!(x)) {                                             \
-                        gf_msg_callingfn ("", GF_LOG_ERROR, 0,          \
-                                          LG_MSG_ASSERTION_FAILED,      \
-                                          "Assertion failed: " #x);     \
-                }                                                       \
-        } while (0)
-#endif
-
-#define GF_UUID_ASSERT(u) \
-        if (gf_uuid_is_null (u))\
-                GF_ASSERT (!"uuid null");
-
-#define GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, op_errno)                 \
-        (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&                 \
-          (op_errno == EEXIST || op_errno == ENOENT))?0:1)              \
-
-union gf_sock_union {
-        struct sockaddr_storage storage;
-        struct sockaddr_in6 sin6;
-        struct sockaddr_in sin;
-        struct sockaddr sa;
-};
-
-#define GF_HIDDEN_PATH ".glusterfs"
-#define GF_UNLINK_PATH GF_HIDDEN_PATH"/unlink"
-#define GF_LANDFILL_PATH GF_HIDDEN_PATH"/landfill"
-
-#define IOV_MIN(n) min(IOV_MAX,n)
-
-#define GF_FOR_EACH_ENTRY_IN_DIR(entry, dir, scr) \
-        do {\
-                entry = NULL;\
-                if (dir) { \
-                        entry = sys_readdir (dir, scr); \
-                        while (entry && (!strcmp (entry->d_name, ".") || \
-                            !fnmatch ("*.tmp", entry->d_name, 0) || \
-                            !strcmp (entry->d_name, ".."))) { \
-                                entry = sys_readdir (dir, scr); \
-                        } \
-                } \
-        } while (0)
-
-static inline void
-iov_free (struct iovec *vector, int count)
-{
-	int i;
-
-	for (i = 0; i < count; i++)
-		FREE (vector[i].iov_base);
-
-	GF_FREE (vector);
-}
-
-
-static inline int
-iov_length (const struct iovec *vector, int count)
-{
-	int     i = 0;
-	size_t  size = 0;
-
-	for (i = 0; i < count; i++)
-		size += vector[i].iov_len;
-
-	return size;
-}
-
-
-static inline struct iovec *
-iov_dup (const struct iovec *vector, int count)
-{
-	int           bytecount = 0;
-	int           i;
-	struct iovec *newvec = NULL;
-
-	bytecount = (count * sizeof (struct iovec));
-	newvec = GF_MALLOC (bytecount, gf_common_mt_iovec);
-	if (!newvec)
-		return NULL;
-
-	for (i = 0; i < count; i++) {
-		newvec[i].iov_len  = vector[i].iov_len;
-		newvec[i].iov_base = vector[i].iov_base;
-	}
-
-	return newvec;
-}
-
-
-static inline int
-iov_subset (struct iovec *orig, int orig_count,
-	    off_t src_offset, off_t dst_offset,
-	    struct iovec *new)
-{
-	int    new_count = 0;
-	int    i;
-	off_t  offset = 0;
-	size_t start_offset = 0;
-	size_t end_offset = 0, origin_iov_len = 0;
-
-
-	for (i = 0; i < orig_count; i++) {
-                origin_iov_len = orig[i].iov_len;
-
-		if ((offset + orig[i].iov_len < src_offset)
-		    || (offset > dst_offset)) {
-			goto not_subset;
-		}
-
-		if (!new) {
-			goto count_only;
-		}
-
-		start_offset = 0;
-		end_offset = orig[i].iov_len;
-
-		if (src_offset >= offset) {
-			start_offset = (src_offset - offset);
-		}
-
-		if (dst_offset <= (offset + orig[i].iov_len)) {
-			end_offset = (dst_offset - offset);
-		}
-
-		new[new_count].iov_base = orig[i].iov_base + start_offset;
-		new[new_count].iov_len = end_offset - start_offset;
-
-	count_only:
-		new_count++;
-
-	not_subset:
-		offset += origin_iov_len;
-	}
-
-	return new_count;
-}
-
-
-static inline void
-iov_unload (char *buf, const struct iovec *vector, int count)
-{
-	int i;
-	int copied = 0;
-
-	for (i = 0; i < count; i++) {
-		memcpy (buf + copied, vector[i].iov_base, vector[i].iov_len);
-		copied += vector[i].iov_len;
-	}
-}
-
-
-static inline size_t
-iov_load (const struct iovec *vector, int count, char *buf, int size)
-{
-	size_t left = size;
-	size_t cp = 0;
-	int    ret = 0;
-	int    i = 0;
-
-	while (left && i < count) {
-		cp = min (vector[i].iov_len, left);
-		if (vector[i].iov_base != buf + (size - left))
-			memcpy (vector[i].iov_base, buf + (size - left), cp);
-		ret += cp;
-		left -= cp;
-		if (left)
-			i++;
-	}
-
-	return ret;
-}
-
-
-static inline size_t
-iov_copy (const struct iovec *dst, int dcnt,
-	  const struct iovec *src, int scnt)
-{
-	size_t  ret = 0;
-	size_t  left = 0;
-	size_t  min_i = 0;
-	int     s_i = 0, s_ii = 0;
-	int     d_i = 0, d_ii = 0;
-
-	ret = min (iov_length (dst, dcnt), iov_length (src, scnt));
-	left = ret;
-
-	while (left) {
-		min_i = min (dst[d_i].iov_len - d_ii, src[s_i].iov_len - s_ii);
-		memcpy (dst[d_i].iov_base + d_ii, src[s_i].iov_base + s_ii,
-			min_i);
-
-		d_ii += min_i;
-		if (d_ii == dst[d_i].iov_len) {
-			d_ii = 0;
-			d_i++;
-		}
-
-		s_ii += min_i;
-		if (s_ii == src[s_i].iov_len) {
-			s_ii = 0;
-			s_i++;
-		}
-
-		left -= min_i;
-	}
-
-	return ret;
-}
-
-
-static inline int
-mem_0filled (const char *buf, size_t size)
-{
-	int i = 0;
-	int ret = 0;
-
-	for (i = 0; i < size; i++) {
-		ret = buf[i];
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-
-static inline int
-iov_0filled (struct iovec *vector, int count)
-{
-	int i = 0;
-	int ret = 0;
-
-	for (i = 0; i < count; i++) {
-		ret = mem_0filled (vector[i].iov_base, vector[i].iov_len);
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-
-static inline void *
-memdup (const void *ptr, size_t size)
-{
-	void *newptr = NULL;
-
-	newptr = GF_MALLOC (size, gf_common_mt_memdup);
-	if (!newptr)
-		return NULL;
-
-	memcpy (newptr, ptr, size);
-	return newptr;
-}
-
-typedef enum {
-        gf_timefmt_default = 0,
-        gf_timefmt_FT = 0,  /* YYYY-MM-DD hh:mm:ss */
-        gf_timefmt_Ymd_T,   /* YYYY/MM-DD-hh:mm:ss */
-        gf_timefmt_bdT,     /* MMM DD hh:mm:ss */
-        gf_timefmt_F_HMS,   /* YYYY-MM-DD hhmmss */
-	gf_timefmt_dirent,
-        gf_timefmt_s,
-        gf_timefmt_last
-} gf_timefmts;
-
-static inline char *
-gf_time_fmt (char *dst, size_t sz_dst, time_t utime, unsigned int fmt)
-{
-        extern void _gf_timestuff (gf_timefmts *, const char ***, const char ***);
-        static gf_timefmts timefmt_last = (gf_timefmts) - 1;
-        static const char **fmts;
-        static const char **zeros;
-        struct tm tm;
-
-        if (timefmt_last == (gf_timefmts) - 1)
-                _gf_timestuff (&timefmt_last, &fmts, &zeros);
-        if (timefmt_last < fmt) fmt = gf_timefmt_default;
-        if (utime && gmtime_r (&utime, &tm) != NULL) {
-                strftime (dst, sz_dst, fmts[fmt], &tm);
-        } else {
-                strncpy (dst, "N/A", sz_dst);
-        }
-        return dst;
-}
-
-/* This function helps us use gfid (unique identity) to generate inode's unique
- * number in glusterfs.
- */
-ino_t
-gfid_to_ino (uuid_t gfid);
-
-int
-mkdir_p (char *path, mode_t mode, gf_boolean_t allow_symlinks);
-/*
- * rounds up nr to power of two. If nr is already a power of two, just returns
- * nr
- */
-
-int
-gf_lstat_dir (const char *path, struct stat *stbuf_in);
-
-int32_t gf_roundup_power_of_two (int32_t nr);
-
-/*
- * rounds up nr to next power of two. If nr is already a power of two, next
- * power of two is returned.
- */
-
-int32_t gf_roundup_next_power_of_two (int32_t nr);
-
-char *gf_trim (char *string);
-int gf_strsplit (const char *str, const char *delim,
-		 char ***tokens, int *token_count);
-int gf_volume_name_validate (const char *volume_name);
-
-int gf_string2long (const char *str, long *n);
-int gf_string2ulong (const char *str, unsigned long *n);
-int gf_string2int (const char *str, int *n);
-int gf_string2uint (const char *str, unsigned int *n);
-int gf_string2double (const char *str, double *n);
-int gf_string2longlong (const char *str, long long *n);
-int gf_string2ulonglong (const char *str, unsigned long long *n);
-
-int gf_string2int8 (const char *str, int8_t *n);
-int gf_string2int16 (const char *str, int16_t *n);
-int gf_string2int32 (const char *str, int32_t *n);
-int gf_string2int64 (const char *str, int64_t *n);
-int gf_string2uint8 (const char *str, uint8_t *n);
-int gf_string2uint16 (const char *str, uint16_t *n);
-int gf_string2uint32 (const char *str, uint32_t *n);
-int gf_string2uint64 (const char *str, uint64_t *n);
-
-int gf_strstr (const char *str, const char *delim, const char *match);
-
-int gf_string2ulong_base10 (const char *str, unsigned long *n);
-int gf_string2uint_base10 (const char *str, unsigned int *n);
-int gf_string2uint8_base10 (const char *str, uint8_t *n);
-int gf_string2uint16_base10 (const char *str, uint16_t *n);
-int gf_string2uint32_base10 (const char *str, uint32_t *n);
-int gf_string2uint64_base10 (const char *str, uint64_t *n);
-int gf_string2bytesize (const char *str, uint64_t *n);
-int gf_string2bytesize_size (const char *str, size_t *n);
-int gf_string2bytesize_uint64 (const char *str, uint64_t *n);
-int gf_string2bytesize_int64 (const char *str, int64_t *n);
-int gf_string2percent_or_bytesize (const char *str, double *n,
-				   gf_boolean_t *is_percent);
-
-int gf_string2boolean (const char *str, gf_boolean_t *b);
-int gf_string2percent (const char *str, double *n);
-int gf_string2time (const char *str, uint32_t *n);
-
-int gf_lockfd (int fd);
-int gf_unlockfd (int fd);
-
-int get_checksum_for_file (int fd, uint32_t *checksum);
-int log_base2 (unsigned long x);
-
-int get_checksum_for_path (char *path, uint32_t *checksum);
-int get_file_mtime (const char *path, time_t *stamp);
-char *gf_resolve_path_parent (const char *path);
-
-char *strtail (char *str, const char *pattern);
-void skipwhite (char **s);
-char *nwstrtail (char *str, char *pattern);
-void skip_word (char **str);
-/* returns a new string with nth word of given string. n>=1 */
-char *get_nth_word (const char *str, int n);
-
-gf_boolean_t mask_match (const uint32_t a, const uint32_t b, const uint32_t m);
-gf_boolean_t gf_is_ip_in_net (const char *network, const char *ip_str);
-char valid_host_name (char *address, int length);
-char valid_ipv4_address (char *address, int length, gf_boolean_t wildcard_acc);
-char valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc);
-char valid_internet_address (char *address, gf_boolean_t wildcard_acc);
-gf_boolean_t valid_mount_auth_address (char *address);
-gf_boolean_t valid_ipv4_subnetwork (const char *address);
-gf_boolean_t gf_sock_union_equal_addr (union gf_sock_union *a,
-                                       union gf_sock_union *b);
-char *gf_rev_dns_lookup (const char *ip);
-
-char *uuid_utoa (uuid_t uuid);
-char *uuid_utoa_r (uuid_t uuid, char *dst);
-char *lkowner_utoa (gf_lkowner_t *lkowner);
-char *lkowner_utoa_r (gf_lkowner_t *lkowner, char *dst, int len);
-char *leaseid_utoa (const char *lease_id);
-gf_boolean_t is_valid_lease_id (const char *lease_id);
-
-void gf_array_insertionsort (void *a, int l, int r, size_t elem_size,
-                             gf_cmp cmp);
-int gf_is_str_int (const char *value);
-
-char *gf_uint64_2human_readable (uint64_t);
-int get_vol_type (int type, int dist_count, int brick_count);
-int validate_brick_name (char *brick);
-char *get_host_name (char *word, char **host);
-char *get_path_name (char *word, char **path);
-void gf_path_strip_trailing_slashes (char *path);
-uint64_t get_mem_size (void);
-int gf_strip_whitespace (char *str, int len);
-int gf_canonicalize_path (char *path);
-char *generate_glusterfs_ctx_id (void);
-char *gf_get_reserved_ports(void);
-int gf_process_reserved_ports (gf_boolean_t ports[], uint32_t ceiling);
-gf_boolean_t
-gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling);
-int gf_get_hostname_from_ip (char *client_ip, char **hostname);
-gf_boolean_t gf_is_local_addr (char *hostname);
-gf_boolean_t gf_is_same_address (char *host1, char *host2);
-void md5_wrapper(const unsigned char *data, size_t len, char *md5);
-int gf_set_timestamp  (const char *src, const char* dest);
-
-int gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
-                      void *(*start_routine)(void *), void *arg);
-int gf_thread_create_detached (pthread_t *thread,
-                      void *(*start_routine)(void *), void *arg);
-
-gf_boolean_t
-gf_is_service_running (char *pidfile, int *pid);
-int
-gf_skip_header_section (int fd, int header_len);
-
-struct iatt;
-struct _dict;
-
-gf_boolean_t
-dht_is_linkfile (struct iatt *buf, struct _dict *dict);
-
-int
-gf_check_log_format (const char *value);
-
-int
-gf_check_logger (const char *value);
-
-gf_boolean_t
-gf_compare_sockaddr (const struct sockaddr *addr1,
-                     const struct sockaddr *addr2);
-
-char *
-gf_backtrace_save (char *buf);
-
-void
-gf_backtrace_done (char *buf);
-
-gf_loglevel_t
-fop_log_level (glusterfs_fop_t fop, int op_errno);
-
-int32_t
-gf_build_absolute_path (char *current_path, char *relative_path, char **path);
-
-int
-recursive_rmdir (const char *delete_path);
-
-int
-gf_get_index_by_elem (char **array, char *elem);
-
-int
-glusterfs_is_local_pathinfo (char *pathinfo, gf_boolean_t *local);
-
-int
-gf_thread_cleanup_xint (pthread_t thread);
-
-ssize_t
-gf_nread (int fd, void *buf, size_t count);
-
-ssize_t
-gf_nwrite (int fd, const void *buf, size_t count);
-
-void _mask_cancellation (void);
-void _unmask_cancellation (void);
-
-gf_boolean_t
-gf_is_zero_filled_stat (struct iatt *buf);
-
-void
-gf_zero_fill_stat (struct iatt *buf);
-
-gf_boolean_t
-gf_is_valid_xattr_namespace (char *k);
-
-const char *
-gf_inode_type_to_str (ia_type_t type);
-
-int32_t
-gf_bits_count (uint64_t n);
-
-int32_t
-gf_bits_index (uint64_t n);
-
-#endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/compat-errno.c b/libglusterfs/src/compat-errno.c
index 3674596ad71..df57e243239 100644
--- a/libglusterfs/src/compat-errno.c
+++ b/libglusterfs/src/compat-errno.c
@@ -10,8 +10,7 @@
 
 #include <stdint.h>
 
-#include "compat-errno.h"
-
+#include "glusterfs/compat-errno.h"
 
 static int32_t gf_error_to_errno_array[1024];
 static int32_t gf_errno_to_error_array[1024];
@@ -20,912 +19,937 @@ static int32_t gf_compat_errno_init_done;
 
 #ifdef GF_SOLARIS_HOST_OS
 static void
-init_compat_errno_arrays ()
+init_compat_errno_arrays()
 {
-/*      ENOMSG  35      / * No message of desired type          */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-        gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-/*      EIDRM   36      / * Identifier removed                  */
-        gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-        gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-
-/*      ECHRNG  37      / * Channel number out of range         */
-        gf_error_to_errno_array[GF_ERROR_CODE_CHRNG] = ECHRNG;
-        gf_errno_to_error_array[ECHRNG] = GF_ERROR_CODE_CHRNG;
-
-/*      EL2NSYNC 38     / * Level 2 not synchronized            */
-        gf_error_to_errno_array[GF_ERROR_CODE_L2NSYNC] = EL2NSYNC;
-        gf_errno_to_error_array[EL2NSYNC] = GF_ERROR_CODE_L2NSYNC;
-
-/*      EL3HLT  39      / * Level 3 halted                      */
-        gf_error_to_errno_array[GF_ERROR_CODE_L3HLT] = EL3HLT;
-        gf_errno_to_error_array[EL3HLT] = GF_ERROR_CODE_L3HLT;
-
-/*      EL3RST  40      / * Level 3 reset                       */
-        gf_error_to_errno_array[GF_ERROR_CODE_L3RST] = EL3RST;
-        gf_errno_to_error_array[EL3RST] = GF_ERROR_CODE_L3RST;
-
-/*      ELNRNG  41      / * Link number out of range            */
-        gf_error_to_errno_array[GF_ERROR_CODE_LNRNG] = ELNRNG;
-        gf_errno_to_error_array[ELNRNG] = GF_ERROR_CODE_LNRNG;
-
-/*      EUNATCH 42      / * Protocol driver not attached                */
-        gf_error_to_errno_array[GF_ERROR_CODE_UNATCH] = EUNATCH;
-        gf_errno_to_error_array[EUNATCH] = GF_ERROR_CODE_UNATCH;
-
-/*      ENOCSI  43      / * No CSI structure available          */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOCSI] = ENOCSI;
-        gf_errno_to_error_array[ENOCSI] = GF_ERROR_CODE_NOCSI;
-
-/*      EL2HLT  44      / * Level 2 halted                      */
-        gf_error_to_errno_array[GF_ERROR_CODE_L2HLT] = EL2HLT;
-        gf_errno_to_error_array[EL2HLT] = GF_ERROR_CODE_L2HLT;
-
-/*      EDEADLK 45      / * Deadlock condition.                 */
-        gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-        gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-
-/*      ENOLCK  46      / * No record locks available.          */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-        gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-/*      ECANCELED 47    / * Operation canceled                  */
-        gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-        gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-/*      ENOTSUP 48      / * Operation not supported             */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTSUPP] = ENOTSUP;
-        gf_errno_to_error_array[ENOTSUP] = GF_ERROR_CODE_NOTSUPP;
-
-/* Filesystem Quotas */
-/*      EDQUOT  49      / * Disc quota exceeded                 */
-        gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-        gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-/* Convergent Error Returns */
-/*      EBADE   50      / * invalid exchange                    */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADE] = EBADE;
-        gf_errno_to_error_array[EBADE] = GF_ERROR_CODE_BADE;
-/*      EBADR   51      / * invalid request descriptor          */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADR] = EBADR;
-        gf_errno_to_error_array[EBADR] = GF_ERROR_CODE_BADR;
-/*      EXFULL  52      / * exchange full                       */
-        gf_error_to_errno_array[GF_ERROR_CODE_XFULL] = EXFULL;
-        gf_errno_to_error_array[EXFULL] = GF_ERROR_CODE_XFULL;
-/*      ENOANO  53      / * no anode                            */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOANO] = ENOANO;
-        gf_errno_to_error_array[ENOANO] = GF_ERROR_CODE_NOANO;
-/*      EBADRQC 54      / * invalid request code                        */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADRQC] = EBADRQC;
-        gf_errno_to_error_array[EBADRQC] = GF_ERROR_CODE_BADRQC;
-/*      EBADSLT 55      / * invalid slot                                */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADSLT] = EBADSLT;
-        gf_errno_to_error_array[EBADSLT] = GF_ERROR_CODE_BADSLT;
-/*      EDEADLOCK 56    / * file locking deadlock error         */
-/* This is same as EDEADLK on linux */
-        gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLOCK;
-        gf_errno_to_error_array[EDEADLOCK] = GF_ERROR_CODE_DEADLK;
-
-/*      EBFONT  57      / * bad font file fmt                   */
-        gf_error_to_errno_array[GF_ERROR_CODE_BFONT] = EBFONT;
-        gf_errno_to_error_array[EBFONT] = GF_ERROR_CODE_BFONT;
-
-/* Interprocess Robust Locks */
-/*      EOWNERDEAD      58      / * process died with the lock */
-        gf_error_to_errno_array[GF_ERROR_CODE_OWNERDEAD] = EOWNERDEAD;
-        gf_errno_to_error_array[EOWNERDEAD] = GF_ERROR_CODE_OWNERDEAD;
-/*      ENOTRECOVERABLE 59      / * lock is not recoverable */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTRECOVERABLE] = ENOTRECOVERABLE;
-        gf_errno_to_error_array[ENOTRECOVERABLE] = GF_ERROR_CODE_NOTRECOVERABLE;
-
-/* stream problems */
-/*      ENOSTR  60      / * Device not a stream                 */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
-        gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
-/*      ENODATA 61      / * no data (for no delay io)           */
-        gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
-        gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
-/*      ETIME   62      / * timer expired                       */
-        gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
-        gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
-/*      ENOSR   63      / * out of streams resources            */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
-        gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
-
-/*      ENONET  64      / * Machine is not on the network       */
-        gf_error_to_errno_array[GF_ERROR_CODE_NONET] = ENONET;
-        gf_errno_to_error_array[ENONET] = GF_ERROR_CODE_NONET;
-/*      ENOPKG  65      / * Package not installed               */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOPKG] = ENOPKG;
-        gf_errno_to_error_array[ENOPKG] = GF_ERROR_CODE_NOPKG;
-/*      EREMOTE 66      / * The object is remote                        */
-        gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-        gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-/*      ENOLINK 67      / * the link has been severed           */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-        gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
-/*      EADV    68      / * advertise error                     */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADV] = EADV;
-        gf_errno_to_error_array[EADV] = GF_ERROR_CODE_ADV;
-/*      ESRMNT  69      / * srmount error                       */
-        gf_error_to_errno_array[GF_ERROR_CODE_SRMNT] = ESRMNT;
-        gf_errno_to_error_array[ESRMNT] = GF_ERROR_CODE_SRMNT;
-
-/*      ECOMM   70      / * Communication error on send         */
-        gf_error_to_errno_array[GF_ERROR_CODE_COMM] = ECOMM;
-        gf_errno_to_error_array[ECOMM] = GF_ERROR_CODE_COMM;
-/*      EPROTO  71      / * Protocol error                      */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-        gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
-
-/* Interprocess Robust Locks */
-/*      ELOCKUNMAPPED   72      / * locked lock was unmapped */
-        gf_error_to_errno_array[GF_ERROR_CODE_LOCKUNMAPPED] = ELOCKUNMAPPED;
-        gf_errno_to_error_array[ELOCKUNMAPPED] = GF_ERROR_CODE_LOCKUNMAPPED;
-
-/*      ENOTACTIVE 73   / * Facility is not active              */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTACTIVE] = ENOTACTIVE;
-        gf_errno_to_error_array[ENOTACTIVE] = GF_ERROR_CODE_NOTACTIVE;
-/*      EMULTIHOP 74    / * multihop attempted                  */
-        gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-        gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
-/*      EBADMSG 77      / * trying to read unreadable message   */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-        gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
-/*      ENAMETOOLONG 78 / * path name is too long               */
-        gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-        gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-/*      EOVERFLOW 79    / * value too large to be stored in data type */
-        gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-        gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-/*      ENOTUNIQ 80     / * given log. name not unique          */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTUNIQ] = ENOTUNIQ;
-        gf_errno_to_error_array[ENOTUNIQ] = GF_ERROR_CODE_NOTUNIQ;
-/*      EBADFD  81      / * f.d. invalid for this operation     */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADFD] = EBADFD;
-        gf_errno_to_error_array[EBADFD] = GF_ERROR_CODE_BADFD;
-/*      EREMCHG 82      / * Remote address changed              */
-        gf_error_to_errno_array[GF_ERROR_CODE_REMCHG] = EREMCHG;
-        gf_errno_to_error_array[EREMCHG] = GF_ERROR_CODE_REMCHG;
-
-/* shared library problems */
-/*      ELIBACC 83      / * Can't access a needed shared lib.   */
-        gf_error_to_errno_array[GF_ERROR_CODE_LIBACC] = ELIBACC;
-        gf_errno_to_error_array[ELIBACC] = GF_ERROR_CODE_LIBACC;
-/*      ELIBBAD 84      / * Accessing a corrupted shared lib.   */
-        gf_error_to_errno_array[GF_ERROR_CODE_LIBBAD] = ELIBBAD;
-        gf_errno_to_error_array[ELIBBAD] = GF_ERROR_CODE_LIBBAD;
-/*      ELIBSCN 85      / * .lib section in a.out corrupted.    */
-        gf_error_to_errno_array[GF_ERROR_CODE_LIBSCN] = ELIBSCN;
-        gf_errno_to_error_array[ELIBSCN] = GF_ERROR_CODE_LIBSCN;
-/*      ELIBMAX 86      / * Attempting to link in too many libs.        */
-        gf_error_to_errno_array[GF_ERROR_CODE_LIBMAX] = ELIBMAX;
-        gf_errno_to_error_array[ELIBMAX] = GF_ERROR_CODE_LIBMAX;
-/*      ELIBEXEC 87     / * Attempting to exec a shared library.        */
-        gf_error_to_errno_array[GF_ERROR_CODE_LIBEXEC] = ELIBEXEC;
-        gf_errno_to_error_array[ELIBEXEC] = GF_ERROR_CODE_LIBEXEC;
-/*      EILSEQ  88      / * Illegal byte sequence.              */
-        gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-        gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-/*      ENOSYS  89      / * Unsupported file system operation   */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-        gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-/*      ELOOP   90      / * Symbolic link loop                  */
-        gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-        gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-/*      ERESTART 91     / * Restartable system call             */
-        gf_error_to_errno_array[GF_ERROR_CODE_RESTART] = ERESTART;
-        gf_errno_to_error_array[ERESTART] = GF_ERROR_CODE_RESTART;
-/*      ESTRPIPE 92     / * if pipe/FIFO, don't sleep in stream head */
-        gf_error_to_errno_array[GF_ERROR_CODE_STRPIPE] = ESTRPIPE;
-        gf_errno_to_error_array[ESTRPIPE] = GF_ERROR_CODE_STRPIPE;
-/*      ENOTEMPTY 93    / * directory not empty                 */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-        gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-/*      EUSERS  94      / * Too many users (for UFS)            */
-        gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-        gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-/* BSD Networking Software */
-        /* argument errors */
-/*      ENOTSOCK        95      / * Socket operation on non-socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-        gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-/*      EDESTADDRREQ    96      / * Destination address required */
-        gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-        gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-/*      EMSGSIZE        97      / * Message too long */
-        gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-        gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-/*      EPROTOTYPE      98      / * Protocol wrong type for socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-        gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-/*      ENOPROTOOPT     99      / * Protocol not available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-        gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-/*      EPROTONOSUPPORT 120     / * Protocol not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-        gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-/*      ESOCKTNOSUPPORT 121     / * Socket type not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-        gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-/*      EOPNOTSUPP      122     / * Operation not supported on socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-        gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-/*      EPFNOSUPPORT    123     / * Protocol family not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-        gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-/*      EAFNOSUPPORT    124     / * Address family not supported by */
-        /* protocol family */
-        gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-        gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
-/*      EADDRINUSE      125     / * Address already in use */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-        gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
-/*      EADDRNOTAVAIL   126     / * Can't assign requested address */
-        /* operational errors */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-        gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
-/*      ENETDOWN        127     / * Network is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-        gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
-/*      ENETUNREACH     128     / * Network is unreachable */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-        gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
-/*      ENETRESET       129     / * Network dropped connection because */
-        /* of reset */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-        gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
-/*      ECONNABORTED    130     / * Software caused connection abort */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-        gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
-/*      ECONNRESET      131     / * Connection reset by peer */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-        gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-/*      ENOBUFS         132     / * No buffer space available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-        gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-/*      EISCONN         133     / * Socket is already connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-        gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-/*      ENOTCONN        134     / * Socket is not connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-        gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-/* XENIX has 135 - 142 */
-/*      ESHUTDOWN       143     / * Can't send after socket shutdown */
-        gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-        gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-/*      ETOOMANYREFS    144     / * Too many references: can't splice */
-        gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-        gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-/*      ETIMEDOUT       145     / * Connection timed out */
-        gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-        gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-/*      ECONNREFUSED    146     / * Connection refused */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-        gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-/*      EHOSTDOWN       147     / * Host is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-        gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-/*      EHOSTUNREACH    148     / * No route to host */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-        gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-/*      EALREADY        149     / * operation already in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-        gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-/*      EINPROGRESS     150     / * operation now in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-        gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-/* SUN Network File System */
-/*      ESTALE          151     / * Stale NFS file handle */
-        gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-        gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-        return ;
+    /*      ENOMSG  35      / * No message of desired type          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
+
+    /*      EIDRM   36      / * Identifier removed                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
+
+    /*      ECHRNG  37      / * Channel number out of range         */
+    gf_error_to_errno_array[GF_ERROR_CODE_CHRNG] = ECHRNG;
+    gf_errno_to_error_array[ECHRNG] = GF_ERROR_CODE_CHRNG;
+
+    /*      EL2NSYNC 38     / * Level 2 not synchronized            */
+    gf_error_to_errno_array[GF_ERROR_CODE_L2NSYNC] = EL2NSYNC;
+    gf_errno_to_error_array[EL2NSYNC] = GF_ERROR_CODE_L2NSYNC;
+
+    /*      EL3HLT  39      / * Level 3 halted                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_L3HLT] = EL3HLT;
+    gf_errno_to_error_array[EL3HLT] = GF_ERROR_CODE_L3HLT;
+
+    /*      EL3RST  40      / * Level 3 reset                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_L3RST] = EL3RST;
+    gf_errno_to_error_array[EL3RST] = GF_ERROR_CODE_L3RST;
+
+    /*      ELNRNG  41      / * Link number out of range            */
+    gf_error_to_errno_array[GF_ERROR_CODE_LNRNG] = ELNRNG;
+    gf_errno_to_error_array[ELNRNG] = GF_ERROR_CODE_LNRNG;
+
+    /*      EUNATCH 42      / * Protocol driver not attached                */
+    gf_error_to_errno_array[GF_ERROR_CODE_UNATCH] = EUNATCH;
+    gf_errno_to_error_array[EUNATCH] = GF_ERROR_CODE_UNATCH;
+
+    /*      ENOCSI  43      / * No CSI structure available          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOCSI] = ENOCSI;
+    gf_errno_to_error_array[ENOCSI] = GF_ERROR_CODE_NOCSI;
+
+    /*      EL2HLT  44      / * Level 2 halted                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_L2HLT] = EL2HLT;
+    gf_errno_to_error_array[EL2HLT] = GF_ERROR_CODE_L2HLT;
+
+    /*      EDEADLK 45      / * Deadlock condition.                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*      ENOLCK  46      / * No record locks available.          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
+
+    /*      ECANCELED 47    / * Operation canceled                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
+
+    /*      ENOTSUP 48      / * Operation not supported             */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSUPP] = ENOTSUP;
+    gf_errno_to_error_array[ENOTSUP] = GF_ERROR_CODE_NOTSUPP;
+
+    /* Filesystem Quotas */
+    /*      EDQUOT  49      / * Disc quota exceeded                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /* Convergent Error Returns */
+    /*      EBADE   50      / * invalid exchange                    */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADE] = EBADE;
+    gf_errno_to_error_array[EBADE] = GF_ERROR_CODE_BADE;
+    /*      EBADR   51      / * invalid request descriptor          */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADR] = EBADR;
+    gf_errno_to_error_array[EBADR] = GF_ERROR_CODE_BADR;
+    /*      EXFULL  52      / * exchange full                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_XFULL] = EXFULL;
+    gf_errno_to_error_array[EXFULL] = GF_ERROR_CODE_XFULL;
+    /*      ENOANO  53      / * no anode                            */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOANO] = ENOANO;
+    gf_errno_to_error_array[ENOANO] = GF_ERROR_CODE_NOANO;
+    /*      EBADRQC 54      / * invalid request code                        */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRQC] = EBADRQC;
+    gf_errno_to_error_array[EBADRQC] = GF_ERROR_CODE_BADRQC;
+    /*      EBADSLT 55      / * invalid slot                                */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADSLT] = EBADSLT;
+    gf_errno_to_error_array[EBADSLT] = GF_ERROR_CODE_BADSLT;
+    /*      EDEADLOCK 56    / * file locking deadlock error         */
+    /* This is same as EDEADLK on linux */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLOCK;
+    gf_errno_to_error_array[EDEADLOCK] = GF_ERROR_CODE_DEADLK;
+
+    /*      EBFONT  57      / * bad font file fmt                   */
+    gf_error_to_errno_array[GF_ERROR_CODE_BFONT] = EBFONT;
+    gf_errno_to_error_array[EBFONT] = GF_ERROR_CODE_BFONT;
+
+    /* Interprocess Robust Locks */
+    /*      EOWNERDEAD      58      / * process died with the lock */
+    gf_error_to_errno_array[GF_ERROR_CODE_OWNERDEAD] = EOWNERDEAD;
+    gf_errno_to_error_array[EOWNERDEAD] = GF_ERROR_CODE_OWNERDEAD;
+    /*      ENOTRECOVERABLE 59      / * lock is not recoverable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTRECOVERABLE] = ENOTRECOVERABLE;
+    gf_errno_to_error_array[ENOTRECOVERABLE] = GF_ERROR_CODE_NOTRECOVERABLE;
+
+    /* stream problems */
+    /*      ENOSTR  60      / * Device not a stream                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
+    gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
+    /*      ENODATA 61      / * no data (for no delay io)           */
+    gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
+    gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
+    /*      ETIME   62      / * timer expired                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
+    gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
+    /*      ENOSR   63      / * out of streams resources            */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
+    gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
+
+    /*      ENONET  64      / * Machine is not on the network       */
+    gf_error_to_errno_array[GF_ERROR_CODE_NONET] = ENONET;
+    gf_errno_to_error_array[ENONET] = GF_ERROR_CODE_NONET;
+    /*      ENOPKG  65      / * Package not installed               */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPKG] = ENOPKG;
+    gf_errno_to_error_array[ENOPKG] = GF_ERROR_CODE_NOPKG;
+    /*      EREMOTE 66      / * The object is remote                        */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+    /*      ENOLINK 67      / * the link has been severed           */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+    /*      EADV    68      / * advertise error                     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADV] = EADV;
+    gf_errno_to_error_array[EADV] = GF_ERROR_CODE_ADV;
+    /*      ESRMNT  69      / * srmount error                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_SRMNT] = ESRMNT;
+    gf_errno_to_error_array[ESRMNT] = GF_ERROR_CODE_SRMNT;
+
+    /*      ECOMM   70      / * Communication error on send         */
+    gf_error_to_errno_array[GF_ERROR_CODE_COMM] = ECOMM;
+    gf_errno_to_error_array[ECOMM] = GF_ERROR_CODE_COMM;
+    /*      EPROTO  71      / * Protocol error                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+
+    /* Interprocess Robust Locks */
+    /*      ELOCKUNMAPPED   72      / * locked lock was unmapped */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOCKUNMAPPED] = ELOCKUNMAPPED;
+    gf_errno_to_error_array[ELOCKUNMAPPED] = GF_ERROR_CODE_LOCKUNMAPPED;
+
+    /*      ENOTACTIVE 73   / * Facility is not active              */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTACTIVE] = ENOTACTIVE;
+    gf_errno_to_error_array[ENOTACTIVE] = GF_ERROR_CODE_NOTACTIVE;
+    /*      EMULTIHOP 74    / * multihop attempted                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
+    /*      EBADMSG 77      / * trying to read unreadable message   */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+    /*      ENAMETOOLONG 78 / * path name is too long               */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+    /*      EOVERFLOW 79    / * value too large to be stored in data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
+    /*      ENOTUNIQ 80     / * given log. name not unique          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTUNIQ] = ENOTUNIQ;
+    gf_errno_to_error_array[ENOTUNIQ] = GF_ERROR_CODE_NOTUNIQ;
+    /*      EBADFD  81      / * f.d. invalid for this operation     */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADFD] = EBADFD;
+    gf_errno_to_error_array[EBADFD] = GF_ERROR_CODE_BADFD;
+    /*      EREMCHG 82      / * Remote address changed              */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMCHG] = EREMCHG;
+    gf_errno_to_error_array[EREMCHG] = GF_ERROR_CODE_REMCHG;
+
+    /* shared library problems */
+    /*      ELIBACC 83      / * Can't access a needed shared lib.   */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBACC] = ELIBACC;
+    gf_errno_to_error_array[ELIBACC] = GF_ERROR_CODE_LIBACC;
+    /*      ELIBBAD 84      / * Accessing a corrupted shared lib.   */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBBAD] = ELIBBAD;
+    gf_errno_to_error_array[ELIBBAD] = GF_ERROR_CODE_LIBBAD;
+    /*      ELIBSCN 85      / * .lib section in a.out corrupted.    */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBSCN] = ELIBSCN;
+    gf_errno_to_error_array[ELIBSCN] = GF_ERROR_CODE_LIBSCN;
+    /*      ELIBMAX 86      / * Attempting to link in too many libs.        */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBMAX] = ELIBMAX;
+    gf_errno_to_error_array[ELIBMAX] = GF_ERROR_CODE_LIBMAX;
+    /*      ELIBEXEC 87     / * Attempting to exec a shared library.        */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBEXEC] = ELIBEXEC;
+    gf_errno_to_error_array[ELIBEXEC] = GF_ERROR_CODE_LIBEXEC;
+    /*      EILSEQ  88      / * Illegal byte sequence.              */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
+    /*      ENOSYS  89      / * Unsupported file system operation   */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
+    /*      ELOOP   90      / * Symbolic link loop                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+    /*      ERESTART 91     / * Restartable system call             */
+    gf_error_to_errno_array[GF_ERROR_CODE_RESTART] = ERESTART;
+    gf_errno_to_error_array[ERESTART] = GF_ERROR_CODE_RESTART;
+    /*      ESTRPIPE 92     / * if pipe/FIFO, don't sleep in stream head */
+    gf_error_to_errno_array[GF_ERROR_CODE_STRPIPE] = ESTRPIPE;
+    gf_errno_to_error_array[ESTRPIPE] = GF_ERROR_CODE_STRPIPE;
+    /*      ENOTEMPTY 93    / * directory not empty                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+    /*      EUSERS  94      / * Too many users (for UFS)            */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /* BSD Networking Software */
+    /* argument errors */
+    /*      ENOTSOCK        95      / * Socket operation on non-socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+    /*      EDESTADDRREQ    96      / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+    /*      EMSGSIZE        97      / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+    /*      EPROTOTYPE      98      / * Protocol wrong type for socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+    /*      ENOPROTOOPT     99      / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+    /*      EPROTONOSUPPORT 120     / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+    /*      ESOCKTNOSUPPORT 121     / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      122     / * Operation not supported on socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+    /*      EPFNOSUPPORT    123     / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+    /*      EAFNOSUPPORT    124     / * Address family not supported by */
+    /* protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+    /*      EADDRINUSE      125     / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+    /*      EADDRNOTAVAIL   126     / * Can't assign requested address */
+    /* operational errors */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+    /*      ENETDOWN        127     / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+    /*      ENETUNREACH     128     / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+    /*      ENETRESET       129     / * Network dropped connection because */
+    /* of reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+    /*      ECONNABORTED    130     / * Software caused connection abort */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+    /*      ECONNRESET      131     / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+    /*      ENOBUFS         132     / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+    /*      EISCONN         133     / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+    /*      ENOTCONN        134     / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+    /* XENIX has 135 - 142 */
+    /*      ESHUTDOWN       143     / * Can't send after socket shutdown */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+    /*      ETOOMANYREFS    144     / * Too many references: can't splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+    /*      ETIMEDOUT       145     / * Connection timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    146     / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+    /*      EHOSTDOWN       147     / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+    /*      EHOSTUNREACH    148     / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+    /*      EALREADY        149     / * operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+    /*      EINPROGRESS     150     / * operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /* SUN Network File System */
+    /*      ESTALE          151     / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    return;
 }
 #endif /* GF_SOLARIS_HOST_OS */
 
 #ifdef GF_DARWIN_HOST_OS
 static void
-init_compat_errno_arrays ()
+init_compat_errno_arrays()
 {
-        /*    EDEADLK         11              / * Resource deadlock would occur */
-        gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-        gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-
-        /*    EAGAIN          35              / * Try Again */
-        gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
-        gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
-
-        /*      EINPROGRESS     36              / * Operation now in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-        gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-        /*      EALREADY        37              / * Operation already in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-        gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-
-        /*      ENOTSOCK        38              / * Socket operation on non-socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-        gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-
-        /*      EDESTADDRREQ    39              / * Destination address required */
-        gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-        gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-
-        /*      EMSGSIZE        40              / * Message too long */
-        gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-        gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-
-        /*      EPROTOTYPE      41              / * Protocol wrong type for socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-        gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-
-        /*      ENOPROTOOPT     42              / * Protocol not available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-        gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-
-        /*      EPROTONOSUPPORT 43              / * Protocol not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-        gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-
-        /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-        gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-        /*      EOPNOTSUPP      45              / * Operation not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-        gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-        /*      EPFNOSUPPORT    46              / * Protocol family not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-        gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-
-        /*      EAFNOSUPPORT    47              / * Address family not supported by protocol family */
-        gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-        gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
-
-        /*      EADDRINUSE      48              / * Address already in use */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-        gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
-
-        /*      EADDRNOTAVAIL   49              / * Can't assign requested address */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-        gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
-
-        /*      ENETDOWN        50              / * Network is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-        gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
-
-        /*      ENETUNREACH     51              / * Network is unreachable */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-        gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
-
-        /*      ENETRESET       52              / * Network dropped connection on reset */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-        gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
-
-        /*      ECONNABORTED    53              / * Software caused connection abort */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-        gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
-
-        /*      ECONNRESET      54              / * Connection reset by peer */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-        gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-
-        /*      ENOBUFS         55              / * No buffer space available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-        gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-
-        /*      EISCONN         56              / * Socket is already connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-        gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-
-        /*      ENOTCONN        57              / * Socket is not connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-        gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-
-        /*      ESHUTDOWN       58              / * Can't send after socket shutdown */
-        gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-        gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-
-        /*      ETOOMANYREFS    59              / * Too many references: can't splice */
-        gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-        gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-
-        /*      ETIMEDOUT       60              / * Operation timed out */
-        gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-        gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-        /*      ECONNREFUSED    61              / * Connection refused */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-        gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-
-        /*      ELOOP           62              / * Too many levels of symbolic links */
-        gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-        gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-
-        /*      ENAMETOOLONG    63              / * File name too long */
-        gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-        gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-
-        /*      EHOSTDOWN       64              / * Host is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-        gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-
-        /*      EHOSTUNREACH    65              / * No route to host */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-        gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-
-        /*      ENOTEMPTY       66              / * Directory not empty */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-        gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-
-        /*      EPROCLIM        67              / * Too many processes */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
-        gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
-
-        /*      EUSERS          68              / * Too many users */
-        gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-        gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-        /*      EDQUOT          69              / * Disc quota exceeded */
-        gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-        gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-        /*      ESTALE          70              / * Stale NFS file handle */
-        gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-        gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-        /*      EREMOTE         71              / * Too many levels of remote in path */
-        gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-        gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-
-        /*      EBADRPC         72              / * RPC struct is bad */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
-        gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
-
-        /*      ERPCMISMATCH    73              / * RPC version wrong */
-        gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
-        gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
-
-        /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
-        gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
-
-        /*      EPROGMISMATCH   75              / * Program version wrong */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
-        gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
-
-        /*      EPROCUNAVAIL    76              / * Bad procedure for program */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
-        gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
-
-        /*      ENOLCK          77              / * No locks available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-        gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-        /*      ENOSYS          78              / * Function not implemented */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-        gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-
-        /*      EFTYPE          79              / * Inappropriate file type or format */
-        gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
-        gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
-
-        /*      EAUTH           80              / * Authentication error */
-        gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
-        gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
-
-        /*      ENEEDAUTH       81              / * Need authenticator */
-        gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-        gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-/* Intelligent device errors */
-/*      EPWROFF         82      / * Device power is off */
-        gf_error_to_errno_array[GF_ERROR_CODE_PWROFF] = EPWROFF;
-        gf_errno_to_error_array[EPWROFF] = GF_ERROR_CODE_PWROFF;
-/*      EDEVERR         83      / * Device error, e.g. paper out */
-        gf_error_to_errno_array[GF_ERROR_CODE_DEVERR] = EDEVERR;
-        gf_errno_to_error_array[EDEVERR] = GF_ERROR_CODE_DEVERR;
-
-        /*      EOVERFLOW       84              / * Value too large to be stored in data type */
-        gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-        gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-
-/* Program loading errors */
-/*   EBADEXEC   85      / * Bad executable */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADEXEC] = EBADEXEC;
-        gf_errno_to_error_array[EBADEXEC] = GF_ERROR_CODE_BADEXEC;
-
-/*   EBADARCH   86      / * Bad CPU type in executable */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADARCH] = EBADARCH;
-        gf_errno_to_error_array[EBADARCH] = GF_ERROR_CODE_BADARCH;
-
-/*   ESHLIBVERS 87      / * Shared library version mismatch */
-        gf_error_to_errno_array[GF_ERROR_CODE_SHLIBVERS] = ESHLIBVERS;
-        gf_errno_to_error_array[ESHLIBVERS] = GF_ERROR_CODE_SHLIBVERS;
-
-/*   EBADMACHO  88      / * Malformed Macho file */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADMACHO] = EBADMACHO;
-        gf_errno_to_error_array[EBADMACHO] = GF_ERROR_CODE_BADMACHO;
+    /*    EDEADLK         11              / * Resource deadlock would occur */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*    EAGAIN          35              / * Try Again */
+    gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
+    gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
+
+    /*      EINPROGRESS     36              / * Operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /*      EALREADY        37              / * Operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+
+    /*      ENOTSOCK        38              / * Socket operation on non-socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+
+    /*      EDESTADDRREQ    39              / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+
+    /*      EMSGSIZE        40              / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+
+    /*      EPROTOTYPE      41              / * Protocol wrong type for socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+
+    /*      ENOPROTOOPT     42              / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+
+    /*      EPROTONOSUPPORT 43              / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+
+    /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      45              / * Operation not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*      EPFNOSUPPORT    46              / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+
+    /*      EAFNOSUPPORT    47              / * Address family not supported by
+     * protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+
+    /*      EADDRINUSE      48              / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+
+    /*      EADDRNOTAVAIL   49              / * Can't assign requested address
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+
+    /*      ENETDOWN        50              / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+
+    /*      ENETUNREACH     51              / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+
+    /*      ENETRESET       52              / * Network dropped connection on
+     * reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+
+    /*      ECONNABORTED    53              / * Software caused connection abort
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+
+    /*      ECONNRESET      54              / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+
+    /*      ENOBUFS         55              / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+
+    /*      EISCONN         56              / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+
+    /*      ENOTCONN        57              / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+
+    /*      ESHUTDOWN       58              / * Can't send after socket shutdown
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+
+    /*      ETOOMANYREFS    59              / * Too many references: can't
+     * splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+
+    /*      ETIMEDOUT       60              / * Operation timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    61              / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+
+    /*      ELOOP           62              / * Too many levels of symbolic
+     * links */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+
+    /*      ENAMETOOLONG    63              / * File name too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+
+    /*      EHOSTDOWN       64              / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+
+    /*      EHOSTUNREACH    65              / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+
+    /*      ENOTEMPTY       66              / * Directory not empty */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+
+    /*      EPROCLIM        67              / * Too many processes */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
+    gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
+
+    /*      EUSERS          68              / * Too many users */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /*      EDQUOT          69              / * Disc quota exceeded */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /*      ESTALE          70              / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    /*      EREMOTE         71              / * Too many levels of remote in
+     * path */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+
+    /*      EBADRPC         72              / * RPC struct is bad */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
+    gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
+
+    /*      ERPCMISMATCH    73              / * RPC version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
+    gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
+
+    /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
+    gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
+
+    /*      EPROGMISMATCH   75              / * Program version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
+    gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
+
+    /*      EPROCUNAVAIL    76              / * Bad procedure for program */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
+    gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
+
+    /*      ENOLCK          77              / * No locks available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
+
+    /*      ENOSYS          78              / * Function not implemented */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
+
+    /*      EFTYPE          79              / * Inappropriate file type or
+     * format */
+    gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
+    gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
+
+    /*      EAUTH           80              / * Authentication error */
+    gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
+    gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
+
+    /*      ENEEDAUTH       81              / * Need authenticator */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+    /* Intelligent device errors */
+    /*      EPWROFF         82      / * Device power is off */
+    gf_error_to_errno_array[GF_ERROR_CODE_PWROFF] = EPWROFF;
+    gf_errno_to_error_array[EPWROFF] = GF_ERROR_CODE_PWROFF;
+    /*      EDEVERR         83      / * Device error, e.g. paper out */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEVERR] = EDEVERR;
+    gf_errno_to_error_array[EDEVERR] = GF_ERROR_CODE_DEVERR;
+
+    /*      EOVERFLOW       84              / * Value too large to be stored in
+     * data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
+
+    /* Program loading errors */
+    /*   EBADEXEC   85      / * Bad executable */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADEXEC] = EBADEXEC;
+    gf_errno_to_error_array[EBADEXEC] = GF_ERROR_CODE_BADEXEC;
+
+    /*   EBADARCH   86      / * Bad CPU type in executable */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADARCH] = EBADARCH;
+    gf_errno_to_error_array[EBADARCH] = GF_ERROR_CODE_BADARCH;
+
+    /*   ESHLIBVERS 87      / * Shared library version mismatch */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHLIBVERS] = ESHLIBVERS;
+    gf_errno_to_error_array[ESHLIBVERS] = GF_ERROR_CODE_SHLIBVERS;
+
+    /*   EBADMACHO  88      / * Malformed Macho file */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMACHO] = EBADMACHO;
+    gf_errno_to_error_array[EBADMACHO] = GF_ERROR_CODE_BADMACHO;
 
 #ifdef EDOOFUS
-        /*    EDOOFUS           88              / * Programming error */
-        gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
-        gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
+    /*    EDOOFUS           88              / * Programming error */
+    gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
+    gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
 #endif
 
-        /*      ECANCELED       89              / * Operation canceled */
-        gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-        gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-        /*   EIDRM              90              / * Identifier removed */
-        gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-        gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-        /*   ENOMSG             91              / * No message of desired type */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-        gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-        /*   EILSEQ             92              / * Illegal byte sequence */
-        gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-        gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-
-        /*   ENOATTR            93              / * Attribute not found */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
-        gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
-
-        /*   EBADMSG            94              / * Bad message */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-        gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
-
-        /*   EMULTIHOP  95              / * Reserved */
-        gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-        gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
-
-        /*      ENODATA         96              / * No message available on STREAM */
-        gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-        gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-
-        /*   ENOLINK            97              / * Reserved */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-        gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
-
-        /*   ENOSR              98              / * No STREAM resources */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
-        gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
-
-        /*   ENOSTR             99              / * Not a STREAM */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
-        gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
-
-/*      EPROTO          100             / * Protocol error */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-        gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
-/*   ETIME              101             / * STREAM ioctl timeout */
-        gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
-        gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
-
-/* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */
-/*      EOPNOTSUPP      102             / * Operation not supported on socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-        gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-/*   ENOPOLICY  103             / * No such policy registered */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOPOLICY] = ENOPOLICY;
-        gf_errno_to_error_array[ENOPOLICY] = GF_ERROR_CODE_NOPOLICY;
-
-        return ;
+    /*      ECANCELED       89              / * Operation canceled */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
+
+    /*   EIDRM              90              / * Identifier removed */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
+    /*   ENOMSG             91              / * No message of desired type */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
+
+    /*   EILSEQ             92              / * Illegal byte sequence */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
+
+    /*   ENOATTR            93              / * Attribute not found */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
+    gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
+
+    /*   EBADMSG            94              / * Bad message */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+
+    /*   EMULTIHOP  95              / * Reserved */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
+
+    /*      ENODATA         96              / * No message available on STREAM
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+
+    /*   ENOLINK            97              / * Reserved */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+
+    /*   ENOSR              98              / * No STREAM resources */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
+    gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
+
+    /*   ENOSTR             99              / * Not a STREAM */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
+    gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
+
+    /*      EPROTO          100             / * Protocol error */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+    /*   ETIME              101             / * STREAM ioctl timeout */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
+    gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
+
+    /* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */
+    /*      EOPNOTSUPP      102             / * Operation not supported on
+     * socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*   ENOPOLICY  103             / * No such policy registered */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPOLICY] = ENOPOLICY;
+    gf_errno_to_error_array[ENOPOLICY] = GF_ERROR_CODE_NOPOLICY;
+
+    return;
 }
 #endif /* GF_DARWIN_HOST_OS */
 
 #ifdef GF_BSD_HOST_OS
 static void
-init_compat_errno_arrays ()
+init_compat_errno_arrays()
 {
-        /* Quite a bit of things changed in FreeBSD - current */
-
-        /*    EAGAIN          35              / * Try Again */
-        gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
-        gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
-
-        /*    EDEADLK         11              / * Resource deadlock would occur */
-        gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-        gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-
-        /*      EINPROGRESS     36              / * Operation now in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-        gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-        /*      EALREADY        37              / * Operation already in progress */
-        gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-        gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-
-        /*      ENOTSOCK        38              / * Socket operation on non-socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-        gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-
-        /*      EDESTADDRREQ    39              / * Destination address required */
-        gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-        gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-
-        /*      EMSGSIZE        40              / * Message too long */
-        gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-        gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-
-        /*      EPROTOTYPE      41              / * Protocol wrong type for socket */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-        gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-
-        /*      ENOPROTOOPT     42              / * Protocol not available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-        gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-
-        /*      EPROTONOSUPPORT 43              / * Protocol not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-        gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-
-        /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-        gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-        /*      EOPNOTSUPP      45              / * Operation not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-        gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-        /*      EPFNOSUPPORT    46              / * Protocol family not supported */
-        gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-        gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-
-        /*      EAFNOSUPPORT    47              / * Address family not supported by protocol family */
-        gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-        gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+    /* Quite a bit of things changed in FreeBSD - current */
+
+    /*    EAGAIN          35              / * Try Again */
+    gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
+    gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
+
+    /*    EDEADLK         11              / * Resource deadlock would occur */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*      EINPROGRESS     36              / * Operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /*      EALREADY        37              / * Operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+
+    /*      ENOTSOCK        38              / * Socket operation on non-socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+
+    /*      EDESTADDRREQ    39              / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+
+    /*      EMSGSIZE        40              / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+
+    /*      EPROTOTYPE      41              / * Protocol wrong type for socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+
+    /*      ENOPROTOOPT     42              / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+
+    /*      EPROTONOSUPPORT 43              / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+
+    /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      45              / * Operation not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*      EPFNOSUPPORT    46              / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+
+    /*      EAFNOSUPPORT    47              / * Address family not supported by
+     * protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+
+    /*      EADDRINUSE      48              / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+
+    /*      EADDRNOTAVAIL   49              / * Can't assign requested address
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+
+    /*      ENETDOWN        50              / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+
+    /*      ENETUNREACH     51              / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+
+    /*      ENETRESET       52              / * Network dropped connection on
+     * reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+
+    /*      ECONNABORTED    53              / * Software caused connection abort
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+
+    /*      ECONNRESET      54              / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+
+    /*      ENOBUFS         55              / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+
+    /*      EISCONN         56              / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+
+    /*      ENOTCONN        57              / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+
+    /*      ESHUTDOWN       58              / * Can't send after socket shutdown
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+
+    /*      ETOOMANYREFS    59              / * Too many references: can't
+     * splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+
+    /*      ETIMEDOUT       60              / * Operation timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    61              / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+
+    /*      ELOOP           62              / * Too many levels of symbolic
+     * links */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+
+    /*      ENAMETOOLONG    63              / * File name too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+
+    /*      EHOSTDOWN       64              / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+
+    /*      EHOSTUNREACH    65              / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+
+    /*      ENOTEMPTY       66              / * Directory not empty */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+
+    /*      EPROCLIM        67              / * Too many processes */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
+    gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
+
+    /*      EUSERS          68              / * Too many users */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /*      EDQUOT          69              / * Disc quota exceeded */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /*      ESTALE          70              / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    /*      EREMOTE         71              / * Too many levels of remote in
+     * path */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+
+    /*      EBADRPC         72              / * RPC struct is bad */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
+    gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
+
+    /*      ERPCMISMATCH    73              / * RPC version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
+    gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
+
+    /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
+    gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
+
+    /*      EPROGMISMATCH   75              / * Program version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
+    gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
+
+    /*      EPROCUNAVAIL    76              / * Bad procedure for program */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
+    gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
+
+    /*      ENOLCK          77              / * No locks available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
 
-        /*      EADDRINUSE      48              / * Address already in use */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-        gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+    /*      ENOSYS          78              / * Function not implemented */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
 
-        /*      EADDRNOTAVAIL   49              / * Can't assign requested address */
-        gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-        gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+    /*      EFTYPE          79              / * Inappropriate file type or
+     * format */
+    gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
+    gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
+
+    /*      EAUTH           80              / * Authentication error */
+    gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
+    gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
+
+    /*      ENEEDAUTH       81              / * Need authenticator */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+
+    /*      EIDRM           82              / * Identifier removed */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
 
-        /*      ENETDOWN        50              / * Network is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-        gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+    /*      ENOMSG          83              / * No message of desired type */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
 
-        /*      ENETUNREACH     51              / * Network is unreachable */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-        gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+    /*      EOVERFLOW       84              / * Value too large to be stored in
+     * data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
 
-        /*      ENETRESET       52              / * Network dropped connection on reset */
-        gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-        gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+    /*      ECANCELED       85              / * Operation canceled */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
 
-        /*      ECONNABORTED    53              / * Software caused connection abort */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-        gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+    /*      EILSEQ          86              / * Illegal byte sequence */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
 
-        /*      ECONNRESET      54              / * Connection reset by peer */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-        gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-
-        /*      ENOBUFS         55              / * No buffer space available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-        gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-
-        /*      EISCONN         56              / * Socket is already connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-        gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-
-        /*      ENOTCONN        57              / * Socket is not connected */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-        gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-
-        /*      ESHUTDOWN       58              / * Can't send after socket shutdown */
-        gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-        gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-
-        /*      ETOOMANYREFS    59              / * Too many references: can't splice */
-        gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-        gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-
-        /*      ETIMEDOUT       60              / * Operation timed out */
-        gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-        gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-        /*      ECONNREFUSED    61              / * Connection refused */
-        gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-        gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-
-        /*      ELOOP           62              / * Too many levels of symbolic links */
-        gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-        gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-
-        /*      ENAMETOOLONG    63              / * File name too long */
-        gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-        gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-
-        /*      EHOSTDOWN       64              / * Host is down */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-        gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-
-        /*      EHOSTUNREACH    65              / * No route to host */
-        gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-        gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-
-        /*      ENOTEMPTY       66              / * Directory not empty */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-        gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-
-        /*      EPROCLIM        67              / * Too many processes */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
-        gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
-
-        /*      EUSERS          68              / * Too many users */
-        gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-        gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-        /*      EDQUOT          69              / * Disc quota exceeded */
-        gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-        gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-        /*      ESTALE          70              / * Stale NFS file handle */
-        gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-        gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-        /*      EREMOTE         71              / * Too many levels of remote in path */
-        gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-        gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-
-        /*      EBADRPC         72              / * RPC struct is bad */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
-        gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
-
-        /*      ERPCMISMATCH    73              / * RPC version wrong */
-        gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
-        gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
-
-        /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
-        gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
-
-        /*      EPROGMISMATCH   75              / * Program version wrong */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
-        gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
-
-        /*      EPROCUNAVAIL    76              / * Bad procedure for program */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
-        gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
-
-        /*      ENOLCK          77              / * No locks available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-        gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-        /*      ENOSYS          78              / * Function not implemented */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-        gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-
-        /*      EFTYPE          79              / * Inappropriate file type or format */
-        gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
-        gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
-
-        /*      EAUTH           80              / * Authentication error */
-        gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
-        gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
-
-        /*      ENEEDAUTH       81              / * Need authenticator */
-        gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-        gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-
-        /*      EIDRM           82              / * Identifier removed */
-        gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-        gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-
-        /*      ENOMSG          83              / * No message of desired type */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-        gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-        /*      EOVERFLOW       84              / * Value too large to be stored in data type */
-        gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-        gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-
-        /*      ECANCELED       85              / * Operation canceled */
-        gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-        gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-        /*      EILSEQ          86              / * Illegal byte sequence */
-        gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-        gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-
-        /*      ENOATTR         87              / * Attribute not found */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
-        gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
+    /*      ENOATTR         87              / * Attribute not found */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
+    gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
 
 #ifdef EDOOFUS
-        /*    EDOOFUS           88              / * Programming error */
-        gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
-        gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
+    /*    EDOOFUS           88              / * Programming error */
+    gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
+    gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
 #endif
 
-        /*      EBADMSG         89              / * Bad message */
-        gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-        gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+    /*      EBADMSG         89              / * Bad message */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
 
 #ifdef __NetBSD__
-        /*      ENODATA         89              / * No message available */
-        gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
-        gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
+    /*      ENODATA         89              / * No message available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
+    gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
 #endif
 
-        /*      EMULTIHOP       90              / * Multihop attempted */
-        gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-        gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
-
-        /*      ENOLINK         91              / * Link has been severed */
-        gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-        gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+    /*      EMULTIHOP       90              / * Multihop attempted */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
 
-        /*      EPROTO          92              / * Protocol error */
-        gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-        gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+    /*      ENOLINK         91              / * Link has been severed */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
 
+    /*      EPROTO          92              / * Protocol error */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
 
-        return ;
+    return;
 }
 #endif /* GF_BSD_HOST_OS */
 
 #ifdef GF_LINUX_HOST_OS
 static void
-init_compat_errno_arrays ()
+init_compat_errno_arrays()
 {
-        /* Things are fine. Everything should work seemlessly on GNU/Linux machines */
-        return ;
+    /* Things are fine. Everything should work seemlessly on GNU/Linux machines
+     */
+    return;
 }
 #endif /* GF_LINUX_HOST_OS */
 
-
 static void
-init_errno_arrays ()
+init_errno_arrays()
 {
-        int i;
-        for (i=0; i < GF_ERROR_CODE_UNKNOWN; i++) {
-                gf_errno_to_error_array[i] = i;
-                gf_error_to_errno_array[i] = i;
-        }
-        /* Now change the order if it needs to be. */
-        init_compat_errno_arrays();
-
-        return;
+    int i;
+    for (i = 0; i < GF_ERROR_CODE_UNKNOWN; i++) {
+        gf_errno_to_error_array[i] = i;
+        gf_error_to_errno_array[i] = i;
+    }
+    /* Now change the order if it needs to be. */
+    init_compat_errno_arrays();
+
+    return;
 }
 
 int32_t
-gf_errno_to_error (int32_t op_errno)
+gf_errno_to_error(int32_t op_errno)
 {
-        if (!gf_compat_errno_init_done) {
-                init_errno_arrays ();
-                gf_compat_errno_init_done = 1;
-        }
+    if (!gf_compat_errno_init_done) {
+        init_errno_arrays();
+        gf_compat_errno_init_done = 1;
+    }
 
-        if ((op_errno > GF_ERROR_CODE_SUCCESS) && (op_errno < GF_ERROR_CODE_UNKNOWN))
-                return gf_errno_to_error_array[op_errno];
+    if ((op_errno > GF_ERROR_CODE_SUCCESS) &&
+        (op_errno < GF_ERROR_CODE_UNKNOWN))
+        return gf_errno_to_error_array[op_errno];
 
-        return op_errno;
+    return op_errno;
 }
 
-
 int32_t
-gf_error_to_errno (int32_t error)
+gf_error_to_errno(int32_t error)
 {
-        if (!gf_compat_errno_init_done) {
-                init_errno_arrays ();
-                gf_compat_errno_init_done = 1;
-        }
+    if (!gf_compat_errno_init_done) {
+        init_errno_arrays();
+        gf_compat_errno_init_done = 1;
+    }
 
-        if ((error > GF_ERROR_CODE_SUCCESS) && (error < GF_ERROR_CODE_UNKNOWN))
-                return gf_error_to_errno_array[error];
+    if ((error > GF_ERROR_CODE_SUCCESS) && (error < GF_ERROR_CODE_UNKNOWN))
+        return gf_error_to_errno_array[error];
 
-        return error;
+    return error;
 }
diff --git a/libglusterfs/src/compat-errno.h b/libglusterfs/src/compat-errno.h
deleted file mode 100644
index 5ee1d18f823..00000000000
--- a/libglusterfs/src/compat-errno.h
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __COMPAT_ERRNO_H__
-#define __COMPAT_ERRNO_H__
-
-#include <errno.h>
-
-#define GF_ERROR_CODE_SUCCESS         0
-#define GF_ERROR_CODE_UNKNOWN         1024
-#define GF_ERRNO_UNKNOWN              1024
-
-#define GF_ERROR_CODE_PERM            1      /* Operation not permitted */
-#define GF_ERROR_CODE_NOENT           2      /* No such file or directory */
-#define GF_ERROR_CODE_SRCH            3      /* No such process */
-#define GF_ERROR_CODE_INTR            4      /* Interrupted system call */
-#define GF_ERROR_CODE_IO              5      /* I/O error */
-#define GF_ERROR_CODE_NXIO            6      /* No such device or address */
-#define GF_ERROR_CODE_2BIG            7      /* Argument list too long */
-#define GF_ERROR_CODE_NOEXEC          8      /* Exec format error */
-#define GF_ERROR_CODE_BADF            9      /* Bad file number */
-#define GF_ERROR_CODE_CHILD          10      /* No child processes */
-#define GF_ERROR_CODE_AGAIN          11      /* Try again */
-#define GF_ERROR_CODE_NOMEM          12      /* Out of memory */
-#define GF_ERROR_CODE_ACCES          13      /* Permission denied */
-#define GF_ERROR_CODE_FAULT          14      /* Bad address */
-#define GF_ERROR_CODE_NOTBLK         15      /* Block device required */
-#define GF_ERROR_CODE_BUSY           16      /* Device or resource busy */
-#define GF_ERROR_CODE_EXIST          17      /* File exists */
-#define GF_ERROR_CODE_XDEV           18      /* Cross-device link */
-#define GF_ERROR_CODE_NODEV          19      /* No such device */
-#define GF_ERROR_CODE_NOTDIR         20      /* Not a directory */
-#define GF_ERROR_CODE_ISDIR          21      /* Is a directory */
-#define GF_ERROR_CODE_INVAL          22      /* Invalid argument */
-#define GF_ERROR_CODE_NFILE          23      /* File table overflow */
-#define GF_ERROR_CODE_MFILE          24      /* Too many open files */
-#define GF_ERROR_CODE_NOTTY          25      /* Not a typewriter */
-#define GF_ERROR_CODE_TXTBSY         26      /* Text file busy */
-#define GF_ERROR_CODE_FBIG           27      /* File too large */
-#define GF_ERROR_CODE_NOSPC          28      /* No space left on device */
-#define GF_ERROR_CODE_SPIPE          29      /* Illegal seek */
-#define GF_ERROR_CODE_ROFS           30      /* Read-only file system */
-#define GF_ERROR_CODE_MLINK          31      /* Too many links */
-#define GF_ERROR_CODE_PIPE           32      /* Broken pipe */
-#define GF_ERROR_CODE_DOM            33      /* Math argument out of domain of func */
-#define GF_ERROR_CODE_RANGE          34      /* Math result not representable */
-#define GF_ERROR_CODE_DEADLK         35      /* Resource deadlock would occur */
-#define GF_ERROR_CODE_NAMETOOLONG    36      /* File name too long */
-#define GF_ERROR_CODE_NOLCK          37      /* No record locks available */
-#define GF_ERROR_CODE_NOSYS          38      /* Function not implemented */
-#define GF_ERROR_CODE_NOTEMPTY       39      /* Directory not empty */
-#define GF_ERROR_CODE_LOOP           40      /* Too many symbolic links encountered */
-
-#define GF_ERROR_CODE_NOMSG          42      /* No message of desired type */
-#define GF_ERROR_CODE_IDRM           43      /* Identifier removed */
-#define GF_ERROR_CODE_CHRNG          44      /* Channel number out of range */
-#define GF_ERROR_CODE_L2NSYNC        45      /* Level 2 not synchronized */
-#define GF_ERROR_CODE_L3HLT          46      /* Level 3 halted */
-#define GF_ERROR_CODE_L3RST          47      /* Level 3 reset */
-#define GF_ERROR_CODE_LNRNG          48      /* Link number out of range */
-#define GF_ERROR_CODE_UNATCH         49      /* Protocol driver not attached */
-#define GF_ERROR_CODE_NOCSI          50      /* No CSI structure available */
-#define GF_ERROR_CODE_L2HLT          51      /* Level 2 halted */
-#define GF_ERROR_CODE_BADE           52      /* Invalid exchange */
-#define GF_ERROR_CODE_BADR           53      /* Invalid request descriptor */
-#define GF_ERROR_CODE_XFULL          54      /* Exchange full */
-#define GF_ERROR_CODE_NOANO          55      /* No anode */
-#define GF_ERROR_CODE_BADRQC         56      /* Invalid request code */
-#define GF_ERROR_CODE_BADSLT         57      /* Invalid slot */
-#define GF_ERROR_CODE_BFONT          59      /* Bad font file format */
-#define GF_ERROR_CODE_NOSTR          60      /* Device not a stream */
-#define GF_ERROR_CODE_NODATA         61      /* No data available */
-#define GF_ERROR_CODE_TIME           62      /* Timer expired */
-#define GF_ERROR_CODE_NOSR           63      /* Out of streams resources */
-#define GF_ERROR_CODE_NONET          64      /* Machine is not on the network */
-#define GF_ERROR_CODE_NOPKG          65      /* Package not installed */
-#define GF_ERROR_CODE_REMOTE         66      /* Object is remote */
-#define GF_ERROR_CODE_NOLINK         67      /* Link has been severed */
-#define GF_ERROR_CODE_ADV            68      /* Advertise error */
-#define GF_ERROR_CODE_SRMNT          69      /* Srmount error */
-#define GF_ERROR_CODE_COMM           70      /* Communication error on send */
-#define GF_ERROR_CODE_PROTO          71      /* Protocol error */
-#define GF_ERROR_CODE_MULTIHOP       72      /* Multihop attempted */
-#define GF_ERROR_CODE_DOTDOT         73      /* RFS specific error */
-#define GF_ERROR_CODE_BADMSG         74      /* Not a data message */
-#define GF_ERROR_CODE_OVERFLOW       75      /* Value too large for defined data type */
-#define GF_ERROR_CODE_NOTUNIQ        76      /* Name not unique on network */
-#define GF_ERROR_CODE_BADFD          77      /* File descriptor in bad state */
-#define GF_ERROR_CODE_REMCHG         78      /* Remote address changed */
-#define GF_ERROR_CODE_LIBACC         79      /* Can not access a needed shared library */
-#define GF_ERROR_CODE_LIBBAD         80      /* Accessing a corrupted shared library */
-#define GF_ERROR_CODE_LIBSCN         81      /* .lib section in a.out corrupted */
-#define GF_ERROR_CODE_LIBMAX         82      /* Attempting to link in too many shared libraries */
-#define GF_ERROR_CODE_LIBEXEC        83      /* Cannot exec a shared library directly */
-#define GF_ERROR_CODE_ILSEQ          84      /* Illegal byte sequence */
-#define GF_ERROR_CODE_RESTART        85      /* Interrupted system call should be restarted */
-#define GF_ERROR_CODE_STRPIPE        86      /* Streams pipe error */
-#define GF_ERROR_CODE_USERS          87      /* Too many users */
-#define GF_ERROR_CODE_NOTSOCK        88      /* Socket operation on non-socket */
-#define GF_ERROR_CODE_DESTADDRREQ    89      /* Destination address required */
-#define GF_ERROR_CODE_MSGSIZE        90      /* Message too long */
-#define GF_ERROR_CODE_PROTOTYPE      91      /* Protocol wrong type for socket */
-#define GF_ERROR_CODE_NOPROTOOPT     92      /* Protocol not available */
-#define GF_ERROR_CODE_PROTONOSUPPORT 93      /* Protocol not supported */
-#define GF_ERROR_CODE_SOCKTNOSUPPORT 94      /* Socket type not supported */
-#define GF_ERROR_CODE_OPNOTSUPP      95      /* Operation not supported on transport endpoint */
-#define GF_ERROR_CODE_PFNOSUPPORT    96      /* Protocol family not supported */
-#define GF_ERROR_CODE_AFNOSUPPORT    97      /* Address family not supported by protocol */
-#define GF_ERROR_CODE_ADDRINUSE      98      /* Address already in use */
-#define GF_ERROR_CODE_ADDRNOTAVAIL   99      /* Cannot assign requested address */
-#define GF_ERROR_CODE_NETDOWN        100     /* Network is down */
-#define GF_ERROR_CODE_NETUNREACH     101     /* Network is unreachable */
-#define GF_ERROR_CODE_NETRESET       102     /* Network dropped connection because of reset */
-#define GF_ERROR_CODE_CONNABORTED    103     /* Software caused connection abort */
-#define GF_ERROR_CODE_CONNRESET      104     /* Connection reset by peer */
-#define GF_ERROR_CODE_NOBUFS         105     /* No buffer space available */
-#define GF_ERROR_CODE_ISCONN         106     /* Transport endpoint is already connected */
-#define GF_ERROR_CODE_NOTCONN        107     /* Transport endpoint is not connected */
-#define GF_ERROR_CODE_SHUTDOWN       108     /* Cannot send after transport endpoint shutdown */
-#define GF_ERROR_CODE_TOOMANYREFS    109     /* Too many references: cannot splice */
-#define GF_ERROR_CODE_TIMEDOUT       110     /* Connection timed out */
-#define GF_ERROR_CODE_CONNREFUSED    111     /* Connection refused */
-#define GF_ERROR_CODE_HOSTDOWN       112     /* Host is down */
-#define GF_ERROR_CODE_HOSTUNREACH    113     /* No route to host */
-#define GF_ERROR_CODE_ALREADY        114     /* Operation already in progress */
-#define GF_ERROR_CODE_INPROGRESS     115     /* Operation now in progress */
-#define GF_ERROR_CODE_ALREADY        114     /* Operation already in progress */
-#define GF_ERROR_CODE_INPROGRESS     115     /* Operation now in progress */
-#define GF_ERROR_CODE_STALE          116     /* Stale NFS file handle */
-#define GF_ERROR_CODE_UCLEAN         117     /* Structure needs cleaning */
-#define GF_ERROR_CODE_NOTNAM         118     /* Not a XENIX named type file */
-#define GF_ERROR_CODE_NAVAIL         119     /* No XENIX semaphores available */
-#define GF_ERROR_CODE_ISNAM          120     /* Is a named type file */
-#define GF_ERROR_CODE_REMOTEIO       121     /* Remote I/O error */
-#define GF_ERROR_CODE_DQUOT          122     /* Quota exceeded */
-#define GF_ERROR_CODE_NOMEDIUM       123     /* No medium found */
-#define GF_ERROR_CODE_MEDIUMTYPE     124     /* Wrong medium type */
-#define GF_ERROR_CODE_CANCELED       125     /* Operation Canceled */
-#define GF_ERROR_CODE_NOKEY          126     /* Required key not available */
-#define GF_ERROR_CODE_KEYEXPIRED     127     /* Key has expired */
-#define GF_ERROR_CODE_KEYREVOKED     128     /* Key has been revoked */
-#define GF_ERROR_CODE_KEYREJECTED    129     /* Key was rejected by service */
-
-/* for robust mutexes */
-#define GF_ERROR_CODE_OWNERDEAD      130     /* Owner died */
-#define GF_ERROR_CODE_NOTRECOVERABLE 131     /* State not recoverable */
-
-
-
-/* Should never be seen by user programs */
-#define GF_ERROR_CODE_RESTARTSYS     512
-#define GF_ERROR_CODE_RESTARTNOINTR  513
-#define GF_ERROR_CODE_RESTARTNOHAND  514     /* restart if no handler.. */
-#define GF_ERROR_CODE_NOIOCTLCMD     515     /* No ioctl command */
-#define GF_ERROR_CODE_RESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */
-
-/* Defined for the NFSv3 protocol */
-#define GF_ERROR_CODE_BADHANDLE      521     /* Illegal NFS file handle */
-#define GF_ERROR_CODE_NOTSYNC        522     /* Update synchronization mismatch */
-#define GF_ERROR_CODE_BADCOOKIE      523     /* Cookie is stale */
-#define GF_ERROR_CODE_NOTSUPP        524     /* Operation is not supported */
-#define GF_ERROR_CODE_TOOSMALL       525     /* Buffer or request is too small */
-#define GF_ERROR_CODE_SERVERFAULT    526     /* An untranslatable error occurred */
-#define GF_ERROR_CODE_BADTYPE        527     /* Type not supported by server */
-#define GF_ERROR_CODE_JUKEBOX        528     /* Request initiated, but will not complete before timeout */
-#define GF_ERROR_CODE_IOCBQUEUED     529     /* iocb queued, will get completion event */
-#define GF_ERROR_CODE_IOCBRETRY      530     /* iocb queued, will trigger a retry */
-
-/* Darwin OS X */
-#define GF_ERROR_CODE_NOPOLICY   701
-#define GF_ERROR_CODE_BADMACHO   702
-#define GF_ERROR_CODE_PWROFF     703
-#define GF_ERROR_CODE_DEVERR     704
-#define GF_ERROR_CODE_BADARCH    705
-#define GF_ERROR_CODE_BADEXEC    706
-#define GF_ERROR_CODE_SHLIBVERS  707
-
-
-
-/* Solaris */
-/*	ENOTACTIVE 73	/ * Facility is not active		*/
-#define GF_ERROR_CODE_NOTACTIVE   801
-/*	ELOCKUNMAPPED	72	/ * locked lock was unmapped */
-#define GF_ERROR_CODE_LOCKUNMAPPED 802
-
-/* BSD system */
-#define GF_ERROR_CODE_PROCLIM	        901		/* Too many processes */
-#define GF_ERROR_CODE_BADRPC		902		/* RPC struct is bad */
-#define GF_ERROR_CODE_RPCMISMATCH	903		/* RPC version wrong */
-#define GF_ERROR_CODE_PROGUNAVAIL	904		/* RPC prog. not avail */
-#define GF_ERROR_CODE_PROGMISMATCH	905		/* Program version wrong */
-#define GF_ERROR_CODE_PROCUNAVAIL	905		/* Bad procedure for program */
-#define GF_ERROR_CODE_FTYPE		906		/* Inappropriate file type or format */
-#define GF_ERROR_CODE_AUTH		907		/* Authentication error */
-#define GF_ERROR_CODE_NEEDAUTH	        908		/* Need authenticator */
-#define GF_ERROR_CODE_DOOFUS		909		/* Programming error */
-
-#define GF_ERROR_CODE_NOATTR		GF_ERROR_CODE_NODATA		/* Attribute not found */
-
-/* Either one of enodata or enoattr will be there in system */
-#ifndef ENOATTR
-#define ENOATTR ENODATA
-#endif /* ENOATTR */
-
-#ifndef ENODATA
-#define ENODATA ENOATTR
-#endif /* ENODATA */
-
-#ifndef EBADFD
-#define EBADFD EBADRPC
-#endif /* EBADFD */
-
-#if !defined(ENODATA)
-/* This happens on FreeBSD.  Value borrowed from Linux. */
-#define ENODATA 61
-#endif
-
-/* These functions are defined for all the OS flags, but content will
- * be different for each OS flag.
- */
-int32_t gf_errno_to_error (int32_t op_errno);
-int32_t gf_error_to_errno (int32_t error);
-
-#endif /* __COMPAT_ERRNO_H__ */
diff --git a/libglusterfs/src/compat.c b/libglusterfs/src/compat.c
index 621ff5f54c9..8a05a30a8fe 100644
--- a/libglusterfs/src/compat.c
+++ b/libglusterfs/src/compat.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdarg.h>
@@ -16,567 +15,604 @@
 #include <sys/types.h>
 #include <dirent.h>
 
-#ifdef GF_SOLARIS_HOST_OS
-#include "logging.h"
-#endif /* GF_SOLARIS_HOST_OS */
-
-#include "compat.h"
-#include "common-utils.h"
-#include "iatt.h"
-#include "inode.h"
-#include "syscall.h"
-#include "run.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/compat.h"
+#include "glusterfs/iatt.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/run.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 #ifdef GF_SOLARIS_HOST_OS
 int
-solaris_fsetxattr(int fd, const char* key, const char *value, size_t size,
+solaris_fsetxattr(int fd, const char *key, const char *value, size_t size,
                   int flags)
 {
-        int attrfd = -1;
-        int ret = 0;
-
-        attrfd = openat (fd, key, flags|O_CREAT|O_WRONLY|O_XATTR, 0777);
-        if (attrfd >= 0) {
-                ftruncate (attrfd, 0);
-                ret = write (attrfd, value, size);
-                close (attrfd);
-        } else {
-                if (errno != ENOENT)
-                        gf_msg ("libglusterfs", GF_LOG_ERROR, errno,
-                                LG_MSG_SET_ATTRIBUTE_FAILED, "Couldn't set "
-                                "extended attribute for %d", fd);
-                return -1;
-        }
-
-        return 0;
+    int attrfd = -1;
+    int ret = 0;
+
+    attrfd = openat(fd, key, flags | O_CREAT | O_WRONLY | O_XATTR, 0777);
+    if (attrfd >= 0) {
+        ftruncate(attrfd, 0);
+        ret = write(attrfd, value, size);
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_ERROR, errno,
+                   LG_MSG_SET_ATTRIBUTE_FAILED,
+                   "Couldn't set "
+                   "extended attribute for %d",
+                   fd);
+        return -1;
+    }
+
+    return 0;
 }
 
-
 int
-solaris_fgetxattr(int fd, const char* key, char *value, size_t size)
+solaris_fgetxattr(int fd, const char *key, char *value, size_t size)
 {
-        int attrfd = -1;
-        int ret = 0;
-
-        attrfd = openat (fd, key, O_RDONLY|O_XATTR);
-        if (attrfd >= 0) {
-                if (size == 0) {
-                        struct stat buf;
-                        fstat (attrfd, &buf);
-                        ret = buf.st_size;
-                } else {
-                        ret = read (attrfd, value, size);
-                }
-                close (attrfd);
+    int attrfd = -1;
+    int ret = 0;
+
+    attrfd = openat(fd, key, O_RDONLY | O_XATTR);
+    if (attrfd >= 0) {
+        if (size == 0) {
+            struct stat buf;
+            fstat(attrfd, &buf);
+            ret = buf.st_size;
         } else {
-                if (errno != ENOENT)
-                        gf_msg ("libglusterfs", GF_LOG_INFO, errno,
-                                LG_MSG_READ_ATTRIBUTE_FAILED, "Couldn't read "
-                                "extended attribute for the file %d", fd);
-                if (errno == ENOENT)
-                        errno = ENODATA;
-                return -1;
+            ret = read(attrfd, value, size);
         }
-
-        return ret;
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_INFO, errno,
+                   LG_MSG_READ_ATTRIBUTE_FAILED,
+                   "Couldn't read "
+                   "extended attribute for the file %d",
+                   fd);
+        if (errno == ENOENT)
+            errno = ENODATA;
+        return -1;
+    }
+
+    return ret;
 }
 
 /* Solaris does not support xattr for symlinks and dev files. Since gfid and
    other trusted attributes are stored as xattrs, we need to provide support for
-   them. A mapped regular file is stored in the /.glusterfs_xattr_inode of the export dir.
-   All xattr ops related to the special files are redirected to this map file.
+   them. A mapped regular file is stored in the /.glusterfs_xattr_inode of the
+   export dir. All xattr ops related to the special files are redirected to this
+   map file.
 */
 
 int
-make_export_path (const char *real_path, char **path)
+make_export_path(const char *real_path, char **path)
 {
-        int     ret = -1;
-        char   *tmp = NULL;
-        char   *export_path = NULL;
-        char   *dup = NULL;
-        char   *ptr = NULL;
-        char   *freeptr = NULL;
-        uuid_t  gfid = {0, };
-
-        export_path = GF_CALLOC (1, sizeof (char) * PATH_MAX, 0);
-        if (!export_path)
-                goto out;
+    int ret = -1;
+    char *tmp = NULL;
+    char *export_path = NULL;
+    char *dup = NULL;
+    char *ptr = NULL;
+    char *freeptr = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    export_path = GF_CALLOC(1, sizeof(char) * PATH_MAX, 0);
+    if (!export_path)
+        goto out;
 
-        dup = gf_strdup (real_path);
-        if (!dup)
-                goto out;
+    dup = gf_strdup(real_path);
+    if (!dup)
+        goto out;
 
-        freeptr = dup;
-        ret = solaris_getxattr ("/", GFID_XATTR_KEY, gfid, 16);
-        /* Return value of getxattr */
+    freeptr = dup;
+    ret = solaris_getxattr("/", GFID_XATTR_KEY, gfid, 16);
+    /* Return value of getxattr */
+    if (ret == 16) {
+        if (__is_root_gfid(gfid)) {
+            strcat(export_path, "/");
+            ret = 0;
+            goto done;
+        }
+    }
+
+    do {
+        ptr = strtok_r(dup, "/", &tmp);
+        if (!ptr)
+            break;
+        strcat(export_path, dup);
+        ret = solaris_getxattr(export_path, GFID_XATTR_KEY, gfid, 16);
         if (ret == 16) {
-                if (__is_root_gfid (gfid)){
-                        strcat (export_path, "/");
-                        ret = 0;
-                        goto done;
-                }
+            if (__is_root_gfid(gfid)) {
+                ret = 0;
+                goto done;
+            }
         }
+        strcat(export_path, "/");
+        dup = tmp;
+    } while (ptr);
 
-        do {
-                ptr = strtok_r (dup, "/", &tmp);
-                if (!ptr)
-                        break;
-                strcat (export_path, dup);
-                ret = solaris_getxattr (export_path, GFID_XATTR_KEY, gfid, 16);
-                if (ret == 16) {
-                        if (__is_root_gfid (gfid)) {
-                                ret = 0;
-                                goto done;
-                        }
-                }
-                strcat (export_path, "/");
-                dup = tmp;
-        } while (ptr);
-
-        goto out;
+    goto out;
 
 done:
-        if (!ret) {
-                *path = export_path;
-        }
+    if (!ret) {
+        *path = export_path;
+    }
 out:
-        GF_FREE (freeptr);
-        if (ret && export_path)
-                GF_FREE (export_path);
+    GF_FREE(freeptr);
+    if (ret && export_path)
+        GF_FREE(export_path);
 
-        return ret;
+    return ret;
 }
 int
-solaris_xattr_resolve_path (const char *real_path, char **path)
+solaris_xattr_resolve_path(const char *real_path, char **path)
 {
-        int                    ret  = -1;
-        char                   *export_path = NULL;
-        char                   xattr_path[PATH_MAX] = {0, };
-        struct stat            lstatbuf = {0, };
-        struct iatt            stbuf = {0, };
-        struct stat            statbuf = {0, };
-
-        ret = lstat (real_path, &lstatbuf);
-        if (ret != 0 )
-                return ret;
-        iatt_from_stat (&stbuf, &lstatbuf);
-        if (IA_ISREG(stbuf.ia_type) || IA_ISDIR(stbuf.ia_type))
-                return -1;
-
-        ret = make_export_path (real_path, &export_path);
-        if (!ret && export_path) {
-                strcat (export_path, "/"GF_SOLARIS_XATTR_DIR);
-                if (lstat (export_path, &statbuf)) {
-                        ret = mkdir (export_path, 0777);
-                        if (ret && (errno != EEXIST)) {
-                                gf_msg_debug (THIS->name, 0, "mkdir failed,"
-                                        " errno: %d", errno);
-                                goto out;
-                        }
-                }
+    int ret = -1;
+    char *export_path = NULL;
+    char xattr_path[PATH_MAX] = {
+        0,
+    };
+    struct stat lstatbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct stat statbuf = {
+        0,
+    };
+
+    ret = lstat(real_path, &lstatbuf);
+    if (ret != 0)
+        return ret;
+    iatt_from_stat(&stbuf, &lstatbuf);
+    if (IA_ISREG(stbuf.ia_type) || IA_ISDIR(stbuf.ia_type))
+        return -1;
+
+    ret = make_export_path(real_path, &export_path);
+    if (!ret && export_path) {
+        strcat(export_path, "/" GF_SOLARIS_XATTR_DIR);
+        if (lstat(export_path, &statbuf)) {
+            ret = mkdir(export_path, 0755);
+            if (ret && (errno != EEXIST)) {
+                gf_msg_debug(THIS->name, 0,
+                             "mkdir failed,"
+                             " errno: %d",
+                             errno);
+                goto out;
+            }
+        }
 
-                snprintf(xattr_path, PATH_MAX, "%s%s%lu", export_path,
-                         "/", stbuf.ia_ino);
+        snprintf(xattr_path, PATH_MAX, "%s%s%lu", export_path, "/",
+                 stbuf.ia_ino);
 
-                ret = lstat (xattr_path, &statbuf);
+        ret = lstat(xattr_path, &statbuf);
 
-                if (ret) {
-                        ret = mknod (xattr_path, S_IFREG|O_WRONLY, 0);
-                        if (ret && (errno != EEXIST)) {
-                                gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                        LG_MSG_FILE_OP_FAILED, "Failed to "
-                                        "create mapped file %s", xattr_path);
-                                goto out;
-                        }
-                }
-                *path = gf_strdup (xattr_path);
+        if (ret) {
+            ret = mknod(xattr_path, S_IFREG | O_WRONLY, 0);
+            if (ret && (errno != EEXIST)) {
+                gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                       "Failed to "
+                       "create mapped file %s",
+                       xattr_path);
+                goto out;
+            }
         }
+        *path = gf_strdup(xattr_path);
+    }
 out:
-        GF_FREE (export_path);
-        if (*path)
-                return 0;
-        else
-                return -1;
+    GF_FREE(export_path);
+    if (*path)
+        return 0;
+    else
+        return -1;
 }
 
 int
-solaris_setxattr(const char *path, const char* key, const char *value,
+solaris_setxattr(const char *path, const char *key, const char *value,
                  size_t size, int flags)
 {
-        int attrfd = -1;
-        int ret = 0;
-        char *mapped_path = NULL;
-
-        ret = solaris_xattr_resolve_path (path, &mapped_path);
-        if (!ret) {
-                attrfd = attropen (mapped_path,  key, flags|O_CREAT|O_WRONLY,
-                                   0777);
-        } else {
-                attrfd = attropen (path, key, flags|O_CREAT|O_WRONLY, 0777);
-        }
-        if (attrfd >= 0) {
-                ftruncate (attrfd, 0);
-                ret = write (attrfd, value, size);
-                close (attrfd);
-                ret = 0;
-        } else {
-                if (errno != ENOENT)
-                        gf_msg ("libglusterfs", GF_LOG_ERROR, errno,
-                                LG_MSG_SET_ATTRIBUTE_FAILED, "Couldn't set "
-                                "extended attribute for %s", path);
-                ret = -1;
-        }
-        GF_FREE (mapped_path);
-        return ret;
+    int attrfd = -1;
+    int ret = 0;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, key, flags | O_CREAT | O_WRONLY, 0777);
+    } else {
+        attrfd = attropen(path, key, flags | O_CREAT | O_WRONLY, 0777);
+    }
+    if (attrfd >= 0) {
+        ftruncate(attrfd, 0);
+        ret = write(attrfd, value, size);
+        close(attrfd);
+        ret = 0;
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_ERROR, errno,
+                   LG_MSG_SET_ATTRIBUTE_FAILED,
+                   "Couldn't set "
+                   "extended attribute for %s",
+                   path);
+        ret = -1;
+    }
+    GF_FREE(mapped_path);
+    return ret;
 }
 
-
 int
 solaris_listxattr(const char *path, char *list, size_t size)
 {
-        int attrdirfd = -1;
-        ssize_t len = 0;
-        DIR *dirptr = NULL;
-        struct dirent *dent = NULL;
-        int newfd = -1;
-        char *mapped_path = NULL;
-        int ret = -1;
-
-        ret = solaris_xattr_resolve_path (path, &mapped_path);
-        if (!ret) {
-                attrdirfd = attropen (mapped_path, ".", O_RDONLY, 0);
-        } else {
-                attrdirfd = attropen (path, ".", O_RDONLY, 0);
-        }
-        if (attrdirfd >= 0) {
-                newfd = dup(attrdirfd);
-                dirptr = fdopendir(newfd);
-                if (dirptr) {
-                        while ((dent = readdir(dirptr))) {
-                                size_t listlen = strlen(dent->d_name);
-                                if (!strcmp(dent->d_name, ".") ||
-                                    !strcmp(dent->d_name, "..")) {
-                                        /* we don't want "." and ".." here */
-                                        continue;
-                                }
-                                if (size == 0) {
-                                        /* return the current size of the list
-                                           of extended attribute names*/
-                                        len += listlen + 1;
-                                } else {
-                                        /* check size and copy entry + null
-                                           into list. */
-                                        if ((len + listlen + 1) > size) {
-                                                errno = ERANGE;
-                                                len = -1;
-                                                break;
-                                        } else {
-                                                strncpy(list + len, dent->d_name, listlen);
-                                                len += listlen;
-                                                list[len] = '\0';
-                                                ++len;
-                                        }
-                                }
-                        }
-
-                        if (closedir(dirptr) == -1) {
-                                close (attrdirfd);
-                                len = -1;
-                                goto out;
-                        }
+    int attrdirfd = -1;
+    ssize_t len = 0;
+    DIR *dirptr = NULL;
+    struct dirent *dent = NULL;
+    int newfd = -1;
+    char *mapped_path = NULL;
+    int ret = -1;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrdirfd = attropen(mapped_path, ".", O_RDONLY, 0);
+    } else {
+        attrdirfd = attropen(path, ".", O_RDONLY, 0);
+    }
+    if (attrdirfd >= 0) {
+        newfd = dup(attrdirfd);
+        dirptr = fdopendir(newfd);
+        if (dirptr) {
+            while ((dent = readdir(dirptr))) {
+                size_t listlen = strlen(dent->d_name);
+                if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
+                    /* we don't want "." and ".." here */
+                    continue;
+                }
+                if (size == 0) {
+                    /* return the current size of the list
+                       of extended attribute names*/
+                    len += listlen + 1;
                 } else {
-                        close (attrdirfd);
+                    /* check size and copy entry + null
+                       into list. */
+                    if ((len + listlen + 1) > size) {
+                        errno = ERANGE;
                         len = -1;
-                        goto out;
+                        break;
+                    } else {
+                        strncpy(list + len, dent->d_name, listlen);
+                        len += listlen;
+                        list[len] = '\0';
+                        ++len;
+                    }
                 }
-                close (attrdirfd);
+            }
+
+            if (closedir(dirptr) == -1) {
+                close(attrdirfd);
+                len = -1;
+                goto out;
+            }
+        } else {
+            close(attrdirfd);
+            len = -1;
+            goto out;
         }
+        close(attrdirfd);
+    }
 out:
-        GF_FREE (mapped_path);
-        return len;
+    GF_FREE(mapped_path);
+    return len;
 }
 
-
 int
 solaris_flistxattr(int fd, char *list, size_t size)
 {
-        int attrdirfd = -1;
-        ssize_t len = 0;
-        DIR *dirptr = NULL;
-        struct dirent *dent = NULL;
-        int newfd = -1;
-
-        attrdirfd = openat (fd, ".", O_RDONLY, 0);
-        if (attrdirfd >= 0) {
-                newfd = dup(attrdirfd);
-                dirptr = fdopendir(newfd);
-                if (dirptr) {
-                        while ((dent = readdir(dirptr))) {
-                                size_t listlen = strlen(dent->d_name);
-                                if (!strcmp(dent->d_name, ".") ||
-                                    !strcmp(dent->d_name, "..")) {
-                                        /* we don't want "." and ".." here */
-                                        continue;
-                                }
-                                if (size == 0) {
-                                        /* return the current size of the list
-                                           of extended attribute names*/
-                                        len += listlen + 1;
-                                } else {
-                                        /* check size and copy entry + null
-                                           into list. */
-                                        if ((len + listlen + 1) > size) {
-                                                errno = ERANGE;
-                                                len = -1;
-                                                break;
-                                        } else {
-                                                strncpy(list + len, dent->d_name, listlen);
-                                                len += listlen;
-                                                list[len] = '\0';
-                                                ++len;
-                                        }
-                                }
-                        }
-
-                        if (closedir(dirptr) == -1) {
-                                close (attrdirfd);
-                                return -1;
-                        }
+    int attrdirfd = -1;
+    ssize_t len = 0;
+    DIR *dirptr = NULL;
+    struct dirent *dent = NULL;
+    int newfd = -1;
+
+    attrdirfd = openat(fd, ".", O_RDONLY, 0);
+    if (attrdirfd >= 0) {
+        newfd = dup(attrdirfd);
+        dirptr = fdopendir(newfd);
+        if (dirptr) {
+            while ((dent = readdir(dirptr))) {
+                size_t listlen = strlen(dent->d_name);
+                if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
+                    /* we don't want "." and ".." here */
+                    continue;
+                }
+                if (size == 0) {
+                    /* return the current size of the list
+                       of extended attribute names*/
+                    len += listlen + 1;
                 } else {
-                        close (attrdirfd);
-                        return -1;
+                    /* check size and copy entry + null
+                       into list. */
+                    if ((len + listlen + 1) > size) {
+                        errno = ERANGE;
+                        len = -1;
+                        break;
+                    } else {
+                        strncpy(list + len, dent->d_name, listlen);
+                        len += listlen;
+                        list[len] = '\0';
+                        ++len;
+                    }
                 }
-                close (attrdirfd);
+            }
+
+            if (closedir(dirptr) == -1) {
+                close(attrdirfd);
+                return -1;
+            }
+        } else {
+            close(attrdirfd);
+            return -1;
         }
-        return len;
+        close(attrdirfd);
+    }
+    return len;
 }
 
-
 int
-solaris_removexattr(const char *path, const char* key)
+solaris_removexattr(const char *path, const char *key)
 {
-        int ret = -1;
-        int attrfd = -1;
-        char *mapped_path = NULL;
-
-        ret = solaris_xattr_resolve_path (path, &mapped_path);
-        if (!ret) {
-                attrfd = attropen (mapped_path, ".", O_RDONLY, 0);
-        } else {
-                attrfd = attropen (path, ".", O_RDONLY, 0);
-        }
-        if (attrfd >= 0) {
-                ret = unlinkat (attrfd, key, 0);
-                close (attrfd);
-        } else {
-                if (errno == ENOENT)
-                        errno = ENODATA;
-                ret = -1;
-        }
-
-        GF_FREE (mapped_path);
-
-        return ret;
+    int ret = -1;
+    int attrfd = -1;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, ".", O_RDONLY, 0);
+    } else {
+        attrfd = attropen(path, ".", O_RDONLY, 0);
+    }
+    if (attrfd >= 0) {
+        ret = unlinkat(attrfd, key, 0);
+        close(attrfd);
+    } else {
+        if (errno == ENOENT)
+            errno = ENODATA;
+        ret = -1;
+    }
+
+    GF_FREE(mapped_path);
+
+    return ret;
 }
 
 int
-solaris_getxattr(const char *path,
-                 const char* key,
-                 char *value,
-                 size_t size)
+solaris_getxattr(const char *path, const char *key, char *value, size_t size)
 {
-        int attrfd = -1;
-        int ret = 0;
-        char *mapped_path = NULL;
-
-        ret = solaris_xattr_resolve_path (path, &mapped_path);
-        if (!ret) {
-                attrfd = attropen (mapped_path, key, O_RDONLY, 0);
+    int attrfd = -1;
+    int ret = 0;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, key, O_RDONLY, 0);
+    } else {
+        attrfd = attropen(path, key, O_RDONLY, 0);
+    }
+
+    if (attrfd >= 0) {
+        if (size == 0) {
+            struct stat buf;
+            fstat(attrfd, &buf);
+            ret = buf.st_size;
         } else {
-                attrfd = attropen (path, key, O_RDONLY, 0);
+            ret = read(attrfd, value, size);
         }
-
-        if (attrfd >= 0) {
-                if (size == 0) {
-                        struct stat buf;
-                        fstat (attrfd, &buf);
-                        ret = buf.st_size;
-                } else {
-                        ret = read (attrfd, value, size);
-                }
-                close (attrfd);
-        } else {
-                if (errno != ENOENT)
-                        gf_msg ("libglusterfs", GF_LOG_INFO, errno,
-                                LG_MSG_READ_ATTRIBUTE_FAILED, "Couldn't read "
-                                "extended attribute for the file %s", path);
-                if (errno == ENOENT)
-                        errno = ENODATA;
-                ret = -1;
-        }
-        GF_FREE (mapped_path);
-        return ret;
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_INFO, errno,
+                   LG_MSG_READ_ATTRIBUTE_FAILED,
+                   "Couldn't read "
+                   "extended attribute for the file %s",
+                   path);
+        if (errno == ENOENT)
+            errno = ENODATA;
+        ret = -1;
+    }
+    GF_FREE(mapped_path);
+    return ret;
 }
 
-
-char* strsep(char** str, const char* delims)
+char *
+strsep(char **str, const char *delims)
 {
-        char* token;
-
-        if (*str==NULL) {
-                /* No more tokens */
-                return NULL;
+    char *token;
+
+    if (*str == NULL) {
+        /* No more tokens */
+        return NULL;
+    }
+
+    token = *str;
+    while (**str != '\0') {
+        if (strchr(delims, **str) != NULL) {
+            **str = '\0';
+            (*str)++;
+            return token;
         }
-
-        token=*str;
-        while (**str!='\0') {
-                if (strchr(delims,**str)!=NULL) {
-                        **str='\0';
-                        (*str)++;
-                        return token;
-                }
-                (*str)++;
-        }
-        /* There is no other token */
-        *str=NULL;
-        return token;
+        (*str)++;
+    }
+    /* There is no other token */
+    *str = NULL;
+    return token;
 }
 
 /* Code comes from libiberty */
 
 int
-vasprintf (char **result, const char *format, va_list args)
+vasprintf(char **result, const char *format, va_list args)
 {
-        return gf_vasprintf(result, format, args);
+    return gf_vasprintf(result, format, args);
 }
 
 int
-asprintf (char **buf, const char *fmt, ...)
+asprintf(char **buf, const char *fmt, ...)
 {
-        int status;
-        va_list ap;
+    int status;
+    va_list ap;
 
-        va_start (ap, fmt);
-        status = vasprintf (buf, fmt, ap);
-        va_end (ap);
-        return status;
+    va_start(ap, fmt);
+    status = vasprintf(buf, fmt, ap);
+    va_end(ap);
+    return status;
 }
 
-int solaris_unlink (const char *path)
+int
+solaris_unlink(const char *path)
 {
-        char *mapped_path = NULL;
-        struct stat     stbuf = {0, };
-        int ret = -1;
-
-        ret = solaris_xattr_resolve_path (path, &mapped_path);
-
-
-        if (!ret && mapped_path) {
-                if (lstat(path, &stbuf)) {
-                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                LG_MSG_FILE_OP_FAILED, "Stat failed on "
-                                "mapped file %s", mapped_path);
-                        goto out;
-                }
-                if (stbuf.st_nlink == 1) {
-                        if(remove (mapped_path))
-                                gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                        LG_MSG_FILE_OP_FAILED, "Failed to "
-                                        "remove mapped file %s", mapped_path);
-                }
-
+    char *mapped_path = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+
+    if (!ret && mapped_path) {
+        if (lstat(path, &stbuf)) {
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                   "Stat failed on "
+                   "mapped file %s",
+                   mapped_path);
+            goto out;
         }
+        if (stbuf.st_nlink == 1) {
+            if (remove(mapped_path))
+                gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                       "Failed to "
+                       "remove mapped file %s",
+                       mapped_path);
+        }
+    }
 
 out:
-        GF_FREE (mapped_path);
+    GF_FREE(mapped_path);
 
-        return  unlink (path);
+    return unlink(path);
 }
 
 int
-solaris_rename (const char *old_path, const char *new_path)
+solaris_rename(const char *old_path, const char *new_path)
 {
-        char *mapped_path = NULL;
-        int ret = -1;
-
-        ret = solaris_xattr_resolve_path (new_path, &mapped_path);
+    char *mapped_path = NULL;
+    int ret = -1;
 
+    ret = solaris_xattr_resolve_path(new_path, &mapped_path);
 
-        if (!ret && mapped_path) {
-                if (!remove (mapped_path))
-                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                LG_MSG_FILE_OP_FAILED, "Failed to remove "
-                                "mapped file %s.", mapped_path);
-                GF_FREE (mapped_path);
-        }
-
-        return rename(old_path, new_path);
+    if (!ret && mapped_path) {
+        if (!remove(mapped_path))
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                   "Failed to remove "
+                   "mapped file %s.",
+                   mapped_path);
+        GF_FREE(mapped_path);
+    }
 
+    return rename(old_path, new_path);
 }
 
 char *
-mkdtemp (char *tempstring)
+mkdtemp(char *tempstring)
 {
-        char *new_string = NULL;
-        int   ret        = 0;
+    char *new_string = NULL;
+    int ret = 0;
 
-        new_string = mkstemp (tempstring);
-        if (!new_string)
-                goto out;
+    new_string = mkstemp(tempstring);
+    if (!new_string)
+        goto out;
 
-        ret = mkdir (new_string, 0700);
-        if (ret < 0)
-                new_string = NULL;
+    ret = mkdir(new_string, 0700);
+    if (ret < 0)
+        new_string = NULL;
 
 out:
-        return new_string;
+    return new_string;
 }
 
 #endif /* GF_SOLARIS_HOST_OS */
 
+#ifdef GF_BSD_HOST_OS
+void
+gf_extattr_list_reshape(char *bsd_list, ssize_t size)
+{
+    /*
+     * the format of bsd_list is
+     *     <attr_len>attr<attr_len>attr...
+     * we try to reformat it as Linux's
+     *     attr<\0>attr<\0>...
+     * */
+    if (NULL == bsd_list || size <= 0)
+        return;
+
+    size_t i = 0, j;
+
+    while (i < size) {
+        size_t attr_len = bsd_list[i];
+
+        for (j = i; j < i + attr_len; ++j)
+            bsd_list[j] = bsd_list[j + 1];
+        bsd_list[j] = '\0';
+
+        i += attr_len + 1;
+        gf_msg_debug("syscall", 0, "syscall debug: %lu", attr_len);
+    }
+}
+#endif /* GF_BSD_HOST_OS */
+
 #ifndef HAVE_STRNLEN
 size_t
 strnlen(const char *string, size_t maxlen)
 {
-        int len = 0;
-        while ((len < maxlen) && string[len])
-                len++;
-        return len;
+    int len = 0;
+    while ((len < maxlen) && string[len])
+        len++;
+    return len;
 }
 #endif /* STRNLEN */
 
 int
-gf_umount_lazy (char *xlname, char *path, int rmdir_flag)
+gf_umount_lazy(char *xlname, char *path, int rmdir_flag)
 {
-        int                   ret               = -1;
-        runner_t              runner            = {0,};
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
 
-        runinit (&runner);
+    runinit(&runner);
 #ifdef GF_LINUX_HOST_OS
-        runner_add_args (&runner, _PATH_UMOUNT, "-l", path, NULL);
+    runner_add_args(&runner, _PATH_UMOUNT, "-l", path, NULL);
 #else
-        if (rmdir_flag)
-                runner_add_args (&runner, SBIN_DIR "/umountd",
-                                 "-r", path, NULL);
-        else
-                runner_add_args (&runner, SBIN_DIR "/umountd",
-                                 path, NULL);
+    if (rmdir_flag)
+        runner_add_args(&runner, SBIN_DIR "/umountd", "-r", path, NULL);
+    else
+        runner_add_args(&runner, SBIN_DIR "/umountd", path, NULL);
 #endif
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_msg (xlname, GF_LOG_ERROR, errno, LG_MSG_UNMOUNT_FAILED,
-                        "Lazy unmount of %s", path);
-        }
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(xlname, GF_LOG_ERROR, errno, LG_MSG_UNMOUNT_FAILED,
+               "Lazy unmount of %s", path);
+    }
 
 #ifdef GF_LINUX_HOST_OS
-        if (!ret && rmdir_flag) {
-                ret = sys_rmdir (path);
-                if (ret)
-                        gf_msg (xlname, GF_LOG_WARNING, errno,
-                                LG_MSG_DIR_OP_FAILED, "rmdir %s", path);
-        }
+    if (!ret && rmdir_flag) {
+        ret = sys_rmdir(path);
+        if (ret)
+            gf_msg(xlname, GF_LOG_WARNING, errno, LG_MSG_DIR_OP_FAILED,
+                   "rmdir %s", path);
+    }
 #endif
 
-        return ret;
+    return ret;
 }
diff --git a/libglusterfs/src/compound-fop-utils.c b/libglusterfs/src/compound-fop-utils.c
deleted file mode 100644
index 03d7b5ba459..00000000000
--- a/libglusterfs/src/compound-fop-utils.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "default-args.h"
-#include "mem-types.h"
-#include "dict.h"
-
-void
-compound_args_cleanup (compound_args_t *args)
-{
-        int i;
-
-        if (!args)
-                return;
-
-        if (args->xdata)
-                dict_unref (args->xdata);
-
-        if (args->req_list) {
-                for (i = 0; i < args->fop_length; i++) {
-                        args_wipe (&args->req_list[i]);
-                }
-        }
-
-        GF_FREE (args->enum_list);
-        GF_FREE (args->req_list);
-        GF_FREE (args);
-}
-
-void
-compound_args_cbk_cleanup (compound_args_cbk_t *args_cbk)
-{
-        int i;
-
-        if (!args_cbk)
-                return;
-
-        if (args_cbk->xdata)
-                dict_unref (args_cbk->xdata);
-
-        if (args_cbk->rsp_list) {
-                for (i = 0; i < args_cbk->fop_length; i++) {
-                        args_cbk_wipe (&args_cbk->rsp_list[i]);
-                }
-        }
-
-        GF_FREE (args_cbk->rsp_list);
-        GF_FREE (args_cbk->enum_list);
-        GF_FREE (args_cbk);
-}
-
-compound_args_cbk_t*
-compound_args_cbk_alloc (int length, dict_t *xdata)
-{
-        int                 i             = 0;
-        compound_args_cbk_t *args_cbk     = NULL;
-
-        args_cbk = GF_CALLOC (1, sizeof (*args_cbk), gf_mt_compound_rsp_t);
-        if (!args_cbk)
-                return NULL;
-
-        args_cbk->fop_length = length;
-
-        args_cbk->rsp_list = GF_CALLOC (length, sizeof (*args_cbk->rsp_list),
-                                        gf_mt_default_args_cbk_t);
-        if (!args_cbk->rsp_list)
-                goto out;
-
-        for (i = 0; i < length; i++) {
-                args_cbk_init (&args_cbk->rsp_list[i]);
-        }
-
-        args_cbk->enum_list = GF_CALLOC (length, sizeof (*args_cbk->enum_list),
-                                        gf_common_mt_int);
-        if (!args_cbk->enum_list)
-                goto out;
-
-        if (xdata) {
-                args_cbk->xdata = dict_copy_with_ref (xdata, NULL);
-                if (!args_cbk->xdata)
-                        goto out;
-        }
-
-        return args_cbk;
-out:
-        compound_args_cbk_cleanup (args_cbk);
-        return NULL;
-}
-
-compound_args_t*
-compound_fop_alloc (int length, glusterfs_compound_fop_t fop, dict_t *xdata)
-{
-        compound_args_t *args     = NULL;
-
-        args = GF_CALLOC (1, sizeof (*args), gf_mt_compound_req_t);
-
-        if (!args)
-                return NULL;
-
-        /* fop_enum can be used by xlators to see which fops are
-         * included as part of compound fop. This will help in checking
-         * for compatibility or support without going through the entire
-         * fop list packed.
-         */
-        args->fop_enum = fop;
-        args->fop_length = length;
-
-        args->enum_list = GF_CALLOC (length, sizeof (*args->enum_list),
-                                     gf_common_mt_int);
-
-        if (!args->enum_list)
-                goto out;
-
-        args->req_list = GF_CALLOC (length, sizeof (*args->req_list),
-                                     gf_mt_default_args_t);
-
-        if (!args->req_list)
-                goto out;
-
-        if (xdata) {
-                args->xdata = dict_copy_with_ref (xdata, args->xdata);
-                if (!args->xdata)
-                        goto out;
-        }
-
-        return args;
-out:
-        compound_args_cleanup (args);
-        return NULL;
-}
diff --git a/libglusterfs/src/compound-fop-utils.h b/libglusterfs/src/compound-fop-utils.h
deleted file mode 100644
index bfd0649aef2..00000000000
--- a/libglusterfs/src/compound-fop-utils.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __COMPOUND_FOP_UTILS_H__
-#define __COMPOUND_FOP_UTILS_H__
-
-#include "defaults.h"
-#include "default-args.h"
-#include "mem-types.h"
-#include "dict.h"
-
-#define COMPOUND_PACK_ARGS(fop, fop_enum, args, counter, params ...) do {    \
-        args->enum_list[counter] = fop_enum;                                 \
-        args_##fop##_store (&args->req_list[counter], params);               \
-} while (0)
-
-compound_args_t*
-compound_fop_alloc (int length, glusterfs_compound_fop_t fop, dict_t *xdata);
-
-void
-compound_args_cleanup (compound_args_t *args);
-
-void
-compound_args_cbk_cleanup (compound_args_cbk_t *args_cbk);
-
-compound_args_cbk_t*
-compound_args_cbk_alloc (int length, dict_t *xdata);
-#endif /* __COMPOUND_FOP_UTILS_H__ */
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
index b009e6270a2..3d890b04ec9 100644
--- a/libglusterfs/src/ctx.c
+++ b/libglusterfs/src/ctx.c
@@ -9,42 +9,89 @@
 */
 
 #include <pthread.h>
-#include "globals.h"
 
-#include "glusterfs.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/glusterfs.h"
+#include "timer-wheel.h"
 
 glusterfs_ctx_t *
-glusterfs_ctx_new ()
+glusterfs_ctx_new()
 {
-        int               ret = 0;
-	glusterfs_ctx_t  *ctx = NULL;
-
-	/* no GF_CALLOC here, gf_acct_mem_set_enable is not
-	   yet decided at this point */
-        ctx = calloc (1, sizeof (*ctx));
-        if (!ctx) {
-                ret = -1;
-                goto out;
-        }
+    glusterfs_ctx_t *ctx = NULL;
+
+    /* no GF_CALLOC here, gf_acct_mem_set_enable is not
+       yet decided at this point */
+    ctx = CALLOC(1, sizeof(*ctx));
+    if (!ctx) {
+        goto out;
+    }
 
-        ctx->mem_acct_enable = gf_global_mem_acct_enable_get();
+    ctx->mem_acct_enable = gf_global_mem_acct_enable_get();
 
-        INIT_LIST_HEAD (&ctx->graphs);
-	INIT_LIST_HEAD (&ctx->mempool_list);
+    INIT_LIST_HEAD(&ctx->graphs);
+    INIT_LIST_HEAD(&ctx->mempool_list);
+    INIT_LIST_HEAD(&ctx->volfile_list);
 
-	ctx->daemon_pipe[0] = -1;
-	ctx->daemon_pipe[1] = -1;
+    ctx->daemon_pipe[0] = -1;
+    ctx->daemon_pipe[1] = -1;
 
-        ctx->log.loglevel = DEFAULT_LOG_LEVEL;
+    ctx->log.loglevel = DEFAULT_LOG_LEVEL;
 
-        /* lock is never destroyed! */
-	ret = LOCK_INIT (&ctx->lock);
-	if (ret) {
-		free (ctx);
-		ctx = NULL;
-	}
+#if defined(RUN_WITH_MEMCHECK)
+    ctx->cmd_args.vgtool = _gf_memcheck;
+#elif defined(RUN_WITH_DRD)
+    ctx->cmd_args.vgtool = _gf_drd;
+#else
+    ctx->cmd_args.vgtool = _gf_none;
+#endif
 
+    /* lock is never destroyed! */
+    if (LOCK_INIT(&ctx->lock)) {
+        free(ctx);
+        ctx = NULL;
+        goto out;
+    }
+
+    GF_ATOMIC_INIT(ctx->stats.max_dict_pairs, 0);
+    GF_ATOMIC_INIT(ctx->stats.total_pairs_used, 0);
+    GF_ATOMIC_INIT(ctx->stats.total_dicts_used, 0);
 out:
-	return ctx;
+    return ctx;
+}
+
+static void
+glusterfs_ctx_tw_destroy(struct gf_ctx_tw *ctx_tw)
+{
+    if (ctx_tw->timer_wheel)
+        gf_tw_cleanup_timers(ctx_tw->timer_wheel);
+
+    GF_FREE(ctx_tw);
 }
 
+struct tvec_base *
+glusterfs_ctx_tw_get(glusterfs_ctx_t *ctx)
+{
+    struct gf_ctx_tw *ctx_tw = NULL;
+
+    LOCK(&ctx->lock);
+    {
+        if (ctx->tw) {
+            ctx_tw = GF_REF_GET(ctx->tw);
+        } else {
+            ctx_tw = GF_CALLOC(1, sizeof(struct gf_ctx_tw),
+                               gf_common_mt_tw_ctx);
+            ctx_tw->timer_wheel = gf_tw_init_timers();
+            GF_REF_INIT(ctx_tw, glusterfs_ctx_tw_destroy);
+            ctx->tw = ctx_tw;
+        }
+    }
+    UNLOCK(&ctx->lock);
+
+    return ctx_tw->timer_wheel;
+}
+
+void
+glusterfs_ctx_tw_put(glusterfs_ctx_t *ctx)
+{
+    GF_REF_PUT(ctx->tw);
+}
diff --git a/libglusterfs/src/daemon.c b/libglusterfs/src/daemon.c
index 348e3ad4083..0a3e5438325 100644
--- a/libglusterfs/src/daemon.c
+++ b/libglusterfs/src/daemon.c
@@ -8,59 +8,58 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <fcntl.h>
 #include <unistd.h>
 #include <stdio.h>
-#include "daemon.h"
+#include "glusterfs/daemon.h"
 
 int
-os_daemon_return (int nochdir, int noclose)
+os_daemon_return(int nochdir, int noclose)
 {
-	pid_t   pid  = -1;
-	int     ret  = -1;
-        FILE    *ptr = NULL;
+    pid_t pid = -1;
+    int ret = -1;
+    FILE *ptr = NULL;
 
-	ret = fork();
-	if (ret)
-		return ret;
+    ret = fork();
+    if (ret)
+        return ret;
 
-	pid = setsid();
+    pid = setsid();
 
-	if (pid == -1) {
-                ret = -1;
-		goto out;
-        }
+    if (pid == -1) {
+        ret = -1;
+        goto out;
+    }
 
-	if (!nochdir)
-		ret = chdir("/");
+    if (!nochdir)
+        ret = chdir("/");
 
-        if (!noclose) {
-                ptr = freopen (DEVNULLPATH, "r", stdin);
-                if (!ptr)
-                        goto out;
+    if (!noclose) {
+        ptr = freopen(DEVNULLPATH, "r", stdin);
+        if (!ptr)
+            goto out;
 
-                ptr = freopen (DEVNULLPATH, "w", stdout);
-                if (!ptr)
-                        goto out;
+        ptr = freopen(DEVNULLPATH, "w", stdout);
+        if (!ptr)
+            goto out;
 
-                ptr = freopen (DEVNULLPATH, "w", stderr);
-                if (!ptr)
-                        goto out;
-	}
+        ptr = freopen(DEVNULLPATH, "w", stderr);
+        if (!ptr)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
 int
-os_daemon (int nochdir, int noclose)
+os_daemon(int nochdir, int noclose)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = os_daemon_return (nochdir, noclose);
-	if (ret <= 0)
-		return ret;
+    ret = os_daemon_return(nochdir, noclose);
+    if (ret <= 0)
+        return ret;
 
-	_exit (0);
+    _exit(0);
 }
diff --git a/libglusterfs/src/default-args.c b/libglusterfs/src/default-args.c
index f40de2dae68..a0ba1cfb299 100644
--- a/libglusterfs/src/default-args.c
+++ b/libglusterfs/src/default-args.c
@@ -13,1597 +13,1639 @@
 #include "config.h"
 #endif
 
-#include "xlator.h"
-#include "defaults.h"
+#include "glusterfs/defaults.h"
 
 int
-args_lookup_store (default_args_t *args, loc_t *loc,
-                   dict_t *xdata)
+args_lookup_store(default_args_t *args, loc_t *loc, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_lookup_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent)
+args_lookup_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, inode_t *inode, struct iatt *buf,
+                      dict_t *xdata, struct iatt *postparent)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_stat_store (default_args_t *args, loc_t *loc, dict_t *xdata)
+args_stat_store(default_args_t *args, loc_t *loc, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_stat_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *buf, dict_t *xdata)
+args_stat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    struct iatt *buf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret == 0)
-                args->stat = *buf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fstat_store (default_args_t *args, fd_t *fd, dict_t *xdata)
+args_fstat_store(default_args_t *args, fd_t *fd, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fstat_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *buf, dict_t *xdata)
+args_fstat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *buf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (buf)
-                args->stat = *buf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (buf)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_truncate_store (default_args_t *args, loc_t *loc, off_t off,
-                     dict_t *xdata)
+args_truncate_store(default_args_t *args, loc_t *loc, off_t off, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->offset = off;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_truncate_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+args_truncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (prebuf)
-                args->prestat = *prebuf;
-        if (postbuf)
-                args->poststat = *postbuf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_ftruncate_store (default_args_t *args, fd_t *fd, off_t off,
-                      dict_t *xdata)
+args_ftruncate_store(default_args_t *args, fd_t *fd, off_t off, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-        args->offset = off;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_ftruncate_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+args_ftruncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (prebuf)
-                args->prestat = *prebuf;
-        if (postbuf)
-                args->poststat = *postbuf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_access_store (default_args_t *args, loc_t *loc, int32_t mask,
-                   dict_t *xdata)
+args_access_store(default_args_t *args, loc_t *loc, int32_t mask, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->mask = mask;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->mask = mask;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_access_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_access_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_readlink_store (default_args_t *args, loc_t *loc, size_t size,
-                     dict_t *xdata)
+args_readlink_store(default_args_t *args, loc_t *loc, size_t size,
+                    dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->size = size;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->size = size;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_readlink_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       const char *path, struct iatt *stbuf, dict_t *xdata)
+args_readlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, const char *path, struct iatt *stbuf,
+                        dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (path)
-                args->buf = gf_strdup (path);
-        if (stbuf)
-                args->stat = *stbuf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (path)
+        args->buf = gf_strdup(path);
+    if (stbuf)
+        args->stat = *stbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_mknod_store (default_args_t *args, loc_t *loc, mode_t mode,
-                  dev_t rdev, mode_t umask, dict_t *xdata)
+args_mknod_store(default_args_t *args, loc_t *loc, mode_t mode, dev_t rdev,
+                 mode_t umask, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->mode = mode;
-        args->rdev = rdev;
-        args->umask = umask;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->rdev = rdev;
+    args->umask = umask;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_mknod_cbk_store (default_args_cbk_t *args, int op_ret,
-                    int32_t op_errno, inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata)
+args_mknod_cbk_store(default_args_cbk_t *args, int op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_mkdir_store (default_args_t *args, loc_t *loc, mode_t mode,
-                  mode_t umask, dict_t *xdata)
+args_mkdir_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+                 dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->mode  = mode;
-        args->umask = umask;
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->umask = umask;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_mkdir_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+args_mkdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_unlink_store (default_args_t *args, loc_t *loc, int xflag, dict_t *xdata)
+args_unlink_store(default_args_t *args, loc_t *loc, int xflag, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->xflag = xflag;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->xflag = xflag;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_unlink_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+args_unlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_rmdir_store (default_args_t *args, loc_t *loc, int flags, dict_t *xdata)
+args_rmdir_store(default_args_t *args, loc_t *loc, int flags, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->flags = flags;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_rmdir_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata)
+args_rmdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_symlink_store (default_args_t *args, const char *linkname, loc_t *loc,
+args_symlink_store(default_args_t *args, const char *linkname, loc_t *loc,
                    mode_t umask, dict_t *xdata)
 {
-        args->linkname = gf_strdup (linkname);
-        args->umask = umask;
-        loc_copy (&args->loc, loc);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->linkname = gf_strdup(linkname);
+    args->umask = umask;
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_symlink_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+args_symlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, inode_t *inode, struct iatt *buf,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_rename_store (default_args_t *args, loc_t *oldloc, loc_t *newloc,
-                   dict_t *xdata)
+args_rename_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                  dict_t *xdata)
 {
-        loc_copy (&args->loc, oldloc);
-        loc_copy (&args->loc2, newloc);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, oldloc);
+    loc_copy(&args->loc2, newloc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_rename_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata)
+args_rename_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *buf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (buf)
-                args->stat = *buf;
-        if (preoldparent)
-                args->preparent = *preoldparent;
-        if (postoldparent)
-                args->postparent = *postoldparent;
-        if (prenewparent)
-                args->preparent2 = *prenewparent;
-        if (postnewparent)
-                args->postparent2 = *postnewparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (buf)
+        args->stat = *buf;
+    if (preoldparent)
+        args->preparent = *preoldparent;
+    if (postoldparent)
+        args->postparent = *postoldparent;
+    if (prenewparent)
+        args->preparent2 = *prenewparent;
+    if (postnewparent)
+        args->postparent2 = *postnewparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_link_store (default_args_t *args, loc_t *oldloc, loc_t *newloc,
-                 dict_t *xdata)
+args_link_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                dict_t *xdata)
 {
-        loc_copy (&args->loc, oldloc);
-        loc_copy (&args->loc2, newloc);
+    loc_copy(&args->loc, oldloc);
+    loc_copy(&args->loc2, newloc);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_link_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
-                   dict_t *xdata)
+args_link_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_create_store (default_args_t *args,
-                  loc_t *loc, int32_t flags, mode_t mode,
+args_create_store(default_args_t *args, loc_t *loc, int32_t flags, mode_t mode,
                   mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->flags = flags;
-        args->mode = mode;
-        args->umask = umask;
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    args->mode = mode;
+    args->umask = umask;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_create_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+args_create_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, fd_t *fd, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (inode)
-                args->inode = inode_ref (inode);
-        if (buf)
-                args->stat = *buf;
-        if (preparent)
-                args->preparent = *preparent;
-        if (postparent)
-                args->postparent = *postparent;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_open_store (default_args_t *args, loc_t *loc, int32_t flags,
-                 fd_t *fd, dict_t *xdata)
+args_open_store(default_args_t *args, loc_t *loc, int32_t flags, fd_t *fd,
+                dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->flags = flags;
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_open_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, dict_t *xdata)
+args_open_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    fd_t *fd, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_readv_store (default_args_t *args, fd_t *fd, size_t size, off_t off,
-                  uint32_t flags, dict_t *xdata)
+args_readv_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                 uint32_t flags, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->size  = size;
-        args->offset  = off;
-        args->flags = flags;
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
+    args->flags = flags;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_readv_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                    int32_t count, struct iatt *stbuf,
-                    struct iobref *iobref, dict_t *xdata)
+args_readv_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iovec *vector, int32_t count, struct iatt *stbuf,
+                     struct iobref *iobref, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret >= 0) {
-                args->vector = iov_dup (vector, count);
-                args->count = count;
-                args->stat = *stbuf;
-                args->iobref = iobref_ref (iobref);
-        }
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        args->vector = iov_dup(vector, count);
+        args->count = count;
+        args->stat = *stbuf;
+        args->iobref = iobref_ref(iobref);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_writev_store (default_args_t *args, fd_t *fd, struct iovec *vector,
-                   int32_t count, off_t off, uint32_t flags,
-                   struct iobref *iobref, dict_t *xdata)
+args_writev_store(default_args_t *args, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t off, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->vector = iov_dup (vector, count);
-        args->count  = count;
-        args->offset = off;
-        args->flags  = flags;
-        args->iobref = iobref_ref (iobref);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->vector = iov_dup(vector, count);
+    args->count = count;
+    args->offset = off;
+    args->flags = flags;
+    args->iobref = iobref_ref(iobref);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_writev_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+args_writev_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret >= 0)
-                args->poststat = *postbuf;
-        if (prebuf)
-                args->prestat = *prebuf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0)
+        args->poststat = *postbuf;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_flush_store (default_args_t *args, fd_t *fd, dict_t *xdata)
+args_put_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+               uint32_t flags, struct iovec *vector, int32_t count, off_t off,
+               struct iobref *iobref, dict_t *xattr, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->umask = umask;
+    args->flags = flags;
+    args->vector = iov_dup(vector, count);
+    args->count = count;
+    args->offset = off;
+    args->iobref = iobref_ref(iobref);
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_flush_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_put_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0)
+        args->stat = *buf;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
+}
+int
+args_flush_store(default_args_t *args, fd_t *fd, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fsync_store (default_args_t *args, fd_t *fd, int32_t datasync,
-                  dict_t *xdata)
+args_flush_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->datasync = datasync;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fsync_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                 dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->datasync = datasync;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fsync_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+args_fsync_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (prebuf)
-                args->prestat = *prebuf;
-        if (postbuf)
-                args->poststat = *postbuf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_opendir_store (default_args_t *args, loc_t *loc, fd_t *fd, dict_t *xdata)
+args_opendir_store(default_args_t *args, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    loc_copy(&args->loc, loc);
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_opendir_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, dict_t *xdata)
+args_opendir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (fd)
-                args->fd = fd_ref (fd);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fsyncdir_store (default_args_t *args, fd_t *fd, int32_t datasync,
-                     dict_t *xdata)
+args_fsyncdir_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                    dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->datasync = datasync;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->datasync = datasync;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 int
-args_fsyncdir_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_fsyncdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_statfs_store (default_args_t *args, loc_t *loc, dict_t *xdata)
+args_statfs_store(default_args_t *args, loc_t *loc, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_statfs_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct statvfs *buf, dict_t *xdata)
+args_statfs_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret == 0)
-                args->statvfs = *buf;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->statvfs = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_setxattr_store (default_args_t *args,
-                     loc_t *loc, dict_t *dict,
-                     int32_t flags, dict_t *xdata)
+args_setxattr_store(default_args_t *args, loc_t *loc, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        /* TODO */
-        if (dict)
-                args->xattr = dict_ref (dict);
-        args->flags = flags;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    /* TODO */
+    if (dict)
+        args->xattr = dict_ref(dict);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_setxattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret,
-                       int32_t op_errno, dict_t *xdata)
+args_setxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_getxattr_store (default_args_t *args,
-                     loc_t *loc, const char *name, dict_t *xdata)
+args_getxattr_store(default_args_t *args, loc_t *loc, const char *name,
+                    dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
+    loc_copy(&args->loc, loc);
 
-        if (name)
-                args->name = gf_strdup (name);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (name)
+        args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_getxattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *dict, dict_t *xdata)
+args_getxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (dict)
-                args->xattr = dict_ref (dict);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (dict)
+        args->xattr = dict_ref(dict);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fsetxattr_store (default_args_t *args,
-                      fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+args_fsetxattr_store(default_args_t *args, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
+    args->fd = fd_ref(fd);
 
-        if (dict)
-                args->xattr = dict_ref (dict);
-        args->flags = flags;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (dict)
+        args->xattr = dict_ref(dict);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fsetxattr_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_fsetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fgetxattr_store (default_args_t *args,
-                      fd_t *fd, const char *name, dict_t *xdata)
+args_fgetxattr_store(default_args_t *args, fd_t *fd, const char *name,
+                     dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
+    args->fd = fd_ref(fd);
 
-        if (name)
-                args->name = gf_strdup (name);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (name)
+        args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fgetxattr_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+args_fgetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (dict)
-                args->xattr = dict_ref (dict);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (dict)
+        args->xattr = dict_ref(dict);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_removexattr_store (default_args_t *args,
-                        loc_t *loc, const char *name, dict_t *xdata)
+args_removexattr_store(default_args_t *args, loc_t *loc, const char *name,
+                       dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->name = gf_strdup (name);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_removexattr_cbk_store (default_args_cbk_t *args,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_removexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fremovexattr_store (default_args_t *args,
-                         fd_t *fd, const char *name, dict_t *xdata)
+args_fremovexattr_store(default_args_t *args, fd_t *fd, const char *name,
+                        dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
-        args->name = gf_strdup (name);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    args->fd = fd_ref(fd);
+    args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fremovexattr_cbk_store (default_args_cbk_t *args,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_fremovexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                            int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_lk_store (default_args_t *args,
-               fd_t *fd, int32_t cmd,
-                struct gf_flock *lock, dict_t *xdata)
+args_lk_store(default_args_t *args, fd_t *fd, int32_t cmd,
+              struct gf_flock *lock, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->cmd = cmd;
-        args->lock = *lock;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->cmd = cmd;
+    args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_lk_cbk_store (default_args_cbk_t *args,
-                 int32_t op_ret, int32_t op_errno,
-                 struct gf_flock *lock, dict_t *xdata)
+args_lk_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                  struct gf_flock *lock, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret == 0)
-                args->lock = *lock;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_inodelk_store (default_args_t *args,
-                    const char *volume, loc_t *loc, int32_t cmd,
-                    struct gf_flock *lock, dict_t *xdata)
+args_inodelk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        if (volume)
-                args->volume = gf_strdup (volume);
+    if (volume)
+        args->volume = gf_strdup(volume);
 
-        loc_copy (&args->loc, loc);
-        args->cmd  = cmd;
-        args->lock = *lock;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    loc_copy(&args->loc, loc);
+    args->cmd = cmd;
+    args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_inodelk_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_inodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret   = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_finodelk_store (default_args_t *args,
-                     const char *volume, fd_t *fd, int32_t cmd,
-                     struct gf_flock *lock, dict_t *xdata)
+args_finodelk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        if (fd)
-                args->fd   = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-        if (volume)
-                args->volume = gf_strdup (volume);
+    if (volume)
+        args->volume = gf_strdup(volume);
 
-        args->cmd  = cmd;
-        args->lock = *lock;
+    args->cmd = cmd;
+    args->lock = *lock;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_finodelk_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_finodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret   = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_entrylk_store (default_args_t *args,
-                    const char *volume, loc_t *loc, const char *name,
-                    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+args_entrylk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   const char *name, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
 {
-        if (volume)
-                args->volume = gf_strdup (volume);
+    if (volume)
+        args->volume = gf_strdup(volume);
 
-        loc_copy (&args->loc, loc);
+    loc_copy(&args->loc, loc);
 
-        args->entrylkcmd = cmd;
-        args->entrylktype = type;
+    args->entrylkcmd = cmd;
+    args->entrylktype = type;
 
-        if (name)
-                args->name = gf_strdup (name);
+    if (name)
+        args->name = gf_strdup(name);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_entrylk_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_entrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret   = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fentrylk_store (default_args_t *args,
-                     const char *volume, fd_t *fd, const char *name,
-                     entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+args_fentrylk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    const char *name, entrylk_cmd cmd, entrylk_type type,
+                    dict_t *xdata)
 {
-        if (volume)
-                args->volume = gf_strdup (volume);
+    if (volume)
+        args->volume = gf_strdup(volume);
 
-        if (fd)
-                args->fd = fd_ref (fd);
-        args->entrylkcmd = cmd;
-        args->entrylktype = type;
-        if (name)
-                args->name = gf_strdup (name);
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->entrylkcmd = cmd;
+    args->entrylktype = type;
+    if (name)
+        args->name = gf_strdup(name);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fentrylk_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_fentrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
 {
-        args->op_ret   = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_readdirp_store (default_args_t *args,
-                     fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        args->fd = fd_ref (fd);
-        args->size = size;
-        args->offset = off;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
-}
-
-int
-args_readdirp_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       gf_dirent_t *entries, dict_t *xdata)
+args_readdirp_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                    dict_t *xdata)
 {
-        gf_dirent_t *stub_entry = NULL, *entry = NULL;
-
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret > 0) {
-                list_for_each_entry (entry, &entries->list, list) {
-                        stub_entry = gf_dirent_for_name (entry->d_name);
-                        if (!stub_entry)
-                                goto out;
-                        stub_entry->d_off = entry->d_off;
-                        stub_entry->d_ino = entry->d_ino;
-                        stub_entry->d_stat = entry->d_stat;
-                        stub_entry->d_type = entry->d_type;
-                        if (entry->inode)
-                                stub_entry->inode = inode_ref (entry->inode);
-                        if (entry->dict)
-                                stub_entry->dict = dict_ref (entry->dict);
-                        list_add_tail (&stub_entry->list,
-                                       &args->entries.list);
-                }
+    args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_readdirp_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+    gf_dirent_t *stub_entry = NULL, *entry = NULL;
+
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            stub_entry = gf_dirent_for_name(entry->d_name);
+            if (!stub_entry)
+                goto out;
+            stub_entry->d_off = entry->d_off;
+            stub_entry->d_ino = entry->d_ino;
+            stub_entry->d_stat = entry->d_stat;
+            stub_entry->d_type = entry->d_type;
+            if (entry->inode)
+                stub_entry->inode = inode_ref(entry->inode);
+            if (entry->dict)
+                stub_entry->dict = dict_ref(entry->dict);
+            list_add_tail(&stub_entry->list, &args->entries.list);
         }
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-args_readdir_store (default_args_t *args,
-                    fd_t *fd, size_t size,
-                    off_t off, dict_t *xdata)
+args_readdir_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                   dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
-        args->size = size;
-        args->offset = off;
+    args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_readdir_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      gf_dirent_t *entries, dict_t *xdata)
+args_readdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        gf_dirent_t *stub_entry = NULL, *entry = NULL;
+    gf_dirent_t *stub_entry = NULL, *entry = NULL;
 
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret > 0) {
-                list_for_each_entry (entry, &entries->list, list) {
-                        stub_entry = gf_dirent_for_name (entry->d_name);
-                        if (!stub_entry)
-                                goto out;
-                        stub_entry->d_off = entry->d_off;
-                        stub_entry->d_ino = entry->d_ino;
-                        stub_entry->d_type = entry->d_type;
-                        list_add_tail (&stub_entry->list,
-                                       &args->entries.list);
-                }
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            stub_entry = gf_dirent_for_name(entry->d_name);
+            if (!stub_entry)
+                goto out;
+            stub_entry->d_off = entry->d_off;
+            stub_entry->d_ino = entry->d_ino;
+            stub_entry->d_type = entry->d_type;
+            list_add_tail(&stub_entry->list, &args->entries.list);
         }
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-args_rchecksum_store (default_args_t *args,
-                      fd_t *fd, off_t offset, int32_t len, dict_t *xdata)
+args_rchecksum_store(default_args_t *args, fd_t *fd, off_t offset, int32_t len,
+                     dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
-        args->offset = offset;
-        args->size    = len;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    args->fd = fd_ref(fd);
+    args->offset = offset;
+    args->size = len;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_rchecksum_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno,
-                        uint32_t weak_checksum, uint8_t *strong_checksum,
-                        dict_t *xdata)
+args_rchecksum_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, uint32_t weak_checksum,
+                         uint8_t *strong_checksum, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret >= 0) {
-                args->weak_checksum =
-                        weak_checksum;
-                args->strong_checksum =
-                        memdup (strong_checksum, MD5_DIGEST_LENGTH);
-        }
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        args->weak_checksum = weak_checksum;
+        args->strong_checksum = gf_memdup(strong_checksum,
+                                          SHA256_DIGEST_LENGTH);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_xattrop_store (default_args_t *args,
-                    loc_t *loc, gf_xattrop_flags_t optype,
-                    dict_t *xattr, dict_t *xdata)
+args_xattrop_store(default_args_t *args, loc_t *loc, gf_xattrop_flags_t optype,
+                   dict_t *xattr, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
+    loc_copy(&args->loc, loc);
 
-        args->optype = optype;
-        args->xattr = dict_ref (xattr);
+    args->optype = optype;
+    args->xattr = dict_ref(xattr);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
-
 int
-args_xattrop_cbk_store (default_args_cbk_t *args, int32_t op_ret,
-                        int32_t op_errno, dict_t *xattr, dict_t *xdata)
+args_xattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        args->op_ret   = op_ret;
-        args->op_errno = op_errno;
-        if (xattr)
-                args->xattr = dict_ref (xattr);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_fxattrop_store (default_args_t *args,
-                     fd_t *fd, gf_xattrop_flags_t optype,
-                     dict_t *xattr, dict_t *xdata)
+args_fxattrop_store(default_args_t *args, fd_t *fd, gf_xattrop_flags_t optype,
+                    dict_t *xattr, dict_t *xdata)
 {
-        args->fd = fd_ref (fd);
+    args->fd = fd_ref(fd);
 
-        args->optype = optype;
-        args->xattr = dict_ref (xattr);
+    args->optype = optype;
+    args->xattr = dict_ref(xattr);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fxattrop_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata)
+args_fxattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xattr)
-                args->xattr = dict_ref (xattr);
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_setattr_store (default_args_t *args,
-                    loc_t *loc, struct iatt *stbuf,
-                    int32_t valid, dict_t *xdata)
+args_setattr_store(default_args_t *args, loc_t *loc, struct iatt *stbuf,
+                   int32_t valid, dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
+    loc_copy(&args->loc, loc);
 
-        if (stbuf)
-                args->stat = *stbuf;
+    if (stbuf)
+        args->stat = *stbuf;
 
-        args->valid = valid;
+    args->valid = valid;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_setattr_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *statpre, struct iatt *statpost,
-                      dict_t *xdata)
+args_setattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (statpre)
-                args->prestat = *statpre;
-        if (statpost)
-                args->poststat = *statpost;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-args_fsetattr_store (default_args_t *args,
-                     fd_t *fd, struct iatt *stbuf,
-                     int32_t valid, dict_t *xdata)
+args_fsetattr_store(default_args_t *args, fd_t *fd, struct iatt *stbuf,
+                    int32_t valid, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-        if (stbuf)
-                args->stat = *stbuf;
+    if (stbuf)
+        args->stat = *stbuf;
 
-        args->valid = valid;
+    args->valid = valid;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 int
-args_fsetattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-                       dict_t *xdata)
+args_fsetattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (statpre)
-                args->prestat = *statpre;
-        if (statpost)
-                args->poststat = *statpost;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_fallocate_store (default_args_t *args, fd_t *fd,
-                      int32_t mode, off_t offset, size_t len, dict_t *xdata)
+args_fallocate_store(default_args_t *args, fd_t *fd, int32_t mode, off_t offset,
+                     size_t len, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-	args->flags = mode;
-	args->offset = offset;
-	args->size = len;
+    args->flags = mode;
+    args->offset = offset;
+    args->size = len;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_fallocate_cbk_store(default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-                       dict_t *xdata)
+args_fallocate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (statpre)
-                args->prestat = *statpre;
-        if (statpost)
-                args->poststat = *statpost;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_discard_store (default_args_t *args, fd_t *fd,
-		    off_t offset, size_t len, dict_t *xdata)
+args_discard_store(default_args_t *args, fd_t *fd, off_t offset, size_t len,
+                   dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-	args->offset = offset;
-	args->size = len;
+    args->offset = offset;
+    args->size = len;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_discard_cbk_store(default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata)
+args_discard_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (statpre)
-                args->prestat = *statpre;
-        if (statpost)
-                args->poststat = *statpost;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_zerofill_store (default_args_t *args, fd_t *fd,
-                     off_t offset, off_t len, dict_t *xdata)
+args_zerofill_store(default_args_t *args, fd_t *fd, off_t offset, off_t len,
+                    dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-        args->offset = offset;
-        args->size = len;
+    args->offset = offset;
+    args->size = len;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_zerofill_cbk_store(default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata)
+args_zerofill_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (statpre)
-                args->prestat = *statpre;
-        if (statpost)
-                args->poststat = *statpost;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_ipc_store (default_args_t *args,
-                int32_t op, dict_t *xdata)
+args_ipc_store(default_args_t *args, int32_t op, dict_t *xdata)
 {
-        args->cmd = op;
+    args->cmd = op;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_ipc_cbk_store (default_args_cbk_t *args,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+args_ipc_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_seek_store (default_args_t *args, fd_t *fd,
-                 off_t offset, gf_seek_what_t what, dict_t *xdata)
+args_seek_store(default_args_t *args, fd_t *fd, off_t offset,
+                gf_seek_what_t what, dict_t *xdata)
 {
-        if (fd)
-                args->fd = fd_ref (fd);
+    if (fd)
+        args->fd = fd_ref(fd);
 
-        args->offset = offset;
-        args->what = what;
+    args->offset = offset;
+    args->what = what;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
-        return 0;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
 }
 
 int
-args_seek_cbk_store (default_args_cbk_t *args, int32_t op_ret,
-                     int32_t op_errno, off_t offset, dict_t *xdata)
+args_seek_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    off_t offset, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        args->offset = offset;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    args->offset = offset;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-args_getactivelk_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      lock_migration_info_t *locklist, dict_t *xdata)
+args_getactivelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, lock_migration_info_t *locklist,
+                           dict_t *xdata)
 {
-        lock_migration_info_t  *stub_entry = NULL, *entry = NULL;
-        int     ret = 0;
+    lock_migration_info_t *stub_entry = NULL, *entry = NULL;
+    int ret = 0;
 
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        /*op_ret needs to carry the number of locks present in the list*/
-        if (op_ret > 0) {
-                list_for_each_entry (entry, &locklist->list, list) {
-                        stub_entry = GF_CALLOC (1, sizeof (*stub_entry),
-                                                gf_common_mt_char);
-                        if (!stub_entry) {
-                               ret = -1;
-                               goto out;
-                        }
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    /*op_ret needs to carry the number of locks present in the list*/
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &locklist->list, list)
+        {
+            stub_entry = GF_CALLOC(1, sizeof(*stub_entry), gf_common_mt_char);
+            if (!stub_entry) {
+                ret = -1;
+                goto out;
+            }
 
-                        INIT_LIST_HEAD (&stub_entry->list);
-                        stub_entry->flock = entry->flock;
+            INIT_LIST_HEAD(&stub_entry->list);
+            stub_entry->flock = entry->flock;
 
-                        stub_entry->lk_flags = entry->lk_flags;
+            stub_entry->lk_flags = entry->lk_flags;
 
-                        stub_entry->client_uid = gf_strdup (entry->client_uid);
-                        if (!stub_entry->client_uid) {
-                                GF_FREE (stub_entry);
-                                ret = -1;
-                                goto out;
-                        }
+            stub_entry->client_uid = gf_strdup(entry->client_uid);
+            if (!stub_entry->client_uid) {
+                GF_FREE(stub_entry);
+                ret = -1;
+                goto out;
+            }
 
-                        list_add_tail (&stub_entry->list,
-                                       &args->locklist.list);
-                }
+            list_add_tail(&stub_entry->list, &args->locklist.list);
         }
+    }
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 out:
-        return ret;
+    return ret;
 }
 
 int
-args_setactivelk_store (default_args_t *args, loc_t *loc,
-                          lock_migration_info_t *locklist, dict_t *xdata)
+args_setactivelk_store(default_args_t *args, loc_t *loc,
+                       lock_migration_info_t *locklist, dict_t *xdata)
 {
-        lock_migration_info_t  *stub_entry = NULL, *entry = NULL;
-        int     ret = 0;
+    lock_migration_info_t *stub_entry = NULL, *entry = NULL;
+    int ret = 0;
 
-        list_for_each_entry (entry, &locklist->list, list) {
-                stub_entry = GF_CALLOC (1, sizeof (*stub_entry),
-                                        gf_common_mt_lock_mig);
-                if (!stub_entry) {
-                       ret = -1;
-                       goto out;
-                }
-
-                INIT_LIST_HEAD (&stub_entry->list);
-                stub_entry->flock = entry->flock;
+    list_for_each_entry(entry, &locklist->list, list)
+    {
+        stub_entry = GF_CALLOC(1, sizeof(*stub_entry), gf_common_mt_lock_mig);
+        if (!stub_entry) {
+            ret = -1;
+            goto out;
+        }
 
-                stub_entry->lk_flags = entry->lk_flags;
+        INIT_LIST_HEAD(&stub_entry->list);
+        stub_entry->flock = entry->flock;
 
-                stub_entry->client_uid = gf_strdup (entry->client_uid);
-                if (!stub_entry->client_uid) {
-                        GF_FREE (stub_entry);
-                        ret = -1;
-                        goto out;
-                }
+        stub_entry->lk_flags = entry->lk_flags;
 
-                list_add_tail (&stub_entry->list,
-                               &args->locklist.list);
+        stub_entry->client_uid = gf_strdup(entry->client_uid);
+        if (!stub_entry->client_uid) {
+            GF_FREE(stub_entry);
+            ret = -1;
+            goto out;
         }
 
-        loc_copy (&args->loc, loc);
+        list_add_tail(&stub_entry->list, &args->locklist.list);
+    }
+
+    loc_copy(&args->loc, loc);
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-args_lease_store (default_args_t *args, loc_t *loc, struct gf_lease *lease,
-                  dict_t *xdata)
+args_lease_store(default_args_t *args, loc_t *loc, struct gf_lease *lease,
+                 dict_t *xdata)
 {
-        loc_copy (&args->loc, loc);
-        args->lease = *lease;
+    loc_copy(&args->loc, loc);
+    args->lease = *lease;
 
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        return;
+    return;
 }
 
 void
-args_lease_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
+args_lease_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
                      struct gf_lease *lease, dict_t *xdata)
 {
-        args->op_ret = op_ret;
-        args->op_errno = op_errno;
-        if (op_ret == 0)
-                args->lease = *lease;
-        if (xdata)
-                args->xdata = dict_ref (xdata);
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->lease = *lease;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 }
 
-void
-args_cbk_wipe (default_args_cbk_t *args_cbk)
+int
+args_icreate_store(default_args_t *args, loc_t *loc, mode_t mode, dict_t *xdata)
 {
-        if (!args_cbk)
-                return;
-        if (args_cbk->inode)
-                inode_unref (args_cbk->inode);
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
 
-        GF_FREE ((char *)args_cbk->buf);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
 
-        GF_FREE (args_cbk->vector);
+int
+args_namelink_store(default_args_t *args, loc_t *loc, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
 
-        if (args_cbk->iobref)
-                iobref_unref (args_cbk->iobref);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
 
-        if (args_cbk->fd)
-                fd_unref (args_cbk->fd);
+int
+args_copy_file_range_store(default_args_t *args, fd_t *fd_in, off64_t off_in,
+                           fd_t *fd_out, off64_t off_out, size_t len,
+                           uint32_t flags, dict_t *xdata)
+{
+    if (fd_in)
+        args->fd = fd_ref(fd_in);
+    if (fd_out)
+        args->fd_dst = fd_ref(fd_out);
+    args->size = len;
+    args->off_in = off_in;
+    args->off_out = off_out;
+    args->flags = flags;
 
-        if (args_cbk->xattr)
-                dict_unref (args_cbk->xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        GF_FREE (args_cbk->strong_checksum);
+    return 0;
+}
 
-        if (args_cbk->xdata)
-                dict_unref (args_cbk->xdata);
+int
+args_copy_file_range_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                               int32_t op_errno, struct iatt *stbuf,
+                               struct iatt *prebuf_dst,
+                               struct iatt *postbuf_dst, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        if (postbuf_dst)
+            args->poststat = *postbuf_dst;
+        if (prebuf_dst)
+            args->prestat = *prebuf_dst;
+        if (stbuf)
+            args->stat = *stbuf;
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
 
-        if (!list_empty (&args_cbk->entries.list))
-                gf_dirent_free (&args_cbk->entries);
+    return 0;
 }
 
 void
-args_wipe (default_args_t *args)
+args_cbk_wipe(default_args_cbk_t *args_cbk)
 {
-        if (!args)
-                return;
+    if (!args_cbk)
+        return;
+    if (args_cbk->inode)
+        inode_unref(args_cbk->inode);
+
+    GF_FREE((char *)args_cbk->buf);
+
+    GF_FREE(args_cbk->vector);
+
+    if (args_cbk->iobref)
+        iobref_unref(args_cbk->iobref);
+
+    if (args_cbk->fd)
+        fd_unref(args_cbk->fd);
 
-        loc_wipe (&args->loc);
+    if (args_cbk->xattr)
+        dict_unref(args_cbk->xattr);
+
+    GF_FREE(args_cbk->strong_checksum);
+
+    if (args_cbk->xdata)
+        dict_unref(args_cbk->xdata);
+
+    if (!list_empty(&args_cbk->entries.list))
+        gf_dirent_free(&args_cbk->entries);
+}
+
+void
+args_wipe(default_args_t *args)
+{
+    if (!args)
+        return;
 
-        loc_wipe (&args->loc2);
+    loc_wipe(&args->loc);
 
-        if (args->fd)
-                fd_unref (args->fd);
+    loc_wipe(&args->loc2);
 
-        GF_FREE ((char *)args->linkname);
+    if (args->fd)
+        fd_unref(args->fd);
 
-	GF_FREE (args->vector);
+    GF_FREE((char *)args->linkname);
 
-        if (args->iobref)
-                iobref_unref (args->iobref);
+    GF_FREE(args->vector);
 
-        if (args->xattr)
-                dict_unref (args->xattr);
+    if (args->iobref)
+        iobref_unref(args->iobref);
 
-        if (args->xdata)
-                dict_unref (args->xdata);
+    if (args->xattr)
+        dict_unref(args->xattr);
 
-	GF_FREE ((char *)args->name);
+    if (args->xdata)
+        dict_unref(args->xdata);
 
-	GF_FREE ((char *)args->volume);
+    GF_FREE((char *)args->name);
 
+    GF_FREE((char *)args->volume);
 }
 
 void
-args_cbk_init (default_args_cbk_t *args_cbk)
+args_cbk_init(default_args_cbk_t *args_cbk)
 {
-        INIT_LIST_HEAD (&args_cbk->entries);
+    INIT_LIST_HEAD(&args_cbk->entries);
 }
diff --git a/libglusterfs/src/default-args.h b/libglusterfs/src/default-args.h
deleted file mode 100644
index a2201dd4703..00000000000
--- a/libglusterfs/src/default-args.h
+++ /dev/null
@@ -1,484 +0,0 @@
-/*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* libglusterfs/src/defaults.h:
-       This file contains definition of default fops and mops functions.
-*/
-
-#ifndef _DEFAULT_ARGS_H
-#define _DEFAULT_ARGS_H
-
-#include "xlator.h"
-
-int
-args_lookup_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent);
-
-
-int
-args_stat_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *buf, dict_t *xdata);
-
-int
-args_fstat_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *buf, dict_t *xdata);
-
-int
-args_truncate_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata);
-
-
-int
-args_ftruncate_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata);
-
-
-int
-args_access_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int
-args_readlink_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       const char *path, struct iatt *stbuf, dict_t *xdata);
-
-int
-args_mknod_cbk_store (default_args_cbk_t *args, int32_t op_ret,
-                    int32_t op_errno, inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-
-int
-args_mkdir_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-int
-args_unlink_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata);
-
-int
-args_rmdir_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-
-int
-args_symlink_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata);
-
-
-int
-args_rename_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata);
-
-int
-args_link_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
-                   dict_t *xdata);
-
-int
-args_create_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata);
-
-int
-args_open_cbk_store (default_args_cbk_t *args,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, dict_t *xdata);
-
-int
-args_readv_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                    int32_t count, struct iatt *stbuf,
-                    struct iobref *iobref, dict_t *xdata);
-
-int
-args_writev_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata);
-
-
-int
-args_flush_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int
-args_fsync_cbk_store (default_args_cbk_t *args,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata);
-
-int
-args_opendir_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, dict_t *xdata);
-
-int
-args_fsyncdir_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_statfs_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct statvfs *buf, dict_t *xdata);
-
-int
-args_setxattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret,
-                       int32_t op_errno, dict_t *xdata);
-
-int
-args_getxattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *dict, dict_t *xdata);
-
-int
-args_fsetxattr_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_fgetxattr_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata);
-
-int
-args_removexattr_cbk_store (default_args_cbk_t *args,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_fremovexattr_cbk_store (default_args_cbk_t *args,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_lk_cbk_store (default_args_cbk_t *args,
-                 int32_t op_ret, int32_t op_errno,
-                 struct gf_flock *lock, dict_t *xdata);
-
-
-int
-args_inodelk_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_finodelk_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_entrylk_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_fentrylk_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int
-args_readdirp_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       gf_dirent_t *entries, dict_t *xdata);
-
-
-int
-args_readdir_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      gf_dirent_t *entries, dict_t *xdata);
-
-
-int
-args_rchecksum_cbk_store (default_args_cbk_t *args,
-                        int32_t op_ret, int32_t op_errno,
-                        uint32_t weak_checksum, uint8_t *strong_checksum,
-                        dict_t *xdata);
-
-
-int
-args_xattrop_cbk_store (default_args_cbk_t *args, int32_t op_ret,
-                        int32_t op_errno, dict_t *xattr, dict_t *xdata);
-
-
-int
-args_fxattrop_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata);
-
-int
-args_setattr_cbk_store (default_args_cbk_t *args,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *statpre, struct iatt *statpost,
-                      dict_t *xdata);
-
-
-int
-args_fsetattr_cbk_store (default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-                       dict_t *xdata);
-
-int
-args_fallocate_cbk_store(default_args_cbk_t *args,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *statpre, struct iatt *statpost,
-                       dict_t *xdata);
-
-int
-args_discard_cbk_store(default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata);
-
-int
-args_zerofill_cbk_store(default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost,
-                     dict_t *xdata);
-
-int
-args_ipc_cbk_store (default_args_cbk_t *args,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int
-args_seek_cbk_store (default_args_cbk_t *args, int32_t op_ret,
-                     int32_t op_errno, off_t offset, dict_t *xdata);
-
-void
-args_lease_cbk_store (default_args_cbk_t *args,
-                     int32_t op_ret, int32_t op_errno,
-                     struct gf_lease *lease, dict_t *xdata);
-
-void
-args_cbk_wipe (default_args_cbk_t *args_cbk);
-
-void
-args_wipe (default_args_t *args);
-
-int
-args_lookup_store (default_args_t *args, loc_t *loc,
-                   dict_t *xdata);
-
-int
-args_stat_store (default_args_t *args, loc_t *loc, dict_t *xdata);
-
-int
-args_fstat_store (default_args_t *args, fd_t *fd, dict_t *xdata);
-
-int
-args_truncate_store (default_args_t *args, loc_t *loc, off_t off,
-                     dict_t *xdata);
-int
-args_ftruncate_store (default_args_t *args, fd_t *fd, off_t off,
-                      dict_t *xdata);
-
-int
-args_access_store (default_args_t *args, loc_t *loc, int32_t mask,
-                   dict_t *xdata);
-
-int
-args_readlink_store (default_args_t *args, loc_t *loc, size_t size,
-                     dict_t *xdata);
-
-int
-args_mknod_store (default_args_t *args, loc_t *loc, mode_t mode,
-                  dev_t rdev, mode_t umask, dict_t *xdata);
-
-int
-args_mkdir_store (default_args_t *args, loc_t *loc, mode_t mode,
-                  mode_t umask, dict_t *xdata);
-
-int
-args_unlink_store (default_args_t *args, loc_t *loc, int xflag, dict_t *xdata);
-
-int
-args_rmdir_store (default_args_t *args, loc_t *loc, int flags, dict_t *xdata);
-
-int
-args_symlink_store (default_args_t *args, const char *linkname, loc_t *loc,
-                   mode_t umask, dict_t *xdata);
-
-int
-args_rename_store (default_args_t *args, loc_t *oldloc, loc_t *newloc,
-                   dict_t *xdata);
-
-int
-args_link_store (default_args_t *args, loc_t *oldloc, loc_t *newloc,
-                 dict_t *xdata);
-
-int
-args_create_store (default_args_t *args,
-                  loc_t *loc, int32_t flags, mode_t mode,
-                  mode_t umask, fd_t *fd, dict_t *xdata);
-
-int
-args_open_store (default_args_t *args, loc_t *loc, int32_t flags,
-                 fd_t *fd, dict_t *xdata);
-
-int
-args_readv_store (default_args_t *args, fd_t *fd, size_t size, off_t off,
-                  uint32_t flags, dict_t *xdata);
-
-int
-args_writev_store (default_args_t *args, fd_t *fd, struct iovec *vector,
-                   int32_t count, off_t off, uint32_t flags,
-                   struct iobref *iobref, dict_t *xdata);
-
-int
-args_flush_store (default_args_t *args, fd_t *fd, dict_t *xdata);
-
-int
-args_fsync_store (default_args_t *args, fd_t *fd, int32_t datasync,
-                  dict_t *xdata);
-
-int
-args_opendir_store (default_args_t *args, loc_t *loc, fd_t *fd, dict_t *xdata);
-
-int
-args_fsyncdir_store (default_args_t *args, fd_t *fd, int32_t datasync,
-                     dict_t *xdata);
-
-int
-args_statfs_store (default_args_t *args, loc_t *loc, dict_t *xdata);
-
-int
-args_setxattr_store (default_args_t *args,
-                     loc_t *loc, dict_t *dict,
-                     int32_t flags, dict_t *xdata);
-
-int
-args_getxattr_store (default_args_t *args,
-                     loc_t *loc, const char *name, dict_t *xdata);
-
-int
-args_fsetxattr_store (default_args_t *args,
-                      fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
-
-int
-args_fgetxattr_store (default_args_t *args,
-                      fd_t *fd, const char *name, dict_t *xdata);
-
-int
-args_removexattr_store (default_args_t *args,
-                        loc_t *loc, const char *name, dict_t *xdata);
-
-int
-args_fremovexattr_store (default_args_t *args,
-                         fd_t *fd, const char *name, dict_t *xdata);
-
-int
-args_lk_store (default_args_t *args,
-               fd_t *fd, int32_t cmd,
-                struct gf_flock *lock, dict_t *xdata);
-
-int
-args_inodelk_store (default_args_t *args,
-                    const char *volume, loc_t *loc, int32_t cmd,
-                    struct gf_flock *lock, dict_t *xdata);
-
-int
-args_finodelk_store (default_args_t *args,
-                     const char *volume, fd_t *fd, int32_t cmd,
-                     struct gf_flock *lock, dict_t *xdata);
-
-int
-args_entrylk_store (default_args_t *args,
-                    const char *volume, loc_t *loc, const char *name,
-                    entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-int
-args_fentrylk_store (default_args_t *args,
-                     const char *volume, fd_t *fd, const char *name,
-                     entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-int
-args_readdirp_store (default_args_t *args,
-                     fd_t *fd, size_t size, off_t off, dict_t *xdata);
-
-int
-args_readdir_store (default_args_t *args,
-                    fd_t *fd, size_t size,
-                    off_t off, dict_t *xdata);
-
-int
-args_rchecksum_store (default_args_t *args,
-                      fd_t *fd, off_t offset, int32_t len, dict_t *xdata);
-
-int
-args_xattrop_store (default_args_t *args,
-                    loc_t *loc, gf_xattrop_flags_t optype,
-                    dict_t *xattr, dict_t *xdata);
-
-int
-args_fxattrop_store (default_args_t *args,
-                     fd_t *fd, gf_xattrop_flags_t optype,
-                     dict_t *xattr, dict_t *xdata);
-
-int
-args_setattr_store (default_args_t *args,
-                    loc_t *loc, struct iatt *stbuf,
-                    int32_t valid, dict_t *xdata);
-
-int
-args_fsetattr_store (default_args_t *args,
-                     fd_t *fd, struct iatt *stbuf,
-                     int32_t valid, dict_t *xdata);
-
-int
-args_fallocate_store (default_args_t *args, fd_t *fd,
-                      int32_t mode, off_t offset, size_t len, dict_t *xdata);
-
-int
-args_discard_store (default_args_t *args, fd_t *fd,
-		    off_t offset, size_t len, dict_t *xdata);
-
-int
-args_zerofill_store (default_args_t *args, fd_t *fd,
-                     off_t offset, off_t len, dict_t *xdata);
-
-int
-args_ipc_store (default_args_t *args,
-                int32_t op, dict_t *xdata);
-
-int
-args_seek_store (default_args_t *args, fd_t *fd,
-                 off_t offset, gf_seek_what_t what, dict_t *xdata);
-
-void
-args_lease_store (default_args_t *args, loc_t *loc, struct gf_lease *lease,
-                  dict_t *xdata);
-
-int
-args_getactivelk_cbk_store (default_args_cbk_t *args,
-                             int32_t op_ret, int32_t op_errno,
-                             lock_migration_info_t *locklist, dict_t *xdata);
-
-int
-args_setactivelk_store (default_args_t *args, loc_t *loc,
-                          lock_migration_info_t *locklist, dict_t *xdata);
-void
-args_cbk_init (default_args_cbk_t *args_cbk);
-#endif /* _DEFAULT_ARGS_H */
diff --git a/libglusterfs/src/defaults-tmpl.c b/libglusterfs/src/defaults-tmpl.c
index b42c377ca31..3cf707f42aa 100644
--- a/libglusterfs/src/defaults-tmpl.c
+++ b/libglusterfs/src/defaults-tmpl.c
@@ -11,7 +11,7 @@
 /* libglusterfs/src/defaults.c:
    This file contains functions, which are used to fill the 'fops', 'cbk'
    structures in the xlator structures, if they are not written. Here, all the
-   function calls are plainly forwared to the first child of the xlator, and
+   function calls are plainly forwarded to the first child of the xlator, and
    all the *_cbk function does plain STACK_UNWIND of the frame, and returns.
 
    This function also implements *_resume () functions, which does same
@@ -25,179 +25,223 @@
 #include "config.h"
 #endif
 
-#include "xlator.h"
-#include "defaults.h"
+#include "glusterfs/xlator.h"
+#include "glusterfs/defaults.h"
 
 #pragma generate
 
 struct xlator_fops _default_fops = {
-        .create = default_create,
-        .open = default_open,
-        .stat = default_stat,
-        .readlink = default_readlink,
-        .mknod = default_mknod,
-        .mkdir = default_mkdir,
-        .unlink = default_unlink,
-        .rmdir = default_rmdir,
-        .symlink = default_symlink,
-        .rename = default_rename,
-        .link = default_link,
-        .truncate = default_truncate,
-        .readv = default_readv,
-        .writev = default_writev,
-        .statfs = default_statfs,
-        .flush = default_flush,
-        .fsync = default_fsync,
-        .setxattr = default_setxattr,
-        .getxattr = default_getxattr,
-        .fsetxattr = default_fsetxattr,
-        .fgetxattr = default_fgetxattr,
-        .removexattr = default_removexattr,
-        .fremovexattr = default_fremovexattr,
-        .opendir = default_opendir,
-        .readdir = default_readdir,
-        .readdirp = default_readdirp,
-        .fsyncdir = default_fsyncdir,
-        .access = default_access,
-        .ftruncate = default_ftruncate,
-        .fstat = default_fstat,
-        .lk = default_lk,
-        .inodelk = default_inodelk,
-        .finodelk = default_finodelk,
-        .entrylk = default_entrylk,
-        .fentrylk = default_fentrylk,
-        .lookup = default_lookup,
-        .rchecksum = default_rchecksum,
-        .xattrop = default_xattrop,
-        .fxattrop = default_fxattrop,
-        .setattr = default_setattr,
-        .fsetattr = default_fsetattr,
-	.fallocate = default_fallocate,
-	.discard = default_discard,
-        .zerofill = default_zerofill,
-        .ipc = default_ipc,
-        .seek = default_seek,
-
-        .getspec = default_getspec,
-        .getactivelk = default_getactivelk,
-        .setactivelk = default_setactivelk,
+    .create = default_create,
+    .open = default_open,
+    .stat = default_stat,
+    .readlink = default_readlink,
+    .mknod = default_mknod,
+    .mkdir = default_mkdir,
+    .unlink = default_unlink,
+    .rmdir = default_rmdir,
+    .symlink = default_symlink,
+    .rename = default_rename,
+    .link = default_link,
+    .truncate = default_truncate,
+    .readv = default_readv,
+    .writev = default_writev,
+    .statfs = default_statfs,
+    .flush = default_flush,
+    .fsync = default_fsync,
+    .setxattr = default_setxattr,
+    .getxattr = default_getxattr,
+    .fsetxattr = default_fsetxattr,
+    .fgetxattr = default_fgetxattr,
+    .removexattr = default_removexattr,
+    .fremovexattr = default_fremovexattr,
+    .opendir = default_opendir,
+    .readdir = default_readdir,
+    .readdirp = default_readdirp,
+    .fsyncdir = default_fsyncdir,
+    .access = default_access,
+    .ftruncate = default_ftruncate,
+    .fstat = default_fstat,
+    .lk = default_lk,
+    .inodelk = default_inodelk,
+    .finodelk = default_finodelk,
+    .entrylk = default_entrylk,
+    .fentrylk = default_fentrylk,
+    .lookup = default_lookup,
+    .rchecksum = default_rchecksum,
+    .xattrop = default_xattrop,
+    .fxattrop = default_fxattrop,
+    .setattr = default_setattr,
+    .fsetattr = default_fsetattr,
+    .fallocate = default_fallocate,
+    .discard = default_discard,
+    .zerofill = default_zerofill,
+    .ipc = default_ipc,
+    .seek = default_seek,
+
+    .getspec = default_getspec,
+    .getactivelk = default_getactivelk,
+    .setactivelk = default_setactivelk,
+    .put = default_put,
+    .icreate = default_icreate,
+    .namelink = default_namelink,
+    .copy_file_range = default_copy_file_range,
 };
 struct xlator_fops *default_fops = &_default_fops;
 
-
 /*
  * Remaining functions don't follow the fop calling conventions, so they're
  * not generated.
  */
 
 int32_t
-default_forget (xlator_t *this, inode_t *inode)
+default_forget(xlator_t *this, inode_t *inode)
 {
-        gf_log_callingfn (this->name, GF_LOG_DEBUG, "xlator does not "
-                          "implement forget_cbk");
-        return 0;
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement forget_cbk");
+    return 0;
 }
 
-
 int32_t
-default_releasedir (xlator_t *this, fd_t *fd)
+default_releasedir(xlator_t *this, fd_t *fd)
 {
-        gf_log_callingfn (this->name, GF_LOG_DEBUG, "xlator does not "
-                          "implement releasedir_cbk");
-        return 0;
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement releasedir_cbk");
+    return 0;
 }
 
 int32_t
-default_release (xlator_t *this, fd_t *fd)
+default_release(xlator_t *this, fd_t *fd)
 {
-        gf_log_callingfn (this->name, GF_LOG_DEBUG, "xlator does not "
-                          "implement release_cbk");
-        return 0;
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement release_cbk");
+    return 0;
 }
 
 /* notify */
 int
-default_notify (xlator_t *this, int32_t event, void *data, ...)
+default_notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        switch (event) {
-        case GF_EVENT_PARENT_UP:
-        case GF_EVENT_PARENT_DOWN:
-        {
-                xlator_list_t *list = this->children;
+    GF_UNUSED int ret = 0;
+    xlator_t *victim = data;
 
-                while (list) {
-                        xlator_notify (list->xlator, event, this);
-                        list = list->next;
-                }
-        }
-        break;
+    glusterfs_graph_t *graph = NULL;
+
+    GF_VALIDATE_OR_GOTO("notify", this, out);
+    graph = this->graph;
+    GF_VALIDATE_OR_GOTO(this->name, graph, out);
+
+    switch (event) {
+        case GF_EVENT_PARENT_UP:
+        case GF_EVENT_PARENT_DOWN: {
+            xlator_list_t *list = this->children;
+
+            while (list) {
+                if (victim && victim->cleanup_starting)
+                    xlator_notify(list->xlator, event, victim);
+                else
+                    xlator_notify(list->xlator, event, this);
+                list = list->next;
+            }
+        } break;
         case GF_EVENT_CHILD_CONNECTING:
-        case GF_EVENT_CHILD_MODIFIED:
         case GF_EVENT_CHILD_DOWN:
         case GF_EVENT_CHILD_UP:
-        case GF_EVENT_AUTH_FAILED:
-        {
-                xlator_list_t *parent = this->parents;
-
-                /*
-                 * Handle case of CHILD_* & AUTH_FAILED event specially, send
-                 * it to fuse.
-                 */
-                if (!parent && this->ctx && this->ctx->master) {
-                        xlator_notify (this->ctx->master, event, this->graph,
-                                       NULL);
-                }
-
-                while (parent) {
-                        if (parent->xlator->init_succeeded)
-                                xlator_notify (parent->xlator, event,
-                                               this, NULL);
-                        parent = parent->next;
+        case GF_EVENT_AUTH_FAILED: {
+            xlator_list_t *parent = this->parents;
+
+            /*
+             * Handle case of CHILD_* & AUTH_FAILED event specially, send
+             * it to fuse.
+             */
+            if (!parent && this->ctx && this->ctx->master) {
+                xlator_notify(this->ctx->master, event, this->graph, NULL);
+            }
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, this, NULL);
+                parent = parent->next;
+            }
+
+            if (event == GF_EVENT_CHILD_DOWN &&
+                !(this->ctx && this->ctx->master) && (graph->top == this)) {
+                /* Make sure this is not a daemon with master xlator */
+                pthread_mutex_lock(&graph->mutex);
+                {
+                    if (graph->parent_down ==
+                        graph_total_client_xlator(graph)) {
+                        graph->used = 0;
+                        pthread_cond_broadcast(&graph->child_down_cond);
+                    }
                 }
+                pthread_mutex_unlock(&graph->mutex);
+            }
+        } break;
+        case GF_EVENT_UPCALL: {
+            xlator_list_t *parent = this->parents;
+
+            if (!parent && this->ctx && this->ctx->master)
+                xlator_notify(this->ctx->master, event, data, NULL);
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, data, NULL);
+                parent = parent->next;
+            }
+        } break;
+        case GF_EVENT_CHILD_PING: {
+            xlator_list_t *parent = this->parents;
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    XLATOR_NOTIFY(ret, parent->xlator, event, this, data);
+                parent = parent->next;
+            }
+        } break;
+        case GF_EVENT_CLEANUP: {
+            xlator_list_t *list = this->children;
+
+            while (list) {
+                xlator_notify(list->xlator, event, this);
+                list = list->next;
+            }
+        } break;
+
+        default: {
+            xlator_list_t *parent = this->parents;
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, this, NULL);
+                parent = parent->next;
+            }
         }
-        break;
-        case GF_EVENT_UPCALL:
-        {
-                xlator_list_t *parent = this->parents;
-
-                if (!parent && this->ctx && this->ctx->master)
-                        xlator_notify (this->ctx->master, event, data, NULL);
-
-                while (parent) {
-                        if (parent->xlator->init_succeeded)
-                                xlator_notify (parent->xlator, event,
-                                               data, NULL);
-                        parent = parent->next;
-                }
-        }
-        break;
-        default:
-        {
-                xlator_list_t *parent = this->parents;
-
-                while (parent) {
-                        if (parent->xlator->init_succeeded)
-                                xlator_notify (parent->xlator, event,
-                                               this, NULL);
-                        parent = parent->next;
-                }
-        }
-        /*
-         * Apparently our picky-about-everything else coding standard allows
-         * adjacent same-indendation-level close braces.  Clearly it has
-         * nothing to do with readability.
-         */
-        }
-
-        return 0;
+            /*
+             * Apparently our picky-about-everything else coding standard allows
+             * adjacent same-indendation-level close braces.  Clearly it has
+             * nothing to do with readability.
+             */
+    }
+out:
+    return 0;
 }
 
 int32_t
-default_mem_acct_init (xlator_t *this)
+default_mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_common_mt_end);
 
-        ret = xlator_mem_acct_init (this, gf_common_mt_end);
+    return ret;
+}
 
-        return ret;
+void
+default_fini(xlator_t *this)
+{
+    if (this && this->private)
+        GF_FREE(this->private);
 }
diff --git a/libglusterfs/src/defaults.h b/libglusterfs/src/defaults.h
deleted file mode 100644
index 50f1909b90b..00000000000
--- a/libglusterfs/src/defaults.h
+++ /dev/null
@@ -1,1300 +0,0 @@
-/*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* libglusterfs/src/defaults.h:
-       This file contains definition of default fops and mops functions.
-*/
-
-#ifndef _DEFAULTS_H
-#define _DEFAULTS_H
-
-#include "xlator.h"
-
-typedef struct {
-        int op_ret;
-        int op_errno;
-        inode_t *inode;
-        struct iatt stat;
-        struct iatt prestat;
-        struct iatt poststat;
-        struct iatt preparent;   /* @preoldparent in rename_cbk */
-        struct iatt postparent;  /* @postoldparent in rename_cbk */
-        struct iatt preparent2;  /* @prenewparent in rename_cbk */
-        struct iatt postparent2; /* @postnewparent in rename_cbk */
-        const char *buf;
-        struct iovec *vector;
-        int count;
-        struct iobref *iobref;
-        fd_t *fd;
-        struct statvfs statvfs;
-        dict_t *xattr;
-        struct gf_flock lock;
-        uint32_t weak_checksum;
-        uint8_t *strong_checksum;
-        dict_t *xdata;
-        gf_dirent_t entries;
-        off_t offset;            /* seek hole/data */
-        int valid; /* If the response is valid or not. For call-stub it is
-                      always valid irrespective of this */
-        struct gf_lease lease;
-        lock_migration_info_t locklist;
-} default_args_cbk_t;
-
-typedef struct {
-        loc_t loc; /* @old in rename(), link() */
-        loc_t loc2; /* @new in rename(), link() */
-        fd_t *fd;
-        off_t offset;
-        int mask;
-        size_t size;
-        mode_t mode;
-        dev_t rdev;
-        mode_t umask;
-        int xflag;
-        int flags;
-        const char *linkname;
-        struct iovec *vector;
-        int count;
-        struct iobref *iobref;
-        int datasync;
-        dict_t *xattr;
-        const char *name;
-        int cmd;
-        struct gf_flock lock;
-        const char *volume;
-        entrylk_cmd entrylkcmd;
-        entrylk_type entrylktype;
-        gf_xattrop_flags_t optype;
-        int valid;
-        struct iatt stat;
-        gf_seek_what_t what;
-        dict_t *xdata;
-        struct gf_lease lease;
-        lock_migration_info_t locklist;
-} default_args_t;
-
-typedef struct {
-        int             fop_enum;
-        unsigned int    fop_length;
-        int             *enum_list;
-        default_args_t  *req_list;
-        dict_t          *xdata;
-} compound_args_t;
-
-typedef struct {
-        int                fop_enum;
-        unsigned int       fop_length;
-        int                *enum_list;
-        default_args_cbk_t *rsp_list;
-        dict_t             *xdata;
-} compound_args_cbk_t;
-
-int32_t default_notify (xlator_t *this,
-                        int32_t event,
-                        void *data,
-                        ...);
-
-int32_t default_forget (xlator_t *this, inode_t *inode);
-
-int32_t default_release (xlator_t *this, fd_t *fd);
-
-int32_t default_releasedir (xlator_t *this, fd_t *fd);
-
-
-extern struct xlator_fops *default_fops;
-
-/* Management Operations */
-
-int32_t default_getspec (call_frame_t *frame,
-                         xlator_t *this,
-                         const char *key,
-                         int32_t flag);
-
-int32_t default_rchecksum (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd, off_t offset,
-                           int32_t len, dict_t *xdata);
-
-/* FileSystem operations */
-int32_t default_lookup (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        dict_t *xdata);
-
-int32_t default_stat (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc, dict_t *xdata);
-
-int32_t default_fstat (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd, dict_t *xdata);
-
-int32_t default_truncate (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          off_t offset, dict_t *xdata);
-
-int32_t default_ftruncate (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           off_t offset, dict_t *xdata);
-
-int32_t default_access (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        int32_t mask, dict_t *xdata);
-
-int32_t default_readlink (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          size_t size, dict_t *xdata);
-
-int32_t default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
-
-int32_t default_mkdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t default_unlink (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t default_rmdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t default_symlink (call_frame_t *frame, xlator_t *this,
-                         const char *linkpath, loc_t *loc, mode_t umask,
-                         dict_t *xdata);
-
-int32_t default_rename (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *oldloc,
-                        loc_t *newloc, dict_t *xdata);
-
-int32_t default_link (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *oldloc,
-                      loc_t *newloc, dict_t *xdata);
-
-int32_t default_create (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int32_t flags, mode_t mode,
-                        mode_t umask, fd_t *fd, dict_t *xdata);
-
-int32_t default_open (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc,
-                      int32_t flags, fd_t *fd,
-                      dict_t *xdata);
-
-int32_t default_readv (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       size_t size,
-                       off_t offset,
-                       uint32_t flags, dict_t *xdata);
-
-int32_t default_writev (call_frame_t *frame,
-                        xlator_t *this,
-                        fd_t *fd,
-                        struct iovec *vector,
-                        int32_t count,
-                        off_t offset,
-                        uint32_t flags,
-                        struct iobref *iobref, dict_t *xdata);
-
-int32_t default_flush (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd, dict_t *xdata);
-
-int32_t default_fsync (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       int32_t datasync, dict_t *xdata);
-
-int32_t default_opendir (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc, fd_t *fd, dict_t *xdata);
-
-int32_t default_fsyncdir (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          int32_t datasync, dict_t *xdata);
-
-int32_t default_statfs (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc, dict_t *xdata);
-
-int32_t default_setxattr (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          dict_t *dict,
-                          int32_t flags, dict_t *xdata);
-
-int32_t default_getxattr (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          const char *name, dict_t *xdata);
-
-int32_t default_fsetxattr (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           dict_t *dict,
-                           int32_t flags, dict_t *xdata);
-
-int32_t default_fgetxattr (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           const char *name, dict_t *xdata);
-
-int32_t default_removexattr (call_frame_t *frame,
-                             xlator_t *this,
-                             loc_t *loc,
-                             const char *name, dict_t *xdata);
-
-int32_t default_fremovexattr (call_frame_t *frame,
-                              xlator_t *this,
-                              fd_t *fd,
-                              const char *name, dict_t *xdata);
-
-int32_t default_lk (call_frame_t *frame,
-                    xlator_t *this,
-                    fd_t *fd,
-                    int32_t cmd,
-                    struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_inodelk (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_finodelk (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_entrylk (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-int32_t default_fentrylk (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-int32_t default_readdir (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off, dict_t *xdata);
-
-int32_t default_readdirp (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off, dict_t *xdata);
-
-int32_t default_xattrop (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         gf_xattrop_flags_t flags,
-                         dict_t *dict, dict_t *xdata);
-
-int32_t default_fxattrop (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          gf_xattrop_flags_t flags,
-                          dict_t *dict, dict_t *xdata);
-
-int32_t default_setattr (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         struct iatt *stbuf,
-                         int32_t valid, dict_t *xdata);
-
-int32_t default_fsetattr (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          struct iatt *stbuf,
-                          int32_t valid, dict_t *xdata);
-
-int32_t default_fallocate(call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          int32_t keep_size, off_t offset,
-                          size_t len, dict_t *xdata);
-
-int32_t default_discard(call_frame_t *frame,
-                        xlator_t *this,
-                        fd_t *fd,
-                        off_t offset,
-                        size_t len, dict_t *xdata);
-
-int32_t default_zerofill(call_frame_t *frame,
-                        xlator_t *this,
-                        fd_t *fd,
-                        off_t offset,
-                        off_t len, dict_t *xdata);
-
-int32_t default_ipc (call_frame_t *frame, xlator_t *this, int32_t op,
-                     dict_t *xdata);
-
-int32_t default_seek (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      off_t offset, gf_seek_what_t what, dict_t *xdata);
-
-int32_t default_lease (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       struct gf_lease *lease, dict_t *xdata);
-
-int32_t
-default_getactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      dict_t *xdata);
-
-int32_t
-default_setactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       lock_migration_info_t *locklist, dict_t *xdata);
-
-/* Resume */
-int32_t default_getspec_resume (call_frame_t *frame,
-                                xlator_t *this,
-                                const char *key,
-                                int32_t flag);
-
-int32_t default_rchecksum_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd, off_t offset,
-                           int32_t len, dict_t *xdata);
-
-/* FileSystem operations */
-int32_t default_lookup_resume (call_frame_t *frame,
-                               xlator_t *this,
-                               loc_t *loc,
-                               dict_t *xdata);
-
-int32_t default_stat_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc, dict_t *xdata);
-
-int32_t default_fstat_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd, dict_t *xdata);
-
-int32_t default_truncate_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          off_t offset, dict_t *xdata);
-
-int32_t default_ftruncate_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           off_t offset, dict_t *xdata);
-
-int32_t default_access_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        int32_t mask, dict_t *xdata);
-
-int32_t default_readlink_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          size_t size, dict_t *xdata);
-
-int32_t default_mknod_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                              mode_t mode, dev_t rdev, mode_t umask,
-                              dict_t *xdata);
-
-int32_t default_mkdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                              mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t default_unlink_resume (call_frame_t *frame,
-                               xlator_t *this,
-                               loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t default_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                              loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t default_symlink_resume (call_frame_t *frame, xlator_t *this,
-                                const char *linkpath, loc_t *loc, mode_t umask,
-                                dict_t *xdata);
-
-int32_t default_rename_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *oldloc,
-                        loc_t *newloc, dict_t *xdata);
-
-int32_t default_link_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *oldloc,
-                      loc_t *newloc, dict_t *xdata);
-
-int32_t default_create_resume (call_frame_t *frame, xlator_t *this,
-                               loc_t *loc, int32_t flags, mode_t mode,
-                               mode_t umask, fd_t *fd, dict_t *xdata);
-
-int32_t default_open_resume (call_frame_t *frame,
-                             xlator_t *this,
-                             loc_t *loc,
-                             int32_t flags, fd_t *fd, dict_t *xdata);
-
-int32_t default_readv_resume (call_frame_t *frame,
-                              xlator_t *this,
-                              fd_t *fd,
-                              size_t size,
-                              off_t offset, uint32_t flags, dict_t *xdata);
-
-int32_t default_writev_resume (call_frame_t *frame,
-                               xlator_t *this,
-                               fd_t *fd,
-                               struct iovec *vector,
-                               int32_t count,
-                               off_t offset, uint32_t flags,
-                               struct iobref *iobref, dict_t *xdata);
-
-int32_t default_flush_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd, dict_t *xdata);
-
-int32_t default_fsync_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       int32_t datasync, dict_t *xdata);
-
-int32_t default_opendir_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc, fd_t *fd, dict_t *xdata);
-
-int32_t default_fsyncdir_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          int32_t datasync, dict_t *xdata);
-
-int32_t default_statfs_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc, dict_t *xdata);
-
-int32_t default_setxattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          dict_t *dict,
-                          int32_t flags, dict_t *xdata);
-
-int32_t default_getxattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          const char *name, dict_t *xdata);
-
-int32_t default_fsetxattr_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           dict_t *dict,
-                           int32_t flags, dict_t *xdata);
-
-int32_t default_fgetxattr_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           const char *name, dict_t *xdata);
-
-int32_t default_removexattr_resume (call_frame_t *frame,
-                             xlator_t *this,
-                             loc_t *loc,
-                             const char *name, dict_t *xdata);
-
-int32_t default_fremovexattr_resume (call_frame_t *frame,
-                                     xlator_t *this,
-                                     fd_t *fd,
-                                     const char *name, dict_t *xdata);
-
-int32_t default_lk_resume (call_frame_t *frame,
-                    xlator_t *this,
-                    fd_t *fd,
-                    int32_t cmd,
-                    struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_inodelk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_finodelk_resume (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock, dict_t *xdata);
-
-int32_t default_entrylk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-int32_t default_fentrylk_resume (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
-
-int32_t default_readdir_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off, dict_t *xdata);
-
-int32_t default_readdirp_resume (call_frame_t *frame,
-                                 xlator_t *this,
-                                 fd_t *fd,
-                                 size_t size, off_t off, dict_t *xdata);
-
-int32_t default_xattrop_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         gf_xattrop_flags_t flags,
-                         dict_t *dict, dict_t *xdata);
-
-int32_t default_fxattrop_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          gf_xattrop_flags_t flags,
-                          dict_t *dict, dict_t *xdata);
-int32_t default_rchecksum_resume (call_frame_t *frame,
-                                  xlator_t *this,
-                                  fd_t *fd, off_t offset,
-                                  int32_t len, dict_t *xdata);
-
-int32_t default_setattr_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         struct iatt *stbuf,
-                         int32_t valid, dict_t *xdata);
-
-int32_t default_fsetattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          struct iatt *stbuf,
-                          int32_t valid, dict_t *xdata);
-
-int32_t default_fallocate_resume(call_frame_t *frame,
-                                 xlator_t *this,
-                                 fd_t *fd,
-                                 int32_t keep_size, off_t offset,
-                                 size_t len, dict_t *xdata);
-
-int32_t default_discard_resume(call_frame_t *frame,
-                               xlator_t *this,
-                               fd_t *fd,
-                               off_t offset,
-                               size_t len, dict_t *xdata);
-
-int32_t default_zerofill_resume(call_frame_t *frame,
-                               xlator_t *this,
-                               fd_t *fd,
-                               off_t offset,
-                               off_t len, dict_t *xdata);
-
-int32_t default_ipc_resume (call_frame_t *frame, xlator_t *this,
-                            int32_t op, dict_t *xdata);
-
-int32_t default_seek_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             off_t offset, gf_seek_what_t what, dict_t *xdata);
-
-int32_t default_lease_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                              struct gf_lease *lease, dict_t *xdata);
-
-int32_t
-default_getactivelk_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            dict_t *xdata);
-
-int32_t
-default_setactivelk_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            lock_migration_info_t *locklist, dict_t *xdata);
-
-/* _cbk_resume */
-
-int32_t
-default_lookup_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           inode_t * inode, struct iatt *buf, dict_t * xdata,
-                           struct iatt *postparent);
-
-int32_t
-default_stat_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         dict_t * xdata);
-
-
-int32_t
-default_truncate_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, struct iatt *prebuf,
-                             struct iatt *postbuf, dict_t * xdata);
-
-int32_t
-default_ftruncate_cbk_resume (call_frame_t * frame, void *cookie,
-                              xlator_t * this, int32_t op_ret,
-                              int32_t op_errno, struct iatt *prebuf,
-                              struct iatt *postbuf, dict_t * xdata);
-
-int32_t
-default_access_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           dict_t * xdata);
-
-int32_t
-default_readlink_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, const char *path,
-                             struct iatt *buf, dict_t * xdata);
-
-
-int32_t
-default_mknod_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, inode_t * inode,
-                          struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t * xdata);
-
-int32_t
-default_mkdir_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, inode_t * inode,
-                          struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t * xdata);
-
-int32_t
-default_unlink_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t * xdata);
-
-int32_t
-default_rmdir_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *preparent, struct iatt *postparent,
-                          dict_t * xdata);
-
-
-int32_t
-default_symlink_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            inode_t * inode, struct iatt *buf,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t * xdata);
-
-
-int32_t
-default_rename_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct iatt *buf, struct iatt *preoldparent,
-                           struct iatt *postoldparent,
-                           struct iatt *prenewparent,
-                           struct iatt *postnewparent, dict_t * xdata);
-
-
-int32_t
-default_link_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, inode_t * inode,
-                         struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t * xdata);
-
-
-int32_t
-default_create_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           fd_t * fd, inode_t * inode, struct iatt *buf,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t * xdata);
-
-int32_t
-default_open_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, fd_t * fd,
-                         dict_t * xdata);
-
-int32_t
-default_readv_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec *vector, int32_t count,
-                          struct iatt *stbuf, struct iobref *iobref,
-                          dict_t * xdata);
-
-
-int32_t
-default_writev_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t * xdata);
-
-
-int32_t
-default_flush_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, dict_t * xdata);
-
-
-
-int32_t
-default_fsync_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *prebuf, struct iatt *postbuf,
-                          dict_t * xdata);
-
-int32_t
-default_fstat_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                          dict_t * xdata);
-
-int32_t
-default_opendir_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            fd_t * fd, dict_t * xdata);
-
-int32_t
-default_fsyncdir_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * xdata);
-
-int32_t
-default_statfs_cbk_resume (call_frame_t * frame, void *cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct statvfs *buf, dict_t * xdata);
-
-
-int32_t
-default_setxattr_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * xdata);
-
-
-int32_t
-default_fsetxattr_cbk_resume (call_frame_t * frame, void *cookie,
-                              xlator_t * this, int32_t op_ret,
-                              int32_t op_errno, dict_t * xdata);
-
-
-
-int32_t
-default_fgetxattr_cbk_resume (call_frame_t * frame, void *cookie,
-                              xlator_t * this, int32_t op_ret,
-                              int32_t op_errno, dict_t * dict,
-                              dict_t * xdata);
-
-
-int32_t
-default_getxattr_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * dict, dict_t * xdata);
-
-int32_t
-default_xattrop_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            dict_t * dict, dict_t * xdata);
-
-int32_t
-default_fxattrop_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * dict, dict_t * xdata);
-
-
-int32_t
-default_removexattr_cbk_resume (call_frame_t * frame, void *cookie,
-                                xlator_t * this, int32_t op_ret,
-                                int32_t op_errno, dict_t * xdata);
-
-int32_t
-default_fremovexattr_cbk_resume (call_frame_t * frame, void *cookie,
-                                 xlator_t * this, int32_t op_ret,
-                                 int32_t op_errno, dict_t * xdata);
-
-int32_t
-default_lk_cbk_resume (call_frame_t * frame, void *cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct gf_flock *lock, dict_t * xdata);
-
-int32_t
-default_inodelk_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            dict_t * xdata);
-
-
-int32_t
-default_finodelk_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * xdata);
-
-int32_t
-default_entrylk_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            dict_t * xdata);
-
-int32_t
-default_fentrylk_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, dict_t * xdata);
-
-
-int32_t
-default_rchecksum_cbk_resume (call_frame_t * frame, void *cookie,
-                              xlator_t * this, int32_t op_ret,
-                              int32_t op_errno, uint32_t weak_checksum,
-                              uint8_t * strong_checksum, dict_t * xdata);
-
-
-int32_t
-default_readdir_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            gf_dirent_t * entries, dict_t * xdata);
-
-
-int32_t
-default_readdirp_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, gf_dirent_t * entries,
-                             dict_t * xdata);
-
-int32_t
-default_setattr_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            struct iatt *statpre, struct iatt *statpost,
-                            dict_t * xdata);
-
-int32_t
-default_fsetattr_cbk_resume (call_frame_t * frame, void *cookie,
-                             xlator_t * this, int32_t op_ret,
-                             int32_t op_errno, struct iatt *statpre,
-                             struct iatt *statpost, dict_t * xdata);
-
-int32_t default_fallocate_cbk_resume (call_frame_t * frame, void *cookie,
-                                      xlator_t * this, int32_t op_ret,
-                                      int32_t op_errno, struct iatt *pre,
-                                      struct iatt *post, dict_t * xdata);
-
-int32_t default_discard_cbk_resume (call_frame_t * frame, void *cookie,
-                                    xlator_t * this, int32_t op_ret,
-                                    int32_t op_errno, struct iatt *pre,
-                                    struct iatt *post, dict_t * xdata);
-
-int32_t default_zerofill_cbk_resume (call_frame_t * frame, void *cookie,
-                                     xlator_t * this, int32_t op_ret,
-                                     int32_t op_errno, struct iatt *pre,
-                                     struct iatt *post, dict_t * xdata);
-int32_t
-default_ipc_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_seek_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, off_t offset,
-                         dict_t *xdata);
-
-int32_t
-default_getspec_cbk_resume (call_frame_t * frame, void *cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            char *spec_data);
-
-int32_t
-default_lease_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct gf_lease *lease, dict_t *xdata);
-
-int32_t
-default_getactivelk_cbk_resume (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno,
-                                lock_migration_info_t *locklist,
-                                dict_t *xdata);
-
-int32_t
-default_setactivelk_cbk_resume (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xdata);
-
-/* _CBK */
-int32_t
-default_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent);
-
-int32_t
-default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata);
-
-
-int32_t
-default_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata);
-
-int32_t
-default_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata);
-
-int32_t
-default_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata);
-
-
-int32_t
-default_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata);
-
-int32_t
-default_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata);
-
-int32_t
-default_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-int32_t
-default_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata);
-
-
-int32_t
-default_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
-
-
-int32_t
-default_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata);
-
-
-int32_t
-default_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata);
-
-
-int32_t
-default_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata);
-
-int32_t
-default_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
-
-int32_t
-default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
-
-
-int32_t
-default_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata);
-
-
-int32_t
-default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-
-int32_t
-default_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata);
-
-int32_t
-default_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata);
-
-int32_t
-default_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
-
-int32_t
-default_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata);
-
-
-int32_t
-default_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int32_t
-default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-
-int32_t
-default_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
-
-
-int32_t
-default_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
-
-int32_t
-default_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
-
-int32_t
-default_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
-
-
-int32_t
-default_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata);
-
-int32_t
-default_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int32_t
-default_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t
-default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int32_t
-default_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum, dict_t *xdata);
-
-
-int32_t
-default_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
-
-
-int32_t
-default_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
-
-int32_t
-default_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost, dict_t *xdata);
-
-int32_t
-default_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost, dict_t *xdata);
-
-int32_t default_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                              struct iatt *post, dict_t *xdata);
-
-int32_t default_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                            struct iatt *post, dict_t *xdata);
-
-int32_t default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                            struct iatt *post, dict_t *xdata);
-
-int32_t default_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-int32_t default_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, off_t offset,
-                          dict_t *xdata);
-
-int32_t
-default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, char *spec_data);
-
-int32_t
-default_lease_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct gf_lease *lease, dict_t *xdata);
-
-int32_t
-default_getactivelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          lock_migration_info_t *locklist,
-                          dict_t *xdata);
-
-int32_t
-default_setactivelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
-
-
-int32_t
-default_lookup_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_stat_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-
-int32_t
-default_truncate_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_ftruncate_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_access_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_readlink_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-
-int32_t
-default_mknod_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_mkdir_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_unlink_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_rmdir_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_symlink_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_rename_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_link_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_create_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_open_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_readv_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_writev_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_flush_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fsync_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fstat_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_opendir_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fsyncdir_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_statfs_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_setxattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fsetxattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fgetxattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_getxattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_xattrop_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fxattrop_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_removexattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fremovexattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_lk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_inodelk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_finodelk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_entrylk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fentrylk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_rchecksum_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_readdir_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_readdirp_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_setattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fsetattr_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_fallocate_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_discard_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_zerofill_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_getspec_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_ipc_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_seek_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_lease_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_getactivelk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_setactivelk_failure_cbk (call_frame_t *frame, int32_t op_errno);
-
-int32_t
-default_mem_acct_init (xlator_t *this);
-#endif /* _DEFAULTS_H */
diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index 96cb9e94bda..1d9be9217a6 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -16,152 +16,227 @@
 #include <limits.h>
 #include <fnmatch.h>
 
-#include "glusterfs.h"
-#include "common-utils.h"
-#include "dict.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "globals.h"
-#include "statedump.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/dict.h"
+#define XXH_INLINE_ALL
+#include "xxhash.h"
+#include "glusterfs/compat.h"
+#include "glusterfs/compat-errno.h"
+#include "glusterfs/byte-order.h"
+#include "glusterfs/statedump.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 struct dict_cmp {
-        dict_t *dict;
-        gf_boolean_t (*value_ignore) (char *k);
+    dict_t *dict;
+    gf_boolean_t (*value_ignore)(char *k);
 };
 
-data_t *
-get_new_data ()
+#define VALIDATE_DATA_AND_LOG(data, type, key, ret_val)                        \
+    do {                                                                       \
+        if (!data || !data->data) {                                            \
+            gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG, \
+                             "data is NULL");                                  \
+            return ret_val;                                                    \
+        }                                                                      \
+        /* Not of the asked type, or old version */                            \
+        if ((data->data_type != type) &&                                       \
+            (data->data_type != GF_DATA_TYPE_STR_OLD)) {                       \
+            gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG, \
+                             "key %s, %s type asked, has %s type", key,        \
+                             data_type_name[type],                             \
+                             data_type_name[data->data_type]);                 \
+        }                                                                      \
+    } while (0)
+
+static data_t *
+get_new_data()
 {
-        data_t *data = NULL;
+    data_t *data = mem_get(THIS->ctx->dict_data_pool);
 
-        data = mem_get0 (THIS->ctx->dict_data_pool);
-        if (!data) {
-                return NULL;
-        }
+    if (!data)
+        return NULL;
+
+    GF_ATOMIC_INIT(data->refcount, 0);
+    data->is_static = _gf_false;
 
-        LOCK_INIT (&data->lock);
-        return data;
+    return data;
 }
 
-dict_t *
-get_new_dict_full (int size_hint)
+static dict_t *
+get_new_dict_full(int size_hint)
 {
-        dict_t *dict = mem_get0 (THIS->ctx->dict_pool);
+    dict_t *dict = mem_get0(THIS->ctx->dict_pool);
 
-        if (!dict) {
-                return NULL;
-        }
-
-        dict->hash_size = size_hint;
-        if (size_hint == 1) {
-                /*
-                 * This is the only case we ever see currently.  If we ever
-                 * need to support resizing the hash table, the resize function
-                 * will have to take into account the possibility that
-                 * "members" is not separately allocated (i.e. don't just call
-                 * realloc() blindly.
-                 */
-                dict->members = &dict->members_internal;
-        }
-        else {
-                /*
-                 * We actually need to allocate space for size_hint *pointers*
-                 * but we actually allocate space for one *structure*.  Since
-                 * a data_pair_t consists of five pointers, we're wasting four
-                 * pointers' worth for N=1, and will overrun what we allocated
-                 * for N>5.  If anybody ever starts using size_hint, we'll need
-                 * to fix this.
-                 */
-                GF_ASSERT (size_hint <=
-                           (sizeof(data_pair_t) / sizeof(data_pair_t *)));
-                dict->members = mem_get0 (THIS->ctx->dict_pair_pool);
-                if (!dict->members) {
-                        mem_put (dict);
-                        return NULL;
-                }
+    if (!dict) {
+        return NULL;
+    }
+
+    dict->hash_size = size_hint;
+    if (size_hint == 1) {
+        /*
+         * This is the only case we ever see currently.  If we ever
+         * need to support resizing the hash table, the resize function
+         * will have to take into account the possibility that
+         * "members" is not separately allocated (i.e. don't just call
+         * realloc() blindly.
+         */
+        dict->members = &dict->members_internal;
+    } else {
+        /*
+         * We actually need to allocate space for size_hint *pointers*
+         * but we actually allocate space for one *structure*.  Since
+         * a data_pair_t consists of five pointers, we're wasting four
+         * pointers' worth for N=1, and will overrun what we allocated
+         * for N>5.  If anybody ever starts using size_hint, we'll need
+         * to fix this.
+         */
+        GF_ASSERT(size_hint <= (sizeof(data_pair_t) / sizeof(data_pair_t *)));
+        dict->members = mem_get0(THIS->ctx->dict_pair_pool);
+        if (!dict->members) {
+            mem_put(dict);
+            return NULL;
         }
+    }
 
-        LOCK_INIT (&dict->lock);
+    dict->free_pair.key = NULL;
+    dict->totkvlen = 0;
+    LOCK_INIT(&dict->lock);
 
-        return dict;
+    return dict;
 }
 
 dict_t *
-get_new_dict (void)
+dict_new(void)
 {
-        return get_new_dict_full (1);
+    dict_t *dict = get_new_dict_full(1);
+
+    if (dict)
+        dict_ref(dict);
+
+    return dict;
 }
 
-dict_t *
-dict_new (void)
+int32_t
+is_data_equal(data_t *one, data_t *two)
 {
-        dict_t *dict = NULL;
+    struct iatt *iatt1, *iatt2;
+    struct mdata_iatt *mdata_iatt1, *mdata_iatt2;
 
-        dict = get_new_dict_full(1);
+    if (!one || !two || !one->data || !two->data) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "input arguments are provided "
+                         "with value data_t as NULL");
+        return -1;
+    }
 
-        if (dict)
-                dict_ref (dict);
+    if (one == two)
+        return 1;
 
-        return dict;
-}
+    if (one->data == two->data)
+        return 1;
 
-int32_t
-is_data_equal (data_t *one,
-               data_t *two)
-{
-        if (!one || !two || !one->data || !two->data) {
-		gf_msg_callingfn ("dict", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG,
-				  "input arguments are provided "
-				  "with value data_t as NULL");
-                return -1;
-	}
+    if (one->data_type != two->data_type) {
+        return 0;
+    }
 
-        if (one == two)
-                return 1;
+    if (one->data_type == GF_DATA_TYPE_IATT) {
+        if ((one->len < sizeof(struct iatt)) ||
+            (two->len < sizeof(struct iatt))) {
+            return 0;
+        }
+
+        iatt1 = (struct iatt *)one->data;
+        iatt2 = (struct iatt *)two->data;
 
-        if (one->len != two->len)
+        /* Two iatt structs are considered equal if main fields are
+         * equal, even if times differ.
+         * TODO: maybe when ctime if fully operational we could
+         *       enforce time matching. */
+        if (iatt1->ia_ino != iatt2->ia_ino) {
+            return 0;
+        }
+        if (iatt1->ia_type != iatt2->ia_type) {
+            return 0;
+        }
+        if ((iatt1->ia_type == IA_IFBLK) || (iatt1->ia_type == IA_IFCHR)) {
+            if (iatt1->ia_rdev != iatt2->ia_rdev) {
                 return 0;
+            }
+        }
+        if (gf_uuid_compare(iatt1->ia_gfid, iatt2->ia_gfid) != 0) {
+            return 0;
+        }
 
-        if (one->data == two->data)
-                return 1;
+        /* TODO: ia_uid, ia_gid, ia_prot and ia_size can be changed
+         *       with some commands. Here we don't have enough
+         *       information to decide if they should match or not. */
+        /*
+                        if ((iatt1->ia_uid != iatt2->ia_uid) ||
+                            (iatt1->ia_gid != iatt2->ia_gid) ||
+                            (st_mode_from_ia(iatt1->ia_prot, iatt1->ia_type) !=
+                                    st_mode_from_ia(iatt2->ia_prot,
+           iatt2->ia_type))) { return 0;
+                        }
+                        if (iatt1->ia_type == IA_IFREG) {
+                                if (iatt1->ia_size != iatt2->ia_size) {
+                                        return 0;
+                                }
+                        }
+        */
+        return 1;
+    }
+    if (one->data_type == GF_DATA_TYPE_MDATA) {
+        if ((one->len < sizeof(struct mdata_iatt)) ||
+            (two->len < sizeof(struct mdata_iatt))) {
+            return 0;
+        }
+        mdata_iatt1 = (struct mdata_iatt *)one->data;
+        mdata_iatt2 = (struct mdata_iatt *)two->data;
+
+        if (mdata_iatt1->ia_atime != mdata_iatt2->ia_atime ||
+            mdata_iatt1->ia_mtime != mdata_iatt2->ia_mtime ||
+            mdata_iatt1->ia_ctime != mdata_iatt2->ia_ctime ||
+            mdata_iatt1->ia_atime_nsec != mdata_iatt2->ia_atime_nsec ||
+            mdata_iatt1->ia_mtime_nsec != mdata_iatt2->ia_mtime_nsec ||
+            mdata_iatt1->ia_ctime_nsec != mdata_iatt2->ia_ctime_nsec) {
+            return 0;
+        }
+        return 1;
+    }
+
+    if (one->len != two->len)
+        return 0;
 
-        if (memcmp (one->data, two->data, one->len) == 0)
-                return 1;
+    if (memcmp(one->data, two->data, one->len) == 0)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 static int
-key_value_cmp (dict_t *one, char *key1, data_t *value1, void *data)
+key_value_cmp(dict_t *one, char *key1, data_t *value1, void *data)
 {
-        struct dict_cmp *cmp = data;
-        dict_t *two = NULL;
-        data_t *value2 = NULL;
+    struct dict_cmp *cmp = data;
+    dict_t *two = cmp->dict;
+    data_t *value2 = dict_get(two, key1);
 
-        two = cmp->dict;
-        value2 = dict_get (two, key1);
+    if (value2) {
+        if (cmp->value_ignore && cmp->value_ignore(key1))
+            return 0;
 
-        if (value2) {
-                if (cmp->value_ignore && cmp->value_ignore (key1))
-                        return 0;
+        if (is_data_equal(value1, value2) == 1)
+            return 0;
+    }
 
-                if (is_data_equal (value1, value2) == 1)
-                        return 0;
-        }
-
-        if (value2 == NULL) {
-                gf_msg_debug (THIS->name, 0, "'%s' found only on one dict",
-                              key1);
-        } else {
-                gf_msg_debug (THIS->name, 0, "'%s' is different in two dicts "
-                              "(%u, %u)", key1, value1->len, value2->len);
-        }
+    if (value2 == NULL) {
+        gf_msg_debug(THIS->name, 0, "'%s' found only on one dict", key1);
+    } else {
+        gf_msg_debug(THIS->name, 0,
+                     "'%s' is different in two dicts "
+                     "(%u, %u)",
+                     key1, value1->len, value2->len);
+    }
 
-        return -1;
+    return -1;
 }
 
 /* If both dicts are NULL then equal. If one of the dicts is NULL but the
@@ -172,1049 +247,1087 @@ key_value_cmp (dict_t *one, char *key1, data_t *value1, void *data)
  * different.
  */
 gf_boolean_t
-are_dicts_equal (dict_t *one, dict_t *two,
-                 gf_boolean_t (*match) (dict_t *d, char *k, data_t *v,
-                                        void *data),
-                 gf_boolean_t (*value_ignore) (char *k))
-{
-        int     num_matches1 = 0;
-        int     num_matches2 = 0;
-        struct  dict_cmp cmp = {0};
-
-        if (one == two)
-                return _gf_true;
-
-        if (!match)
-                match = dict_match_everything;
-
-        cmp.dict = two;
-        cmp.value_ignore = value_ignore;
-        if (!two) {
-                num_matches1 = dict_foreach_match (one, match, NULL,
-                                                   dict_null_foreach_fn, NULL);
-                goto done;
-        } else {
-                num_matches1 = dict_foreach_match (one, match, NULL,
-                                                   key_value_cmp, &cmp);
-        }
+are_dicts_equal(dict_t *one, dict_t *two,
+                gf_boolean_t (*match)(dict_t *d, char *k, data_t *v,
+                                      void *data),
+                gf_boolean_t (*value_ignore)(char *k))
+{
+    int num_matches1 = 0;
+    int num_matches2 = 0;
+    struct dict_cmp cmp = {0};
+
+    if (one == two)
+        return _gf_true;
 
-        if (num_matches1 == -1)
-                return _gf_false;
+    if (!match)
+        match = dict_match_everything;
 
-        if ((num_matches1 == one->count) && (one->count == two->count))
-                return _gf_true;
+    if ((one == NULL) || (two == NULL)) {
+        num_matches1 = dict_foreach_match(one ? one : two, match, NULL,
+                                          dict_null_foreach_fn, NULL);
+        goto done;
+    }
 
-        num_matches2 = dict_foreach_match (two, match, NULL,
-                                           dict_null_foreach_fn, NULL);
-done:
-        /* If the number of matches is same in 'two' then for all the
-         * valid-keys that exist in 'one' the value matched and no extra valid
-         * keys exist in 'two' alone. Otherwise there exists at least one extra
-         * valid-key in 'two' which doesn't exist in 'one' */
-        if (num_matches1 == num_matches2)
-                return _gf_true;
+    cmp.dict = two;
+    cmp.value_ignore = value_ignore;
+    num_matches1 = dict_foreach_match(one, match, NULL, key_value_cmp, &cmp);
+
+    if (num_matches1 == -1)
         return _gf_false;
+
+    if ((num_matches1 == one->count) && (one->count == two->count))
+        return _gf_true;
+
+    num_matches2 = dict_foreach_match(two, match, NULL, dict_null_foreach_fn,
+                                      NULL);
+done:
+    /* If the number of matches is same in 'two' then for all the
+     * valid-keys that exist in 'one' the value matched and no extra valid
+     * keys exist in 'two' alone. Otherwise there exists at least one extra
+     * valid-key in 'two' which doesn't exist in 'one' */
+    if (num_matches1 == num_matches2)
+        return _gf_true;
+    return _gf_false;
 }
 
 void
-data_destroy (data_t *data)
+data_destroy(data_t *data)
 {
-        if (data) {
-                LOCK_DESTROY (&data->lock);
-
-                if (!data->is_static)
-                        GF_FREE (data->data);
+    if (data) {
+        if (!data->is_static)
+            GF_FREE(data->data);
 
-                data->len = 0xbabababa;
-                if (!data->is_const)
-                        mem_put (data);
-        }
+        data->len = 0xbabababa;
+        mem_put(data);
+    }
 }
 
 data_t *
-data_copy (data_t *old)
+data_copy(data_t *old)
 {
-        if (!old) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, 0, LG_MSG_NULL_PTR,
-                                  "old is NULL");
-                return NULL;
-        }
+    if (!old) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, 0, LG_MSG_NULL_PTR,
+                         "old is NULL");
+        return NULL;
+    }
 
-        data_t *newdata = mem_get0 (THIS->ctx->dict_data_pool);
-        if (!newdata) {
-                return NULL;
-        }
+    data_t *newdata = mem_get0(THIS->ctx->dict_data_pool);
+    if (!newdata) {
+        return NULL;
+    }
 
-        if (old) {
-                newdata->len = old->len;
-                if (old->data) {
-                        newdata->data = memdup (old->data, old->len);
-                        if (!newdata->data)
-                                goto err_out;
-                }
-        }
+    newdata->len = old->len;
+    if (old->data) {
+        newdata->data = gf_memdup(old->data, old->len);
+        if (!newdata->data)
+            goto err_out;
+    }
+    newdata->data_type = old->data_type;
 
-        LOCK_INIT (&newdata->lock);
-        return newdata;
+    return newdata;
 
 err_out:
-        mem_put (newdata);
+    mem_put(newdata);
 
-        return NULL;
+    return NULL;
 }
 
+/* Always need to be called under lock
+ * Always this and key variables are not null -
+ * checked by callers.
+ */
 static data_pair_t *
-dict_lookup_common (dict_t *this, char *key)
-{
-        int hashval = 0;
-        if (!this || !key) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG,
-                                  "!this || !key (%s)", key);
-                return NULL;
-        }
-
-        /* If the divisor is 1, the modulo is always 0,
-         * in such case avoid hash calculation.
-         */
-        if (this->hash_size != 1)
-                hashval = SuperFastHash (key, strlen (key)) % this->hash_size;
+dict_lookup_common(const dict_t *this, const char *key, const uint32_t hash)
+{
+    int hashval = 0;
+    data_pair_t *pair;
 
-        data_pair_t *pair;
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1)
+        hashval = hash % this->hash_size;
 
-        for (pair = this->members[hashval]; pair != NULL; pair = pair->hash_next) {
-                if (pair->key && !strcmp (pair->key, key))
-                        return pair;
-        }
+    for (pair = this->members[hashval]; pair != NULL; pair = pair->hash_next) {
+        if (pair->key && (hash == pair->key_hash) && !strcmp(pair->key, key))
+            return pair;
+    }
 
-        return NULL;
+    return NULL;
 }
 
 int32_t
-dict_lookup (dict_t *this, char *key, data_t **data)
+dict_lookup(dict_t *this, char *key, data_t **data)
 {
-        if (!this || !key || !data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!this || !key || "
-                                  "!data");
-                return -1;
-        }
-
-        data_pair_t *tmp = NULL;
-        LOCK (&this->lock);
-        {
-                tmp = dict_lookup_common (this, key);
-        }
-        UNLOCK (&this->lock);
-
-        if (!tmp)
-                return -1;
-
-        *data = tmp->value;
-        return 0;
-}
-
-static int32_t
-dict_set_lk (dict_t *this, char *key, data_t *value, gf_boolean_t replace)
-{
-        int hashval = 0;
-        data_pair_t *pair;
-        char key_free = 0;
-        int tmp = 0;
-        int ret = 0;
-
-        if (!key) {
-                ret = gf_asprintf (&key, "ref:%p", value);
-                if (-1 == ret) {
-                        return -1;
-                }
-                key_free = 1;
-        }
+    if (!this || !key || !data) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !key || "
+                         "!data");
+        return -1;
+    }
 
-        /* If the divisor is 1, the modulo is always 0,
-         * in such case avoid hash calculation.
-         */
-        if (this->hash_size != 1) {
-                tmp = SuperFastHash (key, strlen (key));
-                hashval = (tmp % this->hash_size);
-        }
+    data_pair_t *tmp = NULL;
 
-        /* Search for a existing key if 'replace' is asked for */
-        if (replace) {
-                pair = dict_lookup_common (this, key);
-
-                if (pair) {
-                        data_t *unref_data = pair->value;
-                        pair->value = data_ref (value);
-                        data_unref (unref_data);
-                        if (key_free)
-                                GF_FREE (key);
-                        /* Indicates duplicate key */
-                        return 0;
-                }
-        }
+    uint32_t hash = (uint32_t)XXH64(key, strlen(key), 0);
 
-        if (this->free_pair_in_use) {
-                pair = mem_get0 (THIS->ctx->dict_pair_pool);
-                if (!pair) {
-                        if (key_free)
-                                GF_FREE (key);
-                        return -1;
-                }
-        }
-        else {
-                pair = &this->free_pair;
-                this->free_pair_in_use = _gf_true;
-        }
+    LOCK(&this->lock);
+    {
+        tmp = dict_lookup_common(this, key, hash);
+    }
+    UNLOCK(&this->lock);
 
-        if (key_free) {
-                /* It's ours.  Use it. */
-                pair->key = key;
-                key_free = 0;
-        }
-        else {
-                pair->key = (char *) GF_CALLOC (1, strlen (key) + 1,
-                                                gf_common_mt_char);
-                if (!pair->key) {
-                        if (pair == &this->free_pair) {
-                                this->free_pair_in_use = _gf_false;
-                        }
-                        else {
-                                mem_put (pair);
-                        }
-                        return -1;
-                }
-                strcpy (pair->key, key);
-        }
-        pair->value = data_ref (value);
+    if (!tmp)
+        return -1;
 
-        pair->hash_next = this->members[hashval];
-        this->members[hashval] = pair;
+    *data = tmp->value;
+    return 0;
+}
 
-        pair->next = this->members_list;
-        pair->prev = NULL;
-        if (this->members_list)
-                this->members_list->prev = pair;
-        this->members_list = pair;
-        this->count++;
+static int32_t
+dict_set_lk(dict_t *this, char *key, const int key_len, data_t *value,
+            const uint32_t hash, gf_boolean_t replace)
+{
+    int hashval = 0;
+    data_pair_t *pair;
+    int key_free = 0;
+    uint32_t key_hash;
+    int keylen;
+
+    if (!key) {
+        keylen = gf_asprintf(&key, "ref:%p", value);
+        if (-1 == keylen) {
+            return -1;
+        }
+        key_free = 1;
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    } else {
+        keylen = key_len;
+        key_hash = hash;
+    }
+
+    /* Search for a existing key if 'replace' is asked for */
+    if (replace) {
+        pair = dict_lookup_common(this, key, key_hash);
+        if (pair) {
+            data_t *unref_data = pair->value;
+            pair->value = data_ref(value);
+            this->totkvlen += (value->len - unref_data->len);
+            data_unref(unref_data);
+            if (key_free)
+                GF_FREE(key);
+            /* Indicates duplicate key */
+            return 0;
+        }
+    }
+
+    if (this->free_pair.key) { /* the free_pair is used */
+        pair = mem_get(THIS->ctx->dict_pair_pool);
+        if (!pair) {
+            if (key_free)
+                GF_FREE(key);
+            return -1;
+        }
+    } else { /* assign the pair to the free pair */
+        pair = &this->free_pair;
+    }
+
+    if (key_free) {
+        /* It's ours.  Use it. */
+        pair->key = key;
+        key_free = 0;
+    } else {
+        pair->key = (char *)GF_MALLOC(keylen + 1, gf_common_mt_char);
+        if (!pair->key) {
+            if (pair != &this->free_pair) {
+                mem_put(pair);
+            }
+            return -1;
+        }
+        strcpy(pair->key, key);
+    }
+    pair->key_hash = key_hash;
+    pair->value = data_ref(value);
+    this->totkvlen += (keylen + 1 + value->len);
+
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1) {
+        hashval = (key_hash % this->hash_size);
+    }
+    pair->hash_next = this->members[hashval];
+    this->members[hashval] = pair;
+
+    pair->next = this->members_list;
+    pair->prev = NULL;
+    if (this->members_list)
+        this->members_list->prev = pair;
+    this->members_list = pair;
+    this->count++;
+
+    if (key_free)
+        GF_FREE(key);
+
+    if (this->max_count < this->count)
+        this->max_count = this->count;
+    return 0;
+}
 
-        if (key_free)
-                GF_FREE (key);
-        return 0;
+int32_t
+dict_set(dict_t *this, char *key, data_t *value)
+{
+    if (key)
+        return dict_setn(this, key, strlen(key), value);
+    else
+        return dict_setn(this, NULL, 0, value);
 }
 
 int32_t
-dict_set (dict_t *this,
-          char *key,
-          data_t *value)
+dict_setn(dict_t *this, char *key, const int keylen, data_t *value)
 {
-        int32_t ret;
+    int32_t ret;
+    uint32_t key_hash = 0;
 
-        if (!this || !value) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!this || !value for "
-                                  "key=%s", key);
-                return -1;
-        }
+    if (!this || !value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !value for "
+                         "key=%s",
+                         key);
+        return -1;
+    }
 
-        LOCK (&this->lock);
+    if (key) {
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    }
 
-        ret = dict_set_lk (this, key, value, 1);
+    LOCK(&this->lock);
 
-        UNLOCK (&this->lock);
+    ret = dict_set_lk(this, key, keylen, value, key_hash, 1);
 
-        return ret;
+    UNLOCK(&this->lock);
+
+    return ret;
 }
 
+int32_t
+dict_add(dict_t *this, char *key, data_t *value)
+{
+    if (key)
+        return dict_addn(this, key, strlen(key), value);
+    else
+        return dict_addn(this, NULL, 0, value);
+}
 
 int32_t
-dict_add (dict_t *this, char *key, data_t *value)
+dict_addn(dict_t *this, char *key, const int keylen, data_t *value)
 {
-        int32_t ret;
+    int32_t ret;
+    uint32_t key_hash = 0;
 
-        if (!this || !value) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG,
-                                  "!this || !value for key=%s", key);
-                return -1;
-        }
+    if (!this || !value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !value for key=%s", key);
+        return -1;
+    }
 
-        LOCK (&this->lock);
+    if (key) {
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    }
 
-        ret = dict_set_lk (this, key, value, 0);
+    LOCK(&this->lock);
 
-        UNLOCK (&this->lock);
+    ret = dict_set_lk(this, key, keylen, value, key_hash, 0);
 
-        return ret;
-}
+    UNLOCK(&this->lock);
 
+    return ret;
+}
 
 data_t *
-dict_get (dict_t *this, char *key)
+dict_get(dict_t *this, char *key)
 {
-        data_pair_t *pair;
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", (key) ? key : "()");
+        return NULL;
+    }
 
-        if (!this || !key) {
-                gf_msg_callingfn ("dict", GF_LOG_INFO, EINVAL,
-                                  LG_MSG_INVALID_ARG,
-                                  "!this || key=%s", (key) ? key : "()");
-                return NULL;
-        }
+    return dict_getn(this, key, strlen(key));
+}
 
-        LOCK (&this->lock);
+data_t *
+dict_getn(dict_t *this, char *key, const int keylen)
+{
+    data_pair_t *pair;
+    uint32_t hash;
 
-        pair = dict_lookup_common (this, key);
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", (key) ? key : "()");
+        return NULL;
+    }
 
-        UNLOCK (&this->lock);
+    hash = (uint32_t)XXH64(key, keylen, 0);
 
-        if (pair)
-                return pair->value;
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
+    }
+    UNLOCK(&this->lock);
 
-        return NULL;
+    if (pair)
+        return pair->value;
+
+    return NULL;
 }
 
 int
-dict_key_count (dict_t *this)
+dict_key_count(dict_t *this)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict passed is NULL");
-                return ret;
-        }
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict passed is NULL");
+        return ret;
+    }
 
-        LOCK (&this->lock);
-        {
-                ret = this->count;
-        }
-        UNLOCK (&this->lock);
+    LOCK(&this->lock);
+    {
+        ret = this->count;
+    }
+    UNLOCK(&this->lock);
 
-        return ret;
+    return ret;
 }
 
 void
-dict_del (dict_t *this, char *key)
+dict_del(dict_t *this, char *key)
 {
-        int hashval = 0;
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", key);
+        return;
+    }
 
-        if (!this || !key) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!this || key=%s", key);
-                return;
-        }
+    return dict_deln(this, key, strlen(key));
+}
 
-        LOCK (&this->lock);
+void
+dict_deln(dict_t *this, char *key, const int keylen)
+{
+    int hashval = 0;
+    uint32_t hash;
 
-        /* If the divisor is 1, the modulo is always 0,
-         * in such case avoid hash calculation.
-         */
-        if (this->hash_size != 1)
-                hashval = SuperFastHash (key, strlen (key)) % this->hash_size;
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", key);
+        return;
+    }
 
-        data_pair_t *pair = this->members[hashval];
-        data_pair_t *prev = NULL;
+    hash = (uint32_t)XXH64(key, keylen, 0);
 
-        while (pair) {
-                if (strcmp (pair->key, key) == 0) {
-                        if (prev)
-                                prev->hash_next = pair->hash_next;
-                        else
-                                this->members[hashval] = pair->hash_next;
+    LOCK(&this->lock);
 
-                        data_unref (pair->value);
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1)
+        hashval = hash % this->hash_size;
 
-                        if (pair->prev)
-                                pair->prev->next = pair->next;
-                        else
-                                this->members_list = pair->next;
+    data_pair_t *pair = this->members[hashval];
+    data_pair_t *prev = NULL;
 
-                        if (pair->next)
-                                pair->next->prev = pair->prev;
+    while (pair) {
+        if ((hash == pair->key_hash) && strcmp(pair->key, key) == 0) {
+            if (prev)
+                prev->hash_next = pair->hash_next;
+            else
+                this->members[hashval] = pair->hash_next;
 
-                        GF_FREE (pair->key);
-                        if (pair == &this->free_pair) {
-                                this->free_pair_in_use = _gf_false;
-                        }
-                        else {
-                                mem_put (pair);
-                        }
-                        this->count--;
-                        break;
-                }
+            this->totkvlen -= pair->value->len;
+            data_unref(pair->value);
+
+            if (pair->prev)
+                pair->prev->next = pair->next;
+            else
+                this->members_list = pair->next;
+
+            if (pair->next)
+                pair->next->prev = pair->prev;
 
-                prev = pair;
-                pair = pair->hash_next;
+            this->totkvlen -= (strlen(pair->key) + 1);
+            GF_FREE(pair->key);
+            if (pair == &this->free_pair) {
+                this->free_pair.key = NULL;
+            } else {
+                mem_put(pair);
+            }
+            this->count--;
+            break;
         }
 
-        UNLOCK (&this->lock);
+        prev = pair;
+        pair = pair->hash_next;
+    }
 
-        return;
+    UNLOCK(&this->lock);
+
+    return;
 }
 
 void
-dict_destroy (dict_t *this)
+dict_destroy(dict_t *this)
 {
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return;
+    }
+
+    data_pair_t *pair = this->members_list;
+    data_pair_t *prev = this->members_list;
+    glusterfs_ctx_t *ctx = NULL;
+    uint64_t current_max = 0;
+    uint32_t total_pairs = 0;
+
+    LOCK_DESTROY(&this->lock);
+
+    while (prev) {
+        pair = pair->next;
+        data_unref(prev->value);
+        GF_FREE(prev->key);
+        if (prev != &this->free_pair) {
+            mem_put(prev);
+        } else {
+            this->free_pair.key = NULL;
         }
+        total_pairs++;
+        prev = pair;
+    }
 
-        data_pair_t *pair = this->members_list;
-        data_pair_t *prev = this->members_list;
+    this->totkvlen = 0;
+    if (this->members != &this->members_internal) {
+        mem_put(this->members);
+    }
 
-        LOCK_DESTROY (&this->lock);
+    free(this->extra_stdfree);
 
-        while (prev) {
-                pair = pair->next;
-                data_unref (prev->value);
-                GF_FREE (prev->key);
-                if (prev != &this->free_pair) {
-                        mem_put (prev);
-                }
-                prev = pair;
-        }
+    /* update 'ctx->stats.dict.details' using max_count */
+    ctx = THIS->ctx;
 
-        if (this->members != &this->members_internal) {
-                mem_put (this->members);
-        }
+    /* NOTE: below logic is not totaly race proof */
+    /* thread0 and thread1 gets current_max as 10 */
+    /* thread0 has 'this->max_count as 11 */
+    /* thread1 has 'this->max_count as 20 */
+    /* thread1 goes ahead and sets the max_dict_pairs to 20 */
+    /* thread0 then goes and sets it to 11 */
+    /* As it is for information purpose only, no functionality will be
+       broken by this, but a point to consider about ATOMIC macros. */
+    current_max = GF_ATOMIC_GET(ctx->stats.max_dict_pairs);
+    if (current_max < this->max_count)
+        GF_ATOMIC_INIT(ctx->stats.max_dict_pairs, this->max_count);
 
-        GF_FREE (this->extra_free);
-        free (this->extra_stdfree);
+    GF_ATOMIC_ADD(ctx->stats.total_pairs_used, total_pairs);
+    GF_ATOMIC_INC(ctx->stats.total_dicts_used);
 
-        if (!this->is_static)
-                mem_put (this);
+    mem_put(this);
 
-        return;
+    return;
 }
 
 void
-dict_unref (dict_t *this)
+dict_unref(dict_t *this)
 {
-        int32_t ref;
-
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return;
-        }
-
-        LOCK (&this->lock);
+    uint64_t ref = 0;
 
-        this->refcount--;
-        ref = this->refcount;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return;
+    }
 
-        UNLOCK (&this->lock);
+    ref = GF_ATOMIC_DEC(this->refcount);
 
-        if (!ref)
-                dict_destroy (this);
+    if (!ref)
+        dict_destroy(this);
 }
 
 dict_t *
-dict_ref (dict_t *this)
+dict_ref(dict_t *this)
 {
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return NULL;
-        }
-
-        LOCK (&this->lock);
-
-        this->refcount++;
-
-        UNLOCK (&this->lock);
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return NULL;
+    }
 
-        return this;
+    GF_ATOMIC_INC(this->refcount);
+    return this;
 }
 
 void
-data_unref (data_t *this)
+data_unref(data_t *this)
 {
+    uint64_t ref;
 
-        int32_t ref;
-
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return;
-        }
-
-        LOCK (&this->lock);
-
-        this->refcount--;
-        ref = this->refcount;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "data is NULL");
+        return;
+    }
 
-        UNLOCK (&this->lock);
+    ref = GF_ATOMIC_DEC(this->refcount);
 
-        if (!ref)
-                data_destroy (this);
+    if (!ref)
+        data_destroy(this);
 }
 
 data_t *
-data_ref (data_t *this)
+data_ref(data_t *this)
 {
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return NULL;
-        }
-
-        LOCK (&this->lock);
-
-        this->refcount++;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "data is NULL");
+        return NULL;
+    }
 
-        UNLOCK (&this->lock);
+    GF_ATOMIC_INC(this->refcount);
 
-        return this;
+    return this;
 }
 
 data_t *
-int_to_data (int64_t value)
+int_to_data(int64_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
 
-        ret = gf_asprintf (&data->data, "%"PRId64, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
-        data->len = strlen (data->data) + 1;
+    data->len = gf_asprintf(&data->data, "%" PRId64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_int64 (int64_t value)
+data_from_int64(int64_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRId64, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
-        data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_int32 (int32_t value)
+data_from_int32(int32_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRId32, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId32, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_int16 (int16_t value)
+data_from_int16(int16_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRId16, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId16, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_int8 (int8_t value)
+data_from_int8(int8_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%d", value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%d", value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_uint64 (uint64_t value)
+data_from_uint64(uint64_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRIu64, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
-        return data;
+    return data;
 }
 
-static data_t *
-data_from_double (double value)
+data_t *
+data_from_double(double value)
 {
-        data_t *data = NULL;
-        int     ret  = 0;
-
-        data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
 
-        ret = gf_asprintf (&data->data, "%f", value);
-        if (ret == -1) {
-                return NULL;
-        }
-        data->len = strlen (data->data) + 1;
+    data->len = gf_asprintf(&data->data, "%f", value);
+    if (data->len == -1) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_DOUBLE;
 
-        return data;
+    return data;
 }
 
-
 data_t *
-data_from_uint32 (uint32_t value)
+data_from_uint32(uint32_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRIu32, value);
-        if (-1 == ret) {
-                gf_msg_debug ("dict", 0, "asprintf failed");
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu32, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
-        return data;
+    return data;
 }
 
-
 data_t *
-data_from_uint16 (uint16_t value)
+data_from_uint16(uint16_t value)
 {
-        int     ret = 0;
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data) {
-                return NULL;
-        }
-        ret = gf_asprintf (&data->data, "%"PRIu16, value);
-        if (-1 == ret) {
-                return NULL;
-        }
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu16, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-        data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
-        return data;
+    return data;
 }
 
 static data_t *
-data_from_ptr_common (void *value, gf_boolean_t is_static)
+data_from_ptr_common(void *value, gf_boolean_t is_static)
 {
-        /* it is valid to set 0/NULL as a value, no need to check *value */
+    /* it is valid to set 0/NULL as a value, no need to check *value */
 
-        data_t *data = get_new_data ();
-        if (!data) {
-                return NULL;
-        }
+    data_t *data = get_new_data();
+    if (!data) {
+        return NULL;
+    }
 
-        data->data = value;
-        data->is_static = is_static;
+    data->data = value;
+    data->len = 0;
+    data->is_static = is_static;
 
-        return data;
+    data->data_type = GF_DATA_TYPE_PTR;
+    return data;
 }
 
 data_t *
-str_to_data (char *value)
+str_to_data(char *value)
 {
-        if (!value) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "value is NULL");
-                return NULL;
-        }
-        data_t *data = get_new_data ();
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-        if (!data) {
-                return NULL;
-        }
-        data->len = strlen (value) + 1;
+    return strn_to_data(value, strlen(value));
+}
+
+data_t *
+strn_to_data(char *value, const int vallen)
+{
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
+    data_t *data = get_new_data();
 
-        data->data = value;
-        data->is_static = 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = vallen + 1;
+    data->data_type = GF_DATA_TYPE_STR;
+
+    data->data = value;
+    data->is_static = _gf_true;
 
-        return data;
+    return data;
 }
 
-data_t *
-data_from_dynstr (char *value)
+static data_t *
+data_from_dynstr(char *value)
 {
-        if (!value) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "value is NULL");
-                return NULL;
-        }
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data)
-                return NULL;
-        data->len = strlen (value) + 1;
-        data->data = value;
+    if (!data)
+        return NULL;
+    data->len = strlen(value) + 1;
+    data->data = value;
+    data->data_type = GF_DATA_TYPE_STR;
 
-        return data;
+    return data;
 }
 
 data_t *
-data_from_dynptr (void *value, int32_t len)
+data_from_dynptr(void *value, int32_t len)
 {
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data)
-                return NULL;
+    if (!data)
+        return NULL;
 
-        data->len = len;
-        data->data = value;
+    data->len = len;
+    data->data = value;
+    data->data_type = GF_DATA_TYPE_PTR;
 
-        return data;
+    return data;
 }
 
 data_t *
-bin_to_data (void *value, int32_t len)
+bin_to_data(void *value, int32_t len)
 {
-        if (!value) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "value is NULL");
-                return NULL;
-        }
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-        data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-        if (!data)
-                return NULL;
+    if (!data)
+        return NULL;
 
-        data->is_static = 1;
-        data->len = len;
-        data->data = value;
+    data->is_static = _gf_true;
+    data->len = len;
+    data->data = value;
 
-        return data;
+    return data;
 }
 
+static char *data_type_name[GF_DATA_TYPE_MAX] = {
+    [GF_DATA_TYPE_UNKNOWN] = "unknown",
+    [GF_DATA_TYPE_STR_OLD] = "string-old-version",
+    [GF_DATA_TYPE_INT] = "integer",
+    [GF_DATA_TYPE_UINT] = "unsigned integer",
+    [GF_DATA_TYPE_DOUBLE] = "float",
+    [GF_DATA_TYPE_STR] = "string",
+    [GF_DATA_TYPE_PTR] = "pointer",
+    [GF_DATA_TYPE_GFUUID] = "gf-uuid",
+    [GF_DATA_TYPE_IATT] = "iatt",
+    [GF_DATA_TYPE_MDATA] = "mdata",
+};
+
 int64_t
-data_to_int64 (data_t *data)
+data_to_int64(data_t *data)
 {
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return -1;
-        }
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
 
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    char *endptr = NULL;
+    int64_t value = 0;
 
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-        return (int64_t) strtoull (str, NULL, 0);
-}
+    errno = 0;
+    value = strtoll(data->data, &endptr, 0);
+
+    if (endptr && *endptr != '\0')
+        /* Unrecognized characters at the end of string. */
+        errno = EINVAL;
+    if (errno) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, errno,
+                         LG_MSG_DATA_CONVERSION_ERROR,
+                         "Error in data conversion: '%s' can't "
+                         "be represented as int64_t",
+                         data->data);
+        return -1;
+    }
+    return value;
+}
+
+/* Like above but implies signed range check. */
+
+#define DATA_TO_RANGED_SIGNED(endptr, value, data, type, min, max)             \
+    do {                                                                       \
+        errno = 0;                                                             \
+        value = strtoll(data->data, &endptr, 0);                               \
+        if (endptr && *endptr != '\0')                                         \
+            errno = EINVAL;                                                    \
+        if (errno || value > max || value < min) {                             \
+            gf_msg_callingfn("dict", GF_LOG_WARNING, errno,                    \
+                             LG_MSG_DATA_CONVERSION_ERROR,                     \
+                             "Error in data conversion: '%s' can't "           \
+                             "be represented as " #type,                       \
+                             data->data);                                      \
+            return -1;                                                         \
+        }                                                                      \
+        return (type)value;                                                    \
+    } while (0)
 
 int32_t
-data_to_int32 (data_t *data)
+data_to_int32(data_t *data)
 {
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return -1;
-        }
+    char *endptr = NULL;
+    int64_t value = 0;
 
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
-
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        return strtoul (str, NULL, 0);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int32_t, INT_MIN, INT_MAX);
 }
 
 int16_t
-data_to_int16 (data_t *data)
+data_to_int16(data_t *data)
 {
-        int16_t value = 0;
-
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return -1;
-        }
-
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
-
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    char *endptr = NULL;
+    int64_t value = 0;
 
-        errno = 0;
-        value = strtol (str, NULL, 0);
-
-        if ((value > SHRT_MAX) || (value < SHRT_MIN)) {
-                errno = ERANGE;
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, errno,
-                                  LG_MSG_DATA_CONVERSION_ERROR, "Error in data"
-                                  " conversion: detected overflow");
-                return -1;
-        }
-
-        return (int16_t)value;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int16_t, SHRT_MIN, SHRT_MAX);
 }
 
-
 int8_t
-data_to_int8 (data_t *data)
+data_to_int8(data_t *data)
 {
-        int8_t value = 0;
-
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return -1;
-        }
+    char *endptr = NULL;
+    int64_t value = 0;
 
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
-
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        errno = 0;
-        value = strtol (str, NULL, 0);
-
-        if ((value > SCHAR_MAX) || (value < SCHAR_MIN)) {
-                errno = ERANGE;
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, errno,
-                                  LG_MSG_DATA_CONVERSION_ERROR, "Error in data"
-                                  " conversion: detected overflow");
-                return -1;
-        }
-
-        return (int8_t)value;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int8_t, CHAR_MIN, CHAR_MAX);
 }
 
-
 uint64_t
-data_to_uint64 (data_t *data)
+data_to_uint64(data_t *data)
 {
-        if (!data)
-                return -1;
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
 
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-        return strtoll (str, NULL, 0);
-}
+    errno = 0;
+    value = strtoull(data->data, &endptr, 0);
+
+    if (endptr && *endptr != '\0')
+        errno = EINVAL;
+    if (errno) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, errno,
+                         LG_MSG_DATA_CONVERSION_ERROR,
+                         "Error in data conversion: '%s' can't "
+                         "be represented as uint64_t",
+                         data->data);
+        return -1;
+    }
+    return value;
+}
+
+/* Like above but implies unsigned range check. */
+
+#define DATA_TO_RANGED_UNSIGNED(endptr, value, data, type, max)                \
+    do {                                                                       \
+        errno = 0;                                                             \
+        value = strtoull(data->data, &endptr, 0);                              \
+        if (endptr && *endptr != '\0')                                         \
+            errno = EINVAL;                                                    \
+        if (errno || value > max) {                                            \
+            gf_msg_callingfn("dict", GF_LOG_WARNING, errno,                    \
+                             LG_MSG_DATA_CONVERSION_ERROR,                     \
+                             "Error in data conversion: '%s' can't "           \
+                             "be represented as " #type,                       \
+                             data->data);                                      \
+            return -1;                                                         \
+        }                                                                      \
+        return (type)value;                                                    \
+    } while (0)
 
 uint32_t
-data_to_uint32 (data_t *data)
+data_to_uint32(data_t *data)
 {
-        if (!data)
-                return -1;
-
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        return strtol (str, NULL, 0);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint32_t, UINT_MAX);
 }
 
 uint16_t
-data_to_uint16 (data_t *data)
+data_to_uint16(data_t *data)
 {
-	uint16_t value = 0;
-
-        if (!data)
-                return -1;
-
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-	errno = 0;
-	value = strtol (str, NULL, 0);
-
-	if ((USHRT_MAX - value) < 0) {
-		errno = ERANGE;
-		gf_msg_callingfn ("dict", GF_LOG_WARNING, errno,
-                                  LG_MSG_DATA_CONVERSION_ERROR,
-				  "Error in data conversion: "
-				  "overflow detected");
-		return -1;
-	}
-
-        return (uint16_t)value;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint16_t, USHRT_MAX);
 }
 
 uint8_t
-data_to_uint8 (data_t *data)
+data_to_uint8(data_t *data)
 {
-	uint32_t value = 0;
-
-        if (!data) {
-		gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return -1;
-	}
-
-        char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
-
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-	errno = 0;
-	value = strtol (str, NULL, 0);
-
-	if ((UCHAR_MAX - (uint8_t)value) < 0) {
-		errno = ERANGE;
-		gf_msg_callingfn ("dict", GF_LOG_WARNING, errno,
-                                  LG_MSG_DATA_CONVERSION_ERROR, "data "
-                                  "conversion overflow detected");
-		return -1;
-	}
-
-        return (uint8_t) value;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint8_t, UCHAR_MAX);
 }
 
 char *
-data_to_str (data_t *data)
+data_to_str(data_t *data)
 {
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return NULL;
-        }
-        return data->data;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, "null", NULL);
+    return data->data;
 }
 
 void *
-data_to_ptr (data_t *data)
+data_to_ptr(data_t *data)
 {
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return NULL;
-        }
-        return data->data;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, "null", NULL);
+    return data->data;
 }
 
 void *
-data_to_bin (data_t *data)
+data_to_bin(data_t *data)
 {
-        if (!data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "data is NULL");
-                return NULL;
-        }
-        return data->data;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, "null", NULL);
+    return data->data;
+}
+
+struct iatt *
+data_to_iatt(data_t *data, char *key)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_IATT, key, NULL);
+
+    /* We only check for smaller size. If it's bigger we simply ignore
+     * the extra data. This way it's easy to do changes in the future that
+     * pass more data but are backward compatible (if the initial contents
+     * of the struct are maintained, of course). */
+    if (data->len < sizeof(struct iatt)) {
+        gf_smsg("glusterfs", GF_LOG_ERROR, ENOBUFS, LG_MSG_UNDERSIZED_BUF,
+                "key=%s", key, NULL);
+        return NULL;
+    }
+
+    return (struct iatt *)data->data;
 }
 
 int
-dict_null_foreach_fn (dict_t *d, char *k,
-                      data_t *v, void *tmp)
+dict_null_foreach_fn(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        return 0;
+    return 0;
 }
 
 int
-dict_remove_foreach_fn (dict_t *d, char *k,
-                        data_t *v, void *_tmp)
-{
-        if (!d || !k) {
-                gf_msg ("glusterfs", GF_LOG_WARNING, EINVAL,
-                        LG_MSG_INVALID_ENTRY, "%s is NULL",
-                        d?"key":"dictionary");
-                return -1;
-        }
+dict_remove_foreach_fn(dict_t *d, char *k, data_t *v, void *_tmp)
+{
+    if (!d || !k) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, EINVAL, LG_MSG_KEY_OR_VALUE_NULL,
+                "d=%s", d ? "key" : "dictionary", NULL);
+        return -1;
+    }
 
-        dict_del (d, k);
-        return 0;
+    dict_del(d, k);
+    return 0;
 }
 
 gf_boolean_t
-dict_match_everything (dict_t *d, char *k, data_t *v, void *data)
+dict_match_everything(dict_t *d, char *k, data_t *v, void *data)
 {
-        return _gf_true;
+    return _gf_true;
 }
 
 int
-dict_foreach (dict_t *dict,
-              int (*fn)(dict_t *this,
-                        char *key,
-                        data_t *value,
-                        void *data),
-              void *data)
+dict_foreach(dict_t *dict,
+             int (*fn)(dict_t *this, char *key, data_t *value, void *data),
+             void *data)
 {
-        int     ret = 0;
+    int ret = dict_foreach_match(dict, dict_match_everything, NULL, fn, data);
 
-        ret = dict_foreach_match (dict, dict_match_everything, NULL, fn, data);
-
-        if (ret > 0)
-                ret = 0;
+    if (ret > 0)
+        ret = 0;
 
-        return ret;
+    return ret;
 }
 
 /* return values:
@@ -1223,50 +1336,44 @@ dict_foreach (dict_t *dict,
    +n = n number of matches
 */
 int
-dict_foreach_match (dict_t *dict,
-             gf_boolean_t (*match)(dict_t *this,
-                                char *key,
-                                data_t *value,
-                                void *mdata),
-             void *match_data,
-             int (*action)(dict_t *this,
-                                char *key,
-                                data_t *value,
-                                void *adata),
-              void *action_data)
-{
-        if (!dict || !match || !action) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict|match|action is "
-                                  "NULL");
-                return -1;
-        }
-
-        int          ret   = -1;
-        int          count = 0;
-        data_pair_t *pairs = NULL;
-        data_pair_t *next  = NULL;
-
-        pairs = dict->members_list;
-        while (pairs) {
-                next = pairs->next;
-                if (match (dict, pairs->key, pairs->value, match_data)) {
-                        ret = action (dict, pairs->key, pairs->value,
-                                      action_data);
-                        if (ret < 0)
-                                return ret;
-                        count++;
-                }
-                pairs = next;
+dict_foreach_match(dict_t *dict,
+                   gf_boolean_t (*match)(dict_t *this, char *key, data_t *value,
+                                         void *mdata),
+                   void *match_data,
+                   int (*action)(dict_t *this, char *key, data_t *value,
+                                 void *adata),
+                   void *action_data)
+{
+    if (!dict || !match || !action) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict|match|action is "
+                         "NULL");
+        return -1;
+    }
+
+    int ret = -1;
+    int count = 0;
+    data_pair_t *pairs = dict->members_list;
+    data_pair_t *next = NULL;
+
+    while (pairs) {
+        next = pairs->next;
+        if (match(dict, pairs->key, pairs->value, match_data)) {
+            ret = action(dict, pairs->key, pairs->value, action_data);
+            if (ret < 0)
+                return ret;
+            count++;
         }
+        pairs = next;
+    }
 
-        return count;
+    return count;
 }
 
 static gf_boolean_t
-dict_fnmatch (dict_t *d, char *k, data_t *val, void *match_data)
+dict_fnmatch(dict_t *d, char *k, data_t *val, void *match_data)
 {
-        return (fnmatch (match_data, k, 0) == 0);
+    return (fnmatch(match_data, k, 0) == 0);
 }
 /* return values:
    -1 = failure,
@@ -1274,17 +1381,14 @@ dict_fnmatch (dict_t *d, char *k, data_t *val, void *match_data)
    +n = n number of matches
 */
 int
-dict_foreach_fnmatch (dict_t *dict, char *pattern,
-                      int (*fn)(dict_t *this,
-                                char *key,
-                                data_t *value,
-                                void *data),
-                      void *data)
+dict_foreach_fnmatch(dict_t *dict, char *pattern,
+                     int (*fn)(dict_t *this, char *key, data_t *value,
+                               void *data),
+                     void *data)
 {
-        return dict_foreach_match (dict, dict_fnmatch, pattern, fn, data);
+    return dict_foreach_match(dict, dict_fnmatch, pattern, fn, data);
 }
 
-
 /**
  * dict_keys_join - pack the keys of the dictionary in a buffer.
  *
@@ -1299,89 +1403,85 @@ dict_foreach_fnmatch (dict_t *dict, char *pattern,
  */
 
 int
-dict_keys_join (void *value, int size, dict_t *dict,
-                int (*filter_fn)(char *k))
+dict_keys_join(void *value, int size, dict_t *dict, int (*filter_fn)(char *k))
 {
-	int          len = 0;
-        data_pair_t *pairs = NULL;
-        data_pair_t *next  = NULL;
+    int len = 0;
+    data_pair_t *pairs = dict->members_list;
+    data_pair_t *next = NULL;
 
-        pairs = dict->members_list;
-        while (pairs) {
-                next = pairs->next;
+    while (pairs) {
+        next = pairs->next;
 
-                if (filter_fn && filter_fn (pairs->key)){
-		    pairs = next;
-		    continue;
-		}
+        if (filter_fn && filter_fn(pairs->key)) {
+            pairs = next;
+            continue;
+        }
 
-		if (value && (size > len))
-			strncpy (value + len, pairs->key, size - len);
+        if (value && (size > len))
+            strncpy(value + len, pairs->key, size - len);
 
-                len += (strlen (pairs->key) + 1);
+        len += (strlen(pairs->key) + 1);
 
-                pairs = next;
-        }
+        pairs = next;
+    }
 
-	return len;
+    return len;
 }
 
 static int
-dict_copy_one (dict_t *unused, char *key, data_t *value, void *newdict)
+dict_copy_one(dict_t *unused, char *key, data_t *value, void *newdict)
 {
-        return dict_set ((dict_t *)newdict, key, (value));
+    return dict_set((dict_t *)newdict, key, (value));
 }
 
 dict_t *
-dict_copy (dict_t *dict,
-           dict_t *new)
+dict_copy(dict_t *dict, dict_t *new)
 {
-        if (!dict) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return NULL;
-        }
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return NULL;
+    }
 
-        if (!new)
-                new = get_new_dict_full (dict->hash_size);
+    if (!new)
+        new = get_new_dict_full(dict->hash_size);
 
-        dict_foreach (dict, dict_copy_one, new);
+    dict_foreach(dict, dict_copy_one, new);
 
-        return new;
+    return new;
 }
 
 int
-dict_reset (dict_t *dict)
-{
-        int32_t         ret = -1;
-        if (!dict) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                goto out;
-        }
-        dict_foreach (dict, dict_remove_foreach_fn, NULL);
-        ret = 0;
+dict_reset(dict_t *dict)
+{
+    int32_t ret = -1;
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+    dict_foreach(dict, dict_remove_foreach_fn, NULL);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 dict_t *
-dict_copy_with_ref (dict_t *dict,
-                    dict_t *new)
+dict_copy_with_ref(dict_t *dict, dict_t *new)
 {
-        dict_t *local_new = NULL;
+    dict_t *local_new = NULL;
 
-        GF_VALIDATE_OR_GOTO("dict", dict, fail);
+    GF_VALIDATE_OR_GOTO("dict", dict, fail);
 
-        if (new == NULL) {
-                local_new = dict_new ();
-                GF_VALIDATE_OR_GOTO("dict", local_new, fail);
-                new = local_new;
-        }
+    if (new == NULL) {
+        local_new = dict_new();
+        GF_VALIDATE_OR_GOTO("dict", local_new, fail);
+        new = local_new;
+    }
 
-        dict_foreach (dict, dict_copy_one, new);
+    dict_foreach(dict, dict_copy_one, new);
 fail:
-        return new;
+    return new;
 }
 
 /*
@@ -1395,938 +1495,1292 @@ fail:
  *               -val error, val = errno
  */
 
-
 static int
-dict_get_with_ref (dict_t *this, char *key, data_t **data)
+dict_get_with_refn(dict_t *this, char *key, const int keylen, data_t **data)
 {
-        data_pair_t * pair = NULL;
-        int           ret  = -ENOENT;
+    data_pair_t *pair = NULL;
+    int ret = -ENOENT;
+    uint32_t hash;
 
-        if (!this || !key || !data) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG,
-                                  "dict OR key (%s) is NULL", key);
-                ret = -EINVAL;
-                goto err;
-        }
+    hash = (uint32_t)XXH64(key, keylen, 0);
 
-        LOCK (&this->lock);
-        {
-                pair = dict_lookup_common (this, key);
-        }
-        UNLOCK (&this->lock);
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
 
         if (pair) {
-                ret = 0;
-                *data = data_ref (pair->value);
+            ret = 0;
+            *data = data_ref(pair->value);
         }
+    }
+    UNLOCK(&this->lock);
 
-err:
-        return ret;
+    return ret;
+}
+
+int
+dict_get_with_ref(dict_t *this, char *key, data_t **data)
+{
+    if (!this || !key || !data) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict OR key (%s) is NULL", key);
+        return -EINVAL;
+    }
+
+    return dict_get_with_refn(this, key, strlen(key), data);
 }
 
 static int
-data_to_ptr_common (data_t *data, void **val)
+data_to_ptr_common(data_t *data, void **val)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        *val = data->data;
+    *val = data->data;
 err:
-        return ret;
+    return ret;
 }
 
-
 static int
-data_to_int8_ptr (data_t *data, int8_t *val)
+data_to_int8_ptr(data_t *data, int8_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
-
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    int ret = 0;
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        errno = 0;
-        *val = strtol (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_int16_ptr (data_t *data, int16_t *val)
+data_to_int16_ptr(data_t *data, int16_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
-
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        errno = 0;
-        *val = strtol (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_int32_ptr (data_t *data, int32_t *val)
+data_to_int32_ptr(data_t *data, int32_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        errno = 0;
-        *val = strtol (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_int64_ptr (data_t *data, int64_t *val)
+data_to_int64_ptr(data_t *data, int64_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
-
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        errno = 0;
-        *val = strtoll (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtoll(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_uint16_ptr (data_t *data, uint16_t *val)
+data_to_uint16_ptr(data_t *data, uint16_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
-
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    int ret = 0;
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        errno = 0;
-        *val = strtoul (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtoul(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_uint32_ptr (data_t *data, uint32_t *val)
+data_to_uint32_ptr(data_t *data, uint32_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        errno = 0;
-        *val = strtoul (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtoul(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_uint64_ptr (data_t *data, uint64_t *val)
+data_to_uint64_ptr(data_t *data, uint64_t *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        errno = 0;
-        *val = strtoull (str, NULL, 0);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtoull(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 static int
-data_to_double_ptr (data_t *data, double *val)
+data_to_double_ptr(data_t *data, double *val)
 {
-        int    ret = 0;
-        char * str = NULL;
+    int ret = 0;
 
-        if (!data || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        str = alloca (data->len + 1);
-        if (!str) {
-                ret = -ENOMEM;
-                goto err;
-        }
-        memcpy (str, data->data, data->len);
-        str[data->len] = '\0';
-
-        errno = 0;
-        *val = strtod (str, NULL);
-        if (errno != 0)
-                ret = -errno;
+    errno = 0;
+    *val = strtod(data->data, NULL);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_int8 (dict_t *this, char *key, int8_t *val)
+dict_get_int8(dict_t *this, char *key, int8_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-        ret = data_to_int8_ptr (data, val);
+    ret = data_to_int8_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_int8 (dict_t *this, char *key, int8_t val)
+dict_set_int8(dict_t *this, char *key, int8_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        data = data_from_int8 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    data = data_from_int8(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_int16 (dict_t *this, char *key, int16_t *val)
+dict_get_int16(dict_t *this, char *key, int16_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-        ret = data_to_int16_ptr (data, val);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
+
+    ret = data_to_int16_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
+int
+dict_set_int16(dict_t *this, char *key, int16_t val)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    data = data_from_int16(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+
+err:
+    return ret;
+}
 
 int
-dict_set_int16 (dict_t *this, char *key, int16_t val)
+dict_get_int32n(dict_t *this, char *key, const int keylen, int32_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        data = data_from_int16 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!this || !key || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_get_with_refn(this, key, keylen, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
+
+    ret = data_to_int32_ptr(data, val);
 
 err:
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
 int
-dict_get_int32 (dict_t *this, char *key, int32_t *val)
+dict_get_int32(dict_t *this, char *key, int32_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-        ret = data_to_int32_ptr (data, val);
+    ret = data_to_int32_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
+int
+dict_set_int32n(dict_t *this, char *key, const int keylen, int32_t val)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    data = data_from_int32(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
+
+err:
+    return ret;
+}
 
 int
-dict_set_int32 (dict_t *this, char *key, int32_t val)
+dict_set_int32(dict_t *this, char *key, int32_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_int32(val);
+    int ret = 0;
 
-        data = data_from_int32 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_int64 (dict_t *this, char *key, int64_t *val)
+dict_get_int64(dict_t *this, char *key, int64_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-        ret = data_to_int64_ptr (data, val);
+    ret = data_to_int64_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_int64 (dict_t *this, char *key, int64_t val)
+dict_set_int64(dict_t *this, char *key, int64_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_int64(val);
+    int ret = 0;
 
-        data = data_from_int64 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_uint16 (dict_t *this, char *key, uint16_t *val)
+dict_get_uint16(dict_t *this, char *key, uint16_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
 
-        ret = data_to_uint16_ptr (data, val);
+    ret = data_to_uint16_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_uint16 (dict_t *this, char *key, uint16_t val)
+dict_set_uint16(dict_t *this, char *key, uint16_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_uint16(val);
+    int ret = 0;
 
-        data = data_from_uint16 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_uint32 (dict_t *this, char *key, uint32_t *val)
+dict_get_uint32(dict_t *this, char *key, uint32_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
 
-        ret = data_to_uint32_ptr (data, val);
+    ret = data_to_uint32_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
-
 int
-dict_set_uint32 (dict_t *this, char *key, uint32_t val)
+dict_set_uint32(dict_t *this, char *key, uint32_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_uint32(val);
+    int ret = 0;
 
-        data = data_from_uint32 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_uint64 (dict_t *this, char *key, uint64_t *val)
+dict_get_uint64(dict_t *this, char *key, uint64_t *val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
 
-        ret = data_to_uint64_ptr (data, val);
+    ret = data_to_uint64_ptr(data, val);
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_uint64 (dict_t *this, char *key, uint64_t val)
+dict_set_uint64(dict_t *this, char *key, uint64_t val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_uint64(val);
+    int ret = 0;
 
-        data = data_from_uint64 (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
+/*
+ * dict_check_flag can be used to check a one bit flag in an array of flags
+ * The flag argument indicates the bit position (within the array of bits).
+ * Currently limited to max of 256 flags for a key.
+ * return value,
+ * 1 : flag is set
+ * 0 : flag is not set
+ * <0: Error
+ */
 int
-dict_get_double (dict_t *this, char *key, double *val)
+dict_check_flag(dict_t *this, char *key, int flag)
 {
-        data_t *data = NULL;
-        int     ret  = 0;
+    data_t *data = NULL;
+    int ret = -ENOENT;
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (BIT_VALUE((unsigned char *)(data->data), flag))
+        ret = 1;
+    else
+        ret = 0;
+
+    data_unref(data);
+    return ret;
+}
+
+/*
+ * _dict_modify_flag can be used to set/clear a bit flag in an array of flags
+ * flag: indicates the bit position. limited to max of DICT_MAX_FLAGS.
+ * op: Indicates operation DICT_FLAG_SET / DICT_FLAG_CLEAR
+ */
+static int
+_dict_modify_flag(dict_t *this, char *key, int flag, int op)
+{
+    data_t *data = NULL;
+    int ret = 0;
+    data_pair_t *pair = NULL;
+    char *ptr = NULL;
+    int hashval = 0;
+    uint32_t hash;
+
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict OR key (%s) is NULL", key);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    /*
+     * Using a size of 32 bytes to support max of 256
+     * flags in a single key. This should be suffcient.
+     */
+    GF_ASSERT(flag >= 0 && flag < DICT_MAX_FLAGS);
+
+    hash = (uint32_t)XXH64(key, strlen(key), 0);
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
 
-        if (!this || !key || !val) {
-                ret = -EINVAL;
+        if (pair) {
+            data = pair->value;
+            if (op == DICT_FLAG_SET)
+                BIT_SET((unsigned char *)(data->data), flag);
+            else
+                BIT_CLEAR((unsigned char *)(data->data), flag);
+        } else {
+            ptr = GF_CALLOC(1, DICT_MAX_FLAGS / 8, gf_common_mt_char);
+            if (!ptr) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                        "flag bit array", NULL);
+                ret = -ENOMEM;
                 goto err;
-        }
+            }
+
+            data = data_from_dynptr(ptr, DICT_MAX_FLAGS / 8);
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
+            if (!data) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY, "data",
+                        NULL);
+                GF_FREE(ptr);
+                ret = -ENOMEM;
                 goto err;
+            }
+
+            if (op == DICT_FLAG_SET)
+                BIT_SET((unsigned char *)(data->data), flag);
+            else
+                BIT_CLEAR((unsigned char *)(data->data), flag);
+
+            if (this->free_pair.key) { /* the free pair is in use */
+                pair = mem_get0(THIS->ctx->dict_pair_pool);
+                if (!pair) {
+                    gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                            "dict pair", NULL);
+                    ret = -ENOMEM;
+                    goto err;
+                }
+            } else { /* use the free pair */
+                pair = &this->free_pair;
+            }
+
+            pair->key = (char *)GF_MALLOC(strlen(key) + 1, gf_common_mt_char);
+            if (!pair->key) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                        "dict pair", NULL);
+                ret = -ENOMEM;
+                goto err;
+            }
+            strcpy(pair->key, key);
+            pair->key_hash = hash;
+            pair->value = data_ref(data);
+            this->totkvlen += (strlen(key) + 1 + data->len);
+            hashval = hash % this->hash_size;
+            pair->hash_next = this->members[hashval];
+            this->members[hashval] = pair;
+
+            pair->next = this->members_list;
+            pair->prev = NULL;
+            if (this->members_list)
+                this->members_list->prev = pair;
+            this->members_list = pair;
+            this->count++;
+
+            if (this->max_count < this->count)
+                this->max_count = this->count;
         }
+    }
 
-        ret = data_to_double_ptr (data, val);
+    UNLOCK(&this->lock);
+    return 0;
 
 err:
-        if (data)
-                data_unref (data);
-        return ret;
+    if (key && this)
+        UNLOCK(&this->lock);
+
+    if (pair) {
+        if (pair->key) {
+            GF_FREE(pair->key);
+            pair->key = NULL;
+        }
+        if (pair != &this->free_pair) {
+            mem_put(pair);
+        }
+    }
+
+    if (data)
+        data_destroy(data);
+
+    gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_DICT_SET_FAILED, "key=%s", key,
+            NULL);
+
+    return ret;
 }
 
+/*
+ * Todo:
+ * Add below primitives as needed:
+ * dict_check_flags(this, key, flag...): variadic function to check
+ *                                       multiple flags at a time.
+ * dict_set_flags(this, key, flag...): set multiple flags
+ * dict_clear_flags(this, key, flag...): reset multiple flags
+ */
+
 int
-dict_set_double (dict_t *this, char *key, double val)
+dict_set_flag(dict_t *this, char *key, int flag)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    return _dict_modify_flag(this, key, flag, DICT_FLAG_SET);
+}
 
-        data = data_from_double (val);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+int
+dict_clear_flag(dict_t *this, char *key, int flag)
+{
+    return _dict_modify_flag(this, key, flag, DICT_FLAG_CLEAR);
+}
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+int
+dict_get_double(dict_t *this, char *key, double *val)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_DOUBLE, key, -EINVAL);
+
+    ret = data_to_double_ptr(data, val);
 
 err:
-        return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
 int
-dict_set_static_ptr (dict_t *this, char *key, void *ptr)
+dict_set_double(dict_t *this, char *key, double val)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_double(val);
+    int ret = 0;
 
-        data = data_from_ptr_common (ptr, _gf_true);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_set_dynptr (dict_t *this, char *key, void *ptr, size_t len)
+dict_set_static_ptr(dict_t *this, char *key, void *ptr)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_ptr_common(ptr, _gf_true);
+    int ret = 0;
 
-        data = data_from_dynptr (ptr, len);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-dict_get_ptr (dict_t *this, char *key, void **ptr)
+dict_set_dynptr(dict_t *this, char *key, void *ptr, size_t len)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = data_from_dynptr(ptr, len);
+    int ret = 0;
 
-        if (!this || !key || !ptr) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
-        ret = data_to_ptr_common (data, ptr);
-        if (ret != 0) {
-                goto err;
-        }
+err:
+    return ret;
+}
+
+int
+dict_get_ptr(dict_t *this, char *key, void **ptr)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!ptr) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, -EINVAL);
+
+    ret = data_to_ptr_common(data, ptr);
+    if (ret != 0) {
+        goto err;
+    }
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
 int
-dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!this || !key || !ptr) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!ptr) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret != 0) {
-                goto err;
-        }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	*len = data->len;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, -EINVAL);
 
-        ret = data_to_ptr_common (data, ptr);
-        if (ret != 0) {
-                goto err;
-        }
+    *len = data->len;
+
+    ret = data_to_ptr_common(data, ptr);
+    if (ret != 0) {
+        goto err;
+    }
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
+/* Get string - with known key length */
 int
-dict_set_ptr (dict_t *this, char *key, void *ptr)
+dict_get_strn(dict_t *this, char *key, const int keylen, char **str)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = -EINVAL;
 
-        data = data_from_ptr_common (ptr, _gf_false);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!this || !key || !str) {
+        goto err;
+    }
+    ret = dict_get_with_refn(this, key, keylen, &data);
+    if (ret < 0) {
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, key, -EINVAL);
+
+    *str = data->data;
 
 err:
-        return ret;
-}
+    if (data)
+        data_unref(data);
 
+    return ret;
+}
 
 int
-dict_get_str (dict_t *this, char *key, char **str)
+dict_get_str(dict_t *this, char *key, char **str)
 {
-        data_t * data = NULL;
-        int      ret  = -EINVAL;
+    data_t *data = NULL;
+    int ret = -EINVAL;
 
-        if (!this || !key || !str) {
-                goto err;
-        }
+    if (!str) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret < 0) {
-                goto err;
-        }
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, key, -EINVAL);
 
-        if (!data || !data->data) {
-                goto err;
-        }
-        *str = data->data;
+    *str = data->data;
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
 int
-dict_set_str (dict_t *this, char *key, char *str)
+dict_set_str(dict_t *this, char *key, char *str)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = str_to_data(str);
+    int ret = 0;
 
-        data = str_to_data (str);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-        return ret;
+    return ret;
 }
 
+/* Set string - with known key length */
 int
-dict_set_dynstr_with_alloc (dict_t *this, char *key, const char *str)
+dict_set_strn(dict_t *this, char *key, const int keylen, char *str)
 {
-        char *alloc_str = NULL;
-        int   ret       = -1;
+    data_t *data = NULL;
+    int ret = 0;
 
-        alloc_str = gf_strdup (str);
-        if (!alloc_str)
-                return -1;
+    data = str_to_data(str);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set_dynstr (this, key, alloc_str);
-        if (ret == -EINVAL)
-                GF_FREE (alloc_str);
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
-        return ret;
+err:
+    return ret;
 }
 
+/* Set string - with known key length and known value length */
 int
-dict_set_dynstr (dict_t *this, char *key, char *str)
+dict_set_nstrn(dict_t *this, char *key, const int keylen, char *str,
+               const int vallen)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = strn_to_data(str, vallen);
+    int ret = 0;
 
-        data = data_from_dynstr (str);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_set (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
+    return ret;
+}
+
+int
+dict_set_dynstr_with_alloc(dict_t *this, char *key, const char *str)
+{
+    char *alloc_str = gf_strdup(str);
+    int ret = -1;
+
+    if (!alloc_str)
         return ret;
+
+    ret = dict_set_dynstr(this, key, alloc_str);
+    if (ret == -EINVAL)
+        GF_FREE(alloc_str);
+
+    return ret;
 }
 
 int
-dict_add_dynstr_with_alloc (dict_t *this, char *key, char *str)
+dict_set_dynstr(dict_t *this, char *key, char *str)
 {
-        data_t  *data = NULL;
-        int      ret  = 0;
-        char    *alloc_str = NULL;
+    const int keylen = strlen(key);
+    return dict_set_dynstrn(this, key, keylen, str);
+}
 
-        alloc_str = gf_strdup (str);
-        if (!alloc_str)
-                goto out;
+int
+dict_set_dynstrn(dict_t *this, char *key, const int keylen, char *str)
+{
+    data_t *data = data_from_dynstr(str);
+    int ret = 0;
 
-        data = data_from_dynstr (alloc_str);
-        if (!data) {
-                GF_FREE (alloc_str);
-                ret = -EINVAL;
-                goto out;
-        }
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        ret = dict_add (this, key, data);
-        if (ret < 0)
-                data_destroy (data);
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
-out:
-        return ret;
+err:
+    return ret;
 }
 
+/* This function is called only by the volgen for now.
+   Check how else you can handle it */
+int
+dict_set_option(dict_t *this, char *key, char *str)
+{
+    data_t *data = data_from_dynstr(str);
+    int ret = 0;
+
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    data->data_type = GF_DATA_TYPE_STR_OLD;
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+err:
+    return ret;
+}
 
 int
-dict_get_bin (dict_t *this, char *key, void **bin)
+dict_add_dynstr_with_alloc(dict_t *this, char *key, char *str)
 {
-        data_t * data = NULL;
-        int      ret  = -EINVAL;
+    data_t *data = NULL;
+    int ret = 0;
+    char *alloc_str = gf_strdup(str);
 
-        if (!this || !key || !bin) {
-                goto err;
-        }
+    if (!alloc_str)
+        goto out;
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret < 0) {
-                goto err;
-        }
+    data = data_from_dynstr(alloc_str);
+    if (!data) {
+        GF_FREE(alloc_str);
+        ret = -EINVAL;
+        goto out;
+    }
 
-        if (!data || !data->data) {
-                goto err;
-        }
-        *bin = data->data;
+    ret = dict_add(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+
+out:
+    return ret;
+}
+
+int
+dict_get_bin(dict_t *this, char *key, void **bin)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!bin) {
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, ret);
+
+    *bin = data->data;
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
-
+/********************************************************************
+ *
+ * dict_set_bin_common:
+ *      This is the common function to set key and its value in
+ *      dictionary. Flag(is_static) should be set appropriately based
+ *      on the type of memory type used for value(*ptr). If flag is set
+ *      to false value(*ptr) will be freed using GF_FREE() on destroy.
+ *
+ *******************************************************************/
 static int
-dict_set_bin_common (dict_t *this, char *key, void *ptr, size_t size,
-                     gf_boolean_t is_static)
+dict_set_bin_common(dict_t *this, char *key, void *ptr, size_t size,
+                    gf_boolean_t is_static, gf_dict_data_type_t type)
 {
-        data_t * data = NULL;
-        int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-        if (!ptr || (size > ULONG_MAX)) {
-                ret = -EINVAL;
-                goto err;
-        }
+    if (!ptr || (size > DICT_KEY_VALUE_MAX_SIZE)) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        data = bin_to_data (ptr, size);
-        if (!data) {
-                ret = -EINVAL;
-                goto err;
-        }
+    data = bin_to_data(ptr, size);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-        data->is_static = is_static;
+    data->is_static = is_static;
+    data->data_type = type;
 
-        ret = dict_set (this, key, data);
-        if (ret < 0) {
-                /* don't free data->data, let callers handle it */
-                data->data = NULL;
-                data_destroy (data);
-        }
+    ret = dict_set(this, key, data);
+    if (ret < 0) {
+        /* don't free data->data, let callers handle it */
+        data->data = NULL;
+        data_destroy(data);
+    }
 
 err:
-        return ret;
+    return ret;
 }
 
+/********************************************************************
+ *
+ * dict_set_bin:
+ *      Set key and its value in the dictionary. This function should
+ *      be called if the value is stored in dynamic memory.
+ *
+ *******************************************************************/
 int
-dict_set_bin (dict_t *this, char *key, void *ptr, size_t size)
+dict_set_bin(dict_t *this, char *key, void *ptr, size_t size)
 {
-        return dict_set_bin_common (this, key, ptr, size, _gf_false);
+    return dict_set_bin_common(this, key, ptr, size, _gf_false,
+                               GF_DATA_TYPE_PTR);
 }
 
+/********************************************************************
+ *
+ * dict_set_static_bin:
+ *      Set key and its value in the dictionary. This function should
+ *      be called if the value is stored in static memory.
+ *
+ *******************************************************************/
+int
+dict_set_static_bin(dict_t *this, char *key, void *ptr, size_t size)
+{
+    return dict_set_bin_common(this, key, ptr, size, _gf_true,
+                               GF_DATA_TYPE_PTR);
+}
 
+/*  */
 int
-dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size)
+dict_set_gfuuid(dict_t *this, char *key, uuid_t gfid, bool is_static)
 {
-        return dict_set_bin_common (this, key, ptr, size, _gf_true);
+    return dict_set_bin_common(this, key, gfid, sizeof(uuid_t), is_static,
+                               GF_DATA_TYPE_GFUUID);
 }
 
+int
+dict_get_gfuuid(dict_t *this, char *key, uuid_t *gfid)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!gfid) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_GFUUID, key, -EINVAL);
+
+    memcpy(*gfid, data->data, min(data->len, sizeof(uuid_t)));
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
+
+int
+dict_set_mdata(dict_t *this, char *key, struct mdata_iatt *mdata,
+               bool is_static)
+{
+    return dict_set_bin_common(this, key, mdata, sizeof(struct mdata_iatt),
+                               is_static, GF_DATA_TYPE_MDATA);
+}
+
+int
+dict_get_mdata(dict_t *this, char *key, struct mdata_iatt *mdata)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!mdata) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_MDATA, key, -EINVAL);
+    if (data->len < sizeof(struct mdata_iatt)) {
+        gf_smsg("glusterfs", GF_LOG_ERROR, ENOBUFS, LG_MSG_UNDERSIZED_BUF,
+                "key=%s", key, NULL);
+        ret = -ENOBUFS;
+        goto err;
+    }
+
+    memcpy(mdata, data->data, min(data->len, sizeof(struct mdata_iatt)));
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
+
+int
+dict_set_iatt(dict_t *this, char *key, struct iatt *iatt, bool is_static)
+{
+    return dict_set_bin_common(this, key, iatt, sizeof(struct iatt), is_static,
+                               GF_DATA_TYPE_IATT);
+}
+
+int
+dict_get_iatt(dict_t *this, char *key, struct iatt *iatt)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!iatt) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_IATT, key, -EINVAL);
+
+    memcpy(iatt, data->data, min(data->len, sizeof(struct iatt)));
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
 
 /**
  * dict_get_str_boolean - get a boolean value based on string representation.
@@ -2352,41 +2806,77 @@ dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size)
  */
 
 int
-dict_get_str_boolean (dict_t *this, char *key, int default_val)
+dict_get_str_boolean(dict_t *this, char *key, int default_val)
 {
-        data_t       *data = NULL;
-        gf_boolean_t  boo = _gf_false;
-        int           ret  = 0;
+    data_t *data = NULL;
+    gf_boolean_t boo = _gf_false;
+    int ret = 0;
 
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret < 0) {
-                if (ret == -ENOENT)
-                        ret = default_val;
-                else
-                        ret = -1;
-                goto err;
-        }
-
-        GF_ASSERT (data);
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        if (ret == -ENOENT)
+            ret = default_val;
+        else
+            ret = -1;
+        goto err;
+    }
 
-        if (!data->data) {
-                ret = -1;
-                goto err;
-        }
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-        ret = gf_string2boolean (data->data, &boo);
-        if (ret == -1)
-                goto err;
+    ret = gf_strn2boolean(data->data, data->len - 1, &boo);
+    if (ret == -1)
+        goto err;
 
-        ret = boo;
+    ret = boo;
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
+int
+dict_rename_key(dict_t *this, char *key, char *replace_key)
+{
+    data_pair_t *pair = NULL;
+    int ret = -EINVAL;
+    uint32_t hash;
+    uint32_t replacekey_hash;
+    int replacekey_len;
+
+    /* replacing a key by itself is a NO-OP */
+    if (strcmp(key, replace_key) == 0)
+        return 0;
+
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return ret;
+    }
+
+    hash = (uint32_t)XXH64(key, strlen(key), 0);
+    replacekey_len = strlen(replace_key);
+    replacekey_hash = (uint32_t)XXH64(replace_key, replacekey_len, 0);
+
+    LOCK(&this->lock);
+    {
+        /* no need to data_ref(pair->value), dict_set_lk() does it */
+        pair = dict_lookup_common(this, key, hash);
+        if (!pair)
+            ret = -ENODATA;
+        else
+            ret = dict_set_lk(this, replace_key, replacekey_len, pair->value,
+                              replacekey_hash, 1);
+    }
+    UNLOCK(&this->lock);
+
+    if (!ret)
+        /* only delete the key on success */
+        dict_del(this, key);
+
+    return ret;
+}
 
 /**
  * Serialization format:
@@ -2396,10 +2886,6 @@ err:
  *     4        4         4       <key len>   <value len>
  */
 
-#define DICT_HDR_LEN               4
-#define DICT_DATA_HDR_KEY_LEN      4
-#define DICT_DATA_HDR_VAL_LEN      4
-
 /**
  * dict_serialized_length_lk - return the length of serialized dict. This
  *                             procedure has to be called with this->lock held.
@@ -2410,64 +2896,21 @@ err:
  */
 
 int
-dict_serialized_length_lk (dict_t *this)
+dict_serialized_length_lk(dict_t *this)
 {
-        int ret            = -EINVAL;
-        int count          = 0;
-        int len            = 0;
-        data_pair_t * pair = NULL;
+    int ret = -EINVAL;
+    int count = this->count;
+    const int keyhdrlen = DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN;
 
-        len = DICT_HDR_LEN;
-        count = this->count;
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
 
-        if (count < 0) {
-                gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                        LG_MSG_COUNT_LESS_THAN_ZERO, "count (%d) < 0!", count);
-                goto out;
-        }
-
-        pair = this->members_list;
-
-        while (count) {
-                if (!pair) {
-                        gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                                LG_MSG_COUNT_LESS_THAN_DATA_PAIRS,
-                                "less than count data pairs found!");
-                        goto out;
-                }
-
-                len += DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN;
-
-                if (!pair->key) {
-                        gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                                LG_MSG_NULL_PTR, "pair->key is null!");
-                        goto out;
-                }
-
-                len += strlen (pair->key) + 1  /* for '\0' */;
-
-                if (!pair->value) {
-                        gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                                LG_MSG_NULL_PTR, "pair->value is null!");
-                        goto out;
-                }
-
-                if (pair->value->len < 0) {
-                        gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                                LG_MSG_VALUE_LENGTH_LESS_THAN_ZERO,
-                                "value->len (%d) < 0", pair->value->len);
-                        goto out;
-                }
-
-                len += pair->value->len;
-
-                pair = pair->next;
-                count--;
-        }
-
-        ret = len;
+    ret = DICT_HDR_LEN + this->totkvlen + (count * keyhdrlen);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -2476,92 +2919,80 @@ out:
  *
  * @this: dict to serialize
  * @buf:  buffer to serialize into. This must be
- *        atleast dict_serialized_length (this) large
+ *        at least dict_serialized_length (this) large
  *
  * @return: success: 0
  *          failure: -errno
  */
 
-int
-dict_serialize_lk (dict_t *this, char *buf)
+static int
+dict_serialize_lk(dict_t *this, char *buf)
 {
-        int           ret     = -1;
-        data_pair_t * pair    = NULL;
-        int32_t       count   = 0;
-        int32_t       keylen  = 0;
-        int32_t       vallen  = 0;
-        int32_t       netword = 0;
+    int ret = -1;
+    data_pair_t *pair = this->members_list;
+    int32_t count = this->count;
+    int32_t keylen = 0;
+    int32_t netword = 0;
 
+    if (!buf) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, NULL);
+        goto out;
+    }
 
-        if (!buf) {
-                gf_msg ("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
-                        "buf is null!");
-                goto out;
-        }
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
 
+    netword = hton32(count);
+    memcpy(buf, &netword, sizeof(netword));
+    buf += DICT_HDR_LEN;
 
-        count = this->count;
-        if (count < 0) {
-                gf_msg ("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
-                        "count (%d) < 0!", count);
-                goto out;
+    while (count) {
+        if (!pair) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_PAIRS_LESS_THAN_COUNT,
+                    NULL);
+            goto out;
         }
 
-        netword = hton32 (count);
-        memcpy (buf, &netword, sizeof(netword));
-        buf += DICT_HDR_LEN;
-        pair = this->members_list;
-
-        while (count) {
-                if (!pair) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0,
-                                LG_MSG_PAIRS_LESS_THAN_COUNT,
-                                "less than count data pairs found!");
-                        goto out;
-                }
-
-                if (!pair->key) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR,
-                                "pair->key is null!");
-                        goto out;
-                }
-
-                keylen  = strlen (pair->key);
-                netword = hton32 (keylen);
-                memcpy (buf, &netword, sizeof(netword));
-                buf += DICT_DATA_HDR_KEY_LEN;
+        if (!pair->key) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR, NULL);
+            goto out;
+        }
 
-                if (!pair->value) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0,
-                                LG_MSG_NULL_PTR,
-                                "pair->value is null!");
-                        goto out;
-                }
+        keylen = strlen(pair->key);
+        netword = hton32(keylen);
+        memcpy(buf, &netword, sizeof(netword));
+        buf += DICT_DATA_HDR_KEY_LEN;
 
-                vallen  = pair->value->len;
-                netword = hton32 (vallen);
-                memcpy (buf, &netword, sizeof(netword));
-                buf += DICT_DATA_HDR_VAL_LEN;
+        if (!pair->value) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR, NULL);
+            goto out;
+        }
 
-                memcpy (buf, pair->key, keylen);
-                buf += keylen;
-                *buf++ = '\0';
+        netword = hton32(pair->value->len);
+        memcpy(buf, &netword, sizeof(netword));
+        buf += DICT_DATA_HDR_VAL_LEN;
 
-                if (pair->value->data) {
-                        memcpy (buf, pair->value->data, vallen);
-                        buf += vallen;
-                }
+        memcpy(buf, pair->key, keylen);
+        buf += keylen;
+        *buf++ = '\0';
 
-                pair = pair->next;
-                count--;
+        if (pair->value->data) {
+            memcpy(buf, pair->value->data, pair->value->len);
+            buf += pair->value->len;
         }
 
-        ret = 0;
+        pair = pair->next;
+        count--;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * dict_serialized_length - return the length of serialized dict
  *
@@ -2571,24 +3002,24 @@ out:
  */
 
 int
-dict_serialized_length (dict_t *this)
+dict_serialized_length(dict_t *this)
 {
-        int ret            = -EINVAL;
+    int ret = -EINVAL;
 
-        if (!this) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is null!");
-                goto out;
-        }
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
 
-        LOCK (&this->lock);
-        {
-                ret = dict_serialized_length_lk (this);
-        }
-        UNLOCK (&this->lock);
+    LOCK(&this->lock);
+    {
+        ret = dict_serialized_length_lk(this);
+    }
+    UNLOCK(&this->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -2596,33 +3027,32 @@ out:
  *
  * @this: dict to serialize
  * @buf:  buffer to serialize into. This must be
- *        atleast dict_serialized_length (this) large
+ *        at least dict_serialized_length (this) large
  *
  * @return: success: 0
  *          failure: -errno
  */
 
 int
-dict_serialize (dict_t *this, char *buf)
+dict_serialize(dict_t *this, char *buf)
 {
-        int           ret    = -1;
+    int ret = -1;
 
-        if (!this || !buf) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is null!");
-                goto out;
-        }
+    if (!this || !buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
 
-        LOCK (&this->lock);
-        {
-                ret = dict_serialize_lk (this, buf);
-        }
-        UNLOCK (&this->lock);
+    LOCK(&this->lock);
+    {
+        ret = dict_serialize_lk(this, buf);
+    }
+    UNLOCK(&this->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * dict_unserialize - unserialize a buffer into a dict
  *
@@ -2635,135 +3065,137 @@ out:
  */
 
 int32_t
-dict_unserialize (char *orig_buf, int32_t size, dict_t **fill)
-{
-        char   *buf = NULL;
-        int     ret   = -1;
-        int32_t count = 0;
-        int     i     = 0;
-
-        data_t * value   = NULL;
-        char   * key     = NULL;
-        int32_t  keylen  = 0;
-        int32_t  vallen  = 0;
-        int32_t  hostord = 0;
-
-        buf = orig_buf;
-
-        if (!buf) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "buf is null!");
-                goto out;
-        }
-
-        if (size == 0) {
-                gf_msg_callingfn ("dict", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "size is 0!");
-                goto out;
-        }
-
-        if (!fill) {
-                gf_msg_callingfn ("dict", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "fill is null!");
-                goto out;
-        }
+dict_unserialize(char *orig_buf, int32_t size, dict_t **fill)
+{
+    char *buf = orig_buf;
+    int ret = -1;
+    int32_t count = 0;
+    int i = 0;
+
+    data_t *value = NULL;
+    char *key = NULL;
+    int32_t keylen = 0;
+    int32_t vallen = 0;
+    int32_t hostord = 0;
+
+    if (!buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "buf is null!");
+        goto out;
+    }
+
+    if (size == 0) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "size is 0!");
+        goto out;
+    }
+
+    if (!fill) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "fill is null!");
+        goto out;
+    }
+
+    if (!*fill) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "*fill is null!");
+        goto out;
+    }
+
+    if ((buf + DICT_HDR_LEN) > (orig_buf + size)) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                         "undersized buffer "
+                         "passed. available (%lu) < required (%lu)",
+                         (long)(orig_buf + size), (long)(buf + DICT_HDR_LEN));
+        goto out;
+    }
+
+    memcpy(&hostord, buf, sizeof(hostord));
+    count = ntoh32(hostord);
+    buf += DICT_HDR_LEN;
+
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
+
+    /* count will be set by the dict_set's below */
+    (*fill)->count = 0;
+
+    for (i = 0; i < count; i++) {
+        if ((buf + DICT_DATA_HDR_KEY_LEN) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized "
+                             "buffer passed. available (%lu) < "
+                             "required (%lu)",
+                             (long)(orig_buf + size),
+                             (long)(buf + DICT_DATA_HDR_KEY_LEN));
+            goto out;
+        }
+        memcpy(&hostord, buf, sizeof(hostord));
+        keylen = ntoh32(hostord);
+        buf += DICT_DATA_HDR_KEY_LEN;
+
+        if ((buf + DICT_DATA_HDR_VAL_LEN) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized "
+                             "buffer passed. available (%lu) < "
+                             "required (%lu)",
+                             (long)(orig_buf + size),
+                             (long)(buf + DICT_DATA_HDR_VAL_LEN));
+            goto out;
+        }
+        memcpy(&hostord, buf, sizeof(hostord));
+        vallen = ntoh32(hostord);
+        buf += DICT_DATA_HDR_VAL_LEN;
+
+        if ((keylen < 0) || (vallen < 0)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized length passed "
+                             "key:%d val:%d",
+                             keylen, vallen);
+            goto out;
+        }
+        if ((buf + keylen) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized buffer passed. "
+                             "available (%lu) < required (%lu)",
+                             (long)(orig_buf + size), (long)(buf + keylen));
+            goto out;
+        }
+        key = buf;
+        buf += keylen + 1; /* for '\0' */
+
+        if ((buf + vallen) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized buffer passed. "
+                             "available (%lu) < required (%lu)",
+                             (long)(orig_buf + size), (long)(buf + vallen));
+            goto out;
+        }
+        value = get_new_data();
 
-        if (!*fill) {
-                gf_msg_callingfn ("dict", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "*fill is null!");
-                goto out;
-        }
-
-        if ((buf + DICT_HDR_LEN) > (orig_buf + size)) {
-                gf_msg_callingfn ("dict", GF_LOG_ERROR, 0,
-                                  LG_MSG_UNDERSIZED_BUF, "undersized buffer "
-                                  "passed. available (%lu) < required (%lu)",
-                                  (long)(orig_buf + size),
-                                  (long)(buf + DICT_HDR_LEN));
-                goto out;
-        }
-
-        memcpy (&hostord, buf, sizeof(hostord));
-        count = ntoh32 (hostord);
-        buf += DICT_HDR_LEN;
-
-        if (count < 0) {
-                gf_msg ("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
-                        "count (%d) <= 0", count);
-                goto out;
+        if (!value) {
+            ret = -1;
+            goto out;
         }
+        value->len = vallen;
+        value->data = gf_memdup(buf, vallen);
+        value->data_type = GF_DATA_TYPE_STR_OLD;
+        value->is_static = _gf_false;
+        buf += vallen;
 
-        /* count will be set by the dict_set's below */
-        (*fill)->count = 0;
-
-        for (i = 0; i < count; i++) {
-                if ((buf + DICT_DATA_HDR_KEY_LEN) > (orig_buf + size)) {
-                        gf_msg_callingfn ("dict", GF_LOG_ERROR, 0,
-                                          LG_MSG_UNDERSIZED_BUF, "undersized "
-                                          "buffer passed. available (%lu) < "
-                                          "required (%lu)",
-                                          (long)(orig_buf + size),
-                                          (long)(buf + DICT_DATA_HDR_KEY_LEN));
-                        goto out;
-                }
-                memcpy (&hostord, buf, sizeof(hostord));
-                keylen = ntoh32 (hostord);
-                buf += DICT_DATA_HDR_KEY_LEN;
-
-                if ((buf + DICT_DATA_HDR_VAL_LEN) > (orig_buf + size)) {
-                        gf_msg_callingfn ("dict", GF_LOG_ERROR, 0,
-                                          LG_MSG_UNDERSIZED_BUF, "undersized "
-                                          "buffer passed. available (%lu) < "
-                                          "required (%lu)",
-                                          (long)(orig_buf + size),
-                                          (long)(buf + DICT_DATA_HDR_VAL_LEN));
-                        goto out;
-                }
-                memcpy (&hostord, buf, sizeof(hostord));
-                vallen = ntoh32 (hostord);
-                buf += DICT_DATA_HDR_VAL_LEN;
-
-                if ((buf + keylen) > (orig_buf + size)) {
-                        gf_msg_callingfn ("dict", GF_LOG_ERROR, 0,
-                                          LG_MSG_UNDERSIZED_BUF,
-                                          "undersized buffer passed. "
-                                          "available (%lu) < required (%lu)",
-                                          (long)(orig_buf + size),
-                                          (long)(buf + keylen));
-                        goto out;
-                }
-                key = buf;
-                buf += keylen + 1;  /* for '\0' */
-
-                if ((buf + vallen) > (orig_buf + size)) {
-                        gf_msg_callingfn ("dict", GF_LOG_ERROR, 0,
-                                          LG_MSG_UNDERSIZED_BUF,
-                                          "undersized buffer passed. "
-                                          "available (%lu) < required (%lu)",
-                                          (long)(orig_buf + size),
-                                          (long)(buf + vallen));
-                        goto out;
-                }
-                value = get_new_data ();
-
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                value->len  = vallen;
-                value->data = memdup (buf, vallen);
-                value->is_static = 0;
-                buf += vallen;
-
-                dict_add (*fill, key, value);
-        }
+        ret = dict_addn(*fill, key, keylen, value);
+        if (ret < 0)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * dict_allocate_and_serialize - serialize a dictionary into an allocated buffer
  *
@@ -2776,45 +3208,45 @@ out:
  */
 
 int32_t
-dict_allocate_and_serialize (dict_t *this, char **buf, u_int *length)
+dict_allocate_and_serialize(dict_t *this, char **buf, u_int *length)
 {
-        int           ret    = -EINVAL;
-        ssize_t       len = 0;
+    int ret = -EINVAL;
+    ssize_t len = 0;
 
-        if (!this || !buf) {
-                gf_msg_debug ("dict", 0, "dict OR buf is NULL");
-                goto out;
-        }
+    if (!this || !buf) {
+        gf_msg_debug("dict", 0, "dict OR buf is NULL");
+        goto out;
+    }
 
-        LOCK (&this->lock);
-        {
-                len = dict_serialized_length_lk (this);
-                if (len < 0) {
-                        ret = len;
-                        goto unlock;
-                }
+    LOCK(&this->lock);
+    {
+        len = dict_serialized_length_lk(this);
+        if (len < 0) {
+            ret = len;
+            goto unlock;
+        }
 
-                *buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (*buf == NULL) {
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+        *buf = GF_MALLOC(len, gf_common_mt_char);
+        if (*buf == NULL) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
 
-                ret = dict_serialize_lk (this, *buf);
-                if (ret < 0) {
-                        GF_FREE (*buf);
-                        *buf = NULL;
-                        goto unlock;
-                }
+        ret = dict_serialize_lk(this, *buf);
+        if (ret < 0) {
+            GF_FREE(*buf);
+            *buf = NULL;
+            goto unlock;
+        }
 
-                if (length != NULL) {
-                        *length = len;
-                }
+        if (length != NULL) {
+            *length = len;
         }
+    }
 unlock:
-        UNLOCK (&this->lock);
+    UNLOCK(&this->lock);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -2827,183 +3259,231 @@ out:
  * @delimiter : the delimiter to separate the values
  *
  * @return    : 0 -> success
- *            : -errno -> faliure
+ *            : -errno -> failure
  */
 int
-dict_serialize_value_with_delim_lk (dict_t *this, char *buf, int32_t *serz_len,
-                                    char delimiter)
-{
-        int          ret       = -1;
-        int32_t      count     = 0;
-        int32_t      vallen    = 0;
-        int32_t      total_len = 0;
-        data_pair_t *pair      = NULL;
-
-        if (!buf) {
-                gf_msg ("dict", GF_LOG_ERROR, EINVAL,
-                        LG_MSG_INVALID_ARG, "buf is null");
-                goto out;
-        }
+dict_serialize_value_with_delim_lk(dict_t *this, char *buf, int32_t *serz_len,
+                                   char delimiter)
+{
+    int ret = -1;
+    int32_t count = this->count;
+    int32_t vallen = 0;
+    int32_t total_len = 0;
+    data_pair_t *pair = this->members_list;
 
-        count = this->count;
-        if (count < 0) {
-                gf_msg ("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
-                        "count (%d) < 0", count);
-                goto out;
-        }
+    if (!buf) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, NULL);
+        goto out;
+    }
 
-        pair = this->members_list;
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, "count=%d",
+                count, NULL);
+        goto out;
+    }
 
-        while (count) {
-                if (!pair) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0,
-                                LG_MSG_PAIRS_LESS_THAN_COUNT,
-                                "less than count data pairs found");
-                        goto out;
-                }
+    while (count) {
+        if (!pair) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_PAIRS_LESS_THAN_COUNT,
+                    NULL);
+            goto out;
+        }
 
-                if (!pair->key || !pair->value) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0,
-                                LG_MSG_KEY_OR_VALUE_NULL,
-                                "key or value is null");
-                        goto out;
-                }
+        if (!pair->key || !pair->value) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_KEY_OR_VALUE_NULL, NULL);
+            goto out;
+        }
 
-                if (!pair->value->data) {
-                        gf_msg ("dict", GF_LOG_ERROR, 0,
-                                LG_MSG_NULL_VALUE_IN_DICT,
-                                "null value found in dict");
-                        goto out;
-                }
+        if (!pair->value->data) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_VALUE_IN_DICT, NULL);
+            goto out;
+        }
 
-                vallen = pair->value->len - 1; // length includes \0
-                memcpy (buf, pair->value->data, vallen);
-                buf += vallen;
-                *buf++ = delimiter;
+        vallen = pair->value->len - 1;  // length includes \0
+        memcpy(buf, pair->value->data, vallen);
+        buf += vallen;
+        *buf++ = delimiter;
 
-                total_len += (vallen + 1);
+        total_len += (vallen + 1);
 
-                pair = pair->next;
-                count--;
-        }
+        pair = pair->next;
+        count--;
+    }
 
-        *--buf = '\0'; // remove the last delimiter
-        total_len--;   // adjust the length
-        ret = 0;
+    *--buf = '\0';  // remove the last delimiter
+    total_len--;    // adjust the length
+    ret = 0;
 
-        if (serz_len)
-                *serz_len = total_len;
+    if (serz_len)
+        *serz_len = total_len;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-dict_serialize_value_with_delim (dict_t *this, char *buf, int32_t *serz_len,
-                                 char delimiter)
-{
-        int           ret    = -1;
-
-        if (!this || !buf) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is null!");
-                goto out;
-        }
-
-        LOCK (&this->lock);
-        {
-                ret = dict_serialize_value_with_delim_lk (this, buf, serz_len,
-                                                          delimiter);
-        }
-        UNLOCK (&this->lock);
+dict_serialize_value_with_delim(dict_t *this, char *buf, int32_t *serz_len,
+                                char delimiter)
+{
+    int ret = -1;
+
+    if (!this || !buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
+
+    LOCK(&this->lock);
+    {
+        ret = dict_serialize_value_with_delim_lk(this, buf, serz_len,
+                                                 delimiter);
+    }
+    UNLOCK(&this->lock);
 out:
-        return ret;
+    return ret;
 }
 
 int
-dict_dump_to_str (dict_t *dict, char *dump, int dumpsize, char *format)
+dict_dump_to_str(dict_t *dict, char *dump, int dumpsize, char *format)
 {
-        int          ret                       = 0;
-        int          dumplen                   = 0;
-        data_pair_t *trav                      = NULL;
-
-        for (trav = dict->members_list; trav; trav = trav->next) {
-                ret = snprintf (&dump[dumplen], dumpsize - dumplen,
-                                format, trav->key, trav->value->data);
-                if ((ret == -1) || !ret)
-                        return ret;
+    int ret = 0;
+    int dumplen = 0;
+    data_pair_t *trav = NULL;
 
-                dumplen += ret;
-        }
+    if (!dict)
         return 0;
+
+    for (trav = dict->members_list; trav; trav = trav->next) {
+        ret = snprintf(&dump[dumplen], dumpsize - dumplen, format, trav->key,
+                       trav->value->data);
+        if ((ret == -1) || !ret)
+            return ret;
+
+        dumplen += ret;
+    }
+    return 0;
 }
 
 void
-dict_dump_to_log (dict_t *dict)
-{
-        int          ret                       = -1;
-        char         dump[64*1024]             = {0,};
-        char        *format                    = "(%s:%s)";
+dict_dump_to_log(dict_t *dict)
+{
+    int ret = -1;
+    char *dump = NULL;
+    const int dump_size = 64 * 1024;
+    char *format = "(%s:%s)";
+
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+
+    dump = GF_MALLOC(dump_size, gf_common_mt_char);
+    if (!dump) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                         "dump buffer is NULL");
+        goto out;
+    }
+
+    ret = dict_dump_to_str(dict, dump, dump_size, format);
+    if (ret) {
+        gf_smsg("dict", GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT, NULL);
+        goto out;
+    }
+    gf_smsg("dict", GF_LOG_INFO, 0, LG_MSG_DICT_ERROR, "dict=%p", dict,
+            "dump=%s", dump, NULL);
+out:
+    GF_FREE(dump);
 
-        if (!dict) {
-                gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return;
-        }
+    return;
+}
 
-        ret = dict_dump_to_str (dict, dump, sizeof(dump), format);
-        if (ret) {
-                gf_msg ("dict", GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT,
-                        "Failed to log dictionary");
-                return;
-        }
-        gf_msg_callingfn ("dict", GF_LOG_INFO, 0, LG_MSG_DICT_ERROR,
-                          "dict=%p (%s)", dict, dump);
+void
+dict_dump_to_statedump(dict_t *dict, char *dict_name, char *domain)
+{
+    int ret = -1;
+    char *dump = NULL;
+    const int dump_size = 64 * 1024;
+    char key[4096] = {
+        0,
+    };
+    char *format = "\n\t%s:%s";
+
+    if (!dict) {
+        gf_msg_callingfn(domain, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+
+    dump = GF_MALLOC(dump_size, gf_common_mt_char);
+    if (!dump) {
+        gf_msg_callingfn(domain, GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                         "dump buffer is NULL");
+        goto out;
+    }
+
+    ret = dict_dump_to_str(dict, dump, dump_size, format);
+    if (ret) {
+        gf_smsg(domain, GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT, "name=%s",
+                dict_name, NULL);
+        goto out;
+    }
+    gf_proc_dump_build_key(key, domain, "%s", dict_name);
+    gf_proc_dump_write(key, "%s", dump);
 
-        return;
+out:
+    GF_FREE(dump);
+
+    return;
 }
 
-void
-dict_dump_to_statedump (dict_t *dict, char *dict_name, char *domain)
+dict_t *
+dict_for_key_value(const char *name, const char *value, size_t size,
+                   gf_boolean_t is_static)
 {
-        int          ret                       = -1;
-        char         dump[64*1024]             = {0,};
-        char         key[4096]                 = {0,};
-        char        *format                    = "\n\t%s:%s";
+    dict_t *xattr = dict_new();
+    int ret = 0;
 
-        if (!dict) {
-                gf_msg_callingfn (domain, GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "dict is NULL");
-                return;
-        }
+    if (!xattr)
+        return NULL;
 
-        ret = dict_dump_to_str (dict, dump, sizeof(dump), format);
-        if (ret) {
-                gf_msg (domain, GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT,
-                        "Failed to log dictionary %s", dict_name);
-                return;
-        }
-        gf_proc_dump_build_key (key, domain, dict_name);
-        gf_proc_dump_write (key, "%s", dump);
+    if (is_static)
+        ret = dict_set_static_bin(xattr, (char *)name, (void *)value, size);
+    else
+        ret = dict_set_bin(xattr, (char *)name, (void *)value, size);
 
-        return;
+    if (ret) {
+        dict_destroy(xattr);
+        xattr = NULL;
+    }
+
+    return xattr;
 }
 
-dict_t *
-dict_for_key_value (const char *name, const char *value, size_t size)
+/*
+ * "strings" should be NULL terminated strings array.
+ */
+int
+dict_has_key_from_array(dict_t *dict, char **strings, gf_boolean_t *result)
 {
-	dict_t *xattr = NULL;
-	int     ret = 0;
+    int i = 0;
+    uint32_t hash = 0;
 
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
+    if (!dict || !strings || !result)
+        return -EINVAL;
 
-	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
-	if (ret) {
-		dict_destroy (xattr);
-		xattr = NULL;
-	}
-
-	return xattr;
+    LOCK(&dict->lock);
+    {
+        for (i = 0; strings[i]; i++) {
+            hash = (uint32_t)XXH64(strings[i], strlen(strings[i]), 0);
+            if (dict_lookup_common(dict, strings[i], hash)) {
+                *result = _gf_true;
+                goto unlock;
+            }
+        }
+        *result = _gf_false;
+    }
+unlock:
+    UNLOCK(&dict->lock);
+    return 0;
 }
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
deleted file mode 100644
index a7fb6c78425..00000000000
--- a/libglusterfs/src/dict.h
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _DICT_H
-#define _DICT_H
-
-#include <inttypes.h>
-#include <sys/uio.h>
-#include <pthread.h>
-
-#include "common-utils.h"
-#include "libglusterfs-messages.h"
-
-typedef struct _data data_t;
-typedef struct _dict dict_t;
-typedef struct _data_pair data_pair_t;
-
-
-#define GF_PROTOCOL_DICT_SERIALIZE(this,from_dict,to,len,ope,labl) do { \
-                int    _ret     = 0;                                     \
-                                                                        \
-                if (!from_dict)                                         \
-                        break;                                          \
-                                                                        \
-                _ret = dict_allocate_and_serialize (from_dict, to, &len);\
-                if (_ret < 0) {                                          \
-                        gf_msg (this->name, GF_LOG_WARNING, 0,          \
-                                LG_MSG_DICT_SERIAL_FAILED,            \
-                                "failed to get serialized dict (%s)",   \
-                                (#from_dict));                          \
-                        ope = EINVAL;                                   \
-                        goto labl;                                      \
-                }                                                       \
-        } while (0)
-
-
-#define GF_PROTOCOL_DICT_UNSERIALIZE(xl,to,buff,len,ret,ope,labl) do {  \
-                if (!len)                                               \
-                        break;                                          \
-                to = dict_new();                                        \
-                GF_VALIDATE_OR_GOTO (xl->name, to, labl);               \
-                                                                        \
-                ret = dict_unserialize (buff, len, &to);                \
-                if (ret < 0) {                                          \
-                        gf_msg (xl->name, GF_LOG_WARNING, 0,            \
-                                LG_MSG_DICT_UNSERIAL_FAILED,            \
-                                "failed to unserialize dictionary (%s)", \
-                                (#to));                                 \
-                                                                        \
-                        ope = EINVAL;                                   \
-                        goto labl;                                      \
-                }                                                       \
-                                                                        \
-        } while (0)
-
-struct _data {
-        unsigned char  is_static:1;
-        unsigned char  is_const:1;
-        int32_t        len;
-        char          *data;
-        int32_t        refcount;
-        gf_lock_t      lock;
-};
-
-struct _data_pair {
-        struct _data_pair *hash_next;
-        struct _data_pair *prev;
-        struct _data_pair *next;
-        data_t            *value;
-        char              *key;
-};
-
-struct _dict {
-        unsigned char   is_static:1;
-        int32_t         hash_size;
-        int32_t         count;
-        int32_t         refcount;
-        data_pair_t   **members;
-        data_pair_t    *members_list;
-        char           *extra_free;
-        char           *extra_stdfree;
-        gf_lock_t       lock;
-        data_pair_t    *members_internal;
-        data_pair_t     free_pair;
-        gf_boolean_t    free_pair_in_use;
-};
-
-typedef gf_boolean_t (*dict_match_t) (dict_t *d, char *k, data_t *v,
-                                      void *data);
-
-int32_t is_data_equal (data_t *one, data_t *two);
-void data_destroy (data_t *data);
-
-/* function to set a key/value pair (overwrite existing if matches the key */
-int32_t dict_set (dict_t *this, char *key, data_t *value);
-/* function to set a new key/value pair (without checking for duplicate) */
-int32_t dict_add (dict_t *this, char *key, data_t *value);
-
-data_t *dict_get (dict_t *this, char *key);
-void dict_del (dict_t *this, char *key);
-int dict_reset (dict_t *dict);
-
-int dict_key_count (dict_t *this);
-
-int32_t dict_serialized_length (dict_t *dict);
-int32_t dict_serialize (dict_t *dict, char *buf);
-int32_t dict_unserialize (char *buf, int32_t size, dict_t **fill);
-
-int32_t dict_allocate_and_serialize (dict_t *this, char **buf, u_int *length);
-
-void dict_unref (dict_t *dict);
-dict_t *dict_ref (dict_t *dict);
-data_t *data_ref (data_t *data);
-void data_unref (data_t *data);
-
-int32_t dict_lookup  (dict_t *this, char *key, data_t **data);
-/*
-   TODO: provide converts for different byte sizes, signedness, and void *
- */
-data_t *int_to_data (int64_t value);
-data_t *str_to_data (char *value);
-data_t *data_from_dynstr (char *value);
-data_t *data_from_dynptr (void *value, int32_t len);
-data_t *bin_to_data (void *value, int32_t len);
-data_t *static_str_to_data (char *value);
-data_t *static_bin_to_data (void *value);
-
-int64_t data_to_int64 (data_t *data);
-int32_t data_to_int32 (data_t *data);
-int16_t data_to_int16 (data_t *data);
-int8_t data_to_int8 (data_t *data);
-
-uint64_t data_to_uint64 (data_t *data);
-uint32_t data_to_uint32 (data_t *data);
-uint16_t data_to_uint16 (data_t *data);
-uint8_t data_to_uint8 (data_t *data);
-
-data_t *data_from_int64 (int64_t value);
-data_t *data_from_int32 (int32_t value);
-data_t *data_from_int16 (int16_t value);
-data_t *data_from_int8 (int8_t value);
-
-data_t *data_from_uint64 (uint64_t value);
-data_t *data_from_uint32 (uint32_t value);
-data_t *data_from_uint16 (uint16_t value);
-
-char *data_to_str (data_t *data);
-void *data_to_bin (data_t *data);
-void *data_to_ptr (data_t *data);
-
-data_t * data_copy (data_t *old);
-int dict_foreach (dict_t *this,
-                  int (*fn)(dict_t *this,
-                            char *key,
-                            data_t *value,
-                            void *data),
-                  void *data);
-
-int dict_foreach_fnmatch (dict_t *dict, char *pattern,
-                          int (*fn)(dict_t *this,
-                                    char *key,
-                                    data_t *value,
-                                    void *data),
-                          void *data);
-
-int
-dict_foreach_match (dict_t *dict,
-             gf_boolean_t (*match)(dict_t *this,
-                                char *key,
-                                data_t *value,
-                                void *mdata),
-             void *match_data,
-             int (*action)(dict_t *this,
-                                char *key,
-                                data_t *value,
-                                void *adata),
-              void *action_data);
-
-int dict_null_foreach_fn (dict_t *d, char *k,
-                          data_t *v, void *tmp);
-int dict_remove_foreach_fn (dict_t *d, char *k,
-                            data_t *v, void *tmp);
-dict_t *dict_copy (dict_t *this, dict_t *new);
-dict_t *get_new_dict (void);
-int dict_keys_join (void *value, int size, dict_t *dict,
-                    int (*filter_fn)(char *key));
-
-/* CLEANED UP FUNCTIONS DECLARATIONS */
-GF_MUST_CHECK dict_t *dict_new (void);
-dict_t *dict_copy_with_ref (dict_t *this, dict_t *new);
-
-GF_MUST_CHECK int dict_reset (dict_t *dict);
-
-GF_MUST_CHECK int dict_get_int8 (dict_t *this, char *key, int8_t *val);
-GF_MUST_CHECK int dict_set_int8 (dict_t *this, char *key, int8_t val);
-
-GF_MUST_CHECK int dict_get_int16 (dict_t *this, char *key, int16_t *val);
-GF_MUST_CHECK int dict_set_int16 (dict_t *this, char *key, int16_t val);
-
-GF_MUST_CHECK int dict_get_int32 (dict_t *this, char *key, int32_t *val);
-GF_MUST_CHECK int dict_set_int32 (dict_t *this, char *key, int32_t val);
-
-GF_MUST_CHECK int dict_get_int64 (dict_t *this, char *key, int64_t *val);
-GF_MUST_CHECK int dict_set_int64 (dict_t *this, char *key, int64_t val);
-
-GF_MUST_CHECK int dict_get_uint16 (dict_t *this, char *key, uint16_t *val);
-GF_MUST_CHECK int dict_set_uint16 (dict_t *this, char *key, uint16_t val);
-
-GF_MUST_CHECK int dict_get_uint32 (dict_t *this, char *key, uint32_t *val);
-GF_MUST_CHECK int dict_set_uint32 (dict_t *this, char *key, uint32_t val);
-
-GF_MUST_CHECK int dict_get_uint64 (dict_t *this, char *key, uint64_t *val);
-GF_MUST_CHECK int dict_set_uint64 (dict_t *this, char *key, uint64_t val);
-
-GF_MUST_CHECK int dict_get_double (dict_t *this, char *key, double *val);
-GF_MUST_CHECK int dict_set_double (dict_t *this, char *key, double val);
-
-GF_MUST_CHECK int dict_set_static_ptr (dict_t *this, char *key, void *ptr);
-GF_MUST_CHECK int dict_get_ptr (dict_t *this, char *key, void **ptr);
-GF_MUST_CHECK int dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len);
-GF_MUST_CHECK int dict_set_ptr (dict_t *this, char *key, void *ptr);
-GF_MUST_CHECK int dict_set_dynptr (dict_t *this, char *key, void *ptr, size_t size);
-
-GF_MUST_CHECK int dict_get_bin (dict_t *this, char *key, void **ptr);
-GF_MUST_CHECK int dict_set_bin (dict_t *this, char *key, void *ptr, size_t size);
-GF_MUST_CHECK int dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size);
-
-GF_MUST_CHECK int dict_set_str (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_set_dynstr (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_set_dynstr_with_alloc (dict_t *this, char *key, const char *str);
-GF_MUST_CHECK int dict_add_dynstr_with_alloc (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_get_str (dict_t *this, char *key, char **str);
-
-GF_MUST_CHECK int dict_get_str_boolean (dict_t *this, char *key, int default_val);
-GF_MUST_CHECK int dict_serialize_value_with_delim (dict_t *this, char *buf, int32_t *serz_len,
-                                                    char delimiter);
-void
-dict_dump_to_statedump (dict_t *dict, char *dict_name, char *domain);
-
-void
-dict_dump_to_log (dict_t *dict);
-
-int
-dict_dump_to_str (dict_t *dict, char *dump, int dumpsize, char *format);
-gf_boolean_t
-dict_match_everything (dict_t *d, char *k, data_t *v, void *data);
-
-dict_t *
-dict_for_key_value (const char *name, const char *value, size_t size);
-
-gf_boolean_t
-are_dicts_equal (dict_t *one, dict_t *two,
-                 gf_boolean_t (*match) (dict_t *d, char *k, data_t *v,
-                                        void *data),
-                 gf_boolean_t (*value_ignore) (char *k));
-#endif
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
index 3fd580d9d1a..fb4fb845b40 100644
--- a/libglusterfs/src/event-epoll.c
+++ b/libglusterfs/src/event-epoll.c
@@ -8,762 +8,839 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <sys/poll.h>
 #include <pthread.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <stdlib.h>
 #include <errno.h>
-#include <string.h>
-
-#include "logging.h"
-#include "event.h"
-#include "mem-pool.h"
-#include "common-utils.h"
-#include "syscall.h"
-#include "libglusterfs-messages.h"
 
+#include "glusterfs/gf-event.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 #ifdef HAVE_SYS_EPOLL_H
 #include <sys/epoll.h>
 
-
 struct event_slot_epoll {
-	int fd;
-	int events;
-	int gen;
-	int ref;
-	int do_close;
-	int in_handler;
-	void *data;
-	event_handler_t handler;
-	gf_lock_t lock;
+    int fd;
+    int events;
+    int gen;
+    int idx;
+    gf_atomic_t ref;
+    int do_close;
+    int in_handler;
+    int handled_error;
+    void *data;
+    event_handler_t handler;
+    gf_lock_t lock;
+    struct list_head poller_death;
 };
 
 struct event_thread_data {
-        struct event_pool *event_pool;
-        int    event_index;
+    struct event_pool *event_pool;
+    int event_index;
 };
 
 static struct event_slot_epoll *
-__event_newtable (struct event_pool *event_pool, int table_idx)
+__event_newtable(struct event_pool *event_pool, int table_idx)
 {
-	struct event_slot_epoll *table = NULL;
-	int                      i = -1;
+    struct event_slot_epoll *table = NULL;
+    int i = -1;
 
-	table = GF_CALLOC (sizeof (*table), EVENT_EPOLL_SLOTS,
-			   gf_common_mt_ereg);
-	if (!table)
-		return NULL;
+    table = GF_CALLOC(sizeof(*table), EVENT_EPOLL_SLOTS, gf_common_mt_ereg);
+    if (!table)
+        return NULL;
 
-	for (i = 0; i < EVENT_EPOLL_SLOTS; i++) {
-		table[i].fd = -1;
-		LOCK_INIT (&table[i].lock);
-	}
+    for (i = 0; i < EVENT_EPOLL_SLOTS; i++) {
+        table[i].fd = -1;
+        LOCK_INIT(&table[i].lock);
+        INIT_LIST_HEAD(&table[i].poller_death);
+    }
 
-	event_pool->ereg[table_idx] = table;
-	event_pool->slots_used[table_idx] = 0;
+    event_pool->ereg[table_idx] = table;
+    event_pool->slots_used[table_idx] = 0;
 
-	return table;
+    return table;
 }
 
-
 static int
-__event_slot_alloc (struct event_pool *event_pool, int fd)
+event_slot_ref(struct event_slot_epoll *slot)
 {
-        int  i = 0;
-	int  table_idx = -1;
-	int  gen = -1;
-	struct event_slot_epoll *table = NULL;
-
-	for (i = 0; i < EVENT_EPOLL_TABLES; i++) {
-		switch (event_pool->slots_used[i]) {
-		case EVENT_EPOLL_SLOTS:
-			continue;
-		case 0:
-			if (!event_pool->ereg[i]) {
-				table = __event_newtable (event_pool, i);
-				if (!table)
-					return -1;
-			} else {
-                                table = event_pool->ereg[i];
-                        }
-			break;
-		default:
-			table = event_pool->ereg[i];
-			break;
-		}
-
-		if (table)
-			/* break out of the loop */
-			break;
-	}
-
-	if (!table)
-		return -1;
-
-	table_idx = i;
-
-	for (i = 0; i < EVENT_EPOLL_SLOTS; i++) {
-		if (table[i].fd == -1) {
-			/* wipe everything except bump the generation */
-			gen = table[i].gen;
-			memset (&table[i], 0, sizeof (table[i]));
-			table[i].gen = gen + 1;
-
-			LOCK_INIT (&table[i].lock);
-
-			table[i].fd = fd;
-			event_pool->slots_used[table_idx]++;
-
-			break;
-		}
-	}
-
-	return table_idx * EVENT_EPOLL_SLOTS + i;
-}
+    if (!slot)
+        return -1;
 
+    return GF_ATOMIC_INC(slot->ref);
+}
 
 static int
-event_slot_alloc (struct event_pool *event_pool, int fd)
+__event_slot_alloc(struct event_pool *event_pool, int fd,
+                   char notify_poller_death, struct event_slot_epoll **slot)
 {
-	int  idx = -1;
+    int i = 0;
+    int j = 0;
+    int table_idx = -1;
+    int gen = -1;
+    struct event_slot_epoll *table = NULL;
+
+retry:
+
+    while (i < EVENT_EPOLL_TABLES) {
+        switch (event_pool->slots_used[i]) {
+            case EVENT_EPOLL_SLOTS:
+                break;
+            case 0:
+                if (!event_pool->ereg[i]) {
+                    table = __event_newtable(event_pool, i);
+                    if (!table)
+                        return -1;
+                } else {
+                    table = event_pool->ereg[i];
+                }
+                break;
+            default:
+                table = event_pool->ereg[i];
+                break;
+        }
 
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_slot_alloc (event_pool, fd);
-	}
-	pthread_mutex_unlock (&event_pool->mutex);
+        if (table)
+            /* break out of the loop */
+            break;
+        i++;
+    }
 
-	return idx;
-}
+    if (!table)
+        return -1;
 
+    table_idx = i;
 
+    for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
+        if (table[j].fd == -1) {
+            /* wipe everything except bump the generation */
+            gen = table[j].gen;
+            memset(&table[j], 0, sizeof(table[j]));
+            table[j].gen = gen + 1;
 
-static void
-__event_slot_dealloc (struct event_pool *event_pool, int idx)
-{
-	int                      table_idx = 0;
-	int                      offset = 0;
-	struct event_slot_epoll *table = NULL;
-	struct event_slot_epoll *slot = NULL;
+            LOCK_INIT(&table[j].lock);
+            INIT_LIST_HEAD(&table[j].poller_death);
 
-	table_idx = idx / EVENT_EPOLL_SLOTS;
-	offset = idx % EVENT_EPOLL_SLOTS;
+            table[j].fd = fd;
+            if (notify_poller_death) {
+                table[j].idx = table_idx * EVENT_EPOLL_SLOTS + j;
+                list_add_tail(&table[j].poller_death,
+                              &event_pool->poller_death);
+            }
 
-	table = event_pool->ereg[table_idx];
-	if (!table)
-		return;
+            event_pool->slots_used[table_idx]++;
 
-	slot = &table[offset];
-	slot->gen++;
+            break;
+        }
+    }
+
+    if (j == EVENT_EPOLL_SLOTS) {
+        table = NULL;
+        i++;
+        goto retry;
+    } else {
+        (*slot) = &table[j];
+        event_slot_ref(*slot);
+        return table_idx * EVENT_EPOLL_SLOTS + j;
+    }
+}
+
+static int
+event_slot_alloc(struct event_pool *event_pool, int fd,
+                 char notify_poller_death, struct event_slot_epoll **slot)
+{
+    int idx = -1;
 
-	slot->fd = -1;
-	event_pool->slots_used[table_idx]--;
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_slot_alloc(event_pool, fd, notify_poller_death, slot);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-	return;
+    return idx;
 }
 
+static void
+__event_slot_dealloc(struct event_pool *event_pool, int idx)
+{
+    int table_idx = 0;
+    int offset = 0;
+    struct event_slot_epoll *table = NULL;
+    struct event_slot_epoll *slot = NULL;
+    int fd = -1;
+
+    table_idx = idx / EVENT_EPOLL_SLOTS;
+    offset = idx % EVENT_EPOLL_SLOTS;
+
+    table = event_pool->ereg[table_idx];
+    if (!table)
+        return;
+
+    slot = &table[offset];
+    slot->gen++;
+
+    fd = slot->fd;
+    slot->fd = -1;
+    slot->handled_error = 0;
+    slot->in_handler = 0;
+    list_del_init(&slot->poller_death);
+    if (fd != -1)
+        event_pool->slots_used[table_idx]--;
+
+    return;
+}
 
 static void
-event_slot_dealloc (struct event_pool *event_pool, int idx)
+event_slot_dealloc(struct event_pool *event_pool, int idx)
 {
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		__event_slot_dealloc (event_pool, idx);
-	}
-	pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        __event_slot_dealloc(event_pool, idx);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-	return;
+    return;
 }
 
-
 static struct event_slot_epoll *
-event_slot_get (struct event_pool *event_pool, int idx)
+event_slot_get(struct event_pool *event_pool, int idx)
 {
-	struct event_slot_epoll *slot = NULL;
-	struct event_slot_epoll *table = NULL;
-	int                      table_idx = 0;
-	int                      offset = 0;
-
-	table_idx = idx / EVENT_EPOLL_SLOTS;
-	offset = idx % EVENT_EPOLL_SLOTS;
+    struct event_slot_epoll *slot = NULL;
+    struct event_slot_epoll *table = NULL;
+    int table_idx = 0;
+    int offset = 0;
 
-	table = event_pool->ereg[table_idx];
-	if (!table)
-		return NULL;
+    table_idx = idx / EVENT_EPOLL_SLOTS;
+    offset = idx % EVENT_EPOLL_SLOTS;
 
-	slot = &table[offset];
+    table = event_pool->ereg[table_idx];
+    if (!table)
+        return NULL;
 
-	LOCK (&slot->lock);
-	{
-		slot->ref++;
-	}
-	UNLOCK (&slot->lock);
+    slot = &table[offset];
 
-	return slot;
+    event_slot_ref(slot);
+    return slot;
 }
 
-
 static void
-event_slot_unref (struct event_pool *event_pool, struct event_slot_epoll *slot,
-		  int idx)
+__event_slot_unref(struct event_pool *event_pool, struct event_slot_epoll *slot,
+                   int idx)
 {
-	int ref = -1;
-	int fd = -1;
-	int do_close = 0;
-
-	LOCK (&slot->lock);
-	{
-		ref = --slot->ref;
-		fd = slot->fd;
-		do_close = slot->do_close;
-	}
-	UNLOCK (&slot->lock);
-
-	if (ref)
-		/* slot still alive */
-		goto done;
-
-	event_slot_dealloc (event_pool, idx);
-
-	if (do_close)
-		sys_close (fd);
+    int ref = -1;
+    int fd = -1;
+    int do_close = 0;
+
+    ref = GF_ATOMIC_DEC(slot->ref);
+    if (ref)
+        /* slot still alive */
+        goto done;
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        do_close = slot->do_close;
+        slot->do_close = 0;
+    }
+    UNLOCK(&slot->lock);
+
+    __event_slot_dealloc(event_pool, idx);
+
+    if (do_close)
+        sys_close(fd);
 done:
-	return;
+    return;
 }
 
+static void
+event_slot_unref(struct event_pool *event_pool, struct event_slot_epoll *slot,
+                 int idx)
+{
+    int ref = -1;
+    int fd = -1;
+    int do_close = 0;
+
+    ref = GF_ATOMIC_DEC(slot->ref);
+    if (ref)
+        /* slot still alive */
+        goto done;
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        do_close = slot->do_close;
+        slot->do_close = 0;
+    }
+    UNLOCK(&slot->lock);
+
+    event_slot_dealloc(event_pool, idx);
+
+    if (do_close)
+        sys_close(fd);
+done:
+    return;
+}
 
 static struct event_pool *
-event_pool_new_epoll (int count, int eventthreadcount)
+event_pool_new_epoll(int count, int eventthreadcount)
 {
-        struct event_pool *event_pool = NULL;
-        int                epfd = -1;
+    struct event_pool *event_pool = NULL;
+    int epfd = -1;
 
-        event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
+    event_pool = GF_CALLOC(1, sizeof(*event_pool), gf_common_mt_event_pool);
 
-        if (!event_pool)
-                goto out;
+    if (!event_pool)
+        goto out;
 
-        epfd = epoll_create (count);
-
-        if (epfd == -1) {
-                gf_msg ("epoll", GF_LOG_ERROR, errno,
-                        LG_MSG_EPOLL_FD_CREATE_FAILED, "epoll fd creation "
-                        "failed");
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                event_pool = NULL;
-                goto out;
-        }
+    epfd = epoll_create(count);
 
-        event_pool->fd = epfd;
+    if (epfd == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_CREATE_FAILED,
+                NULL);
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        event_pool = NULL;
+        goto out;
+    }
 
-        event_pool->count = count;
+    event_pool->fd = epfd;
 
-        event_pool->eventthreadcount = eventthreadcount;
+    event_pool->count = count;
+    INIT_LIST_HEAD(&event_pool->poller_death);
+    event_pool->eventthreadcount = eventthreadcount;
+    event_pool->auto_thread_count = 0;
 
-        pthread_mutex_init (&event_pool->mutex, NULL);
+    pthread_mutex_init(&event_pool->mutex, NULL);
 
 out:
-        return event_pool;
+    return event_pool;
 }
 
-
 static void
-__slot_update_events (struct event_slot_epoll *slot, int poll_in, int poll_out)
+__slot_update_events(struct event_slot_epoll *slot, int poll_in, int poll_out)
 {
-	switch (poll_in) {
-	case 1:
-		slot->events |= EPOLLIN;
-		break;
-	case 0:
-		slot->events &= ~EPOLLIN;
-		break;
-	case -1:
-		/* do nothing */
-		break;
-	default:
-		gf_msg ("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_IN,
-			"invalid poll_in value %d", poll_in);
-		break;
-	}
-
-	switch (poll_out) {
-	case 1:
-		slot->events |= EPOLLOUT;
-		break;
-	case 0:
-		slot->events &= ~EPOLLOUT;
-		break;
-	case -1:
-		/* do nothing */
-		break;
-	default:
-		gf_msg ("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_OUT,
-			"invalid poll_out value %d", poll_out);
-		break;
-	}
+    switch (poll_in) {
+        case 1:
+            slot->events |= EPOLLIN;
+            break;
+        case 0:
+            slot->events &= ~EPOLLIN;
+            break;
+        case -1:
+            /* do nothing */
+            break;
+        default:
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_IN,
+                    "value=%d", poll_in, NULL);
+            break;
+    }
+
+    switch (poll_out) {
+        case 1:
+            slot->events |= EPOLLOUT;
+            break;
+        case 0:
+            slot->events &= ~EPOLLOUT;
+            break;
+        case -1:
+            /* do nothing */
+            break;
+        default:
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_OUT,
+                    "value=%d", poll_out, NULL);
+            break;
+    }
 }
 
-
 int
-event_register_epoll (struct event_pool *event_pool, int fd,
-                      event_handler_t handler,
-                      void *data, int poll_in, int poll_out)
+event_register_epoll(struct event_pool *event_pool, int fd,
+                     event_handler_t handler, void *data, int poll_in,
+                     int poll_out, char notify_poller_death)
 {
-        int                 idx = -1;
-        int                 ret = -1;
-        int             destroy = 0;
-        struct epoll_event  epoll_event = {0, };
-        struct event_data  *ev_data = (void *)&epoll_event.data;
-	struct event_slot_epoll *slot = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        /* TODO: Even with the below check, there is a possiblity of race,
-         * What if the destroy mode is set after the check is done.
-         * Not sure of the best way to prevent this race, ref counting
-         * is one possibility.
-         * There is no harm in registering and unregistering the fd
-         * even after destroy mode is set, just that such fds will remain
-         * open until unregister is called, also the events on that fd will be
-         * notified, until one of the poller thread is alive.
-         */
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                destroy = event_pool->destroy;
-        }
-        pthread_mutex_unlock (&event_pool->mutex);
-
-        if (destroy == 1)
-               goto out;
-
-	idx = event_slot_alloc (event_pool, fd);
-	if (idx == -1) {
-		gf_msg ("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND,
-			"could not find slot for fd=%d", fd);
-		return -1;
-	}
-
-	slot = event_slot_get (event_pool, idx);
-
-	assert (slot->fd == fd);
-
-	LOCK (&slot->lock);
-	{
-		/* make epoll 'singleshot', which
-		   means we need to re-add the fd with
-		   epoll_ctl(EPOLL_CTL_MOD) after delivery of every
-		   single event. This assures us that while a poller
-		   thread has picked up and is processing an event,
-		   another poller will not try to pick this at the same
-		   time as well.
-		*/
-
-		slot->events = EPOLLPRI | EPOLLONESHOT;
-		slot->handler = handler;
-		slot->data = data;
-
-		__slot_update_events (slot, poll_in, poll_out);
-
-		epoll_event.events = slot->events;
-		ev_data->idx = idx;
-		ev_data->gen = slot->gen;
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_ADD, fd,
-				 &epoll_event);
-		/* check ret after UNLOCK() to avoid deadlock in
-		   event_slot_unref()
-		*/
-	}
-	UNLOCK (&slot->lock);
-
-	if (ret == -1) {
-		gf_msg ("epoll", GF_LOG_ERROR, errno,
-                        LG_MSG_EPOLL_FD_ADD_FAILED, "failed to add fd(=%d) to "
-                        "epoll fd(=%d)", fd, event_pool->fd);
-		event_slot_unref (event_pool, slot, idx);
-		idx = -1;
-	}
-
-	/* keep slot->ref (do not event_slot_unref) if successful */
+    int idx = -1;
+    int ret = -1;
+    int destroy = 0;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+    struct event_slot_epoll *slot = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    /* TODO: Even with the below check, there is a possibility of race,
+     * What if the destroy mode is set after the check is done.
+     * Not sure of the best way to prevent this race, ref counting
+     * is one possibility.
+     * There is no harm in registering and unregistering the fd
+     * even after destroy mode is set, just that such fds will remain
+     * open until unregister is called, also the events on that fd will be
+     * notified, until one of the poller thread is alive.
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        destroy = event_pool->destroy;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    if (destroy == 1)
+        goto out;
+
+    idx = event_slot_alloc(event_pool, fd, notify_poller_death, &slot);
+    if (idx == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        /* make epoll 'singleshot', which
+           means we need to re-add the fd with
+           epoll_ctl(EPOLL_CTL_MOD) after delivery of every
+           single event. This assures us that while a poller
+           thread has picked up and is processing an event,
+           another poller will not try to pick this at the same
+           time as well.
+        */
+
+        slot->events = EPOLLPRI | EPOLLHUP | EPOLLERR | EPOLLONESHOT;
+        slot->handler = handler;
+        slot->data = data;
+
+        __slot_update_events(slot, poll_in, poll_out);
+
+        epoll_event.events = slot->events;
+        ev_data->idx = idx;
+        ev_data->gen = slot->gen;
+
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_ADD, fd, &epoll_event);
+        /* check ret after UNLOCK() to avoid deadlock in
+           event_slot_unref()
+        */
+    }
+    UNLOCK(&slot->lock);
+
+    if (ret == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_ADD_FAILED,
+                "fd=%d", fd, "epoll_fd=%d", event_pool->fd, NULL);
+        event_slot_unref(event_pool, slot, idx);
+        idx = -1;
+    }
+
+    /* keep slot->ref (do not event_slot_unref) if successful */
 out:
-        return idx;
+    return idx;
 }
 
-
 static int
-event_unregister_epoll_common (struct event_pool *event_pool, int fd,
-			       int idx, int do_close)
+event_unregister_epoll_common(struct event_pool *event_pool, int fd, int idx,
+                              int do_close)
 {
-        int  ret = -1;
-	struct event_slot_epoll *slot = NULL;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-	slot = event_slot_get (event_pool, idx);
-
-	assert (slot->fd == fd);
-
-	LOCK (&slot->lock);
-	{
-                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
-
-                if (ret == -1) {
-                        gf_msg ("epoll", GF_LOG_ERROR, errno,
-                                LG_MSG_EPOLL_FD_DEL_FAILED, "fail to del "
-                                "fd(=%d) from epoll fd(=%d)", fd,
-                                event_pool->fd);
-                        goto unlock;
-                }
-
-		slot->do_close = do_close;
-		slot->gen++; /* detect unregister in dispatch_handler() */
+    int ret = -1;
+    struct event_slot_epoll *slot = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    /* During shutdown, it may happen that a socket registration with
+     * the event sub-system may fail and an rpc_transport_unref() may
+     * be called for such an unregistered socket with idx == -1. This
+     * may cause the following assert(slot->fd == fd) to fail.
+     */
+    if (idx < 0)
+        goto out;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
+
+        if (ret == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_DEL_FAILED,
+                    "fd=%d", fd, "epoll_fd=%d", event_pool->fd, NULL);
+            goto unlock;
         }
+
+        slot->do_close = do_close;
+        slot->gen++; /* detect unregister in dispatch_handler() */
+    }
 unlock:
-	UNLOCK (&slot->lock);
+    UNLOCK(&slot->lock);
 
-	event_slot_unref (event_pool, slot, idx); /* one for event_register() */
-	event_slot_unref (event_pool, slot, idx); /* one for event_slot_get() */
+    event_slot_unref(event_pool, slot, idx); /* one for event_register() */
+    event_slot_unref(event_pool, slot, idx); /* one for event_slot_get() */
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-event_unregister_epoll (struct event_pool *event_pool, int fd, int idx_hint)
+event_unregister_epoll(struct event_pool *event_pool, int fd, int idx_hint)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = event_unregister_epoll_common (event_pool, fd, idx_hint, 0);
+    ret = event_unregister_epoll_common(event_pool, fd, idx_hint, 0);
 
-	return ret;
+    return ret;
 }
 
-
 static int
-event_unregister_close_epoll (struct event_pool *event_pool, int fd,
-			      int idx_hint)
+event_unregister_close_epoll(struct event_pool *event_pool, int fd,
+                             int idx_hint)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = event_unregister_epoll_common (event_pool, fd, idx_hint, 1);
+    ret = event_unregister_epoll_common(event_pool, fd, idx_hint, 1);
 
-	return ret;
+    return ret;
 }
 
-
 static int
-event_select_on_epoll (struct event_pool *event_pool, int fd, int idx,
-                       int poll_in, int poll_out)
+event_select_on_epoll(struct event_pool *event_pool, int fd, int idx,
+                      int poll_in, int poll_out)
 {
-        int ret = -1;
-	struct event_slot_epoll *slot = NULL;
-        struct epoll_event epoll_event = {0, };
-        struct event_data *ev_data = (void *)&epoll_event.data;
-
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-	slot = event_slot_get (event_pool, idx);
-
-	assert (slot->fd == fd);
-
-	LOCK (&slot->lock);
-	{
-		__slot_update_events (slot, poll_in, poll_out);
-
-		epoll_event.events = slot->events;
-		ev_data->idx = idx;
-		ev_data->gen = slot->gen;
-
-		if (slot->in_handler)
-			/* in_handler indicates at least one thread
-			   executing event_dispatch_epoll_handler()
-			   which will perform epoll_ctl(EPOLL_CTL_MOD)
-			   anyways (because of EPOLLET)
-
-			   This not only saves a system call, but also
-			   avoids possibility of another epoll thread
-			   parallely picking up the next event while the
-			   ongoing handler is still in progress (and
-			   resulting in unnecessary contention on
-			   rpc_transport_t->mutex).
-			*/
-			goto unlock;
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, fd,
-				 &epoll_event);
-		if (ret == -1) {
-			gf_msg ("epoll", GF_LOG_ERROR, errno,
-                                LG_MSG_EPOLL_FD_MODIFY_FAILED, "failed to "
-                                "modify fd(=%d) events to %d", fd,
-                                epoll_event.events);
-		}
-	}
+    int ret = -1;
+    struct event_slot_epoll *slot = NULL;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        __slot_update_events(slot, poll_in, poll_out);
+
+        epoll_event.events = slot->events;
+        ev_data->idx = idx;
+        ev_data->gen = slot->gen;
+
+        if (slot->in_handler)
+            /*
+             * in_handler indicates at least one thread
+             * executing event_dispatch_epoll_handler()
+             * which will perform epoll_ctl(EPOLL_CTL_MOD)
+             * anyways (because of EPOLLET)
+             *
+             * This not only saves a system call, but also
+             * avoids possibility of another epoll thread
+             * picking up the next event while the ongoing
+             * handler is still in progress (and resulting
+             * in unnecessary contention on rpc_transport_t->mutex).
+             */
+            goto unlock;
+
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_MOD, fd, &epoll_event);
+        if (ret == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_MODIFY_FAILED,
+                    "fd=%d", fd, "events=%d", epoll_event.events, NULL);
+        }
+    }
 unlock:
-	UNLOCK (&slot->lock);
+    UNLOCK(&slot->lock);
 
-	event_slot_unref (event_pool, slot, idx);
+    event_slot_unref(event_pool, slot, idx);
 
 out:
-        return idx;
+    return idx;
 }
 
-
 static int
-event_dispatch_epoll_handler (struct event_pool *event_pool,
-                              struct epoll_event *event)
+event_dispatch_epoll_handler(struct event_pool *event_pool,
+                             struct epoll_event *event)
 {
-        struct event_data  *ev_data = NULL;
-	struct event_slot_epoll *slot = NULL;
-        event_handler_t     handler = NULL;
-        void               *data = NULL;
-        int                 idx = -1;
-	int                 gen = -1;
-        int                 ret = -1;
-	int                 fd = -1;
-
-	ev_data = (void *)&event->data;
-        handler = NULL;
-        data = NULL;
-
-	idx = ev_data->idx;
-	gen = ev_data->gen;
-
-	slot = event_slot_get (event_pool, idx);
-
-	LOCK (&slot->lock);
-	{
-		fd = slot->fd;
-		if (fd == -1) {
-			gf_msg ("epoll", GF_LOG_ERROR, 0,
-                                LG_MSG_STALE_FD_FOUND, "stale fd found on "
-                                "idx=%d, gen=%d, events=%d, slot->gen=%d",
-				idx, gen, event->events, slot->gen);
-			/* fd got unregistered in another thread */
-			goto pre_unlock;
-		}
-
-		if (gen != slot->gen) {
-			gf_msg ("epoll", GF_LOG_ERROR, 0,
-                                LG_MSG_GENERATION_MISMATCH, "generation "
-                                "mismatch on idx=%d, gen=%d, slot->gen=%d, "
-                                "slot->fd=%d", idx, gen, slot->gen, slot->fd);
-			/* slot was re-used and therefore is another fd! */
-			goto pre_unlock;
-		}
-
-		handler = slot->handler;
-		data = slot->data;
-
-		slot->in_handler++;
-	}
-pre_unlock:
-	UNLOCK (&slot->lock);
-
-        if (!handler)
-		goto out;
-
-	ret = handler (fd, idx, data,
-		       (event->events & (EPOLLIN|EPOLLPRI)),
-		       (event->events & (EPOLLOUT)),
-		       (event->events & (EPOLLERR|EPOLLHUP)));
-
-	LOCK (&slot->lock);
-	{
-		slot->in_handler--;
-
-		if (gen != slot->gen) {
-			/* event_unregister() happened while we were
-			   in handler()
-			*/
-			gf_msg_debug ("epoll", 0, "generation bumped on idx=%d"
-                                      " from gen=%d to slot->gen=%d, fd=%d, "
-				      "slot->fd=%d", idx, gen, slot->gen, fd,
-                                      slot->fd);
-			goto post_unlock;
-		}
-
-		/* This call also picks up the changes made by another
-		   thread calling event_select_on_epoll() while this
-		   thread was busy in handler()
-		*/
-                if (slot->in_handler == 0) {
-                        event->events = slot->events;
-                        ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD,
-                                         fd, event);
-                }
-	}
-post_unlock:
-	UNLOCK (&slot->lock);
-out:
-	event_slot_unref (event_pool, slot, idx);
+    struct event_data *ev_data = NULL;
+    struct event_slot_epoll *slot = NULL;
+    event_handler_t handler = NULL;
+    void *data = NULL;
+    int idx = -1;
+    int gen = -1;
+    int ret = -1;
+    int fd = -1;
+    gf_boolean_t handled_error_previously = _gf_false;
+
+    ev_data = (void *)&event->data;
+    handler = NULL;
+    data = NULL;
+
+    idx = ev_data->idx;
+    gen = ev_data->gen;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "idx=%d", idx,
+                NULL);
+        return -1;
+    }
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        if (fd == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_STALE_FD_FOUND, "idx=%d",
+                    idx, "gen=%d", gen, "events=%d", event->events,
+                    "slot->gen=%d", slot->gen, NULL);
+            /* fd got unregistered in another thread */
+            goto pre_unlock;
+        }
 
-        return ret;
-}
+        if (gen != slot->gen) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_GENERATION_MISMATCH,
+                    "idx=%d", idx, "gen=%d", gen, "slot->gen=%d", slot->gen,
+                    "slot->fd=%d", slot->fd, NULL);
+            /* slot was re-used and therefore is another fd! */
+            goto pre_unlock;
+        }
 
+        handler = slot->handler;
+        data = slot->data;
 
-static void *
-event_dispatch_epoll_worker (void *data)
-{
-        struct epoll_event  event;
-        int                 ret = -1;
-        struct event_thread_data *ev_data = data;
-	struct event_pool  *event_pool;
-        int                 myindex = -1;
-        int                 timetodie = 0;
+        if (slot->in_handler > 0) {
+            /* Another handler is inprogress, skip this one. */
+            handler = NULL;
+            goto pre_unlock;
+        }
 
-        GF_VALIDATE_OR_GOTO ("event", ev_data, out);
+        if (slot->handled_error) {
+            handled_error_previously = _gf_true;
+        } else {
+            slot->handled_error = (event->events & (EPOLLERR | EPOLLHUP));
+            slot->in_handler++;
+        }
+    }
+pre_unlock:
+    UNLOCK(&slot->lock);
 
-        event_pool = ev_data->event_pool;
-        myindex = ev_data->event_index;
+    ret = 0;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    if (!handler)
+        goto out;
 
-        gf_msg ("epoll", GF_LOG_INFO, 0, LG_MSG_STARTED_EPOLL_THREAD, "Started"
-                " thread with index %d", myindex);
+    if (!handled_error_previously) {
+        handler(fd, idx, gen, data, (event->events & (EPOLLIN | EPOLLPRI)),
+                (event->events & (EPOLLOUT)),
+                (event->events & (EPOLLERR | EPOLLHUP)), 0);
+    }
+out:
+    event_slot_unref(event_pool, slot, idx);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                event_pool->activethreadcount++;
-        }
-        pthread_mutex_unlock (&event_pool->mutex);
+    return ret;
+}
 
-	for (;;) {
+static void *
+event_dispatch_epoll_worker(void *data)
+{
+    struct epoll_event event;
+    int ret = -1;
+    struct event_thread_data *ev_data = data;
+    struct event_pool *event_pool;
+    int myindex = -1;
+    int timetodie = 0, gen = 0;
+    struct list_head poller_death_notify;
+    struct event_slot_epoll *slot = NULL, *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", ev_data, out);
+
+    event_pool = ev_data->event_pool;
+    myindex = ev_data->event_index;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    gf_smsg("epoll", GF_LOG_INFO, 0, LG_MSG_STARTED_EPOLL_THREAD, "index=%d",
+            myindex - 1, NULL);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount++;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    for (;;) {
+        if (event_pool->eventthreadcount < myindex) {
+            /* ...time to die, thread count was decreased below
+             * this threads index */
+            /* Start with extra safety at this point, reducing
+             * lock conention in normal case when threads are not
+             * reconfigured always */
+            pthread_mutex_lock(&event_pool->mutex);
+            {
                 if (event_pool->eventthreadcount < myindex) {
-                        /* ...time to die, thread count was decreased below
-                         * this threads index */
-                        /* Start with extra safety at this point, reducing
-                         * lock conention in normal case when threads are not
-                         * reconfigured always */
-                        pthread_mutex_lock (&event_pool->mutex);
-                        {
-                                if (event_pool->eventthreadcount <
-                                    myindex) {
-                                        /* if found true in critical section,
-                                         * die */
-                                        event_pool->pollers[myindex - 1] = 0;
-                                        event_pool->activethreadcount--;
-                                        timetodie = 1;
-                                        pthread_cond_broadcast (&event_pool->cond);
-                                }
-                        }
-                        pthread_mutex_unlock (&event_pool->mutex);
-                        if (timetodie) {
-                                gf_msg ("epoll", GF_LOG_INFO, 0,
-                                        LG_MSG_EXITED_EPOLL_THREAD, "Exited "
-                                        "thread with index %d", myindex);
-                                goto out;
-                        }
+                    while (event_pool->poller_death_sliced) {
+                        pthread_cond_wait(&event_pool->cond,
+                                          &event_pool->mutex);
+                    }
+
+                    INIT_LIST_HEAD(&poller_death_notify);
+                    /* if found true in critical section,
+                     * die */
+                    event_pool->pollers[myindex - 1] = 0;
+                    event_pool->activethreadcount--;
+                    timetodie = 1;
+                    gen = ++event_pool->poller_gen;
+                    list_for_each_entry(slot, &event_pool->poller_death,
+                                        poller_death)
+                    {
+                        event_slot_ref(slot);
+                    }
+
+                    list_splice_init(&event_pool->poller_death,
+                                     &poller_death_notify);
+                    event_pool->poller_death_sliced = 1;
+                    pthread_cond_broadcast(&event_pool->cond);
+                }
+            }
+            pthread_mutex_unlock(&event_pool->mutex);
+            if (timetodie) {
+                list_for_each_entry(slot, &poller_death_notify, poller_death)
+                {
+                    slot->handler(slot->fd, 0, gen, slot->data, 0, 0, 0, 1);
                 }
 
-                ret = epoll_wait (event_pool->fd, &event, 1, -1);
+                pthread_mutex_lock(&event_pool->mutex);
+                {
+                    list_for_each_entry_safe(slot, tmp, &poller_death_notify,
+                                             poller_death)
+                    {
+                        __event_slot_unref(event_pool, slot, slot->idx);
+                    }
+
+                    list_splice(&poller_death_notify,
+                                &event_pool->poller_death);
+                    event_pool->poller_death_sliced = 0;
+                    pthread_cond_broadcast(&event_pool->cond);
+                }
+                pthread_mutex_unlock(&event_pool->mutex);
 
-                if (ret == 0)
-                        /* timeout */
-                        continue;
+                gf_smsg("epoll", GF_LOG_INFO, 0, LG_MSG_EXITED_EPOLL_THREAD,
+                        "index=%d", myindex, NULL);
 
-                if (ret == -1 && errno == EINTR)
-                        /* sys call */
-                        continue;
+                goto out;
+            }
+        }
+
+        ret = epoll_wait(event_pool->fd, &event, 1, -1);
 
-		ret = event_dispatch_epoll_handler (event_pool, &event);
+        if (ret == 0)
+            /* timeout */
+            continue;
+
+        if (ret == -1 && errno == EINTR)
+            /* sys call */
+            continue;
+
+        ret = event_dispatch_epoll_handler(event_pool, &event);
+        if (ret) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_DISPATCH_HANDLER_FAILED,
+                    NULL);
         }
+    }
 out:
-        if (ev_data)
-                GF_FREE (ev_data);
-        return NULL;
+    if (ev_data)
+        GF_FREE(ev_data);
+    return NULL;
 }
 
 /* Attempts to start the # of configured pollers, ensuring at least the first
  * is started in a joinable state */
 static int
-event_dispatch_epoll (struct event_pool *event_pool)
+event_dispatch_epoll(struct event_pool *event_pool)
 {
-	int                       i = 0;
-        pthread_t                 t_id;
-        int                       pollercount = 0;
-	int                       ret = -1;
-        struct event_thread_data *ev_data = NULL;
-
-        /* Start the configured number of pollers */
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                pollercount = event_pool->eventthreadcount;
-
-                /* Set to MAX if greater */
-                if (pollercount > EVENT_MAX_THREADS)
-                        pollercount = EVENT_MAX_THREADS;
-
-                /* Default pollers to 1 in case this is incorrectly set */
-                if (pollercount <= 0)
-                        pollercount = 1;
-
-                event_pool->activethreadcount++;
-
-                for (i = 0; i < pollercount; i++) {
-                        ev_data = GF_CALLOC (1, sizeof (*ev_data),
-                                     gf_common_mt_event_pool);
-                        if (!ev_data) {
-                                if (i == 0) {
-                                        /* Need to suceed creating 0'th
-                                         * thread, to joinable and wait */
-                                        break;
-                                } else {
-                                        /* Inability to create other threads
-                                         * are a lesser evil, and ignored */
-                                        continue;
-                                }
-                        }
-
-                        ev_data->event_pool = event_pool;
-                        ev_data->event_index = i + 1;
-
-                        ret = pthread_create (&t_id, NULL,
-                                              event_dispatch_epoll_worker,
-                                              ev_data);
-                        if (!ret) {
-                                event_pool->pollers[i] = t_id;
-
-                                /* mark all threads other than one in index 0
-                                 * as detachable. Errors can be ignored, they
-                                 * spend their time as zombies if not detched
-                                 * and the thread counts are decreased */
-                                if (i != 0)
-                                        pthread_detach (event_pool->pollers[i]);
-                        } else {
-                                gf_msg ("epoll", GF_LOG_WARNING, 0,
-                                        LG_MSG_START_EPOLL_THREAD_FAILED,
-                                        "Failed to start thread for index %d",
-                                        i);
-                                if (i == 0) {
-                                        GF_FREE (ev_data);
-                                        break;
-                                } else {
-                                        GF_FREE (ev_data);
-                                        continue;
-                                }
-                        }
+    int i = 0;
+    pthread_t t_id;
+    int pollercount = 0;
+    int ret = -1;
+    struct event_thread_data *ev_data = NULL;
+
+    /* Start the configured number of pollers */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        pollercount = event_pool->eventthreadcount;
+
+        /* Set to MAX if greater */
+        if (pollercount > EVENT_MAX_THREADS)
+            pollercount = EVENT_MAX_THREADS;
+
+        /* Default pollers to 1 in case this is incorrectly set */
+        if (pollercount <= 0)
+            pollercount = 1;
+
+        event_pool->activethreadcount++;
+
+        for (i = 0; i < pollercount; i++) {
+            ev_data = GF_CALLOC(1, sizeof(*ev_data), gf_common_mt_event_pool);
+            if (!ev_data) {
+                if (i == 0) {
+                    /* Need to succeed creating 0'th
+                     * thread, to joinable and wait */
+                    break;
+                } else {
+                    /* Inability to create other threads
+                     * are a lesser evil, and ignored */
+                    continue;
+                }
+            }
+
+            ev_data->event_pool = event_pool;
+            ev_data->event_index = i + 1;
+
+            ret = gf_thread_create(&t_id, NULL, event_dispatch_epoll_worker,
+                                   ev_data, "epoll%03hx", i & 0x3ff);
+            if (!ret) {
+                event_pool->pollers[i] = t_id;
+
+                /* mark all threads other than one in index 0
+                 * as detachable. Errors can be ignored, they
+                 * spend their time as zombies if not detched
+                 * and the thread counts are decreased */
+                if (i != 0)
+                    pthread_detach(event_pool->pollers[i]);
+            } else {
+                gf_smsg("epoll", GF_LOG_WARNING, 0,
+                        LG_MSG_START_EPOLL_THREAD_FAILED, "index=%d", i, NULL);
+                if (i == 0) {
+                    GF_FREE(ev_data);
+                    break;
+                } else {
+                    GF_FREE(ev_data);
+                    continue;
                 }
+            }
         }
-        pthread_mutex_unlock (&event_pool->mutex);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-        /* Just wait for the first thread, that is created in a joinable state
-         * and will never die, ensuring this function never returns */
-        if (event_pool->pollers[0] != 0)
-		pthread_join (event_pool->pollers[0], NULL);
+    /* Just wait for the first thread, that is created in a joinable state
+     * and will never die, ensuring this function never returns */
+    if (event_pool->pollers[0] != 0)
+        pthread_join(event_pool->pollers[0], NULL);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                event_pool->activethreadcount--;
-        }
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount--;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-	return ret;
+    return ret;
 }
 
 /**
@@ -776,85 +853,80 @@ event_dispatch_epoll (struct event_pool *event_pool)
  */
 
 static int
-event_pool_dispatched_unlocked (struct event_pool *event_pool)
+event_pool_dispatched_unlocked(struct event_pool *event_pool)
 {
-        return (event_pool->pollers[0] != 0);
-
+    return (event_pool->pollers[0] != 0);
 }
 
-
 int
-event_reconfigure_threads_epoll (struct event_pool *event_pool, int value)
+event_reconfigure_threads_epoll(struct event_pool *event_pool, int value)
 {
-        int                              i;
-        int                              ret = 0;
-        pthread_t                        t_id;
-        int                              oldthreadcount;
-        struct event_thread_data        *ev_data = NULL;
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                /* Reconfigure to 0 threads is allowed only in destroy mode */
-                if (event_pool->destroy == 1) {
-                        value = 0;
-                } else {
-                        /* Set to MAX if greater */
-                        if (value > EVENT_MAX_THREADS)
-                                value = EVENT_MAX_THREADS;
-
-                        /* Default pollers to 1 in case this is set incorrectly */
-                        if (value <= 0)
-                                value = 1;
-                }
+    int i;
+    int ret = 0;
+    pthread_t t_id;
+    int oldthreadcount;
+    struct event_thread_data *ev_data = NULL;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        /* Reconfigure to 0 threads is allowed only in destroy mode */
+        if (event_pool->destroy == 1) {
+            value = 0;
+        } else {
+            /* Set to MAX if greater */
+            if (value > EVENT_MAX_THREADS)
+                value = EVENT_MAX_THREADS;
+
+            /* Default pollers to 1 in case this is set incorrectly */
+            if (value <= 0)
+                value = 1;
+        }
 
-                oldthreadcount = event_pool->eventthreadcount;
-
-                /* Start 'worker' threads as necessary only if event_dispatch()
-                 * was called before. If event_dispatch() was not called, there
-                 * will be no epoll 'worker' threads running yet. */
-
-                if (event_pool_dispatched_unlocked(event_pool)
-                    && (oldthreadcount < value)) {
-                        /* create more poll threads */
-                        for (i = oldthreadcount; i < value; i++) {
-                                /* Start a thread if the index at this location
-                                 * is a 0, so that the older thread is confirmed
-                                 * as dead */
-                                if (event_pool->pollers[i] == 0) {
-                                        ev_data = GF_CALLOC (1,
-                                                      sizeof (*ev_data),
-                                                      gf_common_mt_event_pool);
-                                        if (!ev_data) {
-                                                continue;
-                                        }
-
-                                        ev_data->event_pool = event_pool;
-                                        ev_data->event_index = i + 1;
-
-                                        ret = pthread_create (&t_id, NULL,
-                                                event_dispatch_epoll_worker,
-                                                ev_data);
-                                        if (ret) {
-                                                gf_msg ("epoll", GF_LOG_WARNING,
-                                                        0,
-                                                        LG_MSG_START_EPOLL_THREAD_FAILED,
-                                                        "Failed to start thread"
-                                                        " for index %d", i);
-                                                GF_FREE (ev_data);
-                                        } else {
-                                                pthread_detach (t_id);
-                                                event_pool->pollers[i] = t_id;
-                                        }
-                                }
-                        }
+        oldthreadcount = event_pool->eventthreadcount;
+
+        /* Start 'worker' threads as necessary only if event_dispatch()
+         * was called before. If event_dispatch() was not called, there
+         * will be no epoll 'worker' threads running yet. */
+
+        if (event_pool_dispatched_unlocked(event_pool) &&
+            (oldthreadcount < value)) {
+            /* create more poll threads */
+            for (i = oldthreadcount; i < value; i++) {
+                /* Start a thread if the index at this location
+                 * is a 0, so that the older thread is confirmed
+                 * as dead */
+                if (event_pool->pollers[i] == 0) {
+                    ev_data = GF_CALLOC(1, sizeof(*ev_data),
+                                        gf_common_mt_event_pool);
+                    if (!ev_data) {
+                        continue;
+                    }
+
+                    ev_data->event_pool = event_pool;
+                    ev_data->event_index = i + 1;
+
+                    ret = gf_thread_create(&t_id, NULL,
+                                           event_dispatch_epoll_worker, ev_data,
+                                           "epoll%03hx", i & 0x3ff);
+                    if (ret) {
+                        gf_smsg("epoll", GF_LOG_WARNING, 0,
+                                LG_MSG_START_EPOLL_THREAD_FAILED, "index=%d", i,
+                                NULL);
+                        GF_FREE(ev_data);
+                    } else {
+                        pthread_detach(t_id);
+                        event_pool->pollers[i] = t_id;
+                    }
                 }
-
-                /* if value decreases, threads will terminate, themselves */
-                event_pool->eventthreadcount = value;
+            }
         }
-        pthread_mutex_unlock (&event_pool->mutex);
 
-        return 0;
+        /* if value decreases, threads will terminate, themselves */
+        event_pool->eventthreadcount = value;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return 0;
 }
 
 /* This function is the destructor for the event_pool data structure
@@ -862,43 +934,99 @@ event_reconfigure_threads_epoll (struct event_pool *event_pool, int value)
  * else will lead to crashes.
  */
 static int
-event_pool_destroy_epoll (struct event_pool *event_pool)
+event_pool_destroy_epoll(struct event_pool *event_pool)
 {
-        int ret = 0, i = 0, j = 0;
-        struct event_slot_epoll *table = NULL;
-
-        ret = sys_close (event_pool->fd);
-
-        for (i = 0; i < EVENT_EPOLL_TABLES; i++) {
-                if (event_pool->ereg[i]) {
-                        table = event_pool->ereg[i];
-                        event_pool->ereg[i] = NULL;
-                                for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
-                                        LOCK_DESTROY (&table[j].lock);
-                                }
-                        GF_FREE (table);
-                }
+    int ret = 0, i = 0, j = 0;
+    struct event_slot_epoll *table = NULL;
+
+    ret = sys_close(event_pool->fd);
+
+    for (i = 0; i < EVENT_EPOLL_TABLES; i++) {
+        if (event_pool->ereg[i]) {
+            table = event_pool->ereg[i];
+            event_pool->ereg[i] = NULL;
+            for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
+                LOCK_DESTROY(&table[j].lock);
+            }
+            GF_FREE(table);
+        }
+    }
+
+    pthread_mutex_destroy(&event_pool->mutex);
+    pthread_cond_destroy(&event_pool->cond);
+
+    GF_FREE(event_pool->evcache);
+    GF_FREE(event_pool->reg);
+    GF_FREE(event_pool);
+
+    return ret;
+}
+
+static int
+event_handled_epoll(struct event_pool *event_pool, int fd, int idx, int gen)
+{
+    struct event_slot_epoll *slot = NULL;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+    int ret = 0;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        slot->in_handler--;
+
+        if (gen != slot->gen) {
+            /* event_unregister() happened while we were
+               in handler()
+            */
+            gf_msg_debug("epoll", 0,
+                         "generation bumped on idx=%d"
+                         " from gen=%d to slot->gen=%d, fd=%d, "
+                         "slot->fd=%d",
+                         idx, gen, slot->gen, fd, slot->fd);
+            goto unlock;
         }
 
-        pthread_mutex_destroy (&event_pool->mutex);
-        pthread_cond_destroy (&event_pool->cond);
+        /* This call also picks up the changes made by another
+           thread calling event_select_on_epoll() while this
+           thread was busy in handler()
+        */
+        if (slot->in_handler == 0) {
+            epoll_event.events = slot->events;
+            ev_data->idx = idx;
+            ev_data->gen = gen;
+
+            ret = epoll_ctl(event_pool->fd, EPOLL_CTL_MOD, fd, &epoll_event);
+        }
+    }
+unlock:
+    UNLOCK(&slot->lock);
 
-        GF_FREE (event_pool->evcache);
-        GF_FREE (event_pool->reg);
-        GF_FREE (event_pool);
+    event_slot_unref(event_pool, slot, idx);
 
-        return ret;
+    return ret;
 }
 
 struct event_ops event_ops_epoll = {
-        .new                       = event_pool_new_epoll,
-        .event_register            = event_register_epoll,
-        .event_select_on           = event_select_on_epoll,
-        .event_unregister          = event_unregister_epoll,
-        .event_unregister_close    = event_unregister_close_epoll,
-        .event_dispatch            = event_dispatch_epoll,
-        .event_reconfigure_threads = event_reconfigure_threads_epoll,
-        .event_pool_destroy        = event_pool_destroy_epoll
+    .new = event_pool_new_epoll,
+    .event_register = event_register_epoll,
+    .event_select_on = event_select_on_epoll,
+    .event_unregister = event_unregister_epoll,
+    .event_unregister_close = event_unregister_close_epoll,
+    .event_dispatch = event_dispatch_epoll,
+    .event_reconfigure_threads = event_reconfigure_threads_epoll,
+    .event_pool_destroy = event_pool_destroy_epoll,
+    .event_handled = event_handled_epoll,
 };
 
 #endif
diff --git a/libglusterfs/src/event-history.c b/libglusterfs/src/event-history.c
index 95484a4322b..379fed866be 100644
--- a/libglusterfs/src/event-history.c
+++ b/libglusterfs/src/event-history.c
@@ -8,76 +8,75 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "event-history.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/event-history.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 eh_t *
-eh_new (size_t buffer_size, gf_boolean_t use_buffer_once,
-        void (*destroy_buffer_data) (void *data))
+eh_new(size_t buffer_size, gf_boolean_t use_buffer_once,
+       void (*destroy_buffer_data)(void *data))
 {
-        eh_t *history = NULL;
-        buffer_t *buffer = NULL;
+    eh_t *history = NULL;
+    buffer_t *buffer = NULL;
 
-        history = GF_CALLOC (1, sizeof (eh_t), gf_common_mt_eh_t);
-        if (!history) {
-                goto out;
-        }
+    history = GF_CALLOC(1, sizeof(eh_t), gf_common_mt_eh_t);
+    if (!history) {
+        goto out;
+    }
 
-        buffer = cb_buffer_new (buffer_size, use_buffer_once,
-                                destroy_buffer_data);
-        if (!buffer) {
-                GF_FREE (history);
-                history = NULL;
-                goto out;
-        }
+    buffer = cb_buffer_new(buffer_size, use_buffer_once, destroy_buffer_data);
+    if (!buffer) {
+        GF_FREE(history);
+        history = NULL;
+        goto out;
+    }
 
-        history->buffer = buffer;
+    history->buffer = buffer;
 
-        pthread_mutex_init (&history->lock, NULL);
+    pthread_mutex_init(&history->lock, NULL);
 out:
-        return history;
+    return history;
 }
 
 void
-eh_dump (eh_t *history, void *data,
-         int (dump_fn) (circular_buffer_t *buffer, void *data))
+eh_dump(eh_t *history, void *data,
+        int(dump_fn)(circular_buffer_t *buffer, void *data))
 {
-        if (!history) {
-                gf_msg_debug ("event-history", 0, "history is NULL");
-                goto out;
-        }
+    if (!history) {
+        gf_msg_debug("event-history", 0, "history is NULL");
+        goto out;
+    }
 
-        cb_buffer_dump (history->buffer, data, dump_fn);
+    cb_buffer_dump(history->buffer, data, dump_fn);
 
 out:
-        return;
+    return;
 }
 
 int
-eh_save_history (eh_t *history, void *data)
+eh_save_history(eh_t *history, void *data)
 {
-        int   ret = -1;
+    int ret = -1;
 
-        ret = cb_add_entry_buffer (history->buffer, data);
+    ret = cb_add_entry_buffer(history->buffer, data);
 
-        return ret;
+    return ret;
 }
 
 int
-eh_destroy (eh_t *history)
+eh_destroy(eh_t *history)
 {
-        if (!history) {
-                gf_msg ("event-history", GF_LOG_INFO, 0, LG_MSG_INVALID_ARG,
-                        "history for the xlator is NULL");
-                return -1;
-        }
+    if (!history) {
+        gf_msg("event-history", GF_LOG_INFO, 0, LG_MSG_INVALID_ARG,
+               "history for the xlator is NULL");
+        return -1;
+    }
 
-        cb_buffer_destroy (history->buffer);
-        history->buffer = NULL;
+    cb_buffer_destroy(history->buffer);
+    history->buffer = NULL;
 
-        pthread_mutex_destroy (&history->lock);
+    pthread_mutex_destroy(&history->lock);
 
-        GF_FREE (history);
+    GF_FREE(history);
 
-        return 0;
+    return 0;
 }
diff --git a/libglusterfs/src/event-history.h b/libglusterfs/src/event-history.h
deleted file mode 100644
index de5d47cdfee..00000000000
--- a/libglusterfs/src/event-history.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _EH_H
-#define _EH_H
-
-#include "mem-types.h"
-#include "circ-buff.h"
-
-struct event_hist
-{
-        buffer_t *buffer;
-        pthread_mutex_t lock;
-};
-
-typedef struct event_hist eh_t;
-
-void
-eh_dump (eh_t *event , void *data,
-         int (fn) (circular_buffer_t *buffer, void *data));
-
-eh_t *
-eh_new (size_t buffer_size, gf_boolean_t use_buffer_once,
-        void (*destroy_data) (void *data));
-
-int
-eh_save_history (eh_t *history, void *string);
-
-int
-eh_destroy (eh_t *history);
-
-#endif /* _EH_H */
diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c
index 2006e33d33b..2cba963f096 100644
--- a/libglusterfs/src/event-poll.c
+++ b/libglusterfs/src/event-poll.c
@@ -16,486 +16,465 @@
 #include <errno.h>
 #include <string.h>
 
-#include "logging.h"
-#include "event.h"
-#include "mem-pool.h"
-#include "common-utils.h"
-#include "syscall.h"
-#include "libglusterfs-messages.h"
-
-
+#include "glusterfs/logging.h"
+#include "glusterfs/gf-event.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 struct event_slot_poll {
-	int fd;
-	int events;
-	void *data;
-	event_handler_t handler;
+    int fd;
+    int events;
+    void *data;
+    event_handler_t handler;
 };
 
-
 static int
-event_register_poll (struct event_pool *event_pool, int fd,
-                     event_handler_t handler,
-                     void *data, int poll_in, int poll_out);
+event_register_poll(struct event_pool *event_pool, int fd,
+                    event_handler_t handler, void *data, int poll_in,
+                    int poll_out, char notify_poller_death);
 
-
-static int
-__flush_fd (int fd, int idx, void *data,
-            int poll_in, int poll_out, int poll_err)
+static void
+__flush_fd(int fd, int idx, int gen, void *data, int poll_in, int poll_out,
+           int poll_err, char event_thread_died)
 {
-        char buf[64];
-        int ret = -1;
+    char buf[64];
+    int ret = -1;
 
-        if (!poll_in)
-                return ret;
+    if (!poll_in)
+        return;
 
-        do {
-                ret = sys_read (fd, buf, 64);
-                if (ret == -1 && errno != EAGAIN) {
-                        gf_msg ("poll", GF_LOG_ERROR, errno,
-                                LG_MSG_FILE_OP_FAILED, "read on %d returned "
-                                "error", fd);
-                }
-        } while (ret == 64);
+    do {
+        ret = sys_read(fd, buf, 64);
+        if (ret == -1 && errno != EAGAIN) {
+            gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_READ_FILE_FAILED,
+                    "fd=%d", fd, NULL);
+        }
+    } while (ret == 64);
 
-        return ret;
+    return;
 }
 
-
 static int
-__event_getindex (struct event_pool *event_pool, int fd, int idx)
+__event_getindex(struct event_pool *event_pool, int fd, int idx)
 {
-        int  ret = -1;
-        int  i = 0;
+    int ret = -1;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        /* lookup in used space based on index provided */
-        if (idx > -1 && idx < event_pool->used) {
-                if (event_pool->reg[idx].fd == fd) {
-                        ret = idx;
-                        goto out;
-                }
+    /* lookup in used space based on index provided */
+    if (idx > -1 && idx < event_pool->used) {
+        if (event_pool->reg[idx].fd == fd) {
+            ret = idx;
+            goto out;
         }
+    }
 
-        /* search in used space, if lookup fails */
-        for (i = 0; i < event_pool->used; i++) {
-                if (event_pool->reg[i].fd == fd) {
-                        ret = i;
-                        break;
-                }
+    /* search in used space, if lookup fails */
+    for (i = 0; i < event_pool->used; i++) {
+        if (event_pool->reg[i].fd == fd) {
+            ret = i;
+            break;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 static struct event_pool *
-event_pool_new_poll (int count, int eventthreadcount)
+event_pool_new_poll(int count, int eventthreadcount)
 {
-        struct event_pool *event_pool = NULL;
-        int                ret = -1;
+    struct event_pool *event_pool = NULL;
+    int ret = -1;
 
-        event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
+    event_pool = GF_CALLOC(1, sizeof(*event_pool), gf_common_mt_event_pool);
 
-        if (!event_pool)
-                return NULL;
+    if (!event_pool)
+        return NULL;
 
-        event_pool->count = count;
-        event_pool->reg = GF_CALLOC (event_pool->count,
-                                     sizeof (*event_pool->reg),
-                                     gf_common_mt_reg);
+    event_pool->count = count;
+    event_pool->reg = GF_CALLOC(event_pool->count, sizeof(*event_pool->reg),
+                                gf_common_mt_reg);
 
-        if (!event_pool->reg) {
-                GF_FREE (event_pool);
-                return NULL;
-        }
+    if (!event_pool->reg) {
+        GF_FREE(event_pool);
+        return NULL;
+    }
 
-        pthread_mutex_init (&event_pool->mutex, NULL);
+    pthread_mutex_init(&event_pool->mutex, NULL);
 
-        ret = pipe (event_pool->breaker);
+    ret = pipe(event_pool->breaker);
 
-        if (ret == -1) {
-                gf_msg ("poll", GF_LOG_ERROR, errno, LG_MSG_PIPE_CREATE_FAILED,
-                        "pipe creation failed");
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_PIPE_CREATE_FAILED, NULL);
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
 
-        ret = fcntl (event_pool->breaker[0], F_SETFL, O_NONBLOCK);
-        if (ret == -1) {
-                gf_msg ("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED,
-                        "could not set pipe to non blocking mode");
-                sys_close (event_pool->breaker[0]);
-                sys_close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
+    ret = fcntl(event_pool->breaker[0], F_SETFL, O_NONBLOCK);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED, NULL);
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
 
-        ret = fcntl (event_pool->breaker[1], F_SETFL, O_NONBLOCK);
-        if (ret == -1) {
-                gf_msg ("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED,
-                        "could not set pipe to non blocking mode");
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
 
-                sys_close (event_pool->breaker[0]);
-                sys_close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
+    ret = fcntl(event_pool->breaker[1], F_SETFL, O_NONBLOCK);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED, NULL);
 
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
 
-        ret = event_register_poll (event_pool, event_pool->breaker[0],
-                                   __flush_fd, NULL, 1, 0);
-        if (ret == -1) {
-                gf_msg ("poll", GF_LOG_ERROR, 0, LG_MSG_REGISTER_PIPE_FAILED,
-                        "could not register pipe fd with poll event loop");
-                sys_close (event_pool->breaker[0]);
-                sys_close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    ret = event_register_poll(event_pool, event_pool->breaker[0], __flush_fd,
+                              NULL, 1, 0, 0);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_REGISTER_PIPE_FAILED, NULL);
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
 
-        if (eventthreadcount > 1) {
-                gf_msg ("poll", GF_LOG_INFO, 0,
-                        LG_MSG_POLL_IGNORE_MULTIPLE_THREADS, "Currently poll "
-                        "does not use multiple event processing threads, "
-                        "thread count (%d) ignored", eventthreadcount);
-        }
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
 
-        return event_pool;
-}
+    if (eventthreadcount > 1) {
+        gf_smsg("poll", GF_LOG_INFO, 0, LG_MSG_POLL_IGNORE_MULTIPLE_THREADS,
+                "count=%d", eventthreadcount, NULL);
+    }
 
+    /* although, eventhreadcount for poll implementation is always
+     * going to be 1, eventthreadcount needs to be set to 1 so that
+     * rpcsvc_request_handler() thread scaling works flawlessly in
+     * both epoll and poll models
+     */
+    event_pool->eventthreadcount = 1;
+
+    return event_pool;
+}
 
 static int
-event_register_poll (struct event_pool *event_pool, int fd,
-                     event_handler_t handler,
-                     void *data, int poll_in, int poll_out)
+event_register_poll(struct event_pool *event_pool, int fd,
+                    event_handler_t handler, void *data, int poll_in,
+                    int poll_out, char notify_poller_death)
 {
-        int idx = -1;
+    int idx = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                if (event_pool->count == event_pool->used)
-                {
-                        event_pool->count += 256;
-
-                        event_pool->reg = GF_REALLOC (event_pool->reg,
-                                                      event_pool->count *
-                                                      sizeof (*event_pool->reg));
-                        if (!event_pool->reg)
-                                goto unlock;
-                }
-
-                idx = event_pool->used++;
-
-                event_pool->reg[idx].fd = fd;
-                event_pool->reg[idx].events = POLLPRI;
-                event_pool->reg[idx].handler = handler;
-                event_pool->reg[idx].data = data;
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_msg ("poll", GF_LOG_ERROR, 0,
-                                LG_MSG_INVALID_POLL_IN,
-                                "invalid poll_in value %d", poll_in);
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_msg ("poll", GF_LOG_ERROR, 0,
-                                LG_MSG_INVALID_POLL_OUT,
-                                "invalid poll_out value %d", poll_out);
-                        break;
-                }
-
-                event_pool->changed = 1;
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        if (event_pool->count == event_pool->used) {
+            event_pool->count += 256;
+
+            event_pool->reg = GF_REALLOC(
+                event_pool->reg, event_pool->count * sizeof(*event_pool->reg));
+            if (!event_pool->reg)
+                goto unlock;
+        }
+
+        idx = event_pool->used++;
+
+        event_pool->reg[idx].fd = fd;
+        event_pool->reg[idx].events = POLLPRI;
+        event_pool->reg[idx].handler = handler;
+        event_pool->reg[idx].data = data;
+
+        switch (poll_in) {
+            case 1:
+                event_pool->reg[idx].events |= POLLIN;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLIN;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_IN,
+                        "value=%d", poll_in, NULL);
+                break;
+        }
 
+        switch (poll_out) {
+            case 1:
+                event_pool->reg[idx].events |= POLLOUT;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLOUT;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_OUT,
+                        "value=%d", poll_out, NULL);
+                break;
         }
+
+        event_pool->changed = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_unlock(&event_pool->mutex);
 
 out:
-        return idx;
+    return idx;
 }
 
-
 static int
-event_unregister_poll (struct event_pool *event_pool, int fd, int idx_hint)
+event_unregister_poll(struct event_pool *event_pool, int fd, int idx_hint)
 {
-        int idx = -1;
+    int idx = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_msg ("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                event_pool->reg[idx] =  event_pool->reg[--event_pool->used];
-                event_pool->changed = 1;
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, fd, idx_hint);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    fd, "idx_hint=%d", idx_hint, NULL);
+            errno = ENOENT;
+            goto unlock;
         }
+
+        event_pool->reg[idx] = event_pool->reg[--event_pool->used];
+        event_pool->changed = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_unlock(&event_pool->mutex);
 
 out:
-        return idx;
+    return idx;
 }
 
-
 static int
-event_unregister_close_poll (struct event_pool *event_pool, int fd,
-			     int idx_hint)
+event_unregister_close_poll(struct event_pool *event_pool, int fd, int idx_hint)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = event_unregister_poll (event_pool, fd, idx_hint);
+    ret = event_unregister_poll(event_pool, fd, idx_hint);
 
-	sys_close (fd);
+    sys_close(fd);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-event_select_on_poll (struct event_pool *event_pool, int fd, int idx_hint,
-                      int poll_in, int poll_out)
+event_select_on_poll(struct event_pool *event_pool, int fd, int idx_hint,
+                     int poll_in, int poll_out)
 {
-        int idx = -1;
+    int idx = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_msg ("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        /* TODO: log error */
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        /* TODO: log error */
-                        break;
-                }
-
-                if (poll_in + poll_out > -2)
-                        event_pool->changed = 1;
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, fd, idx_hint);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    fd, "idx_hint=%d", idx_hint, NULL);
+            errno = ENOENT;
+            goto unlock;
         }
+
+        switch (poll_in) {
+            case 1:
+                event_pool->reg[idx].events |= POLLIN;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLIN;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                /* TODO: log error */
+                break;
+        }
+
+        switch (poll_out) {
+            case 1:
+                event_pool->reg[idx].events |= POLLOUT;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLOUT;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                /* TODO: log error */
+                break;
+        }
+
+        if (poll_in + poll_out > -2)
+            event_pool->changed = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_unlock(&event_pool->mutex);
 
 out:
-        return idx;
+    return idx;
 }
 
-
 static int
-event_dispatch_poll_handler (struct event_pool *event_pool,
-                             struct pollfd *ufds, int i)
+event_dispatch_poll_handler(struct event_pool *event_pool, struct pollfd *ufds,
+                            int i)
 {
-        event_handler_t  handler = NULL;
-        void            *data = NULL;
-        int              idx = -1;
-        int              ret = 0;
-
-        handler = NULL;
-        data    = NULL;
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, ufds[i].fd, i);
-
-                if (idx == -1) {
-                        gf_msg ("poll", GF_LOG_ERROR, 0,
-                                LG_MSG_INDEX_NOT_FOUND, "index not found for "
-                                "fd=%d (idx_hint=%d)", ufds[i].fd, i);
-                        goto unlock;
-                }
-
-                handler = event_pool->reg[idx].handler;
-                data = event_pool->reg[idx].data;
+    event_handler_t handler = NULL;
+    void *data = NULL;
+    int idx = -1;
+    int ret = 0;
+
+    handler = NULL;
+    data = NULL;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, ufds[i].fd, i);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    ufds[i].fd, "idx_hint=%d", i, NULL);
+            goto unlock;
         }
+
+        handler = event_pool->reg[idx].handler;
+        data = event_pool->reg[idx].data;
+    }
 unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_unlock(&event_pool->mutex);
 
-        if (handler)
-                ret = handler (ufds[i].fd, idx, data,
-                               (ufds[i].revents & (POLLIN|POLLPRI)),
-                               (ufds[i].revents & (POLLOUT)),
-                               (ufds[i].revents & (POLLERR|POLLHUP|POLLNVAL)));
+    if (handler)
+        handler(ufds[i].fd, idx, 0, data,
+                (ufds[i].revents & (POLLIN | POLLPRI)),
+                (ufds[i].revents & (POLLOUT)),
+                (ufds[i].revents & (POLLERR | POLLHUP | POLLNVAL)), 0);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-event_dispatch_poll_resize (struct event_pool *event_pool,
-                            struct pollfd *ufds, int size)
+event_dispatch_poll_resize(struct event_pool *event_pool, struct pollfd *ufds,
+                           int size)
 {
-        int              i = 0;
+    int i = 0;
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                if (event_pool->changed == 0) {
-                        goto unlock;
-                }
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        if (event_pool->changed == 0) {
+            goto unlock;
+        }
 
-                if (event_pool->used > event_pool->evcache_size) {
-                        GF_FREE (event_pool->evcache);
+        if (event_pool->used > event_pool->evcache_size) {
+            GF_FREE(event_pool->evcache);
 
-                        event_pool->evcache = ufds = NULL;
+            event_pool->evcache = ufds = NULL;
 
-                        event_pool->evcache_size = event_pool->used;
+            event_pool->evcache_size = event_pool->used;
 
-                        ufds = GF_CALLOC (sizeof (struct pollfd),
-                                          event_pool->evcache_size,
-                                          gf_common_mt_pollfd);
-                        if (!ufds)
-                                goto unlock;
-                        event_pool->evcache = ufds;
-                }
+            ufds = GF_CALLOC(sizeof(struct pollfd), event_pool->evcache_size,
+                             gf_common_mt_pollfd);
+            if (!ufds)
+                goto unlock;
+            event_pool->evcache = ufds;
+        }
 
-                for (i = 0; i < event_pool->used; i++) {
-                        ufds[i].fd = event_pool->reg[i].fd;
-                        ufds[i].events = event_pool->reg[i].events;
-                        ufds[i].revents = 0;
-                }
+        if (ufds == NULL) {
+            goto unlock;
+        }
 
-                size = i;
+        for (i = 0; i < event_pool->used; i++) {
+            ufds[i].fd = event_pool->reg[i].fd;
+            ufds[i].events = event_pool->reg[i].events;
+            ufds[i].revents = 0;
         }
+
+        size = i;
+    }
 unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_unlock(&event_pool->mutex);
 
-        return size;
+    return size;
 }
 
-
 static int
-event_dispatch_poll (struct event_pool *event_pool)
+event_dispatch_poll(struct event_pool *event_pool)
 {
-        struct pollfd   *ufds = NULL;
-        int              size = 0;
-        int              i = 0;
-        int              ret = -1;
+    struct pollfd *ufds = NULL;
+    int size = 0;
+    int i = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        pthread_mutex_lock (&event_pool->mutex);
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount = 1;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    while (1) {
+        pthread_mutex_lock(&event_pool->mutex);
         {
-                event_pool->activethreadcount = 1;
+            if (event_pool->destroy == 1) {
+                event_pool->activethreadcount = 0;
+                pthread_cond_broadcast(&event_pool->cond);
+                pthread_mutex_unlock(&event_pool->mutex);
+                return 0;
+            }
         }
-        pthread_mutex_unlock (&event_pool->mutex);
-
-        while (1) {
-                pthread_mutex_lock (&event_pool->mutex);
-                {
-                        if (event_pool->destroy == 1) {
-                                event_pool->activethreadcount = 0;
-                                pthread_cond_broadcast (&event_pool->cond);
-                                pthread_mutex_unlock (&event_pool->mutex);
-                                return 0;
-                        }
-                }
-                pthread_mutex_unlock (&event_pool->mutex);
-
-                size = event_dispatch_poll_resize (event_pool, ufds, size);
-                ufds = event_pool->evcache;
-
-                ret = poll (ufds, size, 1);
-
-                if (ret == 0)
-                        /* timeout */
-                        continue;
-
-                if (ret == -1 && errno == EINTR)
-                        /* sys call */
-                        continue;
-
-                for (i = 0; i < size; i++) {
-                        if (!ufds[i].revents)
-                                continue;
-
-                        event_dispatch_poll_handler (event_pool, ufds, i);
-                }
+        pthread_mutex_unlock(&event_pool->mutex);
+
+        size = event_dispatch_poll_resize(event_pool, ufds, size);
+        ufds = event_pool->evcache;
+
+        ret = poll(ufds, size, 1);
+
+        if (ret == 0)
+            /* timeout */
+            continue;
+
+        if (ret == -1 && errno == EINTR)
+            /* sys call */
+            continue;
+
+        for (i = 0; i < size; i++) {
+            if (!ufds[i].revents)
+                continue;
+
+            event_dispatch_poll_handler(event_pool, ufds, i);
         }
+    }
 
 out:
-        return -1;
+    return -1;
 }
 
 int
-event_reconfigure_threads_poll (struct event_pool *event_pool, int value)
+event_reconfigure_threads_poll(struct event_pool *event_pool, int value)
 {
-        /* No-op for poll */
+    /* No-op for poll */
 
-        return 0;
+    return 0;
 }
 
 /* This function is the destructor for the event_pool data structure
@@ -503,33 +482,32 @@ event_reconfigure_threads_poll (struct event_pool *event_pool, int value)
  * else will lead to crashes.
  */
 static int
-event_pool_destroy_poll (struct event_pool *event_pool)
+event_pool_destroy_poll(struct event_pool *event_pool)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = sys_close (event_pool->breaker[0]);
-        if (ret)
-                return ret;
+    ret = sys_close(event_pool->breaker[0]);
+    if (ret)
+        return ret;
 
-        ret = sys_close (event_pool->breaker[1]);
-        if (ret)
-                return ret;
+    ret = sys_close(event_pool->breaker[1]);
+    if (ret)
+        return ret;
 
-        event_pool->breaker[0] = event_pool->breaker[1] = -1;
+    event_pool->breaker[0] = event_pool->breaker[1] = -1;
 
-        GF_FREE (event_pool->reg);
-        GF_FREE (event_pool);
+    GF_FREE(event_pool->reg);
+    GF_FREE(event_pool);
 
-        return ret;
+    return ret;
 }
 
 struct event_ops event_ops_poll = {
-        .new                    = event_pool_new_poll,
-        .event_register         = event_register_poll,
-        .event_select_on        = event_select_on_poll,
-        .event_unregister       = event_unregister_poll,
-        .event_unregister_close = event_unregister_close_poll,
-        .event_dispatch         = event_dispatch_poll,
-        .event_reconfigure_threads = event_reconfigure_threads_poll,
-        .event_pool_destroy     = event_pool_destroy_poll
-};
+    .new = event_pool_new_poll,
+    .event_register = event_register_poll,
+    .event_select_on = event_select_on_poll,
+    .event_unregister = event_unregister_poll,
+    .event_unregister_close = event_unregister_close_poll,
+    .event_dispatch = event_dispatch_poll,
+    .event_reconfigure_threads = event_reconfigure_threads_poll,
+    .event_pool_destroy = event_pool_destroy_poll};
diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c
index 09ecce1599d..402c253ca25 100644
--- a/libglusterfs/src/event.c
+++ b/libglusterfs/src/event.c
@@ -16,262 +16,290 @@
 #include <errno.h>
 #include <string.h>
 
-#include "logging.h"
-#include "event.h"
-#include "mem-pool.h"
-#include "common-utils.h"
-#include "libglusterfs-messages.h"
-#include "syscall.h"
-
-
+#include "glusterfs/gf-event.h"
+#include "glusterfs/timespec.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/libglusterfs-messages.h"
+#include "glusterfs/syscall.h"
 
 struct event_pool *
-event_pool_new (int count, int eventthreadcount)
+gf_event_pool_new(int count, int eventthreadcount)
 {
-        struct event_pool *event_pool = NULL;
-	extern struct event_ops event_ops_poll;
+    struct event_pool *event_pool = NULL;
+    extern struct event_ops event_ops_poll;
 
 #ifdef HAVE_SYS_EPOLL_H
-	extern struct event_ops event_ops_epoll;
+    extern struct event_ops event_ops_epoll;
 
-        event_pool = event_ops_epoll.new (count, eventthreadcount);
+    event_pool = event_ops_epoll.new(count, eventthreadcount);
 
-        if (event_pool) {
-                event_pool->ops = &event_ops_epoll;
-        } else {
-                gf_msg ("event", GF_LOG_WARNING, 0, LG_MSG_FALLBACK_TO_POLL,
-                        "falling back to poll based event handling");
-        }
+    if (event_pool) {
+        event_pool->ops = &event_ops_epoll;
+    } else {
+        gf_msg("event", GF_LOG_WARNING, 0, LG_MSG_FALLBACK_TO_POLL,
+               "falling back to poll based event handling");
+    }
 #endif
 
-        if (!event_pool) {
-                event_pool = event_ops_poll.new (count, eventthreadcount);
+    if (!event_pool) {
+        event_pool = event_ops_poll.new(count, eventthreadcount);
 
-                if (event_pool)
-                        event_pool->ops = &event_ops_poll;
-        }
+        if (event_pool)
+            event_pool->ops = &event_ops_poll;
+    }
 
-        return event_pool;
+    return event_pool;
 }
 
-
 int
-event_register (struct event_pool *event_pool, int fd,
-                event_handler_t handler,
-                void *data, int poll_in, int poll_out)
+gf_event_register(struct event_pool *event_pool, int fd,
+                  event_handler_t handler, void *data, int poll_in,
+                  int poll_out, char notify_poller_death)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        ret = event_pool->ops->event_register (event_pool, fd, handler, data,
-                                               poll_in, poll_out);
+    ret = event_pool->ops->event_register(
+        event_pool, fd, handler, data, poll_in, poll_out, notify_poller_death);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-event_unregister (struct event_pool *event_pool, int fd, int idx)
+gf_event_unregister(struct event_pool *event_pool, int fd, int idx)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        ret = event_pool->ops->event_unregister (event_pool, fd, idx);
+    ret = event_pool->ops->event_unregister(event_pool, fd, idx);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-event_unregister_close (struct event_pool *event_pool, int fd, int idx)
+gf_event_unregister_close(struct event_pool *event_pool, int fd, int idx)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        ret = event_pool->ops->event_unregister_close (event_pool, fd, idx);
+    ret = event_pool->ops->event_unregister_close(event_pool, fd, idx);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-event_select_on (struct event_pool *event_pool, int fd, int idx_hint,
-                 int poll_in, int poll_out)
+gf_event_select_on(struct event_pool *event_pool, int fd, int idx_hint,
+                   int poll_in, int poll_out)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        ret = event_pool->ops->event_select_on (event_pool, fd, idx_hint,
-                                                poll_in, poll_out);
+    ret = event_pool->ops->event_select_on(event_pool, fd, idx_hint, poll_in,
+                                           poll_out);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-event_dispatch (struct event_pool *event_pool)
+gf_event_dispatch(struct event_pool *event_pool)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        ret = event_pool->ops->event_dispatch (event_pool);
-        if (ret)
-                goto out;
+    ret = event_pool->ops->event_dispatch(event_pool);
+    if (ret)
+        goto out;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-event_reconfigure_threads (struct event_pool *event_pool, int value)
+gf_event_reconfigure_threads(struct event_pool *event_pool, int value)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        /* call event refresh function */
-        ret = event_pool->ops->event_reconfigure_threads (event_pool,
-                                                          value);
+    /* call event refresh function */
+    ret = event_pool->ops->event_reconfigure_threads(event_pool, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-event_pool_destroy (struct event_pool *event_pool)
+gf_event_pool_destroy(struct event_pool *event_pool)
 {
-        int ret = -1;
-        int destroy = 0, activethreadcount = 0;
+    int ret = -1;
+    int destroy = 0, activethreadcount = 0;
 
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                destroy = event_pool->destroy;
-                activethreadcount = event_pool->activethreadcount;
-        }
-        pthread_mutex_unlock (&event_pool->mutex);
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        destroy = event_pool->destroy;
+        activethreadcount = event_pool->activethreadcount;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-        if (!destroy || (activethreadcount > 0))
-                goto out;
+    if (!destroy || (activethreadcount > 0)) {
+        goto out;
+    }
 
-        ret = event_pool->ops->event_pool_destroy (event_pool);
+    ret = event_pool->ops->event_pool_destroy(event_pool);
 out:
-        return ret;
+    return ret;
 }
 
-int
-poller_destroy_handler (int fd, int idx, void *data,
-                       int poll_out, int poll_in, int poll_err)
+void
+poller_destroy_handler(int fd, int idx, int gen, void *data, int poll_out,
+                       int poll_in, int poll_err, char event_thread_exit)
 {
-        int readfd = -1;
-        char buf = '\0';
+    struct event_destroy_data *destroy = NULL;
+    int readfd = -1;
+    char buf = '\0';
 
-        readfd = *(int *)data;
-        if (readfd < 0)
-                return -1;
+    destroy = data;
+    readfd = destroy->readfd;
+    if (readfd < 0) {
+        goto out;
+    }
 
-        while (sys_read (readfd, &buf, 1) > 0) {
-        }
-        return 0;
+    while (sys_read(readfd, &buf, 1) > 0) {
+    }
+
+out:
+    gf_event_handled(destroy->pool, fd, idx, gen);
+
+    return;
 }
 
 /* This function destroys all the poller threads.
- * Note: to be called before event_pool_destroy is called.
+ * Note: to be called before gf_event_pool_destroy is called.
  * The order in which cleaning is performed:
  * - Register a pipe fd(this is for waking threads in poll()/epoll_wait())
- * - Set the destroy mode, which this no new event registration will succede
- * - Reconfigure the thread count to 0(this will succede only in destroy mode)
+ * - Set the destroy mode, which this no new event registration will succeed
+ * - Reconfigure the thread count to 0(this will succeed only in destroy mode)
  * - Wake up all the threads in poll() or epoll_wait(), so that they can
  *   destroy themselves.
  * - Wait for the thread to join(which will happen only after all the other
  *   threads are destroyed)
  */
 int
-event_dispatch_destroy (struct event_pool *event_pool)
+gf_event_dispatch_destroy(struct event_pool *event_pool)
 {
-        int  ret     = -1;
-        int  fd[2]   = {-1};
-        int  idx     = -1;
-        int  flags   = 0;
-        struct timespec   sleep_till = {0, };
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        ret = pipe (fd);
-        if (ret < 0)
-                goto out;
-
-        /* Make the read end of the pipe nonblocking */
-        flags = fcntl(fd[0], F_GETFL);
-        flags |= O_NONBLOCK;
-        ret = fcntl(fd[0], F_SETFL, flags);
-        if (ret < 0)
-                goto out;
-
-        /* Make the write end of the pipe nonblocking */
-        flags = fcntl(fd[1], F_GETFL);
-        flags |= O_NONBLOCK;
-        fcntl(fd[1], F_SETFL, flags);
-        if (ret < 0)
-                goto out;
-
-        /* From the main thread register an event on the pipe fd[0],
+    int ret = -1, threadcount = 0;
+    int fd[2] = {-1};
+    int idx = -1;
+    int flags = 0;
+    struct timespec sleep_till = {
+        0,
+    };
+    struct event_destroy_data data = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    ret = pipe(fd);
+    if (ret < 0)
+        goto out;
+
+    /* Make the read end of the pipe nonblocking */
+    flags = fcntl(fd[0], F_GETFL);
+    flags |= O_NONBLOCK;
+    ret = fcntl(fd[0], F_SETFL, flags);
+    if (ret < 0)
+        goto out;
+
+    /* Make the write end of the pipe nonblocking */
+    flags = fcntl(fd[1], F_GETFL);
+    flags |= O_NONBLOCK;
+    ret = fcntl(fd[1], F_SETFL, flags);
+    if (ret < 0)
+        goto out;
+
+    data.pool = event_pool;
+    data.readfd = fd[1];
+
+    /* From the main thread register an event on the pipe fd[0],
+     */
+    idx = gf_event_register(event_pool, fd[0], poller_destroy_handler, &data, 1,
+                            0, 0);
+    if (idx < 0)
+        goto out;
+
+    /* Enter the destroy mode first, set this before reconfiguring to 0
+     * threads, to prevent further reconfigure to thread count > 0.
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        threadcount = event_pool->eventthreadcount;
+        event_pool->destroy = 1;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    ret = gf_event_reconfigure_threads(event_pool, 0);
+    if (ret < 0)
+        goto out;
+
+    /* Write something onto the write end of the pipe(fd[1]) so that
+     * poll wakes up and calls the handler, poller_destroy_handler()
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        /* Write to pipe(fd[1]) and then wait for 1 second or until
+         * a poller thread that is dying, broadcasts. Make sure we
+         * do not loop forever by limiting to 10 retries
          */
-        idx = event_register (event_pool, fd[0], poller_destroy_handler,
-                              &fd[1], 1, 0);
-        if (idx < 0)
-                goto out;
-
-        /* Enter the destroy mode first, set this before reconfiguring to 0
-         * threads, to prevent further reconfigure to thread count > 0.
-         */
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                event_pool->destroy = 1;
+        int retry = 0;
+
+        while (event_pool->activethreadcount > 0 &&
+               (retry++ < (threadcount + 10))) {
+            if (sys_write(fd[1], "dummy", 6) == -1) {
+                break;
+            }
+            timespec_now_realtime(&sleep_till);
+            sleep_till.tv_sec += 1;
+            ret = pthread_cond_timedwait(&event_pool->cond, &event_pool->mutex,
+                                         &sleep_till);
+            if (ret) {
+                gf_msg_debug("event", 0,
+                             "thread cond-timedwait failed "
+                             "active-thread-count: %d, "
+                             "retry: %d",
+                             event_pool->activethreadcount, retry);
+            }
         }
-        pthread_mutex_unlock (&event_pool->mutex);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-        ret = event_reconfigure_threads (event_pool, 0);
-        if (ret < 0)
-                goto out;
+    ret = gf_event_unregister(event_pool, fd[0], idx);
 
-        /* Write something onto the write end of the pipe(fd[1]) so that
-         * poll wakes up and calls the handler, poller_destroy_handler()
-         */
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                /* Write to pipe(fd[1]) and then wait for 1 second or until
-                 * a poller thread that is dying, broadcasts. Make sure we
-                 * do not loop forever by limiting to 10 retries
-                 */
-                int retry = 0;
-
-                while (event_pool->activethreadcount > 0 && retry++ < 10) {
-                        if (sys_write (fd[1], "dummy", 6) == -1)
-                                break;
-                        sleep_till.tv_sec = time (NULL) + 1;
-                        ret = pthread_cond_timedwait (&event_pool->cond,
-                                                      &event_pool->mutex,
-                                                      &sleep_till);
-                }
-        }
-        pthread_mutex_unlock (&event_pool->mutex);
+out:
+    if (fd[0] != -1)
+        sys_close(fd[0]);
+    if (fd[1] != -1)
+        sys_close(fd[1]);
+
+    return ret;
+}
 
-        ret = event_unregister (event_pool, fd[0], idx);
+int
+gf_event_handled(struct event_pool *event_pool, int fd, int idx, int gen)
+{
+    int ret = 0;
 
- out:
-        if (fd[0] != -1)
-                sys_close (fd[0]);
-        if (fd[1] != -1)
-                sys_close (fd[1]);
+    if (event_pool->ops->event_handled)
+        ret = event_pool->ops->event_handled(event_pool, fd, idx, gen);
 
-        return ret;
+    return ret;
 }
diff --git a/libglusterfs/src/event.h b/libglusterfs/src/event.h
deleted file mode 100644
index b01ef24bb8e..00000000000
--- a/libglusterfs/src/event.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _EVENT_H_
-#define _EVENT_H_
-
-#include <pthread.h>
-
-struct event_pool;
-struct event_ops;
-struct event_slot_poll;
-struct event_slot_epoll;
-struct event_data {
-	int idx;
-	int gen;
-} __attribute__ ((__packed__, __may_alias__));
-
-
-typedef int (*event_handler_t) (int fd, int idx, void *data,
-				int poll_in, int poll_out, int poll_err);
-
-#define EVENT_EPOLL_TABLES 1024
-#define EVENT_EPOLL_SLOTS 1024
-#define EVENT_MAX_THREADS  32
-
-struct event_pool {
-	struct event_ops *ops;
-
-	int fd;
-	int breaker[2];
-
-	int count;
-	struct event_slot_poll  *reg;
-	struct event_slot_epoll *ereg[EVENT_EPOLL_TABLES];
-	int slots_used[EVENT_EPOLL_TABLES];
-
-	int used;
-	int changed;
-
-	pthread_mutex_t mutex;
-	pthread_cond_t cond;
-
-	void *evcache;
-	int evcache_size;
-
-        /* NOTE: Currently used only when event processing is done using
-         * epoll. */
-        int eventthreadcount; /* number of event threads to execute. */
-        pthread_t pollers[EVENT_MAX_THREADS]; /* poller thread_id store,
-                                                     * and live status */
-        int destroy;
-        int activethreadcount;
-};
-
-struct event_ops {
-        struct event_pool * (*new) (int count, int eventthreadcount);
-
-        int (*event_register) (struct event_pool *event_pool, int fd,
-                               event_handler_t handler,
-                               void *data, int poll_in, int poll_out);
-
-        int (*event_select_on) (struct event_pool *event_pool, int fd, int idx,
-                                int poll_in, int poll_out);
-
-        int (*event_unregister) (struct event_pool *event_pool, int fd, int idx);
-
-        int (*event_unregister_close) (struct event_pool *event_pool, int fd,
-				       int idx);
-
-        int (*event_dispatch) (struct event_pool *event_pool);
-
-        int (*event_reconfigure_threads) (struct event_pool *event_pool,
-                                          int newcount);
-        int (*event_pool_destroy) (struct event_pool *event_pool);
-};
-
-struct event_pool *event_pool_new (int count, int eventthreadcount);
-int event_select_on (struct event_pool *event_pool, int fd, int idx,
-		     int poll_in, int poll_out);
-int event_register (struct event_pool *event_pool, int fd,
-		    event_handler_t handler,
-		    void *data, int poll_in, int poll_out);
-int event_unregister (struct event_pool *event_pool, int fd, int idx);
-int event_unregister_close (struct event_pool *event_pool, int fd, int idx);
-int event_dispatch (struct event_pool *event_pool);
-int event_reconfigure_threads (struct event_pool *event_pool, int value);
-int event_pool_destroy (struct event_pool *event_pool);
-int event_dispatch_destroy (struct event_pool *event_pool);
-#endif /* _EVENT_H_ */
diff --git a/libglusterfs/src/events.c b/libglusterfs/src/events.c
index b7b513eb39a..33157549897 100644
--- a/libglusterfs/src/events.c
+++ b/libglusterfs/src/events.c
@@ -17,106 +17,120 @@
 #include <stdarg.h>
 #include <string.h>
 #include <netinet/in.h>
-#include <arpa/inet.h>
 #include <netdb.h>
 
-#include "syscall.h"
-#include "mem-pool.h"
-#include "glusterfs.h"
-#include "globals.h"
-#include "events.h"
-
+#include "glusterfs/syscall.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/events.h"
 
 #define EVENT_HOST "127.0.0.1"
 #define EVENT_PORT 24009
 
-
 int
-gf_event (eventtypes_t event, char *fmt, ...)
+_gf_event(eventtypes_t event, const char *fmt, ...)
 {
-        int                ret                   = 0;
-        int                sock                  = -1;
-        char              *eventstr              = NULL;
-        struct             sockaddr_in server;
-        va_list            arguments;
-        char              *msg                   = NULL;
-        glusterfs_ctx_t   *ctx                   = NULL;
-        struct hostent    *host_data;
-        char              *host                  = NULL;
-
-        /* Global context */
-        ctx = THIS->ctx;
-
-        if (event < 0 || event >= EVENT_LAST) {
-                ret = EVENT_ERROR_INVALID_INPUTS;
-                goto out;
-        }
-
-        /* Initialize UDP socket */
-        sock = socket (AF_INET, SOCK_DGRAM, 0);
-        if (sock < 0) {
-                ret = EVENT_ERROR_SOCKET;
-                goto out;
-        }
-
-        /* Get Host name to send message */
-        if (ctx && ctx->cmd_args.volfile_server) {
-                /* If it is client code then volfile_server is set
-                   use that information to push the events. */
-                host_data = gethostbyname (ctx->cmd_args.volfile_server);
-                if (host_data == NULL) {
-                        ret = EVENT_ERROR_RESOLVE;
-                        goto out;
-                }
-                host = inet_ntoa (*(struct in_addr *)(host_data->h_addr));
-        } else {
-                /* Localhost, Use the defined IP for localhost */
-                host = EVENT_HOST;
+    int ret = 0;
+    int sock = -1;
+    char *eventstr = NULL;
+    va_list arguments;
+    char *msg = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char *host = NULL;
+    struct addrinfo hints;
+    struct addrinfo *result = NULL;
+    struct addrinfo *iter_result_ptr = NULL;
+    xlator_t *this = THIS;
+    char *volfile_server_transport = NULL;
+
+    /* Global context */
+    ctx = this->ctx;
+
+    if (event < 0 || event >= EVENT_LAST) {
+        ret = EVENT_ERROR_INVALID_INPUTS;
+        goto out;
+    }
+
+    if (ctx) {
+        volfile_server_transport = ctx->cmd_args.volfile_server_transport;
+    }
+    if (!volfile_server_transport) {
+        volfile_server_transport = "tcp";
+    }
+
+    /* host = NULL returns localhost */
+    if (ctx && ctx->cmd_args.volfile_server &&
+        (strcmp(volfile_server_transport, "unix"))) {
+        /* If it is client code then volfile_server is set
+           use that information to push the events. */
+        host = ctx->cmd_args.volfile_server;
+    }
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
+    hints.ai_socktype = SOCK_DGRAM;
+    hints.ai_flags = AI_ADDRCONFIG;
+
+    if ((getaddrinfo(host, TOSTRING(EVENT_PORT), &hints, &result)) != 0) {
+        ret = EVENT_ERROR_RESOLVE;
+        goto out;
+    }
+
+    // iterate over the result and break when socket creation is success.
+    for (iter_result_ptr = result; iter_result_ptr != NULL;
+         iter_result_ptr = iter_result_ptr->ai_next) {
+        sock = socket(iter_result_ptr->ai_family, iter_result_ptr->ai_socktype,
+                      iter_result_ptr->ai_protocol);
+        if (sock != -1) {
+            break;
         }
-
-        /* Socket Configurations */
-        server.sin_family = AF_INET;
-        server.sin_port = htons (EVENT_PORT);
-        server.sin_addr.s_addr = inet_addr (host);
-        memset (&server.sin_zero, '\0', sizeof (server.sin_zero));
-
-        va_start (arguments, fmt);
-        ret = gf_vasprintf (&msg, fmt, arguments);
-        va_end (arguments);
-
-        if (ret < 0) {
-                ret = EVENT_ERROR_INVALID_INPUTS;
-                goto out;
-        }
-
-        ret = gf_asprintf (&eventstr, "%u %d %s",
-                            (unsigned)time(NULL), event, msg);
-
-        if (ret <= 0) {
-                ret = EVENT_ERROR_MSG_FORMAT;
-                goto out;
-        }
-
-        /* Send Message */
-        if (sendto (sock, eventstr, strlen (eventstr),
-                    0, (struct sockaddr *)&server, sizeof (server)) <= 0) {
-                ret = EVENT_ERROR_SEND;
-        }
-
-        ret = EVENT_SEND_OK;
-
- out:
-        if (sock >= 0) {
-                sys_close (sock);
-        }
-
-        /* Allocated by gf_vasprintf */
-        if (msg)
-                GF_FREE (msg);
-
-        /* Allocated by gf_asprintf */
-        if (eventstr)
-                GF_FREE (eventstr);
-
-        return ret;
+    }
+    /*
+     * If none of the addrinfo structures lead to a successful socket
+     * creation, socket creation has failed.
+     */
+    if (sock < 0) {
+        ret = EVENT_ERROR_SOCKET;
+        goto out;
+    }
+
+    va_start(arguments, fmt);
+    ret = gf_vasprintf(&msg, fmt, arguments);
+    va_end(arguments);
+
+    if (ret < 0) {
+        ret = EVENT_ERROR_INVALID_INPUTS;
+        goto out;
+    }
+
+    ret = gf_asprintf(&eventstr, "%u %d %s", (unsigned)gf_time(), event, msg);
+    GF_FREE(msg);
+    if (ret <= 0) {
+        ret = EVENT_ERROR_MSG_FORMAT;
+        goto out;
+    }
+
+    /* Send Message */
+    if (sendto(sock, eventstr, strlen(eventstr), 0, result->ai_addr,
+               result->ai_addrlen) <= 0) {
+        ret = EVENT_ERROR_SEND;
+        goto out;
+    }
+
+    ret = EVENT_SEND_OK;
+
+out:
+    if (sock >= 0) {
+        sys_close(sock);
+    }
+
+    /* Allocated by gf_asprintf */
+    if (eventstr)
+        GF_FREE(eventstr);
+
+    if (result)
+        freeaddrinfo(result);
+
+    return ret;
 }
diff --git a/libglusterfs/src/events.h b/libglusterfs/src/events.h
deleted file mode 100644
index 6b280a0066f..00000000000
--- a/libglusterfs/src/events.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __EVENTS_H__
-#define __EVENTS_H__
-
-#include "eventtypes.h"
-
-#ifdef USE_EVENTS
-int
-gf_event (eventtypes_t event, char *fmt, ...);
-#else
-static inline int
-gf_event (eventtypes_t event, char *fmt, ...)
-{
-        return 0;
-}
-#endif /* USE_EVENTS */
-
-#endif /* __EVENTS_H__ */
diff --git a/libglusterfs/src/fd-lk.c b/libglusterfs/src/fd-lk.c
index 358cf3b616a..c2d34f81c9c 100644
--- a/libglusterfs/src/fd-lk.c
+++ b/libglusterfs/src/fd-lk.c
@@ -8,480 +8,426 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "fd-lk.h"
-#include "common-utils.h"
-#include "libglusterfs-messages.h"
+#include "glusterfs/fd-lk.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 int32_t
-_fd_lk_delete_lock (fd_lk_ctx_node_t *lock)
+_fd_lk_delete_lock(fd_lk_ctx_node_t *lock)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("fd-lk", lock, out);
+    GF_VALIDATE_OR_GOTO("fd-lk", lock, out);
 
-        list_del_init (&lock->next);
+    list_del_init(&lock->next);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-_fd_lk_destroy_lock (fd_lk_ctx_node_t *lock)
+_fd_lk_destroy_lock(fd_lk_ctx_node_t *lock)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("fd-lk", lock, out);
+    GF_VALIDATE_OR_GOTO("fd-lk", lock, out);
 
-        GF_FREE (lock);
+    GF_FREE(lock);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-_fd_lk_destroy_lock_list (fd_lk_ctx_t *lk_ctx)
+_fd_lk_destroy_lock_list(fd_lk_ctx_t *lk_ctx)
 {
-        int               ret     = -1;
-        fd_lk_ctx_node_t *lk      = NULL;
-        fd_lk_ctx_node_t *tmp     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, out);
-
-        list_for_each_entry_safe (lk, tmp, &lk_ctx->lk_list, next) {
-                _fd_lk_delete_lock (lk);
-                _fd_lk_destroy_lock (lk);
-        }
-        ret = 0;
+    int ret = -1;
+    fd_lk_ctx_node_t *lk = NULL;
+    fd_lk_ctx_node_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, out);
+
+    list_for_each_entry_safe(lk, tmp, &lk_ctx->lk_list, next)
+    {
+        _fd_lk_delete_lock(lk);
+        _fd_lk_destroy_lock(lk);
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-fd_lk_ctx_unref (fd_lk_ctx_t *lk_ctx)
+fd_lk_ctx_unref(fd_lk_ctx_t *lk_ctx)
 {
-        int ref = -1;
+    int ref = -1;
 
-        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, err);
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, err);
 
-        LOCK (&lk_ctx->lock);
-        {
-                ref = --lk_ctx->ref;
-                if (ref < 0)
-                        GF_ASSERT (!ref);
-                if (ref == 0)
-                        _fd_lk_destroy_lock_list (lk_ctx);
-        }
-        UNLOCK (&lk_ctx->lock);
+    ref = GF_ATOMIC_DEC(lk_ctx->ref);
+    if (ref < 0)
+        GF_ASSERT(!ref);
+    if (ref == 0)
+        _fd_lk_destroy_lock_list(lk_ctx);
 
-        if (ref == 0) {
-                LOCK_DESTROY (&lk_ctx->lock);
-                GF_FREE (lk_ctx);
-        }
+    if (ref == 0) {
+        LOCK_DESTROY(&lk_ctx->lock);
+        GF_FREE(lk_ctx);
+    }
 
-        return 0;
+    return 0;
 err:
-        return -1;
-}
-
-fd_lk_ctx_t *
-_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
-{
-        if (!lk_ctx) {
-                gf_msg_callingfn ("fd-lk", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return NULL;
-        }
-
-        ++lk_ctx->ref;
-
-        return lk_ctx;
-}
-
-fd_lk_ctx_t *
-fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
-{
-        fd_lk_ctx_t *new_lk_ctx = NULL;
-
-        if (!lk_ctx) {
-                gf_msg_callingfn ("fd-lk", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return NULL;
-        }
-
-        LOCK (&lk_ctx->lock);
-        {
-                new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
-        }
-        UNLOCK (&lk_ctx->lock);
-
-        return new_lk_ctx;
+    return -1;
 }
 
 fd_lk_ctx_t *
-fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx)
+fd_lk_ctx_ref(fd_lk_ctx_t *lk_ctx)
 {
-        int         ret         = -1;
-        fd_lk_ctx_t *new_lk_ctx = NULL;
-
-        if (!lk_ctx) {
-                goto out;
-        }
-
-        ret = TRY_LOCK (&lk_ctx->lock);
-        if (ret)
-                goto out;
+    if (!lk_ctx) {
+        gf_msg_callingfn("fd-lk", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return NULL;
+    }
 
-        new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
-        UNLOCK (&lk_ctx->lock);
+    GF_ATOMIC_INC(lk_ctx->ref);
 
-out:
-        return new_lk_ctx;
+    return lk_ctx;
 }
 
 fd_lk_ctx_t *
-fd_lk_ctx_create ()
+fd_lk_ctx_create()
 {
-        fd_lk_ctx_t *fd_lk_ctx = NULL;
+    fd_lk_ctx_t *fd_lk_ctx = NULL;
 
-        fd_lk_ctx = GF_CALLOC (1, sizeof (fd_lk_ctx_t),
-                               gf_common_mt_fd_lk_ctx_t);
-        if (!fd_lk_ctx)
-                goto out;
+    fd_lk_ctx = GF_CALLOC(1, sizeof(fd_lk_ctx_t), gf_common_mt_fd_lk_ctx_t);
+    if (!fd_lk_ctx)
+        goto out;
 
-        INIT_LIST_HEAD (&fd_lk_ctx->lk_list);
+    INIT_LIST_HEAD(&fd_lk_ctx->lk_list);
 
-        LOCK_INIT (&fd_lk_ctx->lock);
+    LOCK_INIT(&fd_lk_ctx->lock);
 
-        fd_lk_ctx = fd_lk_ctx_ref (fd_lk_ctx);
+    fd_lk_ctx = fd_lk_ctx_ref(fd_lk_ctx);
 out:
-        return fd_lk_ctx;
+    return fd_lk_ctx;
 }
 
 int
-_fd_lk_insert_lock (fd_lk_ctx_t *lk_ctx,
-                    fd_lk_ctx_node_t *lock)
+_fd_lk_insert_lock(fd_lk_ctx_t *lk_ctx, fd_lk_ctx_node_t *lock)
 {
-        list_add_tail (&lock->next, &lk_ctx->lk_list);
-        return 0;
+    list_add_tail(&lock->next, &lk_ctx->lk_list);
+    return 0;
 }
 
 static off_t
-_fd_lk_get_lock_len (off_t start, off_t end)
+_fd_lk_get_lock_len(off_t start, off_t end)
 {
-        if (end == LLONG_MAX)
-                return 0;
-        else
-                return (end - start + 1);
+    if (end == LLONG_MAX)
+        return 0;
+    else
+        return (end - start + 1);
 }
 
 fd_lk_ctx_node_t *
-fd_lk_ctx_node_new (int32_t cmd, struct gf_flock *flock)
+fd_lk_ctx_node_new(int32_t cmd, struct gf_flock *flock)
 {
-        fd_lk_ctx_node_t  *new_lock  = NULL;
+    fd_lk_ctx_node_t *new_lock = NULL;
 
-        /* TODO: get from mem-pool */
-        new_lock = GF_CALLOC (1, sizeof (fd_lk_ctx_node_t),
-                              gf_common_mt_fd_lk_ctx_node_t);
-        if (!new_lock)
-                goto out;
+    /* TODO: get from mem-pool */
+    new_lock = GF_CALLOC(1, sizeof(fd_lk_ctx_node_t),
+                         gf_common_mt_fd_lk_ctx_node_t);
+    if (!new_lock)
+        goto out;
 
-        new_lock->cmd = cmd;
+    new_lock->cmd = cmd;
 
-        if (flock) {
-                new_lock->fl_type  = flock->l_type;
-                new_lock->fl_start = flock->l_start;
+    if (flock) {
+        new_lock->fl_type = flock->l_type;
+        new_lock->fl_start = flock->l_start;
 
-                if (flock->l_len == 0)
-                        new_lock->fl_end = LLONG_MAX;
-                else
-                        new_lock->fl_end = flock->l_start + flock->l_len - 1;
+        if (flock->l_len == 0)
+            new_lock->fl_end = LLONG_MAX;
+        else
+            new_lock->fl_end = flock->l_start + flock->l_len - 1;
 
-                memcpy (&new_lock->user_flock, flock,
-                        sizeof (struct gf_flock));
-        }
+        memcpy(&new_lock->user_flock, flock, sizeof(struct gf_flock));
+    }
 
-        INIT_LIST_HEAD (&new_lock->next);
+    INIT_LIST_HEAD(&new_lock->next);
 out:
-        return new_lock;
+    return new_lock;
 }
 
 int32_t
-_fd_lk_delete_unlck_locks (fd_lk_ctx_t *lk_ctx)
+_fd_lk_delete_unlck_locks(fd_lk_ctx_t *lk_ctx)
 {
-        int32_t            ret   = -1;
-        fd_lk_ctx_node_t  *tmp   = NULL;
-        fd_lk_ctx_node_t  *lk    = NULL;
+    int32_t ret = -1;
+    fd_lk_ctx_node_t *tmp = NULL;
+    fd_lk_ctx_node_t *lk = NULL;
 
-        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, out);
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, out);
 
-        list_for_each_entry_safe (lk, tmp, &lk_ctx->lk_list, next) {
-                if (lk->fl_type == F_UNLCK) {
-                        _fd_lk_delete_lock (lk);
-                        _fd_lk_destroy_lock (lk);
-                }
+    list_for_each_entry_safe(lk, tmp, &lk_ctx->lk_list, next)
+    {
+        if (lk->fl_type == F_UNLCK) {
+            _fd_lk_delete_lock(lk);
+            _fd_lk_destroy_lock(lk);
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-fd_lk_overlap (fd_lk_ctx_node_t *l1,
-               fd_lk_ctx_node_t *l2)
+fd_lk_overlap(fd_lk_ctx_node_t *l1, fd_lk_ctx_node_t *l2)
 {
-        if (l1->fl_end >= l2->fl_start &&
-            l2->fl_end >= l1->fl_start)
-                return 1;
+    if (l1->fl_end >= l2->fl_start && l2->fl_end >= l1->fl_start)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 fd_lk_ctx_node_t *
-_fd_lk_add_locks (fd_lk_ctx_node_t *l1,
-                  fd_lk_ctx_node_t *l2)
+_fd_lk_add_locks(fd_lk_ctx_node_t *l1, fd_lk_ctx_node_t *l2)
 {
-        fd_lk_ctx_node_t  *sum = NULL;
+    fd_lk_ctx_node_t *sum = NULL;
 
-        sum = fd_lk_ctx_node_new (0, NULL);
-        if (!sum)
-                goto out;
+    sum = fd_lk_ctx_node_new(0, NULL);
+    if (!sum)
+        goto out;
 
-        sum->fl_start = min (l1->fl_start, l2->fl_start);
-        sum->fl_end   = max (l1->fl_end, l2->fl_end);
+    sum->fl_start = min(l1->fl_start, l2->fl_start);
+    sum->fl_end = max(l1->fl_end, l2->fl_end);
 
-        sum->user_flock.l_start = sum->fl_start;
-        sum->user_flock.l_len   = _fd_lk_get_lock_len (sum->fl_start,
-                                                       sum->fl_end);
+    sum->user_flock.l_start = sum->fl_start;
+    sum->user_flock.l_len = _fd_lk_get_lock_len(sum->fl_start, sum->fl_end);
 out:
-        return sum;
+    return sum;
 }
 
 /* Subtract two locks */
 struct _values {
-        fd_lk_ctx_node_t *locks[3];
+    fd_lk_ctx_node_t *locks[3];
 };
 
 int32_t
-_fd_lk_sub_locks (struct _values *v,
-                  fd_lk_ctx_node_t *big,
-                  fd_lk_ctx_node_t *small)
+_fd_lk_sub_locks(struct _values *v, fd_lk_ctx_node_t *big,
+                 fd_lk_ctx_node_t *small)
 {
-        int32_t  ret  = -1;
-
-        if ((big->fl_start == small->fl_start) &&
-            (big->fl_end   == small->fl_end)) {
-                /* both edges coincide with big */
-                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
-                if (!v->locks[0])
-                        goto out;
-
-                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
-
-                v->locks[0]->fl_type            = small->fl_type;
-                v->locks[0]->user_flock.l_type = small->fl_type;
-        } else if ((small->fl_start > big->fl_start) &&
-                   (small->fl_end   < big->fl_end)) {
-                /* small lock is completely inside big lock,
-                   break it down into 3 different locks. */
-                v->locks[0] = fd_lk_ctx_node_new (big->cmd, NULL);
-                if (!v->locks[0])
-                        goto out;
-
-                v->locks[1] = fd_lk_ctx_node_new (small->cmd, NULL);
-                if (!v->locks[1])
-                        goto out;
-
-                v->locks[2] = fd_lk_ctx_node_new (big->cmd, NULL);
-                if (!v->locks[2])
-                        goto out;
-
-                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
-                v->locks[0]->fl_end = small->fl_start - 1;
-                v->locks[0]->user_flock.l_len =
-                        _fd_lk_get_lock_len (v->locks[0]->fl_start,
-                                             v->locks[0]->fl_end);
-
-                memcpy (v->locks[1], small, sizeof (fd_lk_ctx_node_t));
-
-                memcpy (v->locks[2], big, sizeof (fd_lk_ctx_node_t));
-                v->locks[2]->fl_start = small->fl_end + 1;
-                v->locks[2]->user_flock.l_len =
-                        _fd_lk_get_lock_len (v->locks[2]->fl_start,
-                                             v->locks[2]->fl_end);
-        } else if (small->fl_start == big->fl_start) {
-                /* One of the ends co-incide, break the
-                   locks into two separate parts */
-                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
-                if (!v->locks[0])
-                        goto out;
-
-                v->locks[1] = fd_lk_ctx_node_new (big->cmd, NULL);
-                if (!v->locks[1])
-                        goto out;
-
-                memcpy (v->locks[0], small, sizeof (fd_lk_ctx_node_t));
-
-                memcpy (v->locks[1], big, sizeof (fd_lk_ctx_node_t));
-                v->locks[1]->fl_start = small->fl_end + 1;
-                v->locks[1]->user_flock.l_start = small->fl_end + 1;
-        } else if (small->fl_end == big->fl_end) {
-                /* One of the ends co-incide, break the
-                   locks into two separate parts */
-                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
-                if (!v->locks[0])
-                        goto out;
-
-                v->locks[1] = fd_lk_ctx_node_new (big->cmd, NULL);
-                if (!v->locks[1])
-                        goto out;
-
-                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
-                v->locks[0]->fl_end = small->fl_start - 1;
-                v->locks[0]->user_flock.l_len =
-                        _fd_lk_get_lock_len (v->locks[0]->fl_start,
-                                             v->locks[0]->fl_end);
-
-                memcpy (v->locks[1], small, sizeof (fd_lk_ctx_node_t));
-        } else {
-                /* We should never come to this case */
-                GF_ASSERT (!"Invalid case");
-        }
-        ret = 0;
+    int32_t ret = -1;
+
+    if ((big->fl_start == small->fl_start) && (big->fl_end == small->fl_end)) {
+        /* both edges coincide with big */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+
+        v->locks[0]->fl_type = small->fl_type;
+        v->locks[0]->user_flock.l_type = small->fl_type;
+    } else if ((small->fl_start > big->fl_start) &&
+               (small->fl_end < big->fl_end)) {
+        /* small lock is completely inside big lock,
+           break it down into 3 different locks. */
+        v->locks[0] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        v->locks[2] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[2])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[0]->fl_end = small->fl_start - 1;
+        v->locks[0]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[0]->fl_start, v->locks[0]->fl_end);
+
+        memcpy(v->locks[1], small, sizeof(fd_lk_ctx_node_t));
+
+        memcpy(v->locks[2], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[2]->fl_start = small->fl_end + 1;
+        v->locks[2]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[2]->fl_start, v->locks[2]->fl_end);
+    } else if (small->fl_start == big->fl_start) {
+        /* One of the ends co-incide, break the
+           locks into two separate parts */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        memcpy(v->locks[0], small, sizeof(fd_lk_ctx_node_t));
+
+        memcpy(v->locks[1], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[1]->fl_start = small->fl_end + 1;
+        v->locks[1]->user_flock.l_start = small->fl_end + 1;
+    } else if (small->fl_end == big->fl_end) {
+        /* One of the ends co-incide, break the
+           locks into two separate parts */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[0]->fl_end = small->fl_start - 1;
+        v->locks[0]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[0]->fl_start, v->locks[0]->fl_end);
+
+        memcpy(v->locks[1], small, sizeof(fd_lk_ctx_node_t));
+    } else {
+        /* We should never come to this case */
+        GF_ASSERT(!"Invalid case");
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static void
-_fd_lk_insert_and_merge (fd_lk_ctx_t *lk_ctx,
-                         fd_lk_ctx_node_t *lock)
+_fd_lk_insert_and_merge(fd_lk_ctx_t *lk_ctx, fd_lk_ctx_node_t *lock)
 {
-        int32_t               ret     = -1;
-        int32_t               i       = 0;
-        fd_lk_ctx_node_t     *entry   = NULL;
-        fd_lk_ctx_node_t     *t       = NULL;
-        fd_lk_ctx_node_t     *sum     = NULL;
-        struct _values        v       = {.locks = {0, 0, 0 }};
-
-        list_for_each_entry_safe (entry, t, &lk_ctx->lk_list, next) {
-                if (!fd_lk_overlap (entry, lock))
-                        continue;
-
-                if (entry->fl_type == lock->fl_type) {
-                        sum = _fd_lk_add_locks (entry, lock);
-                        if (!sum)
-                                return;
-                        sum->fl_type = entry->fl_type;
-                        sum->user_flock.l_type = entry->fl_type;
-                        _fd_lk_delete_lock (entry);
-                        _fd_lk_destroy_lock (entry);
-                        _fd_lk_destroy_lock (lock);
-                        _fd_lk_insert_and_merge (lk_ctx, sum);
-                        return;
-                } else {
-                        sum = _fd_lk_add_locks (entry, lock);
-                        sum->fl_type = lock->fl_type;
-                        sum->user_flock.l_type = lock->fl_type;
-                        ret = _fd_lk_sub_locks (&v, sum, lock);
-                        if (ret)
-                                return;
-                        _fd_lk_delete_lock (entry);
-                        _fd_lk_destroy_lock (entry);
-
-                        _fd_lk_delete_lock (lock);
-                        _fd_lk_destroy_lock (lock);
-
-                        _fd_lk_destroy_lock (sum);
-
-                        for (i = 0; i < 3; i++) {
-                                if (!v.locks[i])
-                                        continue;
-
-                                INIT_LIST_HEAD (&v.locks[i]->next);
-                                _fd_lk_insert_and_merge (lk_ctx, v.locks[i]);
-                        }
-                        _fd_lk_delete_unlck_locks (lk_ctx);
-                        return;
-                }
-        }
-
-        /* no conflicts, so just insert */
-        if (lock->fl_type != F_UNLCK) {
-                _fd_lk_insert_lock (lk_ctx, lock);
+    int32_t ret = -1;
+    int32_t i = 0;
+    fd_lk_ctx_node_t *entry = NULL;
+    fd_lk_ctx_node_t *t = NULL;
+    fd_lk_ctx_node_t *sum = NULL;
+    struct _values v = {.locks = {0, 0, 0}};
+
+    list_for_each_entry_safe(entry, t, &lk_ctx->lk_list, next)
+    {
+        if (!fd_lk_overlap(entry, lock))
+            continue;
+
+        if (entry->fl_type == lock->fl_type) {
+            sum = _fd_lk_add_locks(entry, lock);
+            if (!sum)
+                return;
+            sum->fl_type = entry->fl_type;
+            sum->user_flock.l_type = entry->fl_type;
+            _fd_lk_delete_lock(entry);
+            _fd_lk_destroy_lock(entry);
+            _fd_lk_destroy_lock(lock);
+            _fd_lk_insert_and_merge(lk_ctx, sum);
+            return;
         } else {
-                _fd_lk_destroy_lock (lock);
+            sum = _fd_lk_add_locks(entry, lock);
+            sum->fl_type = lock->fl_type;
+            sum->user_flock.l_type = lock->fl_type;
+            ret = _fd_lk_sub_locks(&v, sum, lock);
+            if (ret)
+                return;
+            _fd_lk_delete_lock(entry);
+            _fd_lk_destroy_lock(entry);
+
+            _fd_lk_delete_lock(lock);
+            _fd_lk_destroy_lock(lock);
+
+            _fd_lk_destroy_lock(sum);
+
+            for (i = 0; i < 3; i++) {
+                if (!v.locks[i])
+                    continue;
+
+                INIT_LIST_HEAD(&v.locks[i]->next);
+                _fd_lk_insert_and_merge(lk_ctx, v.locks[i]);
+            }
+            _fd_lk_delete_unlck_locks(lk_ctx);
+            return;
         }
+    }
+
+    /* no conflicts, so just insert */
+    if (lock->fl_type != F_UNLCK) {
+        _fd_lk_insert_lock(lk_ctx, lock);
+    } else {
+        _fd_lk_destroy_lock(lock);
+    }
 }
 
 static void
-print_lock_list (fd_lk_ctx_t *lk_ctx)
+print_lock_list(fd_lk_ctx_t *lk_ctx)
 {
-        fd_lk_ctx_node_t    *lk     = NULL;
-
-        gf_msg_debug ("fd-lk", 0, "lock list:");
-
-        list_for_each_entry (lk, &lk_ctx->lk_list, next)
-                gf_msg_debug ("fd-lk", 0, "owner = %s, cmd = %s fl_type = %s,"
-                              " fs_start = %"PRId64", fs_end = %"PRId64", "
-                              "user_flock: l_type = %s, l_start = %"PRId64", "
-                              "l_len = %"PRId64", ",
-                              lkowner_utoa (&lk->user_flock.l_owner),
-                              get_lk_cmd (lk->cmd), get_lk_type (lk->fl_type),
-                              lk->fl_start, lk->fl_end,
-                              get_lk_type (lk->user_flock.l_type),
-                              lk->user_flock.l_start, lk->user_flock.l_len);
+    fd_lk_ctx_node_t *lk = NULL;
+
+    gf_msg_debug("fd-lk", 0, "lock list:");
+
+    list_for_each_entry(lk, &lk_ctx->lk_list, next)
+        gf_msg_debug("fd-lk", 0,
+                     "owner = %s, cmd = %s fl_type = %s,"
+                     " fs_start = %" PRId64 ", fs_end = %" PRId64
+                     ", "
+                     "user_flock: l_type = %s, l_start = %" PRId64
+                     ", "
+                     "l_len = %" PRId64 ", ",
+                     lkowner_utoa(&lk->user_flock.l_owner), get_lk_cmd(lk->cmd),
+                     get_lk_type(lk->fl_type), lk->fl_start, lk->fl_end,
+                     get_lk_type(lk->user_flock.l_type), lk->user_flock.l_start,
+                     lk->user_flock.l_len);
 }
 
 int
-fd_lk_insert_and_merge (fd_t *fd, int32_t cmd,
-                        struct gf_flock *flock)
+fd_lk_insert_and_merge(fd_t *fd, int32_t cmd, struct gf_flock *flock)
 {
-        int32_t              ret      = -1;
-        fd_lk_ctx_t         *lk_ctx   = NULL;
-        fd_lk_ctx_node_t    *lk     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("fd-lk", fd, out);
-        GF_VALIDATE_OR_GOTO ("fd-lk", flock, out);
-
-        lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
-        lk     = fd_lk_ctx_node_new (cmd, flock);
-
-        gf_msg_debug ("fd-lk", 0, "new lock request: owner = %s, fl_type = %s"
-                      ", fs_start = %"PRId64", fs_end = %"PRId64", user_flock:"
-                      " l_type = %s, l_start = %"PRId64", l_len = %"PRId64,
-                      lkowner_utoa (&flock->l_owner),
-                      get_lk_type (lk->fl_type), lk->fl_start, lk->fl_end,
-                      get_lk_type (lk->user_flock.l_type),
-                      lk->user_flock.l_start, lk->user_flock.l_len);
-
-        LOCK (&lk_ctx->lock);
-        {
-                _fd_lk_insert_and_merge (lk_ctx, lk);
-                print_lock_list (lk_ctx);
-        }
-        UNLOCK (&lk_ctx->lock);
-
-        fd_lk_ctx_unref (lk_ctx);
-
-        ret = 0;
+    int32_t ret = -1;
+    fd_lk_ctx_t *lk_ctx = NULL;
+    fd_lk_ctx_node_t *lk = NULL;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", fd, out);
+    GF_VALIDATE_OR_GOTO("fd-lk", flock, out);
+
+    lk_ctx = fd_lk_ctx_ref(fd->lk_ctx);
+    lk = fd_lk_ctx_node_new(cmd, flock);
+
+    gf_msg_debug("fd-lk", 0,
+                 "new lock request: owner = %s, fl_type = %s"
+                 ", fs_start = %" PRId64 ", fs_end = %" PRId64
+                 ", user_flock:"
+                 " l_type = %s, l_start = %" PRId64 ", l_len = %" PRId64,
+                 lkowner_utoa(&flock->l_owner), get_lk_type(lk->fl_type),
+                 lk->fl_start, lk->fl_end, get_lk_type(lk->user_flock.l_type),
+                 lk->user_flock.l_start, lk->user_flock.l_len);
+
+    LOCK(&lk_ctx->lock);
+    {
+        _fd_lk_insert_and_merge(lk_ctx, lk);
+        print_lock_list(lk_ctx);
+    }
+    UNLOCK(&lk_ctx->lock);
+
+    fd_lk_ctx_unref(lk_ctx);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 gf_boolean_t
-fd_lk_ctx_empty (fd_lk_ctx_t *lk_ctx)
+fd_lk_ctx_empty(fd_lk_ctx_t *lk_ctx)
 {
-	gf_boolean_t verdict = _gf_true;
+    gf_boolean_t verdict = _gf_true;
 
-	if (!lk_ctx)
-		return _gf_true;
+    if (!lk_ctx)
+        return _gf_true;
 
-	LOCK (&lk_ctx->lock);
-	{
-		verdict = list_empty (&lk_ctx->lk_list);
-	}
-	UNLOCK (&lk_ctx->lock);
+    LOCK(&lk_ctx->lock);
+    {
+        verdict = list_empty(&lk_ctx->lk_list);
+    }
+    UNLOCK(&lk_ctx->lock);
 
-	return verdict;
+    return verdict;
 }
diff --git a/libglusterfs/src/fd-lk.h b/libglusterfs/src/fd-lk.h
deleted file mode 100644
index 51f62991681..00000000000
--- a/libglusterfs/src/fd-lk.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _FD_LK_H
-#define _FD_LK_H
-
-#include "fd.h"
-#include "locking.h"
-#include "list.h"
-#include "logging.h"
-#include "mem-pool.h"
-#include "mem-types.h"
-#include "glusterfs.h"
-#include "common-utils.h"
-
-#define get_lk_type(type)                                               \
-        type == F_UNLCK ? "F_UNLCK" : (type == F_RDLCK ? "F_RDLCK" : "F_WRLCK")
-
-#define get_lk_cmd(cmd)                                                 \
-        cmd == F_SETLKW ? "F_SETLKW" : (cmd == F_SETLK ? "F_SETLK" : "F_GETLK")
-
-struct _fd;
-
-struct fd_lk_ctx {
-        struct list_head lk_list;
-        int   ref;
-        gf_lock_t lock;
-};
-typedef struct fd_lk_ctx fd_lk_ctx_t;
-
-struct fd_lk_ctx_node {
-        int32_t            cmd;
-        struct gf_flock    user_flock;
-        off_t              fl_start;
-        off_t              fl_end;
-        short              fl_type;
-        struct list_head   next;
-};
-typedef struct fd_lk_ctx_node fd_lk_ctx_node_t;
-
-fd_lk_ctx_t *
-_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
-
-fd_lk_ctx_t *
-fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
-
-fd_lk_ctx_t *
-fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx);
-
-fd_lk_ctx_t *
-fd_lk_ctx_create (void);
-
-int
-fd_lk_insert_and_merge (struct _fd *lk_ctx, int32_t cmd,
-                        struct gf_flock *flock);
-
-int
-fd_lk_ctx_unref (fd_lk_ctx_t *lk_ctx);
-
-gf_boolean_t
-fd_lk_ctx_empty (fd_lk_ctx_t *lk_ctx);
-
-#endif /* _FD_LK_H */
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 118f876c396..62606e91164 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -8,1250 +8,1196 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "fd.h"
-#include "glusterfs.h"
-#include "inode.h"
-#include "dict.h"
-#include "statedump.h"
-#include "libglusterfs-messages.h"
-
+#include "glusterfs/fd.h"
+#include <errno.h>     // for EINVAL, errno, ENOMEM
+#include <inttypes.h>  // for PRIu64
+#include <stdint.h>    // for UINT32_MAX
+#include <string.h>    // for NULL, memcpy, memset, size_t
+#include "glusterfs/statedump.h"
 
 static int
-gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr);
-
+gf_fd_fdtable_expand(fdtable_t *fdtable, uint32_t nr);
 
 fd_t *
-__fd_ref (fd_t *fd);
+__fd_ref(fd_t *fd);
 
 static int
-gf_fd_chain_fd_entries (fdentry_t *entries, uint32_t startidx,
-                        uint32_t endcount)
+gf_fd_chain_fd_entries(fdentry_t *entries, uint32_t startidx, uint32_t endcount)
 {
-        uint32_t        i = 0;
+    uint32_t i = 0;
 
-        if (!entries) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!entries");
-                return -1;
-        }
+    if (!entries) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!entries");
+        return -1;
+    }
 
-        /* Chain only till the second to last entry because we want to
-         * ensure that the last entry has GF_FDTABLE_END.
-         */
-        for (i = startidx; i < (endcount - 1); i++)
-                entries[i].next_free = i + 1;
+    /* Chain only till the second to last entry because we want to
+     * ensure that the last entry has GF_FDTABLE_END.
+     */
+    for (i = startidx; i < (endcount - 1); i++)
+        entries[i].next_free = i + 1;
 
-        /* i has already been incremented up to the last entry. */
-        entries[i].next_free = GF_FDTABLE_END;
+    /* i has already been incremented up to the last entry. */
+    entries[i].next_free = GF_FDTABLE_END;
 
-        return 0;
+    return 0;
 }
 
-
 static int
-gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr)
+gf_fd_fdtable_expand(fdtable_t *fdtable, uint32_t nr)
 {
-        fdentry_t   *oldfds = NULL;
-        uint32_t     oldmax_fds = -1;
-        int          ret = -1;
-
-        if (fdtable == NULL || nr > UINT32_MAX) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                ret = EINVAL;
-                goto out;
-        }
-
-        nr /= (1024 / sizeof (fdentry_t));
-        nr = gf_roundup_next_power_of_two (nr + 1);
-        nr *= (1024 / sizeof (fdentry_t));
-
-        oldfds = fdtable->fdentries;
-        oldmax_fds = fdtable->max_fds;
-
-        fdtable->fdentries = GF_CALLOC (nr, sizeof (fdentry_t),
-                                        gf_common_mt_fdentry_t);
-        if (!fdtable->fdentries) {
-                ret = ENOMEM;
-                goto out;
-        }
-        fdtable->max_fds = nr;
-
-        if (oldfds) {
-                uint32_t cpy = oldmax_fds * sizeof (fdentry_t);
-                memcpy (fdtable->fdentries, oldfds, cpy);
-        }
-
-        gf_fd_chain_fd_entries (fdtable->fdentries, oldmax_fds,
-                                fdtable->max_fds);
-
-        /* Now that expansion is done, we must update the fd list
-         * head pointer so that the fd allocation functions can continue
-         * using the expanded table.
-         */
-        fdtable->first_free = oldmax_fds;
-        GF_FREE (oldfds);
-        ret = 0;
+    fdentry_t *oldfds = NULL;
+    uint32_t oldmax_fds = -1;
+    int ret = -1;
+
+    if (fdtable == NULL || nr > UINT32_MAX) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        ret = EINVAL;
+        goto out;
+    }
+
+    nr /= (1024 / sizeof(fdentry_t));
+    nr = gf_roundup_next_power_of_two(nr + 1);
+    nr *= (1024 / sizeof(fdentry_t));
+
+    oldfds = fdtable->fdentries;
+    oldmax_fds = fdtable->max_fds;
+
+    fdtable->fdentries = GF_CALLOC(nr, sizeof(fdentry_t),
+                                   gf_common_mt_fdentry_t);
+    if (!fdtable->fdentries) {
+        ret = ENOMEM;
+        goto out;
+    }
+    fdtable->max_fds = nr;
+
+    if (oldfds) {
+        uint32_t cpy = oldmax_fds * sizeof(fdentry_t);
+        memcpy(fdtable->fdentries, oldfds, cpy);
+    }
+
+    gf_fd_chain_fd_entries(fdtable->fdentries, oldmax_fds, fdtable->max_fds);
+
+    /* Now that expansion is done, we must update the fd list
+     * head pointer so that the fd allocation functions can continue
+     * using the expanded table.
+     */
+    fdtable->first_free = oldmax_fds;
+    GF_FREE(oldfds);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 fdtable_t *
-gf_fd_fdtable_alloc (void)
+gf_fd_fdtable_alloc(void)
 {
-        fdtable_t *fdtable = NULL;
+    fdtable_t *fdtable = NULL;
 
-        fdtable = GF_CALLOC (1, sizeof (*fdtable), gf_common_mt_fdtable_t);
-        if (!fdtable)
-                return NULL;
+    fdtable = GF_CALLOC(1, sizeof(*fdtable), gf_common_mt_fdtable_t);
+    if (!fdtable)
+        return NULL;
 
-        pthread_mutex_init (&fdtable->lock, NULL);
+    pthread_rwlock_init(&fdtable->lock, NULL);
 
-        pthread_mutex_lock (&fdtable->lock);
-        {
-                gf_fd_fdtable_expand (fdtable, 0);
-        }
-        pthread_mutex_unlock (&fdtable->lock);
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        gf_fd_fdtable_expand(fdtable, 0);
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        return fdtable;
+    return fdtable;
 }
 
-
 static fdentry_t *
-__gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+__gf_fd_fdtable_get_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t       *fdentries = NULL;
+    fdentry_t *fdentries = NULL;
 
-        if (count == NULL) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!count");
-                goto out;
-        }
+    if (count == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!count");
+        goto out;
+    }
 
-        fdentries = fdtable->fdentries;
-        fdtable->fdentries = GF_CALLOC (fdtable->max_fds, sizeof (fdentry_t),
-                                        gf_common_mt_fdentry_t);
-        gf_fd_chain_fd_entries (fdtable->fdentries, 0, fdtable->max_fds);
-        *count = fdtable->max_fds;
+    fdentries = fdtable->fdentries;
+    fdtable->fdentries = GF_CALLOC(fdtable->max_fds, sizeof(fdentry_t),
+                                   gf_common_mt_fdentry_t);
+    gf_fd_chain_fd_entries(fdtable->fdentries, 0, fdtable->max_fds);
+    *count = fdtable->max_fds;
 
 out:
-        return fdentries;
+    return fdentries;
 }
 
-
 fdentry_t *
-gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+gf_fd_fdtable_get_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t       *entries = NULL;
+    fdentry_t *entries = NULL;
 
-        if (fdtable) {
-                pthread_mutex_lock (&fdtable->lock);
-                {
-                        entries = __gf_fd_fdtable_get_all_fds (fdtable, count);
-                }
-                pthread_mutex_unlock (&fdtable->lock);
+    if (fdtable) {
+        pthread_rwlock_wrlock(&fdtable->lock);
+        {
+            entries = __gf_fd_fdtable_get_all_fds(fdtable, count);
         }
+        pthread_rwlock_unlock(&fdtable->lock);
+    }
 
-        return entries;
+    return entries;
 }
 
-
 static fdentry_t *
-__gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count)
+__gf_fd_fdtable_copy_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t *fdentries = NULL;
-        int        i         = 0;
+    fdentry_t *fdentries = NULL;
+    int i = 0;
 
-        if (count == NULL) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!count");
-                goto out;
-        }
+    if (count == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!count");
+        goto out;
+    }
 
-        fdentries = GF_CALLOC (fdtable->max_fds, sizeof (fdentry_t),
-                               gf_common_mt_fdentry_t);
-        if (fdentries == NULL) {
-                goto out;
-        }
+    fdentries = GF_CALLOC(fdtable->max_fds, sizeof(fdentry_t),
+                          gf_common_mt_fdentry_t);
+    if (fdentries == NULL) {
+        goto out;
+    }
 
-        *count = fdtable->max_fds;
+    *count = fdtable->max_fds;
 
-        for (i = 0; i < fdtable->max_fds; i++) {
-                if (fdtable->fdentries[i].fd != NULL) {
-                        fdentries[i].fd = fd_ref (fdtable->fdentries[i].fd);
-                }
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (fdtable->fdentries[i].fd != NULL) {
+            fdentries[i].fd = fd_ref(fdtable->fdentries[i].fd);
         }
+    }
 
 out:
-        return fdentries;
+    return fdentries;
 }
 
-
 fdentry_t *
-gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count)
+gf_fd_fdtable_copy_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t *entries = NULL;
+    fdentry_t *entries = NULL;
 
-        if (fdtable) {
-                pthread_mutex_lock (&fdtable->lock);
-                {
-                        entries = __gf_fd_fdtable_copy_all_fds (fdtable, count);
-                }
-                pthread_mutex_unlock (&fdtable->lock);
+    if (fdtable) {
+        pthread_rwlock_rdlock(&fdtable->lock);
+        {
+            entries = __gf_fd_fdtable_copy_all_fds(fdtable, count);
         }
+        pthread_rwlock_unlock(&fdtable->lock);
+    }
 
-        return entries;
+    return entries;
 }
 
-
 void
-gf_fd_fdtable_destroy (fdtable_t *fdtable)
+gf_fd_fdtable_destroy(fdtable_t *fdtable)
 {
-        struct list_head  list = {0, };
-        fd_t             *fd = NULL;
-        fdentry_t        *fdentries = NULL;
-        uint32_t          fd_count = 0;
-        int32_t           i = 0;
-
-        INIT_LIST_HEAD (&list);
-
-        if (!fdtable) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!fdtable");
-                return;
-        }
-
-        pthread_mutex_lock (&fdtable->lock);
-        {
-                fdentries = __gf_fd_fdtable_get_all_fds (fdtable, &fd_count);
-                GF_FREE (fdtable->fdentries);
-        }
-        pthread_mutex_unlock (&fdtable->lock);
-
-        if (fdentries != NULL) {
-                for (i = 0; i < fd_count; i++) {
-                        fd = fdentries[i].fd;
-                        if (fd != NULL) {
-                                fd_unref (fd);
-                        }
-                }
-
-                GF_FREE (fdentries);
-                pthread_mutex_destroy (&fdtable->lock);
-                GF_FREE (fdtable);
-        }
+    struct list_head list = {
+        0,
+    };
+    fd_t *fd = NULL;
+    fdentry_t *fdentries = NULL;
+    uint32_t fd_count = 0;
+    int32_t i = 0;
+
+    INIT_LIST_HEAD(&list);
+
+    if (!fdtable) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!fdtable");
+        return;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        fdentries = __gf_fd_fdtable_get_all_fds(fdtable, &fd_count);
+        GF_FREE(fdtable->fdentries);
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
+
+    if (fdentries != NULL) {
+        for (i = 0; i < fd_count; i++) {
+            fd = fdentries[i].fd;
+            if (fd != NULL) {
+                fd_unref(fd);
+            }
+        }
+
+        GF_FREE(fdentries);
+        pthread_rwlock_destroy(&fdtable->lock);
+        GF_FREE(fdtable);
+    }
 }
 
-
 int
-gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr)
+gf_fd_unused_get(fdtable_t *fdtable, fd_t *fdptr)
 {
-        int32_t         fd = -1;
-        fdentry_t       *fde = NULL;
-        int             error;
-        int             alloc_attempts = 0;
-
-        if (fdtable == NULL || fdptr == NULL) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return EINVAL;
-        }
-
-        pthread_mutex_lock (&fdtable->lock);
-        {
-        fd_alloc_try_again:
-                if (fdtable->first_free != GF_FDTABLE_END) {
-                        fde = &fdtable->fdentries[fdtable->first_free];
-                        fd = fdtable->first_free;
-                        fdtable->first_free = fde->next_free;
-                        fde->next_free = GF_FDENTRY_ALLOCATED;
-                        fde->fd = fdptr;
-                } else {
-                        /* If this is true, there is something
-                         * seriously wrong with our data structures.
-                         */
-                        if (alloc_attempts >= 2) {
-                                gf_msg ("fd", GF_LOG_ERROR, 0,
-                                        LG_MSG_EXPAND_FD_TABLE_FAILED,
-                                        "multiple attempts to expand fd table"
-                                        " have failed.");
-                                goto out;
-                        }
-                        error = gf_fd_fdtable_expand (fdtable,
-                                                      fdtable->max_fds + 1);
-                        if (error) {
-                                gf_msg ("fd", GF_LOG_ERROR, error,
-                                        LG_MSG_EXPAND_FD_TABLE_FAILED,
-                                        "Cannot expand fdtable");
-                                goto out;
-                        }
-                        ++alloc_attempts;
-                        /* At this point, the table stands expanded
-                         * with the first_free referring to the first
-                         * free entry in the new set of fdentries that
-                         * have just been allocated. That means, the
-                         * above logic should just work.
-                         */
-                        goto fd_alloc_try_again;
-                }
-        }
+    int32_t fd = -1;
+    fdentry_t *fde = NULL;
+    int error;
+    int alloc_attempts = 0;
+
+    if (fdtable == NULL || fdptr == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return EINVAL;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+    fd_alloc_try_again:
+        if (fdtable->first_free != GF_FDTABLE_END) {
+            fde = &fdtable->fdentries[fdtable->first_free];
+            fd = fdtable->first_free;
+            fdtable->first_free = fde->next_free;
+            fde->next_free = GF_FDENTRY_ALLOCATED;
+            fde->fd = fdptr;
+        } else {
+            /* If this is true, there is something
+             * seriously wrong with our data structures.
+             */
+            if (alloc_attempts >= 2) {
+                gf_msg("fd", GF_LOG_ERROR, 0, LG_MSG_EXPAND_FD_TABLE_FAILED,
+                       "multiple attempts to expand fd table"
+                       " have failed.");
+                goto out;
+            }
+            error = gf_fd_fdtable_expand(fdtable, fdtable->max_fds + 1);
+            if (error) {
+                gf_msg("fd", GF_LOG_ERROR, error, LG_MSG_EXPAND_FD_TABLE_FAILED,
+                       "Cannot expand fdtable");
+                goto out;
+            }
+            ++alloc_attempts;
+            /* At this point, the table stands expanded
+             * with the first_free referring to the first
+             * free entry in the new set of fdentries that
+             * have just been allocated. That means, the
+             * above logic should just work.
+             */
+            goto fd_alloc_try_again;
+        }
+    }
 out:
-        pthread_mutex_unlock (&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        return fd;
+    return fd;
 }
 
-
 void
-gf_fd_put (fdtable_t *fdtable, int32_t fd)
+gf_fd_put(fdtable_t *fdtable, int32_t fd)
 {
-        fd_t *fdptr = NULL;
-        fdentry_t *fde = NULL;
-
-        if (fd == GF_ANON_FD_NO)
-                return;
+    fd_t *fdptr = NULL;
+    fdentry_t *fde = NULL;
 
-        if (fdtable == NULL || fd < 0) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return;
-        }
+    if (fd == GF_ANON_FD_NO)
+        return;
 
-        if (!(fd < fdtable->max_fds)) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return;
-        }
+    if (fdtable == NULL || fd < 0) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
 
-        pthread_mutex_lock (&fdtable->lock);
-        {
-                fde = &fdtable->fdentries[fd];
-                /* If the entry is not allocated, put operation must return
-                 * without doing anything.
-                 * This has the potential of masking out any bugs in a user of
-                 * fd that ends up calling gf_fd_put twice for the same fd or
-                 * for an unallocated fd, but it is a price we have to pay for
-                 * ensuring sanity of our fd-table.
-                 */
-                if (fde->next_free != GF_FDENTRY_ALLOCATED)
-                        goto unlock_out;
-                fdptr = fde->fd;
-                fde->fd = NULL;
-                fde->next_free = fdtable->first_free;
-                fdtable->first_free = fd;
-        }
+    if (!(fd < fdtable->max_fds)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        fde = &fdtable->fdentries[fd];
+        /* If the entry is not allocated, put operation must return
+         * without doing anything.
+         * This has the potential of masking out any bugs in a user of
+         * fd that ends up calling gf_fd_put twice for the same fd or
+         * for an unallocated fd, but it is a price we have to pay for
+         * ensuring sanity of our fd-table.
+         */
+        if (fde->next_free != GF_FDENTRY_ALLOCATED)
+            goto unlock_out;
+        fdptr = fde->fd;
+        fde->fd = NULL;
+        fde->next_free = fdtable->first_free;
+        fdtable->first_free = fd;
+    }
 unlock_out:
-        pthread_mutex_unlock (&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        if (fdptr) {
-                fd_unref (fdptr);
-        }
+    if (fdptr) {
+        fd_unref(fdptr);
+    }
 }
 
-
 void
-gf_fdptr_put (fdtable_t *fdtable, fd_t *fd)
+gf_fdptr_put(fdtable_t *fdtable, fd_t *fd)
 {
-        fdentry_t *fde   = NULL;
-        int32_t    i     = 0;
+    fdentry_t *fde = NULL;
+    int32_t i = 0;
 
-        if ((fdtable == NULL) || (fd == NULL)) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return;
-        }
-
-        pthread_mutex_lock (&fdtable->lock);
-        {
-                for (i = 0; i < fdtable->max_fds; i++) {
-                        if (fdtable->fdentries[i].fd == fd) {
-                                fde = &fdtable->fdentries[i];
-                                break;
-                        }
-                }
+    if ((fdtable == NULL) || (fd == NULL)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
 
-                if (fde == NULL) {
-                        gf_msg_callingfn ("fd", GF_LOG_WARNING, 0,
-                                          LG_MSG_FD_NOT_FOUND_IN_FDTABLE,
-                                          "fd (%p) is not present in fdtable",
-                                          fd);
-                        goto unlock_out;
-                }
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        for (i = 0; i < fdtable->max_fds; i++) {
+            if (fdtable->fdentries[i].fd == fd) {
+                fde = &fdtable->fdentries[i];
+                break;
+            }
+        }
 
-                /* If the entry is not allocated, put operation must return
-                 * without doing anything.
-                 * This has the potential of masking out any bugs in a user of
-                 * fd that ends up calling gf_fd_put twice for the same fd or
-                 * for an unallocated fd, but it is a price we have to pay for
-                 * ensuring sanity of our fd-table.
-                 */
-                if (fde->next_free != GF_FDENTRY_ALLOCATED)
-                        goto unlock_out;
-                fde->fd = NULL;
-                fde->next_free = fdtable->first_free;
-                fdtable->first_free = i;
+        if (fde == NULL) {
+            gf_msg_callingfn("fd", GF_LOG_WARNING, 0,
+                             LG_MSG_FD_NOT_FOUND_IN_FDTABLE,
+                             "fd (%p) is not present in fdtable", fd);
+            goto unlock_out;
         }
+
+        /* If the entry is not allocated, put operation must return
+         * without doing anything.
+         * This has the potential of masking out any bugs in a user of
+         * fd that ends up calling gf_fd_put twice for the same fd or
+         * for an unallocated fd, but it is a price we have to pay for
+         * ensuring sanity of our fd-table.
+         */
+        if (fde->next_free != GF_FDENTRY_ALLOCATED)
+            goto unlock_out;
+        fde->fd = NULL;
+        fde->next_free = fdtable->first_free;
+        fdtable->first_free = i;
+    }
 unlock_out:
-        pthread_mutex_unlock (&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        if ((fd != NULL) && (fde != NULL)) {
-                fd_unref (fd);
-        }
+    if ((fd != NULL) && (fde != NULL)) {
+        fd_unref(fd);
+    }
 }
 
-
 fd_t *
-gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd)
+gf_fd_fdptr_get(fdtable_t *fdtable, int64_t fd)
 {
-        fd_t *fdptr = NULL;
+    fd_t *fdptr = NULL;
 
-        if (fdtable == NULL || fd < 0) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
+    if (fdtable == NULL || fd < 0) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
 
-        if (!(fd < fdtable->max_fds)) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
+    if (!(fd < fdtable->max_fds)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
 
-        pthread_mutex_lock (&fdtable->lock);
-        {
-                fdptr = fdtable->fdentries[fd].fd;
-                if (fdptr) {
-                        fd_ref (fdptr);
-                }
+    pthread_rwlock_rdlock(&fdtable->lock);
+    {
+        fdptr = fdtable->fdentries[fd].fd;
+        if (fdptr) {
+            fd_ref(fdptr);
         }
-        pthread_mutex_unlock (&fdtable->lock);
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        return fdptr;
+    return fdptr;
 }
 
-
 fd_t *
-__fd_ref (fd_t *fd)
+__fd_ref(fd_t *fd)
 {
-        ++fd->refcount;
+    GF_ATOMIC_INC(fd->refcount);
 
-        return fd;
+    return fd;
 }
 
-
 fd_t *
-fd_ref (fd_t *fd)
+fd_ref(fd_t *fd)
 {
-        fd_t *refed_fd = NULL;
-
-        if (!fd) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "null fd");
-                return NULL;
-        }
+    if (!fd) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "null fd");
+        return NULL;
+    }
 
-        LOCK (&fd->inode->lock);
-        refed_fd = __fd_ref (fd);
-        UNLOCK (&fd->inode->lock);
+    GF_ATOMIC_INC(fd->refcount);
 
-        return refed_fd;
+    return fd;
 }
 
-
-fd_t *
-__fd_unref (fd_t *fd)
+static void
+fd_destroy(fd_t *fd, gf_boolean_t bound)
 {
-        GF_ASSERT (fd->refcount);
-
-        --fd->refcount;
-
-        return fd;
+    xlator_t *xl = NULL;
+    int i = 0;
+    xlator_t *old_THIS = NULL;
+
+    if (fd == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        goto out;
+    }
+
+    if (fd->inode == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, 0, LG_MSG_FD_INODE_NULL,
+                         "fd->inode is NULL");
+        goto out;
+    }
+    if (!fd->_ctx)
+        goto out;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        for (i = 0; i < fd->xl_count; i++) {
+            if (fd->_ctx[i].key) {
+                xl = fd->_ctx[i].xl_key;
+                old_THIS = THIS;
+                THIS = xl;
+                if (!xl->call_cleanup && xl->cbks->releasedir)
+                    xl->cbks->releasedir(xl, fd);
+                THIS = old_THIS;
+            }
+        }
+    } else {
+        for (i = 0; i < fd->xl_count; i++) {
+            if (fd->_ctx[i].key) {
+                xl = fd->_ctx[i].xl_key;
+                old_THIS = THIS;
+                THIS = xl;
+                if (!xl->call_cleanup && xl->cbks->release)
+                    xl->cbks->release(xl, fd);
+                THIS = old_THIS;
+            }
+        }
+    }
+
+    LOCK_DESTROY(&fd->lock);
+
+    GF_FREE(fd->_ctx);
+    if (bound) {
+        /*Decrease the count only after close happens on file*/
+        LOCK(&fd->inode->lock);
+        {
+            fd->inode->fd_count--;
+        }
+        UNLOCK(&fd->inode->lock);
+    }
+    inode_unref(fd->inode);
+    fd->inode = NULL;
+    fd_lk_ctx_unref(fd->lk_ctx);
+    mem_put(fd);
+out:
+    return;
 }
 
-
-static void
-fd_destroy (fd_t *fd, gf_boolean_t bound)
+void
+fd_close(fd_t *fd)
 {
-        xlator_t    *xl = NULL;
-        int          i = 0;
-        xlator_t    *old_THIS = NULL;
+    xlator_t *xl, *old_THIS;
 
-        if (fd == NULL){
-                gf_msg_callingfn ("xlator", GF_LOG_ERROR,  EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                goto out;
-        }
+    old_THIS = THIS;
 
-        if (fd->inode == NULL){
-                gf_msg_callingfn ("xlator", GF_LOG_ERROR, 0,
-                                  LG_MSG_FD_INODE_NULL,
-                                  "fd->inode is NULL");
-                goto out;
-        }
-        if (!fd->_ctx)
-                goto out;
+    for (xl = fd->inode->table->xl->graph->first; xl != NULL; xl = xl->next) {
+        if (!xl->call_cleanup) {
+            THIS = xl;
 
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                for (i = 0; i <  fd->xl_count; i++) {
-                        if (fd->_ctx[i].key) {
-                                xl = fd->_ctx[i].xl_key;
-                                old_THIS = THIS;
-                                THIS = xl;
-                                if (xl->cbks->releasedir)
-                                        xl->cbks->releasedir (xl, fd);
-                                THIS = old_THIS;
-                        }
+            if (IA_ISDIR(fd->inode->ia_type)) {
+                if (xl->cbks->fdclosedir != NULL) {
+                    xl->cbks->fdclosedir(xl, fd);
                 }
-        } else {
-                for (i = 0; i < fd->xl_count; i++) {
-                        if (fd->_ctx[i].key) {
-                                xl = fd->_ctx[i].xl_key;
-                                old_THIS = THIS;
-                                THIS = xl;
-                                if (xl->cbks->release)
-                                        xl->cbks->release (xl, fd);
-                                THIS = old_THIS;
-                        }
+            } else {
+                if (xl->cbks->fdclose != NULL) {
+                    xl->cbks->fdclose(xl, fd);
                 }
+            }
         }
+    }
 
-        LOCK_DESTROY (&fd->lock);
-
-        GF_FREE (fd->_ctx);
-        if (bound) {
-                /*Decrease the count only after close happens on file*/
-                LOCK (&fd->inode->lock);
-                {
-                        fd->inode->fd_count--;
-                }
-                UNLOCK (&fd->inode->lock);
-        }
-        inode_unref (fd->inode);
-        fd->inode = NULL;
-        fd_lk_ctx_unref (fd->lk_ctx);
-        mem_put (fd);
-out:
-        return;
+    THIS = old_THIS;
 }
 
-
 void
-fd_unref (fd_t *fd)
+fd_unref(fd_t *fd)
 {
-        int32_t refcount = 0;
-        gf_boolean_t bound = _gf_false;
-
-        if (!fd) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "fd is NULL");
-                return;
-        }
+    int32_t refcount = 0;
+    gf_boolean_t bound = _gf_false;
 
-        LOCK (&fd->inode->lock);
-        {
-                __fd_unref (fd);
-                refcount = fd->refcount;
-                if (refcount == 0) {
-                        if (!list_empty (&fd->inode_list)) {
-                                list_del_init (&fd->inode_list);
-                                bound = _gf_true;
-                        }
-                }
-
-        }
-        UNLOCK (&fd->inode->lock);
+    if (!fd) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "fd is NULL");
+        return;
+    }
 
+    LOCK(&fd->inode->lock);
+    {
+        refcount = GF_ATOMIC_DEC(fd->refcount);
         if (refcount == 0) {
-                fd_destroy (fd, bound);
+            if (!list_empty(&fd->inode_list)) {
+                list_del_init(&fd->inode_list);
+                fd->inode->active_fd_count--;
+                bound = _gf_true;
+            }
         }
+    }
+    UNLOCK(&fd->inode->lock);
 
-        return ;
-}
+    if (refcount == 0) {
+        fd_destroy(fd, bound);
+    }
 
+    return;
+}
 
-fd_t *
-__fd_bind (fd_t *fd)
+static fd_t *
+__fd_bind(fd_t *fd)
 {
-        list_del_init (&fd->inode_list);
-        list_add (&fd->inode_list, &fd->inode->fd_list);
-        fd->inode->fd_count++;
+    list_del_init(&fd->inode_list);
+    list_add(&fd->inode_list, &fd->inode->fd_list);
+    fd->inode->fd_count++;
+    fd->inode->active_fd_count++;
 
-        return fd;
+    return fd;
 }
 
-
 fd_t *
-fd_bind (fd_t *fd)
+fd_bind(fd_t *fd)
 {
-        if (!fd || !fd->inode) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!fd || !fd->inode");
-                return NULL;
-        }
+    if (!fd || !fd->inode) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "!fd || !fd->inode");
+        return NULL;
+    }
 
-        LOCK (&fd->inode->lock);
-        {
-                fd = __fd_bind (fd);
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        fd = __fd_bind(fd);
+    }
+    UNLOCK(&fd->inode->lock);
 
-        return fd;
+    return fd;
 }
 
-
 static fd_t *
-__fd_create (inode_t *inode, uint64_t pid)
+fd_allocate(inode_t *inode, uint64_t pid)
 {
-        fd_t *fd = NULL;
+    fd_t *fd;
 
-        if (inode == NULL) {
-                gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return NULL;
-        }
-
-        fd = mem_get0 (inode->table->fd_mem_pool);
-        if (!fd)
-                goto out;
-
-        fd->xl_count = inode->table->xl->graph->xl_count + 1;
-
-        fd->_ctx = GF_CALLOC (1, (sizeof (struct _fd_ctx) * fd->xl_count),
-                              gf_common_mt_fd_ctx);
-        if (!fd->_ctx)
-                goto free_fd;
-
-        fd->lk_ctx = fd_lk_ctx_create ();
-        if (!fd->lk_ctx)
-                goto free_fd_ctx;
+    if (inode == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return NULL;
+    }
 
-        fd->inode = inode_ref (inode);
+    fd = mem_get0(inode->table->fd_mem_pool);
+    if (fd == NULL) {
+        return NULL;
+    }
+
+    fd->xl_count = inode->table->xl->graph->xl_count + 1;
+
+    fd->_ctx = GF_CALLOC(1, (sizeof(struct _fd_ctx) * fd->xl_count),
+                         gf_common_mt_fd_ctx);
+    if (fd->_ctx == NULL) {
+        goto failed;
+    }
+
+    fd->lk_ctx = fd_lk_ctx_create();
+    if (fd->lk_ctx != NULL) {
+        /* We need to take a reference from the inode, but we cannot do it
+         * here because this function can be called with the inode lock taken
+         * and inode_ref() takes the inode's table lock. This is the reverse
+         * of the logical lock acquisition order and can cause a deadlock. So
+         * we simply assign the inode here and we delefate the inode reference
+         * responsibility to the caller (when this function succeeds and the
+         * inode lock is released). This is safe because the caller must hold
+         * a reference of the inode to use it, so it's guaranteed that the
+         * number of references won't reach 0 before the caller finishes.
+         *
+         * TODO: minimize use of locks in favor of atomic operations to avoid
+         *       these dependencies. */
+        fd->inode = inode;
         fd->pid = pid;
-        INIT_LIST_HEAD (&fd->inode_list);
-
-        LOCK_INIT (&fd->lock);
-out:
+        INIT_LIST_HEAD(&fd->inode_list);
+        LOCK_INIT(&fd->lock);
+        GF_ATOMIC_INIT(fd->refcount, 1);
         return fd;
+    }
 
-free_fd_ctx:
-        GF_FREE (fd->_ctx);
-free_fd:
-        mem_put (fd);
+    GF_FREE(fd->_ctx);
 
-        return NULL;
-}
+failed:
+    mem_put(fd);
 
+    return NULL;
+}
 
 fd_t *
-fd_create (inode_t *inode, pid_t pid)
+fd_create_uint64(inode_t *inode, uint64_t pid)
 {
-        fd_t *fd = NULL;
-
-        fd = __fd_create (inode, (uint64_t)pid);
-        if (!fd)
-                goto out;
+    fd_t *fd;
 
-        fd = fd_ref (fd);
+    fd = fd_allocate(inode, pid);
+    if (fd != NULL) {
+        /* fd_allocate() doesn't get a reference from the inode. We need to
+         * take it here in case of success. */
+        inode_ref(inode);
+    }
 
-out:
-        return fd;
+    return fd;
 }
 
 fd_t *
-fd_create_uint64 (inode_t *inode, uint64_t pid)
+fd_create(inode_t *inode, pid_t pid)
 {
-        fd_t *fd = NULL;
-
-        fd = __fd_create (inode, pid);
-        if (!fd)
-                goto out;
-
-        fd = fd_ref (fd);
-
-out:
-        return fd;
+    return fd_create_uint64(inode, (uint64_t)pid);
 }
 
-
 static fd_t *
-__fd_lookup (inode_t *inode, uint64_t pid)
+__fd_lookup(inode_t *inode, uint64_t pid)
 {
-        fd_t *iter_fd = NULL;
-        fd_t *fd = NULL;
-
-        if (list_empty (&inode->fd_list))
-                return NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *fd = NULL;
 
+    if (list_empty(&inode->fd_list))
+        return NULL;
 
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if (iter_fd->anonymous)
-                        /* If someone was interested in getting an
-                           anonymous fd (or was OK getting an anonymous fd),
-                           they can as well call fd_anonymous() directly */
-                        continue;
+    list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+    {
+        if (iter_fd->anonymous)
+            /* If someone was interested in getting an
+               anonymous fd (or was OK getting an anonymous fd),
+               they can as well call fd_anonymous() directly */
+            continue;
 
-                if (!pid || iter_fd->pid == pid) {
-                        fd = __fd_ref (iter_fd);
-                        break;
-                }
+        if (!pid || iter_fd->pid == pid) {
+            fd = __fd_ref(iter_fd);
+            break;
         }
+    }
 
-        return fd;
+    return fd;
 }
 
-
 fd_t *
-fd_lookup (inode_t *inode, pid_t pid)
+fd_lookup(inode_t *inode, pid_t pid)
 {
-        fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-        if (!inode) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!inode");
-                return NULL;
-        }
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
 
-        LOCK (&inode->lock);
-        {
-                fd = __fd_lookup (inode, (uint64_t)pid);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup(inode, (uint64_t)pid);
+    }
+    UNLOCK(&inode->lock);
 
-        return fd;
+    return fd;
 }
 
 fd_t *
-fd_lookup_uint64 (inode_t *inode, uint64_t pid)
+fd_lookup_uint64(inode_t *inode, uint64_t pid)
 {
-        fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-        if (!inode) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!inode");
-                return NULL;
-        }
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
 
-        LOCK (&inode->lock);
-        {
-                fd = __fd_lookup (inode, pid);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup(inode, pid);
+    }
+    UNLOCK(&inode->lock);
 
-        return fd;
+    return fd;
 }
 
 static fd_t *
-__fd_lookup_anonymous (inode_t *inode, int32_t flags)
+__fd_lookup_anonymous(inode_t *inode, int32_t flags)
 {
-        fd_t *iter_fd = NULL;
-        fd_t *fd = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *fd = NULL;
 
-        if (list_empty (&inode->fd_list))
-                return NULL;
+    if (list_empty(&inode->fd_list))
+        return NULL;
 
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if ((iter_fd->anonymous) && (flags == iter_fd->flags)) {
-                        fd = __fd_ref (iter_fd);
-                        break;
-                }
+    list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+    {
+        if ((iter_fd->anonymous) && (flags == iter_fd->flags)) {
+            fd = __fd_ref(iter_fd);
+            break;
         }
+    }
 
-        return fd;
+    return fd;
 }
 
-static fd_t *
-__fd_anonymous (inode_t *inode, int32_t flags)
+fd_t *
+fd_anonymous_with_flags(inode_t *inode, int32_t flags)
 {
-        fd_t *fd = NULL;
-
-        fd = __fd_lookup_anonymous (inode, flags);
+    fd_t *fd = NULL;
+    bool ref = false;
 
-        /* if (fd); then we already have increased the refcount in
-           __fd_lookup_anonymous(), so no need of one more fd_ref().
-           if (!fd); then both create and bind wont bump up the ref
-           count, so we have to call fd_ref() after bind. */
-        if (!fd) {
-                fd = __fd_create (inode, 0);
+    LOCK(&inode->lock);
 
-                if (!fd)
-                        return NULL;
+    fd = __fd_lookup_anonymous(inode, flags);
 
-                fd->anonymous = _gf_true;
-                fd->flags = GF_ANON_FD_FLAGS|flags;
+    /* if (fd); then we already have increased the refcount in
+       __fd_lookup_anonymous(), so no need of one more fd_ref().
+       if (!fd); then both create and bind won't bump up the ref
+       count, so we have to call fd_ref() after bind. */
+    if (fd == NULL) {
+        fd = fd_allocate(inode, 0);
+        if (fd != NULL) {
+            fd->anonymous = _gf_true;
+            fd->flags = GF_ANON_FD_FLAGS | (flags & O_DIRECT);
 
-                __fd_bind (fd);
+            __fd_bind(fd);
 
-                __fd_ref (fd);
+            ref = true;
         }
+    }
 
-        return fd;
-}
-
+    UNLOCK(&inode->lock);
 
-fd_t *
-fd_anonymous (inode_t *inode)
-{
-        fd_t *fd = NULL;
-
-        LOCK (&inode->lock);
-        {
-                fd = __fd_anonymous (inode, GF_ANON_FD_FLAGS);
-        }
-        UNLOCK (&inode->lock);
+    if (ref) {
+        /* fd_allocate() doesn't get a reference from the inode. We need to
+         * take it here in case of success. */
+        inode_ref(inode);
+    }
 
-        return fd;
+    return fd;
 }
 
 fd_t *
-fd_anonymous_with_flags (inode_t *inode, int32_t flags)
+fd_anonymous(inode_t *inode)
 {
-        fd_t *fd = NULL;
-
-        LOCK (&inode->lock);
-        {
-                if (flags & O_DIRECT)
-                        flags = GF_ANON_FD_FLAGS | O_DIRECT;
-                else
-                        flags = GF_ANON_FD_FLAGS;
-
-                fd = __fd_anonymous (inode, flags);
-        }
-        UNLOCK (&inode->lock);
-
-        return fd;
+    return fd_anonymous_with_flags(inode, 0);
 }
 
-fd_t*
-fd_lookup_anonymous (inode_t *inode, int32_t flags)
+fd_t *
+fd_lookup_anonymous(inode_t *inode, int32_t flags)
 {
-        fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-        if (!inode) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "!inode");
-                return NULL;
-        }
-
-        LOCK (&inode->lock);
-        {
-                fd = __fd_lookup_anonymous (inode, flags);
-        }
-        UNLOCK (&inode->lock);
-        return fd;
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
+
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup_anonymous(inode, flags);
+    }
+    UNLOCK(&inode->lock);
+    return fd;
 }
 
 gf_boolean_t
-fd_is_anonymous (fd_t *fd)
+fd_is_anonymous(fd_t *fd)
 {
-        return (fd && fd->anonymous);
+    return (fd && fd->anonymous);
 }
 
-
 uint8_t
-fd_list_empty (inode_t *inode)
+fd_list_empty(inode_t *inode)
 {
-        uint8_t empty = 0;
+    uint8_t empty = 0;
 
-        LOCK (&inode->lock);
-        {
-                empty = list_empty (&inode->fd_list);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        empty = list_empty(&inode->fd_list);
+    }
+    UNLOCK(&inode->lock);
 
-        return empty;
+    return empty;
 }
 
-
 int
-__fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
+__fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value)
 {
-        int             index   = 0, new_xl_count = 0;
-        int             ret     = 0;
-        int             set_idx = -1;
-        void           *begin   = NULL;
-        size_t          diff    = 0;
-        struct _fd_ctx *tmp     = NULL;
-
-	if (!fd || !xlator)
-		return -1;
-
-        for (index = 0; index < fd->xl_count; index++) {
-                if (!fd->_ctx[index].key) {
-                        if (set_idx == -1)
-                                set_idx = index;
-                        /* dont break, to check if key already exists
-                           further on */
-                }
-                if (fd->_ctx[index].xl_key == xlator) {
-                        set_idx = index;
-                        break;
-                }
+    int index = 0, new_xl_count = 0;
+    int ret = 0;
+    int set_idx = -1;
+    void *begin = NULL;
+    size_t diff = 0;
+    struct _fd_ctx *tmp = NULL;
+
+    if (!fd || !xlator)
+        return -1;
+
+    for (index = 0; index < fd->xl_count; index++) {
+        if (!fd->_ctx[index].key) {
+            if (set_idx == -1)
+                set_idx = index;
+            /* don't break, to check if key already exists
+               further on */
         }
+        if (fd->_ctx[index].xl_key == xlator) {
+            set_idx = index;
+            break;
+        }
+    }
 
-        if (set_idx == -1) {
-                set_idx = fd->xl_count;
+    if (set_idx == -1) {
+        set_idx = fd->xl_count;
 
-                new_xl_count = fd->xl_count + xlator->graph->xl_count;
+        new_xl_count = fd->xl_count + xlator->graph->xl_count;
 
-                tmp = GF_REALLOC (fd->_ctx,
-                                  (sizeof (struct _fd_ctx)
-                                   * new_xl_count));
-                if (tmp == NULL) {
-                        ret = -1;
-                        goto out;
-                }
+        tmp = GF_REALLOC(fd->_ctx, (sizeof(struct _fd_ctx) * new_xl_count));
+        if (tmp == NULL) {
+            ret = -1;
+            goto out;
+        }
 
-                fd->_ctx = tmp;
+        fd->_ctx = tmp;
 
-                begin = fd->_ctx;
-                begin += (fd->xl_count * sizeof (struct _fd_ctx));
+        begin = fd->_ctx;
+        begin += (fd->xl_count * sizeof(struct _fd_ctx));
 
-                diff = (new_xl_count - fd->xl_count )
-                        * sizeof (struct _fd_ctx);
+        diff = (new_xl_count - fd->xl_count) * sizeof(struct _fd_ctx);
 
-                memset (begin, 0, diff);
+        memset(begin, 0, diff);
 
-                fd->xl_count = new_xl_count;
-        }
+        fd->xl_count = new_xl_count;
+    }
 
-        fd->_ctx[set_idx].xl_key = xlator;
-        fd->_ctx[set_idx].value1  = value;
+    fd->_ctx[set_idx].xl_key = xlator;
+    fd->_ctx[set_idx].value1 = value;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
+fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value)
 {
-        int ret = 0;
+    int ret = 0;
 
-	if (!fd || !xlator) {
-                gf_msg_callingfn ("fd", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "%p %p", fd, xlator);
-		return -1;
-        }
+    if (!fd || !xlator) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "%p %p", fd, xlator);
+        return -1;
+    }
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_set (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_set(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 int
-__fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value)
+__fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int index = 0;
-        int ret = 0;
+    int index = 0;
+    int ret = 0;
 
-        if (!fd || !xlator)
-                return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        for (index = 0; index < fd->xl_count; index++) {
-                if (fd->_ctx[index].xl_key == xlator)
-                        break;
-        }
+    for (index = 0; index < fd->xl_count; index++) {
+        if (fd->_ctx[index].xl_key == xlator)
+            break;
+    }
 
-        if (index == fd->xl_count) {
-                ret = -1;
-                goto out;
-        }
+    if (index == fd->xl_count) {
+        ret = -1;
+        goto out;
+    }
 
-        if (value)
-                *value = fd->_ctx[index].value1;
+    if (value)
+        *value = fd->_ctx[index].value1;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value)
+fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!fd || !xlator)
-                return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 int
-__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value)
+__fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int index = 0;
-        int ret = 0;
+    int index = 0;
+    int ret = 0;
 
-        if (!fd || !xlator)
-                return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        for (index = 0; index < fd->xl_count; index++) {
-                if (fd->_ctx[index].xl_key == xlator)
-                        break;
-        }
+    for (index = 0; index < fd->xl_count; index++) {
+        if (fd->_ctx[index].xl_key == xlator)
+            break;
+    }
 
-        if (index == fd->xl_count) {
-                ret = -1;
-                goto out;
-        }
+    if (index == fd->xl_count) {
+        ret = -1;
+        goto out;
+    }
 
-        if (value)
-                *value = fd->_ctx[index].value1;
+    if (value)
+        *value = fd->_ctx[index].value1;
 
-        fd->_ctx[index].key   = 0;
-        fd->_ctx[index].value1 = 0;
+    fd->_ctx[index].key = 0;
+    fd->_ctx[index].value1 = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value)
+fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!fd || !xlator)
-                return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_del (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_del(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 void
-fd_dump (fd_t *fd, char *prefix)
+fd_dump(fd_t *fd, char *prefix)
 {
-        char        key[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
 
-        if (!fd)
-                return;
+    if (!fd)
+        return;
 
-        memset(key, 0, sizeof(key));
-        gf_proc_dump_write("pid", "%llu", fd->pid);
-        gf_proc_dump_write("refcount", "%d", fd->refcount);
-        gf_proc_dump_write("flags", "%d", fd->flags);
-
-        if (fd->inode) {
-                gf_proc_dump_build_key (key, "inode", NULL);
-                gf_proc_dump_add_section(key);
-                inode_dump (fd->inode, key);
-        }
+    gf_proc_dump_write("pid", "%" PRIu64, fd->pid);
+    gf_proc_dump_write("refcount", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(fd->refcount));
+    gf_proc_dump_write("flags", "%d", fd->flags);
 
+    if (fd->inode) {
+        gf_proc_dump_build_key(key, "inode", NULL);
+        gf_proc_dump_add_section("%s", key);
+        inode_dump(fd->inode, key);
+    }
 }
 
-
 void
-fdentry_dump (fdentry_t *fdentry, char *prefix)
+fdentry_dump(fdentry_t *fdentry, char *prefix)
 {
-        if (!fdentry)
-                return;
+    if (!fdentry)
+        return;
 
-        if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
-                return;
+    if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
+        return;
 
-        if (fdentry->fd)
-                fd_dump(fdentry->fd, prefix);
+    if (fdentry->fd)
+        fd_dump(fdentry->fd, prefix);
 }
 
-
 void
-fdtable_dump (fdtable_t *fdtable, char *prefix)
+fdtable_dump(fdtable_t *fdtable, char *prefix)
 {
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-        int     ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    int ret = -1;
 
-        if (!fdtable)
-                return;
+    if (!fdtable)
+        return;
 
-        ret = pthread_mutex_trylock (&fdtable->lock);
+    ret = pthread_rwlock_tryrdlock(&fdtable->lock);
+    if (ret)
+        goto out;
 
-        if (ret)
-                goto out;
+    gf_proc_dump_build_key(key, prefix, "refcount");
+    gf_proc_dump_write(key, "%d", fdtable->refcount);
+    gf_proc_dump_build_key(key, prefix, "maxfds");
+    gf_proc_dump_write(key, "%d", fdtable->max_fds);
+    gf_proc_dump_build_key(key, prefix, "first_free");
+    gf_proc_dump_write(key, "%d", fdtable->first_free);
 
-        memset(key, 0, sizeof(key));
-        gf_proc_dump_build_key(key, prefix, "refcount");
-        gf_proc_dump_write(key, "%d", fdtable->refcount);
-        gf_proc_dump_build_key(key, prefix, "maxfds");
-        gf_proc_dump_write(key, "%d", fdtable->max_fds);
-        gf_proc_dump_build_key(key, prefix, "first_free");
-        gf_proc_dump_write(key, "%d", fdtable->first_free);
-
-        for ( i = 0 ; i < fdtable->max_fds; i++) {
-                if (GF_FDENTRY_ALLOCATED ==
-                    fdtable->fdentries[i].next_free) {
-                        gf_proc_dump_build_key(key, prefix, "fdentry[%d]", i);
-                        gf_proc_dump_add_section(key);
-                        fdentry_dump(&fdtable->fdentries[i], key);
-                }
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (GF_FDENTRY_ALLOCATED == fdtable->fdentries[i].next_free) {
+            gf_proc_dump_build_key(key, prefix, "fdentry[%d]", i);
+            gf_proc_dump_add_section("%s", key);
+            fdentry_dump(&fdtable->fdentries[i], key);
         }
+    }
 
-        pthread_mutex_unlock(&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
 out:
-        if (ret != 0)
-                gf_proc_dump_write ("Unable to dump the fdtable",
-                                    "(Lock acquistion failed) %p", fdtable);
-        return;
+    if (ret != 0)
+        gf_proc_dump_write("Unable to dump the fdtable",
+                           "(Lock acquistion failed) %p", fdtable);
+    return;
 }
 
-
 void
-fd_ctx_dump (fd_t *fd, char *prefix)
+fd_ctx_dump(fd_t *fd, char *prefix)
 {
-        struct _fd_ctx *fd_ctx = NULL;
-        xlator_t       *xl     = NULL;
-        int    i               = 0;
-
-
-        if ((fd == NULL) || (fd->_ctx == NULL)) {
-                goto out;
-        }
-
-        LOCK (&fd->lock);
-        {
-                if (fd->_ctx != NULL) {
-                        fd_ctx = GF_CALLOC (fd->xl_count, sizeof (*fd_ctx),
-                                            gf_common_mt_fd_ctx);
-                        if (fd_ctx == NULL) {
-                                goto unlock;
-                        }
-
-                        for (i = 0; i < fd->xl_count; i++) {
-                                fd_ctx[i] = fd->_ctx[i];
-                        }
-                }
-        }
+    struct _fd_ctx *fd_ctx = NULL;
+    xlator_t *xl = NULL;
+    int i = 0;
+
+    if ((fd == NULL) || (fd->_ctx == NULL)) {
+        goto out;
+    }
+
+    LOCK(&fd->lock);
+    {
+        if (fd->_ctx != NULL) {
+            fd_ctx = GF_CALLOC(fd->xl_count, sizeof(*fd_ctx),
+                               gf_common_mt_fd_ctx);
+            if (fd_ctx == NULL) {
+                goto unlock;
+            }
+
+            for (i = 0; i < fd->xl_count; i++) {
+                fd_ctx[i] = fd->_ctx[i];
+            }
+        }
+    }
 unlock:
-        UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
-        if (fd_ctx == NULL) {
-                goto out;
-        }
+    if (fd_ctx == NULL) {
+        goto out;
+    }
 
-        for (i = 0; i < fd->xl_count; i++) {
-                if (fd_ctx[i].xl_key) {
-                        xl = (xlator_t *)(long)fd_ctx[i].xl_key;
-                        if (xl->dumpops && xl->dumpops->fdctx)
-                                xl->dumpops->fdctx (xl, fd);
-                }
+    for (i = 0; i < fd->xl_count; i++) {
+        if (fd_ctx[i].xl_key) {
+            xl = (xlator_t *)(long)fd_ctx[i].xl_key;
+            if (xl->dumpops && xl->dumpops->fdctx)
+                xl->dumpops->fdctx(xl, fd);
         }
+    }
 
 out:
-        GF_FREE (fd_ctx);
+    GF_FREE(fd_ctx);
 
-        return;
+    return;
 }
 
 void
-fdentry_dump_to_dict (fdentry_t *fdentry, char *prefix, dict_t *dict,
-                      int *openfds)
+fdentry_dump_to_dict(fdentry_t *fdentry, char *prefix, dict_t *dict,
+                     int *openfds)
 {
-        char    key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int     ret = -1;
-
-        if (!fdentry)
-                return;
-        if (!dict)
-                return;
-
-        if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
-                return;
-
-        if (fdentry->fd) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pid", prefix);
-                ret = dict_set_int32 (dict, key, fdentry->fd->pid);
-                if (ret)
-                        return;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.refcount", prefix);
-                ret = dict_set_int32 (dict, key, fdentry->fd->refcount);
-                if (ret)
-                        return;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.flags", prefix);
-                ret = dict_set_int32 (dict, key, fdentry->fd->flags);
-
-                (*openfds)++;
-        }
-        return;
-}
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int ret = -1;
 
-void
-fdtable_dump_to_dict (fdtable_t *fdtable, char *prefix, dict_t *dict)
-{
-        char    key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int     i = 0;
-        int     openfds = 0;
-        int     ret = -1;
+    if (!fdentry)
+        return;
+    if (!dict)
+        return;
 
-        if (!fdtable)
-                return;
-        if (!dict)
-                return;
+    if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
+        return;
 
-        ret = pthread_mutex_trylock (&fdtable->lock);
+    if (fdentry->fd) {
+        snprintf(key, sizeof(key), "%s.pid", prefix);
+        ret = dict_set_uint64(dict, key, fdentry->fd->pid);
         if (ret)
-                goto out;
+            return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.fdtable.refcount", prefix);
-        ret = dict_set_int32 (dict, key, fdtable->refcount);
+        snprintf(key, sizeof(key), "%s.refcount", prefix);
+        ret = dict_set_int32(dict, key, GF_ATOMIC_GET(fdentry->fd->refcount));
         if (ret)
-                goto out;
+            return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.fdtable.maxfds", prefix);
-        ret = dict_set_uint32 (dict, key, fdtable->max_fds);
+        snprintf(key, sizeof(key), "%s.flags", prefix);
+        ret = dict_set_int32(dict, key, fdentry->fd->flags);
         if (ret)
-                goto out;
+            return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.fdtable.firstfree", prefix);
-        ret = dict_set_int32 (dict, key, fdtable->first_free);
-        if (ret)
-                goto out;
+        (*openfds)++;
+    }
+    return;
+}
 
-        for (i = 0; i < fdtable->max_fds; i++) {
-                if (GF_FDENTRY_ALLOCATED ==
-                    fdtable->fdentries[i].next_free) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.fdtable.fdentry%d",
-                                  prefix, i);
-                        fdentry_dump_to_dict (&fdtable->fdentries[i], key,
-                                              dict, &openfds);
-                }
+void
+fdtable_dump_to_dict(fdtable_t *fdtable, char *prefix, dict_t *dict)
+{
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int i = 0;
+    int openfds = 0;
+    int ret = -1;
+
+    if (!fdtable)
+        return;
+    if (!dict)
+        return;
+
+    ret = pthread_rwlock_tryrdlock(&fdtable->lock);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.fdtable.refcount", prefix);
+    ret = dict_set_int32(dict, key, fdtable->refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.fdtable.maxfds", prefix);
+    ret = dict_set_uint32(dict, key, fdtable->max_fds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.fdtable.firstfree", prefix);
+    ret = dict_set_int32(dict, key, fdtable->first_free);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (GF_FDENTRY_ALLOCATED == fdtable->fdentries[i].next_free) {
+            snprintf(key, sizeof(key), "%s.fdtable.fdentry%d", prefix, i);
+            fdentry_dump_to_dict(&fdtable->fdentries[i], key, dict, &openfds);
         }
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.fdtable.openfds", prefix);
-        ret = dict_set_int32 (dict, key, openfds);
+    snprintf(key, sizeof(key), "%s.fdtable.openfds", prefix);
+    ret = dict_set_int32(dict, key, openfds);
+    if (ret)
+        goto out;
 
 out:
-        pthread_mutex_unlock (&fdtable->lock);
-        return;
+    pthread_rwlock_unlock(&fdtable->lock);
+    return;
 }
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
deleted file mode 100644
index 31f494a7e8f..00000000000
--- a/libglusterfs/src/fd.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _FD_H
-#define _FD_H
-
-#include "list.h"
-#include <sys/types.h>
-#include <unistd.h>
-#include "glusterfs.h"
-#include "locking.h"
-#include "fd-lk.h"
-#include "common-utils.h"
-
-#define GF_ANON_FD_NO -2
-#define GF_ANON_FD_FLAGS (O_RDWR|O_LARGEFILE)
-
-struct _inode;
-struct _dict;
-struct fd_lk_ctx;
-
-struct _fd_ctx {
-        union {
-                uint64_t  key;
-                void     *xl_key;
-        };
-        union {
-                uint64_t  value1;
-                void     *ptr1;
-        };
-};
-
-struct _fd {
-        uint64_t          pid;
-	int32_t           flags;
-        int32_t           refcount;
-        struct list_head  inode_list;
-        struct _inode    *inode;
-        gf_lock_t         lock; /* used ONLY for manipulating
-                                   'struct _fd_ctx' array (_ctx).*/
-	struct _fd_ctx   *_ctx;
-        int               xl_count; /* Number of xl referred in this fd */
-        struct fd_lk_ctx *lk_ctx;
-        gf_boolean_t      anonymous; /* fd which does not have counterpart open
-                                        fd on backend (server for client, posix
-                                        for server). */
-};
-typedef struct _fd fd_t;
-
-
-struct fd_table_entry {
-        fd_t    *fd;
-        int     next_free;
-};
-typedef struct fd_table_entry fdentry_t;
-
-
-struct _fdtable {
-        int             refcount;
-        uint32_t        max_fds;
-        pthread_mutex_t lock;
-        fdentry_t       *fdentries;
-        int             first_free;
-};
-typedef struct _fdtable fdtable_t;
-
-
-/* Signifies no more entries in the fd table. */
-#define GF_FDTABLE_END  -1
-
-/* This is used to invalidated
- * the next_free value in an fdentry that has been allocated
- */
-#define GF_FDENTRY_ALLOCATED    -2
-
-#include "logging.h"
-#include "xlator.h"
-
-
-void
-gf_fd_put (fdtable_t *fdtable, int32_t fd);
-
-
-fd_t *
-gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd);
-
-
-fdtable_t *
-gf_fd_fdtable_alloc (void);
-
-
-int
-gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr);
-
-
-fdentry_t *
-gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count);
-
-
-void
-gf_fd_fdtable_destroy (fdtable_t *fdtable);
-
-
-fd_t *
-__fd_ref (fd_t *fd);
-
-
-fd_t *
-fd_ref (fd_t *fd);
-
-
-void
-fd_unref (fd_t *fd);
-
-
-fd_t *
-fd_create (struct _inode *inode, pid_t pid);
-
-fd_t *
-fd_create_uint64 (struct _inode *inode, uint64_t pid);
-
-fd_t *
-fd_lookup (struct _inode *inode, pid_t pid);
-
-fd_t *
-fd_lookup_uint64 (struct _inode *inode, uint64_t pid);
-
-fd_t*
-fd_lookup_anonymous (inode_t *inode, int32_t flags);
-
-fd_t *
-fd_anonymous (inode_t *inode);
-
-fd_t *
-fd_anonymous_with_flags (inode_t *inode, int32_t flags);
-
-gf_boolean_t
-fd_is_anonymous (fd_t *fd);
-
-
-uint8_t
-fd_list_empty (struct _inode *inode);
-
-
-fd_t *
-fd_bind (fd_t *fd);
-
-
-int
-fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value);
-
-
-int
-fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-int
-fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-int
-__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-int
-__fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value);
-
-
-int
-__fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-void
-fd_ctx_dump (fd_t *fd, char *prefix);
-
-fdentry_t *
-gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count);
-
-
-void
-gf_fdptr_put (fdtable_t *fdtable, fd_t *fd);
-
-#endif /* _FD_H */
diff --git a/libglusterfs/src/gen-defaults.py b/libglusterfs/src/gen-defaults.py
index f8e76d02ef4..e31d3a9fe8a 100644..100755
--- a/libglusterfs/src/gen-defaults.py
+++ b/libglusterfs/src/gen-defaults.py
@@ -1,5 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import sys
 from generator import ops, fop_subs, cbk_subs, generate
 
@@ -60,21 +61,21 @@ default_@NAME@ (
 """
 
 def gen_defaults ():
-	for name in ops.iterkeys():
-		print generate(FAILURE_CBK_TEMPLATE,name,cbk_subs)
-	for name in ops.iterkeys():
-		print generate(CBK_RESUME_TEMPLATE,name,cbk_subs)
-	for name in ops.iterkeys():
-		print generate(CBK_TEMPLATE,name,cbk_subs)
-	for name in ops.iterkeys():
-		print generate(RESUME_TEMPLATE,name,fop_subs)
-	for name in ops.iterkeys():
-		print generate(FOP_TEMPLATE,name,fop_subs)
+	for name in list(ops.keys()):
+		print(generate(FAILURE_CBK_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(CBK_RESUME_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(CBK_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(RESUME_TEMPLATE, name, fop_subs))
+	for name in list(ops.keys()):
+		print(generate(FOP_TEMPLATE, name, fop_subs))
 
-for l in open(sys.argv[1],'r').readlines():
+for l in open(sys.argv[1], 'r').readlines():
 	if l.find('#pragma generate') != -1:
-		print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
+		print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
 		gen_defaults()
-		print "/* END GENERATED CODE */"
+		print("/* END GENERATED CODE */")
 	else:
-		print l[:-1]
+		print(l[:-1])
diff --git a/libglusterfs/src/generator.py b/libglusterfs/src/generator.py
index 21f5cf7d387..5b7aa4764a0 100755
--- a/libglusterfs/src/generator.py
+++ b/libglusterfs/src/generator.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import string
 
@@ -123,6 +123,8 @@ ops['fstat'] = (
 ops['fsync'] = (
 	('fop-arg',	'fd',			'fd_t *'),
 	('fop-arg',	'flags',		'int32_t'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
 	('fop-arg',	'xdata',		'dict_t *'),
 	('cbk-arg',	'prebuf',		'struct iatt *'),
 	('cbk-arg',	'postbuf',		'struct iatt *'),
@@ -142,6 +144,8 @@ ops['writev'] = (
 	('fop-arg',	'off',			'off_t',			'offset'),
 	('fop-arg',	'flags',		'uint32_t',			'flags'),
 	('fop-arg',	'iobref',		'struct iobref *'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
 	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
 	('cbk-arg',	'prebuf',		'struct iatt *'),
 	('cbk-arg',	'postbuf',		'struct iatt *'),
@@ -154,6 +158,7 @@ ops['readv'] = (
 	('fop-arg',	'size',			'size_t'),
 	('fop-arg',	'offset',		'off_t'),
 	('fop-arg',	'flags',		'uint32_t'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
 	('fop-arg',	'xdata',		'dict_t *'),
 	('cbk-arg',	'vector',		'struct iovec *'),
 	('cbk-arg',	'count',		'int32_t'),
@@ -208,9 +213,9 @@ ops['rename'] = (
 	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
 	('cbk-arg',	'buf',			'struct iatt *'),
 	('cbk-arg',	'preoldparent',	'struct iatt *'),
-	('cbk-arg',	'postoldparent','struct iatt *'),
+	('cbk-arg',	'postoldparent', 'struct iatt *'),
 	('cbk-arg',	'prenewparent',	'struct iatt *'),
-	('cbk-arg',	'postnewparent','struct iatt *'),
+	('cbk-arg',	'postnewparent', 'struct iatt *'),
 	('cbk-arg',	'xdata',		'dict_t *'),
 	('journal',	'entry-op'),
 )
@@ -298,6 +303,8 @@ ops['access'] = (
 ops['ftruncate'] = (
 	('fop-arg',	'fd',			'fd_t *',				'fd'),
 	('fop-arg',	'offset',		'off_t',				'offset'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
 	('fop-arg',	'xdata',		'dict_t *',				'xdata'),
 	('cbk-arg',	'prebuf',		'struct iatt *'),
 	('cbk-arg',	'postbuf',		'struct iatt *'),
@@ -515,8 +522,9 @@ ops['zerofill'] = (
 
 ops['ipc'] = (
 	('fop-arg',	'op',			'int32_t'),
-	('fop-arg',	'xdata',		'dict_t *'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
 	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
 )
 
 ops['seek'] = (
@@ -535,135 +543,183 @@ ops['getspec'] = (
 )
 
 ops['lease'] = (
-        ('fop-arg',     'loc',                  'loc_t *'),
-        ('fop-arg',     'lease',                'struct gf_lease *'),
-        ('fop-arg',     'xdata',                'dict_t *'),
-        ('cbk-arg',     'lease',                'struct gf_lease *'),
-        ('cbk-arg',     'xdata',                'dict_t *'),
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'lease',                'struct gf_lease *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'lease',                'struct gf_lease *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
 )
 
 ops['getactivelk'] = (
-        ('fop-arg',     'loc',                  'loc_t *'),
-        ('fop-arg',     'xdata',                'dict_t *'),
-        ('cbk-arg',     'locklist',             'lock_migration_info_t *'),
-        ('cbk-arg',     'xdata',                'dict_t *'),
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'locklist',             'lock_migration_info_t *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
 )
 
 ops['setactivelk'] = (
-        ('fop-arg',     'loc',                  'loc_t *'),
-        ('fop-arg',     'locklist',             'lock_migration_info_t *'),
-        ('fop-arg',     'xdata',                'dict_t *'),
-        ('cbk-arg',     'xdata',                'dict_t *'),
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'locklist',             'lock_migration_info_t *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['put'] = (
+	('fop-arg',     'loc',                  'loc_t *',                      'loc'),
+	('fop-arg',     'mode',                 'mode_t',                       'mode'),
+	('fop-arg',     'umask',                'mode_t',                       'umask'),
+	('fop-arg',     'flags',                'uint32_t',                     'flags'),
+	('fop-arg',     'vector',               'struct iovec *',               'vector'),
+	('fop-arg',     'count',                'int32_t'),
+	('fop-arg',     'off',                  'off_t',                        'offset'),
+	('fop-arg',     'iobref',               'struct iobref *'),
+	('fop-arg',     'dict',                 'dict_t *',                     'xattr'),
+	('fop-arg',     'xdata',                'dict_t *',                     'xdata'),
+	('cbk-arg',     'inode',                'inode_t *'),
+	('cbk-arg',     'buf',                  'struct iatt *'),
+	('cbk-arg',     'preparent',            'struct iatt *'),
+	('cbk-arg',     'postparent',           'struct iatt *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['icreate'] = (
+	('fop-arg',     'loc',                   'loc_t *'),
+	('fop-arg',     'mode',                  'mode_t'),
+	('fop-arg',     'xdata',                 'dict_t *'),
+	('cbk-arg',     'inode',                 'inode_t *'),
+	('cbk-arg',     'buf',                   'struct iatt *'),
+	('cbk-arg',     'xdata',                 'dict_t *'),
+)
+
+ops['namelink'] = (
+	('fop-arg',     'loc',                   'loc_t *'),
+	('fop-arg',     'xdata',                 'dict_t *'),
+	('cbk-arg',     'prebuf',                'struct iatt *'),
+	('cbk-arg',     'postbuf',               'struct iatt *'),
+	('cbk-arg',     'xdata',                 'dict_t *'),
+)
+
+ops['copy_file_range'] = (
+        ('fop-arg',     'fd_in',                 'fd_t *'),
+        ('fop-arg',     'off_in',                'off64_t '),
+        ('fop-arg',     'fd_out',                'fd_t *'),
+        ('fop-arg',     'off_out',               'off64_t '),
+        ('fop-arg',     'len',                   'size_t'),
+        ('fop-arg',     'flags',                 'uint32_t'),
+        ('fop-arg',     'xdata',                 'dict_t *'),
+        ('cbk-arg',     'stbuf',                 'struct iatt *'),
+        ('cbk-arg',     'prebuf_dst',            'struct iatt *'),
+        ('cbk-arg',     'postbuf_dst',           'struct iatt *'),
+        ('cbk-arg',     'xdata',                 'dict_t *'),
 )
-
 #####################################################################
 xlator_cbks['forget'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'inode',       'inode_t *'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
 	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['release'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'fd',          'fd_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['releasedir'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'fd',          'fd_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['invalidate'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'inode',       'inode_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['client_destroy'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'client',      'client_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'client',      'client_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['client_disconnect'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'client',      'client_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'client',      'client_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_cbks['ictxmerge'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'fd',          'fd_t *'),
-        ('fn-arg',      'inode',       'inode_t *'),
-        ('fn-arg',      'linked_inode', 'inode_t *'),
-        ('ret-val',     'void',        ''),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
+	('fn-arg',      'linked_inode', 'inode_t *'),
+	('ret-val',     'void',        ''),
 )
 
 #####################################################################
 xlator_dumpops['priv'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['inode'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['fd'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['inodectx'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'ino',         'inode_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'ino',         'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['fdctx'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'fd',          'fd_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['priv_to_dict'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'dict',        'dict_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['inode_to_dict'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'dict',        'dict_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['fd_to_dict'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'dict',        'dict_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['inodectx_to_dict'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'ino',         'inode_t *'),
-        ('fn-arg',      'dict',        'dict_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'ino',         'inode_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['fdctx_to_dict'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('fn-arg',      'fd',          'fd_t *'),
-        ('fn-arg',      'dict',        'dict_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 xlator_dumpops['history'] = (
-        ('fn-arg',      'this',        'xlator_t *'),
-        ('ret-val',     'int32_t',     '0'),
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
 )
 
 def get_error_arg (type_str):
@@ -671,45 +727,50 @@ def get_error_arg (type_str):
 		return "NULL"
 	return "-1"
 
-def get_subs (names, types):
+def get_subs (names, types, cbktypes=None):
 	sdict = {}
-	sdict["@SHORT_ARGS@"] = string.join(names,", ")
+	sdict["@SHORT_ARGS@"] = ', '.join(names)
 	# Convert two separate tuples to one of (name, type) sub-tuples.
-	as_tuples = zip(types,names)
+	as_tuples = list(zip(types, names))
 	# Convert each sub-tuple into a "type name" string.
-	as_strings = map(string.join,as_tuples)
+	as_strings = [' '.join(item) for item in as_tuples]
 	# Join all of those into one big string.
-	sdict["@LONG_ARGS@"] = string.join(as_strings,",\n\t")
+	sdict["@LONG_ARGS@"] = ',\n\t'.join(as_strings)
 	# So much more readable than string.join(map(string.join,zip(...))))
-	sdict["@ERROR_ARGS@"] = string.join(map(get_error_arg,types),", ")
+	sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, types)))
+	if cbktypes is not None:
+		sdict["@CBK_ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, cbktypes)))
 	return sdict
 
 def generate (tmpl, name, subs):
-	text = tmpl.replace("@NAME@",name)
+	text = tmpl.replace("@NAME@", name)
 	if name == "writev":
 		# More spurious inconsistency.
-		text = text.replace("@UPNAME@","WRITE")
+		text = text.replace("@UPNAME@", "WRITE")
+	elif name == "readv":
+		text = text.replace("@UPNAME@", "READ")
 	else:
-		text = text.replace("@UPNAME@",name.upper())
-	for old, new in subs[name].iteritems():
-		text = text.replace(old,new)
+		text = text.replace("@UPNAME@", name.upper())
+	for old, new in subs[name].items():
+		text = text.replace(old, new)
 	# TBD: reindent/reformat the result for maximum readability.
-	return  text
+	return text
 
 fop_subs = {}
 cbk_subs = {}
 
-for name, args in ops.iteritems():
+for name, args in ops.items():
 
 	# Create the necessary substitution strings for fops.
 	arg_names = [ a[1] for a in args if a[0] == 'fop-arg']
 	arg_types = [ a[2] for a in args if a[0] == 'fop-arg']
-	fop_subs[name] = get_subs(arg_names,arg_types)
+	cbk_types = [ a[2] for a in args if a[0] == 'cbk-arg']
+	fop_subs[name] = get_subs(arg_names, arg_types, cbk_types)
 
 	# Same thing for callbacks.
 	arg_names = [ a[1] for a in args if a[0] == 'cbk-arg']
 	arg_types = [ a[2] for a in args if a[0] == 'cbk-arg']
-	cbk_subs[name] = get_subs(arg_names,arg_types)
+	cbk_subs[name] = get_subs(arg_names, arg_types)
 
 	# Callers can add other subs to these tables, or even create their
 	# own tables, using these same techniques, and then pass the result
diff --git a/libglusterfs/src/gf-dirent.c b/libglusterfs/src/gf-dirent.c
index d0df80e2886..a809efc97ef 100644
--- a/libglusterfs/src/gf-dirent.c
+++ b/libglusterfs/src/gf-dirent.c
@@ -8,237 +8,232 @@
   cases as published by the Free Software Foundation.
 */
 
-
-
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>
-#include "compat.h"
-#include "xlator.h"
-#include "syncop.h"
+#include "glusterfs/compat.h"
+#include "glusterfs/syncop.h"
 
 #define ONE 1ULL
 #define PRESENT_D_OFF_BITS 63
 #define BACKEND_D_OFF_BITS 63
 #define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
 #define MASK (~0ULL)
-#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
+#define SHIFT_BITS (max(0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 #define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
 
 static uint64_t
-bits_for (uint64_t num)
+bits_for(uint64_t num)
 {
-	uint64_t bits = 0, ctrl = 1;
+    uint64_t bits = 0, ctrl = 1;
 
-	while (ctrl < num) {
-		ctrl *= 2;
-		bits++;
-	}
+    while (ctrl < num) {
+        ctrl *= 2;
+        bits++;
+    }
 
-	return bits;
+    return bits;
 }
 
 int
-gf_deitransform(xlator_t *this,
-                uint64_t offset)
+gf_deitransform(xlator_t *this, uint64_t offset)
 {
-        int         cnt = 0;
-        int         max = 0;
-        int         max_bits = 0;
-        uint64_t    off_mask = 0;
-        uint64_t    host_mask = 0;
-
-        max = glusterfs_get_leaf_count(this->graph);
-
-	if (max == 1) {
-		cnt = 0;
-		goto out;
-	}
-
-        if (offset & TOP_BIT) {
-                /* HUGE d_off */
-                max_bits = bits_for (max);
-                off_mask = (MASK << max_bits);
-                host_mask = ~(off_mask);
-
-                cnt = offset & host_mask;
-	} else {
-                /* small d_off */
-                cnt = offset % max;
-        }
+    int cnt = 0;
+    int max = 0;
+    int max_bits = 0;
+    uint64_t off_mask = 0;
+    uint64_t host_mask = 0;
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        cnt = 0;
+        goto out;
+    }
+
+    if (offset & TOP_BIT) {
+        /* HUGE d_off */
+        max_bits = bits_for(max);
+        off_mask = (MASK << max_bits);
+        host_mask = ~(off_mask);
+
+        cnt = offset & host_mask;
+    } else {
+        /* small d_off */
+        cnt = offset % max;
+    }
 out:
-        return cnt;
+    return cnt;
 }
 
 uint64_t
-gf_dirent_orig_offset(xlator_t *this,
-                      uint64_t offset)
+gf_dirent_orig_offset(xlator_t *this, uint64_t offset)
 {
-        int         max = 0;
-        int         max_bits = 0;
-        uint64_t    off_mask = 0;
-        uint64_t    orig_offset;
-
-        max = glusterfs_get_leaf_count(this->graph);
-
-	if (max == 1) {
-                orig_offset = offset;
-		goto out;
-	}
-
-        if (offset & TOP_BIT) {
-                /* HUGE d_off */
-                max_bits = bits_for (max);
-                off_mask = (MASK << max_bits);
-                orig_offset = ((offset & ~TOP_BIT) & off_mask) << SHIFT_BITS;
-	} else {
-                /* small d_off */
-                orig_offset = offset / max;
-        }
+    int max = 0;
+    int max_bits = 0;
+    uint64_t off_mask = 0;
+    uint64_t orig_offset;
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        orig_offset = offset;
+        goto out;
+    }
+
+    if (offset & TOP_BIT) {
+        /* HUGE d_off */
+        max_bits = bits_for(max);
+        off_mask = (MASK << max_bits);
+        orig_offset = ((offset & ~TOP_BIT) & off_mask) << SHIFT_BITS;
+    } else {
+        /* small d_off */
+        orig_offset = offset / max;
+    }
 out:
-        return orig_offset;
+    return orig_offset;
 }
 
 int
-gf_itransform (xlator_t *this, uint64_t x, uint64_t *y_p, int client_id)
+gf_itransform(xlator_t *this, uint64_t x, uint64_t *y_p, int client_id)
 {
-        int         max = 0;
-        uint64_t    y = 0;
-        uint64_t    hi_mask = 0;
-        uint64_t    off_mask = 0;
-        int         max_bits = 0;
-
-        if (x == ((uint64_t) -1)) {
-                y = (uint64_t) -1;
-                goto out;
-        }
-
-        if (!x) {
-                y = 0;
-                goto out;
-        }
-
-        max = glusterfs_get_leaf_count(this->graph);
-
-	if (max == 1) {
-		y = x;
-		goto out;
-	}
-
-        max_bits = bits_for (max);
-
-        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
-
-        if (x & hi_mask) {
-                /* HUGE d_off */
-                off_mask = MASK << max_bits;
-                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | client_id;
-        } else {
-                /* small d_off */
-                y = ((x * max) + client_id);
-        }
+    int max = 0;
+    uint64_t y = 0;
+    uint64_t hi_mask = 0;
+    uint64_t off_mask = 0;
+    int max_bits = 0;
+
+    if (x == ((uint64_t)-1)) {
+        y = (uint64_t)-1;
+        goto out;
+    }
+
+    if (!x) {
+        y = 0;
+        goto out;
+    }
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        y = x;
+        goto out;
+    }
+
+    max_bits = bits_for(max);
+
+    hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
+
+    if (x & hi_mask) {
+        /* HUGE d_off */
+        off_mask = MASK << max_bits;
+        y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | client_id;
+    } else {
+        /* small d_off */
+        y = ((x * max) + client_id);
+    }
 
 out:
-        if (y_p)
-                *y_p = y;
+    if (y_p)
+        *y_p = y;
 
-        return 0;
+    return 0;
 }
 
 gf_dirent_t *
-gf_dirent_for_name (const char *name)
+gf_dirent_for_name(const char *name)
 {
-        gf_dirent_t *gf_dirent = NULL;
+    gf_dirent_t *gf_dirent = NULL;
 
-        /* TODO: use mem-pool */
-        gf_dirent = GF_CALLOC (gf_dirent_size (name), 1,
-                               gf_common_mt_gf_dirent_t);
-        if (!gf_dirent)
-                return NULL;
+    /* TODO: use mem-pool */
+    gf_dirent = GF_CALLOC(gf_dirent_size(name), 1, gf_common_mt_gf_dirent_t);
+    if (!gf_dirent)
+        return NULL;
 
-        INIT_LIST_HEAD (&gf_dirent->list);
-        strcpy (gf_dirent->d_name, name);
+    INIT_LIST_HEAD(&gf_dirent->list);
+    strcpy(gf_dirent->d_name, name);
 
-        gf_dirent->d_off = 0;
-        gf_dirent->d_ino = -1;
-        gf_dirent->d_type = 0;
-        gf_dirent->d_len = strlen (name);
+    gf_dirent->d_off = 0;
+    gf_dirent->d_ino = -1;
+    gf_dirent->d_type = 0;
+    gf_dirent->d_len = strlen(name);
 
-        return gf_dirent;
+    return gf_dirent;
 }
 
 void
-gf_dirent_entry_free (gf_dirent_t *entry)
+gf_dirent_entry_free(gf_dirent_t *entry)
 {
-        if (!entry)
-                return;
+    if (!entry)
+        return;
 
-        if (entry->dict)
-                dict_unref (entry->dict);
-        if (entry->inode)
-                inode_unref (entry->inode);
+    if (entry->dict)
+        dict_unref(entry->dict);
+    if (entry->inode)
+        inode_unref(entry->inode);
 
-        list_del_init (&entry->list);
-        GF_FREE (entry);
+    list_del_init(&entry->list);
+    GF_FREE(entry);
 }
 
 void
-gf_dirent_free (gf_dirent_t *entries)
+gf_dirent_free(gf_dirent_t *entries)
 {
-        gf_dirent_t *entry = NULL;
-        gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
 
-        if (!entries)
-                return;
+    if (!entries)
+        return;
 
-        if (list_empty (&entries->list))
-                return;
+    if (list_empty(&entries->list))
+        return;
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                gf_dirent_entry_free (entry);
-        }
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        gf_dirent_entry_free(entry);
+    }
 }
 
 gf_dirent_t *
-entry_copy (gf_dirent_t *source)
+entry_copy(gf_dirent_t *source)
 {
-        gf_dirent_t *sink = NULL;
+    gf_dirent_t *sink = NULL;
 
-        sink = gf_dirent_for_name (source->d_name);
-        if (!sink)
-                return NULL;
+    sink = gf_dirent_for_name(source->d_name);
+    if (!sink)
+        return NULL;
 
-        sink->d_off = source->d_off;
-        sink->d_ino = source->d_ino;
-        sink->d_type = source->d_type;
-        sink->d_stat = source->d_stat;
-        sink->d_len  = source->d_len;
+    sink->d_off = source->d_off;
+    sink->d_ino = source->d_ino;
+    sink->d_type = source->d_type;
+    sink->d_stat = source->d_stat;
+    sink->d_len = source->d_len;
 
-	if (source->inode)
-		sink->inode = inode_ref (source->inode);
+    if (source->inode)
+        sink->inode = inode_ref(source->inode);
 
-	if (source->dict)
-		sink->dict = dict_ref (source->dict);
-        return sink;
+    if (source->dict)
+        sink->dict = dict_ref(source->dict);
+    return sink;
 }
 
 void
-gf_link_inode_from_dirent (xlator_t *this, inode_t *parent, gf_dirent_t *entry)
+gf_link_inode_from_dirent(xlator_t *this, inode_t *parent, gf_dirent_t *entry)
 {
-        inode_t     *link_inode = NULL;
-        inode_t     *tmp        = NULL;
-
-        if (!entry->inode)
-                return;
-        link_inode = inode_link (entry->inode, parent,
-                                 entry->d_name, &entry->d_stat);
-        if (!link_inode)
-                return;
-
-        inode_lookup (link_inode);
-        tmp = entry->inode;
-        entry->inode = link_inode;
-        inode_unref (tmp);
+    inode_t *link_inode = NULL;
+    inode_t *tmp = NULL;
+
+    if (!entry->inode)
+        return;
+    link_inode = inode_link(entry->inode, parent, entry->d_name,
+                            &entry->d_stat);
+    if (!link_inode)
+        return;
+
+    inode_lookup(link_inode);
+    tmp = entry->inode;
+    entry->inode = link_inode;
+    inode_unref(tmp);
 }
 
 /* TODO: Currently, with this function, we will be breaking the
@@ -247,55 +242,60 @@ gf_link_inode_from_dirent (xlator_t *this, inode_t *parent, gf_dirent_t *entry)
    Need more thoughts before finalizing this function
 */
 int
-gf_link_inodes_from_dirent (xlator_t *this, inode_t *parent,
-                            gf_dirent_t *entries)
+gf_link_inodes_from_dirent(xlator_t *this, inode_t *parent,
+                           gf_dirent_t *entries)
 {
-        gf_dirent_t *entry      = NULL;
+    gf_dirent_t *entry = NULL;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                gf_link_inode_from_dirent (this, parent, entry);
-        }
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        gf_link_inode_from_dirent(this, parent, entry);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-gf_fill_iatt_for_dirent (gf_dirent_t *entry, inode_t *parent, xlator_t *subvol)
+gf_fill_iatt_for_dirent(gf_dirent_t *entry, inode_t *parent, xlator_t *subvol)
 {
-        loc_t                   loc             = {0, };
-        int                     ret             = -1;
-        char                   *path            = NULL;
-        struct iatt             iatt            = {0,};
-
-        loc.inode = inode_grep (parent->table, parent, entry->d_name);
-        if (!loc.inode) {
-                loc.inode = inode_new (parent->table);
-                gf_uuid_copy (loc.inode->gfid, entry->d_stat.ia_gfid);
-        }
-
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.name = entry->d_name;
-        loc.parent = inode_ref (parent);
-        ret = inode_path (loc.inode, entry->d_name, &path);
-        loc.path = path;
-        if (ret < 0)
-                goto out;
-
-        ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-        if (ret)
-                goto out;
-
-        entry->d_stat = iatt;
-        entry->inode = inode_ref (loc.inode);
-        /* We don't need to link inode here, because as part of readdirp_cbk
-         * we will link all dirents.
-         *
-         * Since we did a proper lookup, we don't need to set need_lookup
-         * flag.
-         */
-
-        ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    loc.inode = inode_grep(parent->table, parent, entry->d_name);
+    if (!loc.inode) {
+        loc.inode = inode_new(parent->table);
+        gf_uuid_copy(loc.inode->gfid, entry->d_stat.ia_gfid);
+    }
+
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = entry->d_name;
+    loc.parent = inode_ref(parent);
+    ret = inode_path(loc.parent, entry->d_name, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+
+    entry->d_stat = iatt;
+    entry->inode = inode_ref(loc.inode);
+    /* We don't need to link inode here, because as part of readdirp_cbk
+     * we will link all dirents.
+     *
+     * Since we did a proper lookup, we don't need to set need_lookup
+     * flag.
+     */
+
+    ret = 0;
 out:
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
diff --git a/libglusterfs/src/gf-dirent.h b/libglusterfs/src/gf-dirent.h
deleted file mode 100644
index caa09c1728e..00000000000
--- a/libglusterfs/src/gf-dirent.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _GF_DIRENT_H
-#define _GF_DIRENT_H
-
-#include "iatt.h"
-#include "inode.h"
-
-#define gf_dirent_size(name) (sizeof (gf_dirent_t) + strlen (name) + 1)
-
-int
-gf_deitransform(xlator_t *this, uint64_t y);
-
-int
-gf_itransform (xlator_t *this, uint64_t x, uint64_t *y_p, int client_id);
-
-uint64_t
-gf_dirent_orig_offset (xlator_t *this, uint64_t offset);
-
-
-struct _dir_entry_t {
-        struct _dir_entry_t *next;
-	char                *name;
-	char                *link;
-	struct iatt          buf;
-};
-
-
-struct _gf_dirent_t {
-	union {
-		struct list_head             list;
-		struct {
-			struct _gf_dirent_t *next;
-			struct _gf_dirent_t *prev;
-		};
-	};
-	uint64_t                             d_ino;
-	uint64_t                             d_off;
-	uint32_t                             d_len;
-	uint32_t                             d_type;
-        struct iatt                          d_stat;
-        dict_t                              *dict;
-        inode_t                             *inode;
-	char                                 d_name[];
-};
-
-#define DT_ISDIR(mode) (mode == DT_DIR)
-
-gf_dirent_t *gf_dirent_for_name (const char *name);
-gf_dirent_t *entry_copy (gf_dirent_t *source);
-void gf_dirent_entry_free (gf_dirent_t *entry);
-void gf_dirent_free (gf_dirent_t *entries);
-int gf_link_inodes_from_dirent (xlator_t *this, inode_t *parent,
-                                gf_dirent_t *entries);
-int
-gf_fill_iatt_for_dirent (gf_dirent_t *entry, inode_t *parent,
-                         xlator_t *subvol);
-
-void
-gf_link_inode_from_dirent (xlator_t *this, inode_t *parent, gf_dirent_t *entry);
-#endif /* _GF_DIRENT_H */
diff --git a/libglusterfs/src/gfdb/Makefile.am b/libglusterfs/src/gfdb/Makefile.am
deleted file mode 100644
index 3931e694c24..00000000000
--- a/libglusterfs/src/gfdb/Makefile.am
+++ /dev/null
@@ -1,37 +0,0 @@
-libgfdb_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
-	$(SQLITE_CFLAGS) -DDATADIR=\"$(localstatedir)\"
-
-libgfdb_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src \
-	-I$(top_builddir)/rpc/xdr/src \
-	-DDATADIR=\"$(localstatedir)\"
-
-libgfdb_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(SQLITE_LIBS) $(UUID_LIBS)
-
-libgfdb_la_LDFLAGS = $(GF_LDFLAGS) -version-info $(LIBGLUSTERFS_LT_VERSION)
-
-libgfdbdir = $(includedir)/glusterfs/gfdb
-
-if BUILD_GFDB
- lib_LTLIBRARIES = libgfdb.la
-endif
-
-CONTRIB_BUILDDIR = $(top_builddir)/contrib
-
-libgfdb_la_SOURCES = gfdb_data_store.c gfdb_data_store_helper.c \
-	gfdb_sqlite3_helper.c gfdb_sqlite3.c
-
-noinst_HEADERS =  gfdb_data_store.h gfdb_data_store_types.h \
-	gfdb_sqlite3_helper.h gfdb_sqlite3.h gfdb_mem-types.h \
-	gfdb_data_store_helper.h
-
-libgfdb_HEADERS = gfdb_data_store.h gfdb_data_store_types.h \
-	gfdb_data_store_helper.h gfdb_sqlite3.h gfdb_mem-types.h \
-	gfdb_sqlite3_helper.h
-
-CLEANFILES =
-
-$(top_builddir)/libglusterfs/src/libglusterfs.la:
-	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/libglusterfs/src/gfdb/gfdb_data_store.c b/libglusterfs/src/gfdb/gfdb_data_store.c
deleted file mode 100644
index cb567503fa3..00000000000
--- a/libglusterfs/src/gfdb/gfdb_data_store.c
+++ /dev/null
@@ -1,879 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3.h"
-#include "gfdb_data_store.h"
-#include "list.h"
-#include "libglusterfs-messages.h"
-
-/******************************************************************************
- *
- *                     Database Connection utils/internals
- *
- * ****************************************************************************/
-
-/* GFDB Connection Node:
- * ~~~~~~~~~~~~~~~~~~~~
- * Represents the connection to the database while using libgfdb
- * The connection node is not thread safe as far as fini_db is concerned.
- * You can use a single connection node
- * to do multithreaded db operations like insert/delete/find of records.
- * But you need to wait for all the operating threads to complete i.e
- * pthread_join() and then do fini_db() to kill the connection node.
- * gfdb_conn_node_t is an opaque structure.
- * */
-struct gfdb_conn_node_t {
-        gfdb_connection_t       gfdb_connection;
-        struct list_head        conn_list;
-};
-
-
-/*
- * db_conn_list is the circular linked list which
- * will have all the database connections for the process
- *
- * */
-static gfdb_conn_node_t *db_conn_list;
-
-/*
- * db_conn_mutex is the mutex for db_conn_list
- *
- * */
-static pthread_mutex_t db_conn_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-
-/*Checks the sanity of the connection node*/
-#define CHECK_CONN_NODE(_conn_node)\
-do {\
-        GF_ASSERT (_conn_node);\
-        GF_ASSERT (_conn_node->gfdb_connection.gf_db_connection);\
-} while (0)
-
-/* Checks the sanity of the connection node and goto */
-#define CHECK_CONN_NODE_GOTO(_conn_node, label)\
-do {\
-        if (!_conn_node) {\
-                goto label;\
-        };\
-        if (!_conn_node->gfdb_connection.gf_db_connection) {\
-                goto label;\
-        };\
-} while (0)
-
-/*Check if the conn node is first in the list*/
-#define IS_FIRST_NODE(db_conn_list, _conn_node)\
-        ((_conn_node == db_conn_list) ? _gf_true : _gf_false)
-
-
-/*Check if the conn node is the only node in the list*/
-#define IS_THE_ONLY_NODE(_conn_node)\
-((_conn_node->conn_list.next == _conn_node->conn_list.prev)\
-        ? _gf_true : _gf_false)
-
-
-
-/*Internal Function: Adds connection node to the end of
- * the db connection list.*/
-static int
-add_connection_node (gfdb_conn_node_t *_conn_node) {
-        int ret = -1;
-
-        GF_ASSERT (_conn_node);
-
-        /*Lock the list*/
-        ret = pthread_mutex_lock (&db_conn_mutex);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ret,
-                        LG_MSG_LOCK_LIST_FAILED, "Failed lock db connection "
-                        "list %s", strerror(ret));
-                ret = -1;
-                goto out;
-        }
-
-        if (db_conn_list == NULL) {
-                db_conn_list = _conn_node;
-        } else {
-                list_add_tail (&_conn_node->conn_list,
-                                &db_conn_list->conn_list);
-        }
-
-        /*unlock the list*/
-        ret = pthread_mutex_unlock (&db_conn_mutex);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ret,
-                        LG_MSG_UNLOCK_LIST_FAILED, "Failed unlock db "
-                        "connection list %s", strerror(ret));
-                ret = -1;
-                /*TODO What if the unlock fails.
-                * Will it lead to deadlock?
-                * Most of the gluster code
-                * no check for unlock or destory of mutex!*/
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/*Internal Function:
- * Delete connection node from the list*/
-static int
-delete_conn_node (gfdb_conn_node_t *_conn_node)
-{
-        int ret = -1;
-
-        GF_ASSERT (_conn_node);
-
-        /*Lock of the list*/
-        ret = pthread_mutex_lock (&db_conn_mutex);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ret,
-                        LG_MSG_LOCK_LIST_FAILED, "Failed lock on db connection"
-                        " list %s", strerror(ret));
-                goto out;
-        }
-
-        /*Remove the connection object from list*/
-        if (IS_THE_ONLY_NODE(_conn_node)) {
-                db_conn_list = NULL;
-                GF_FREE (_conn_node);
-        } else {
-                if (IS_FIRST_NODE(db_conn_list, _conn_node)) {
-                        db_conn_list = list_entry (db_conn_list->conn_list.next,
-                                                gfdb_conn_node_t, conn_list);
-                }
-                list_del(&_conn_node->conn_list);
-                GF_FREE (_conn_node);
-        }
-
-        /*Release the list lock*/
-        ret =  pthread_mutex_unlock (&db_conn_mutex);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_WARNING, ret,
-                        LG_MSG_UNLOCK_LIST_FAILED, "Failed unlock on db "
-                        "connection list %s", strerror(ret));
-                /*TODO What if the unlock fails.
-                * Will it lead to deadlock?
-                * Most of the gluster code
-                * no check for unlock or destory of mutex!*/
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/*Internal function: Used initialize/map db operation of
- * specified type of db plugin*/
-static int
-init_db_operations (gfdb_db_type_t       gfdb_db_type,
-                    gfdb_db_operations_t *gfdb_db_operations)
-{
-
-        int ret = -1;
-
-        GF_ASSERT (gfdb_db_operations);
-
-        /*Clear the gfdb_db_operations*/
-        gfdb_db_operations = memset(gfdb_db_operations, 0,
-                                        sizeof(*gfdb_db_operations));
-        switch (gfdb_db_type) {
-        case GFDB_SQLITE3:
-                gf_sqlite3_fill_db_operations (gfdb_db_operations);
-                ret = 0;
-                break;
-        case GFDB_HYPERDEX:
-        case GFDB_HASH_FILE_STORE:
-        case GFDB_ROCKS_DB:
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_UNSUPPORTED_PLUGIN, "Plugin not supported");
-                break;
-        case GFDB_INVALID_DB:
-        case GFDB_DB_END:
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_INVALID_DB_TYPE, "Invalid DB Type");
-                break;
-        }
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      LIBGFDB API Functions
- *
- * ****************************************************************************/
-
-
-/*Libgfdb API Function: Used to initialize a db connection
- *                      (Constructor function for db connection object)
- * Arguments:
- *      args         :  Dictionary containing database specific parameters
- *                      eg: For sqlite3, pagesize, cachesize, db name, db path
-                        etc
- *      gfdb_db_type :  Type of data base used i.e sqlite or hyperdex etc
- * Returns : if successful return the GFDB Connection node to the caller or
- *          NULL in case of failure*/
-gfdb_conn_node_t *
-init_db (dict_t *args, gfdb_db_type_t gfdb_db_type)
-{
-        int ret                                 = -1;
-        gfdb_conn_node_t *_conn_node            = NULL;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-
-        /*Create data base connection object*/
-        _conn_node = GF_CALLOC (1, sizeof(gfdb_conn_node_t),
-                                        gf_mt_db_conn_node_t);
-        if (!_conn_node) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
-                        LG_MSG_NO_MEMORY, "Failed mem alloc for "
-                        "gfdb_conn_node_t");
-                goto alloc_failed;
-        }
-
-        /*Init the list component of db conneciton object*/
-        INIT_LIST_HEAD (&_conn_node->conn_list);
-
-
-        /*Add created connection node to the list*/
-        ret = add_connection_node (_conn_node);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_ADD_TO_LIST_FAILED, "Failed to add connection "
-                        "node to list");
-                goto _conn_failed;
-        }
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-
-        /*init the db ops object of db connection object*/
-        ret = init_db_operations(gfdb_db_type, db_operations_t);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_INIT_DB_FAILED, "Failed initializing database "
-                        "operation failed.");
-                ret = -1;
-                goto init_db_failed;
-        }
-
-        /*Calling the init_db_op of the respected db type*/
-        GF_ASSERT (db_operations_t->init_db_op);
-        ret = db_operations_t->init_db_op (args, &_conn_node->gfdb_connection.
-                                           gf_db_connection);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_INIT_DB_FAILED, "Failed initializing database");
-                ret = -1;
-                goto init_db_failed;
-        }
-        _conn_node->gfdb_connection.gfdb_db_type = gfdb_db_type;
-        ret = 0;
-
-        return _conn_node;
-
-        /*****Error Handling********/
-        /* If init_db_operations or init_db of plugin failed delete
-        * conn node from the list.
-        * connection node will be free by delete_conn_node*/
-init_db_failed:
-        ret = delete_conn_node (_conn_node);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DELETE_FROM_LIST_FAILED, "Failed deleting "
-                        "connection node from list");
-        }
-        return NULL;
-        /*if adding to the list failed free connection node*/
-_conn_failed:
-        GF_FREE (_conn_node);
-        /*if allocation failed*/
-alloc_failed:
-        return NULL;
-        /*****Error Handling********/
-}
-
-
-
-
-
-/*Libgfdb API Function: Used to terminate/de-initialize db connection
- *                      (Destructor function for db connection object)
- * Arguments:
- *      _conn_node  :  GFDB Connection node
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-fini_db (gfdb_conn_node_t *_conn_node)
-{
-        int ret                                 = -1;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-
-        CHECK_CONN_NODE_GOTO (_conn_node, empty);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-
-        GF_ASSERT (db_operations_t->fini_db_op);
-
-        ret = db_operations_t->fini_db_op(&_conn_node->gfdb_connection.
-                                          gf_db_connection);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_CLOSE_CONNECTION_FAILED, "Failed close the db "
-                        "connection");
-                goto out;
-        }
-
-        ret = delete_conn_node (_conn_node);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DELETE_FROM_LIST_FAILED, "Failed deleting "
-                        "connection node from list");
-        }
-empty:
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-
-
-
-/*Libgfdb API Function: Used to insert/update records in the database
- *                      NOTE: In current gfdb_sqlite plugin we use that
- *                      same function to delete the record. Set the
- *                      gfdb_fop_path to GFDB_FOP_UNDEL to delete the
- *                      link of inode from GF_FLINK_TB and
- *                      GFDB_FOP_UNDEL_ALL to delete all the records from
- *                      GF_FLINK_TB and GF_FILE_TB.
- *                      TODO: Should seperate this function into the
- *                      delete_record function
- *                      Refer CTR Xlator features/changetimerecorder for usage
- * Arguments:
- *      _conn_node     :  GFDB Connection node
- *      gfdb_db_record :  Record to be inserted/updated
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-insert_record (gfdb_conn_node_t *_conn_node,
-               gfdb_db_record_t *gfdb_db_record)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->insert_record_op) {
-
-                ret = db_operations_t->insert_record_op (gf_db_connection,
-                                                         gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_INSERT_OR_UPDATE_FAILED, "Insert/Update"
-                                " operation failed");
-                }
-        }
-
-        return ret;
-}
-
-
-
-
-/*Libgfdb API Function: Used to delete record from the database
- *                      NOTE: In the current gfdb_sqlite3 plugin
- *                      implementation this function is dummy.
- *                      Use the insert_record function.
- *                      Refer CTR Xlator features/changetimerecorder for usage
- * Arguments:
- *      _conn_node     :  GFDB Connection node
- *      gfdb_db_record :  Record to be deleted
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-delete_record (gfdb_conn_node_t *_conn_node,
-               gfdb_db_record_t *gfdb_db_record)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->delete_record_op) {
-
-                ret = db_operations_t->delete_record_op (gf_db_connection,
-                                                         gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DELETE_FAILED, "Delete operation "
-                                "failed");
-                }
-
-        }
-
-        return ret;
-}
-
-/*Libgfdb API Function: Compact the database.
- *
- * Arguments:
- *      _conn_node                      :  GFDB Connection node
- *      _compact_active                 :  Is compaction currently on?
- *      _compact_mode_switched          :  Was the compaction switch flipped?
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-compact_db (gfdb_conn_node_t *_conn_node, gf_boolean_t _compact_active,
-            gf_boolean_t _compact_mode_switched)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->compact_db_op) {
-
-                ret = db_operations_t->compact_db_op (gf_db_connection,
-                                                      _compact_active,
-                                                      _compact_mode_switched);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_COMPACT_FAILED, "Compaction operation "
-                                "failed");
-                }
-
-        }
-
-        return ret;
-}
-
-
-
-
-
-/*Libgfdb API Function: Query all the records from the database
- * Arguments:
- *      _conn_node      : GFDB Connection node
- *      query_callback  : Call back function that will be called
- *                        for every record found
- *      _query_cbk_args : Custom argument passed for the call back
- *                        function query_callback
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-find_all (gfdb_conn_node_t      *_conn_node,
-          gf_query_callback_t   query_callback,
-          void                  *_query_cbk_args)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->find_all_op) {
-                ret = db_operations_t->find_all_op (gf_db_connection,
-                                                    query_callback,
-                                                    _query_cbk_args);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED, "Find all operation "
-                                "failed");
-                }
-
-        }
-
-        return ret;
-}
-
-
-
-/*Libgfdb API Function: Query records/files that have not changed/accessed
- *                      from a time in past to current time
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are not
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-find_unchanged_for_time(gfdb_conn_node_t        *_conn_node,
-                        gf_query_callback_t     query_callback,
-                        void                    *_query_cbk_args,
-                        gfdb_time_t             *for_time)
-{
-
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->find_unchanged_for_time_op) {
-
-                ret = db_operations_t->find_unchanged_for_time_op
-                                (gf_db_connection, query_callback,
-                                _query_cbk_args, for_time);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED, "Find unchanged "
-                                "operation failed");
-                }
-
-        }
-
-        return ret;
-}
-
-/*Libgfdb API Function: Query records/files that have changed/accessed from a
- *                      time in past to current time
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-find_recently_changed_files(gfdb_conn_node_t    *_conn_node,
-                            gf_query_callback_t query_callback,
-                            void                *_query_cbk_args,
-                            gfdb_time_t         *from_time)
-{
-
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->find_recently_changed_files_op) {
-
-                ret =  db_operations_t->find_recently_changed_files_op (
-                                gf_db_connection, query_callback,
-                                _query_cbk_args, from_time);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED,
-                                "Find changed operation failed");
-                }
-
-        }
-
-        return ret;
-
-}
-
-/*Libgfdb API Function: Query records/files that have not changed/accessed
- *                      from a time in past to current time, with
- *                      a desired frequency
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are not
- *                                changed/accessed
- *      write_freq_thresold     : Desired Write Frequency lower limit
- *      read_freq_thresold      : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-find_unchanged_for_time_freq(gfdb_conn_node_t *_conn_node,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *for_time,
-                                        int write_freq_thresold,
-                                        int read_freq_thresold,
-                                        gf_boolean_t _clear_counters)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->find_unchanged_for_time_freq_op) {
-
-                ret = db_operations_t->find_unchanged_for_time_freq_op(
-                                gf_db_connection, query_callback,
-                                _query_cbk_args, for_time,
-                                write_freq_thresold, read_freq_thresold,
-                                _clear_counters);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED,
-                                "Find unchanged with freq operation failed");
-                }
-
-        }
-
-        return ret;
-}
-
-/*Libgfdb API Function: Query records/files that have changed/accessed from a
- *                      time in past to current time, with
- *                      a desired frequency
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are
- *                                changed/accessed
- *      write_freq_thresold     : Desired Write Frequency lower limit
- *      read_freq_thresold      : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-find_recently_changed_files_freq(gfdb_conn_node_t *_conn_node,
-                                gf_query_callback_t query_callback,
-                                void *_query_cbk_args,
-                                gfdb_time_t *from_time,
-                                int write_freq_thresold,
-                                int read_freq_thresold,
-                                gf_boolean_t _clear_counters)
-{
-
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations_t   = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(_conn_node);
-
-        db_operations_t = &_conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = _conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations_t->find_recently_changed_files_freq_op) {
-
-                ret =  db_operations_t->find_recently_changed_files_freq_op(
-                                gf_db_connection, query_callback,
-                                _query_cbk_args, from_time,
-                                write_freq_thresold, read_freq_thresold,
-                                _clear_counters);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED,
-                                "Find changed with freq operation failed");
-                }
-
-        }
-
-        return ret;
-
-}
-
-
-
-/*Libgfdb API Function: Clear the heat for all the files
- *
- *  Arguments:
- *    conn_node              : GFDB Connection node
- *
- * Returns : if successful return 0 or
- *           -ve value in case of failure
- **/
-
-int
-clear_files_heat (gfdb_conn_node_t *conn_node)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations     = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(conn_node);
-
-        db_operations = &conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations->clear_files_heat_op) {
-                ret =  db_operations->clear_files_heat_op (gf_db_connection);
-                if (ret) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_INSERT_OR_UPDATE_FAILED,
-                                "Clear files heat operation failed");
-                }
-        }
-
-        return ret;
-}
-
-
-/* Libgfdb API Function: Function to extract version of the db
- * Input:
- * gfdb_conn_node_t *conn_node        : GFDB Connection node
- * char **version  : the version is extracted as a string and will be stored in
- *                   this variable. The freeing of the memory should be done by
- *                   the caller.
- * Return:
- *      On success return the lenght of the version string that is
- *      extracted.
- *      On failure return -1
- * */
-int
-get_db_version (gfdb_conn_node_t *conn_node, char **version)
-{
-        int ret                                 = 0;
-        gfdb_db_operations_t *db_operations     = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(conn_node);
-
-        db_operations = &conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations->get_db_version) {
-                ret =  db_operations->get_db_version (gf_db_connection,
-                                                      version);
-                if (ret < 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED,
-                                "Get version failed");
-                }
-        }
-
-        return ret;
-}
-
-int
-get_db_params (gfdb_conn_node_t *conn_node, char *param_key,
-                char **param_value)
-{
-        int ret                                 = -1;
-        gfdb_db_operations_t *db_operations     = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(conn_node);
-
-        db_operations = &conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations->get_db_params) {
-                ret =  db_operations->get_db_params (gf_db_connection,
-                                                     param_key,
-                                                     param_value);
-                if (ret < 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_FIND_OP_FAILED,
-                                "Get setting failed");
-                }
-        }
-
-        return ret;
-}
-
-
-int
-set_db_params (gfdb_conn_node_t *conn_node, char *param_key,
-                char *param_value)
-{
-        int ret                                 = -1;
-        gfdb_db_operations_t *db_operations     = NULL;
-        void *gf_db_connection                  = NULL;
-
-        CHECK_CONN_NODE(conn_node);
-
-        db_operations = &conn_node->gfdb_connection.gfdb_db_operations;
-        gf_db_connection = conn_node->gfdb_connection.gf_db_connection;
-
-        if (db_operations->set_db_params) {
-                ret =  db_operations->set_db_params (gf_db_connection,
-                                                     param_key,
-                                                     param_value);
-                if (ret < 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_INSERT_OR_UPDATE_FAILED,
-                                "Failed to set database setting");
-                }
-        }
-
-        return ret;
-}
-
-
-
-
-static const
-char *get_db_path_key()
-{
-        return GFDB_SQL_PARAM_DBPATH;
-}
-
-void get_gfdb_methods (gfdb_methods_t *methods)
-{
-        methods->init_db = init_db;
-        methods->fini_db = fini_db;
-        methods->find_unchanged_for_time = find_unchanged_for_time;
-        methods->find_recently_changed_files = find_recently_changed_files;
-        methods->find_unchanged_for_time_freq = find_unchanged_for_time_freq;
-        methods->find_recently_changed_files_freq =
-                                               find_recently_changed_files_freq;
-        methods->clear_files_heat = clear_files_heat;
-        methods->get_db_version = get_db_version;
-        methods->get_db_params = get_db_params;
-        methods->set_db_params = set_db_params;
-        methods->get_db_path_key = get_db_path_key;
-
-        /* Query Record related functions */
-        methods->gfdb_query_record_new = gfdb_query_record_new;
-        methods->gfdb_query_record_free = gfdb_query_record_free;
-        methods->gfdb_add_link_to_query_record = gfdb_add_link_to_query_record;
-        methods->gfdb_write_query_record = gfdb_write_query_record;
-        methods->gfdb_read_query_record = gfdb_read_query_record;
-
-        /* Link info related functions */
-        methods->gfdb_link_info_new = gfdb_link_info_new;
-        methods->gfdb_link_info_free = gfdb_link_info_free;
-
-        /* Compaction related functions */
-        methods->compact_db = compact_db;
-}
-
diff --git a/libglusterfs/src/gfdb/gfdb_data_store.h b/libglusterfs/src/gfdb/gfdb_data_store.h
deleted file mode 100644
index 0aac4611153..00000000000
--- a/libglusterfs/src/gfdb/gfdb_data_store.h
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GFDB_DATA_STORE_H
-#define __GFDB_DATA_STORE_H
-
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-
-#include "gfdb_data_store_types.h"
-
-#define GFDB_IPC_CTR_KEY "gfdb.ipc-ctr-op"
-
-/*
- * CTR IPC OPERATIONS
- *
- *
- */
-#define GFDB_IPC_CTR_QUERY_OPS "gfdb.ipc-ctr-query-op"
-#define GFDB_IPC_CTR_CLEAR_OPS "gfdb.ipc-ctr-clear-op"
-#define GFDB_IPC_CTR_GET_DB_PARAM_OPS "gfdb.ipc-ctr-get-db-parm"
-#define GFDB_IPC_CTR_GET_DB_VERSION_OPS "gfdb.ipc-ctr-get-db-version"
-#define GFDB_IPC_CTR_SET_COMPACT_PRAGMA "gfdb.ipc-ctr-set-compact-pragma"
-/*
- * CTR IPC INPUT/OUTPUT
- *
- *
- */
-#define GFDB_IPC_CTR_GET_QFILE_PATH "gfdb.ipc-ctr-get-qfile-path"
-#define GFDB_IPC_CTR_GET_QUERY_PARAMS "gfdb.ipc-ctr-get-query-parms"
-#define GFDB_IPC_CTR_RET_QUERY_COUNT "gfdb.ipc-ctr-ret-rec-count"
-#define GFDB_IPC_CTR_GET_DB_KEY "gfdb.ipc-ctr-get-params-key"
-#define GFDB_IPC_CTR_RET_DB_VERSION "gfdb.ipc-ctr-ret-db-version"
-
-/*
- * gfdb ipc ctr params for query
- *
- *
- */
-typedef struct gfdb_ipc_ctr_params {
-        gf_boolean_t is_promote;
-        int write_freq_threshold;
-        int read_freq_threshold;
-        gfdb_time_t time_stamp;
-} gfdb_ipc_ctr_params_t;
-
-
-/* GFDB Connection Node:
- * ~~~~~~~~~~~~~~~~~~~~
- * Represents the connection to the database while using libgfdb
- * The connection node is not thread safe as far as fini_db is concerned.
- * You can use a single connection node
- * to do multithreaded db operations like insert/delete/find of records.
- * But you need to wait for all the operating threads to complete i.e
- * pthread_join() and then do fini_db() to kill the connection node.
- * gfdb_conn_node_t is an opaque structure.
- * */
-typedef struct gfdb_conn_node_t gfdb_conn_node_t;
-
-
-
-
-/*Libgfdb API Function: Used to initialize db connection
- * Arguments:
- *      args         :  Dictionary containing database specific parameters
- *                      eg: For sqlite3, pagesize, cachesize, db name, db path
-                        etc
- *      gfdb_db_type :  Type of data base used i.e sqlite or hyperdex etc
- * Returns : if successful return the GFDB Connection Node to the caller or
- *          NULL value in case of failure*/
-gfdb_conn_node_t *
-init_db(dict_t *arg, gfdb_db_type_t db_type);
-
-typedef gfdb_conn_node_t * (*init_db_t) (dict_t *args,
-                                         gfdb_db_type_t gfdb_db_type);
-
-
-
-
-/*Libgfdb API Function: Used to terminate/de-initialize db connection
- *                      (Destructor function for db connection object)
- * Arguments:
- *      _conn_node  :  DB Connection Index of the DB Connection
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-fini_db(gfdb_conn_node_t *);
-
-typedef int (*fini_db_t) (gfdb_conn_node_t *_conn_node);
-
-
-
-/*Libgfdb API Function: Used to insert/updated records in the database
- *                      NOTE: In current gfdb_sqlite plugin we use that
- *                      same function to delete the record. Set the
- *                      gfdb_fop_path to GFDB_FOP_UNDEL to delete the
- *                      link of inode from GF_FLINK_TB and
- *                      GFDB_FOP_UNDEL_ALL to delete all the records from
- *                      GF_FLINK_TB and GF_FILE_TB.
- *                      TODO: Should seperate this function into the
- *                      delete_record function
- *                      Refer CTR Xlator features/changetimerecorder for usage
- * Arguments:
- *      _conn_node     :  GFDB Connection node
- *      gfdb_db_record :  Record to be inserted/updated
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-insert_record(gfdb_conn_node_t *, gfdb_db_record_t *gfdb_db_record);
-
-
-
-
-/*Libgfdb API Function: Used to delete record from the database
- *                      NOTE: In the current gfdb_sqlite3 plugin
- *                      implementation this function is dummy.
- *                      Use the insert_record function.
- *                      Refer CTR Xlator features/changetimerecorder for usage
- * Arguments:
- *      _conn_node     :  GFDB Connection node
- *      gfdb_db_record :  Record to be deleted
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-delete_record(gfdb_conn_node_t *, gfdb_db_record_t *gfdb_db_record);
-
-
-
-
-
-/*Libgfdb API Function: Query all the records from the database
- * Arguments:
- *      _conn_node      : GFDB Connection node
- *      query_callback  : Call back function that will be called
- *                        for every record found
- *      _query_cbk_args : Custom argument passed for the call back
- *                        function query_callback
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int find_all(gfdb_conn_node_t *, gf_query_callback_t query_callback,
-                void *_query_cbk_args);
-
-
-
-
-
-/*Libgfdb API Function: Query records/files that have not changed/accessed
- *                      from a time in past to current time
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are not
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int find_unchanged_for_time(gfdb_conn_node_t *,
-                        gf_query_callback_t query_callback,
-                        void *_query_cbk_args, gfdb_time_t *for_time);
-
-typedef int (*find_unchanged_for_time_t) (gfdb_conn_node_t *_conn_node,
-                                          gf_query_callback_t query_callback,
-                                          void *_query_cbk_args,
-                                          gfdb_time_t *for_time);
-
-
-
-
-/*Libgfdb API Function: Query records/files that have changed/accessed from a
- *                      time in past to current time
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int find_recently_changed_files(gfdb_conn_node_t *_conn,
-                gf_query_callback_t query_callback, void *_query_cbk_args,
-                gfdb_time_t *from_time);
-
-typedef int (*find_recently_changed_files_t) (gfdb_conn_node_t *_conn_node,
-                                              gf_query_callback_t query_callback,
-                                              void *_query_cbk_args,
-                                              gfdb_time_t *from_time);
-
-
-
-
-/*Libgfdb API Function: Query records/files that have not changed/accessed
- *                      from a time in past to current time, with
- *                      a desired frequency
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are not
- *                                changed/accessed
- *      write_freq_thresold     : Desired Write Frequency lower limit
- *      read_freq_thresold      : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int find_unchanged_for_time_freq(gfdb_conn_node_t *_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *for_time,
-                                        int write_freq_thresold,
-                                        int read_freq_thresold,
-                                        gf_boolean_t _clear_counters);
-
-typedef int (*find_unchanged_for_time_freq_t) (gfdb_conn_node_t *_conn_node,
-                                               gf_query_callback_t query_callback,
-                                               void *_query_cbk_args,
-                                               gfdb_time_t *for_time,
-                                               int write_freq_thresold,
-                                               int read_freq_thresold,
-                                               gf_boolean_t _clear_counters);
-
-
-
-
-/*Libgfdb API Function: Query records/files that have changed/accessed from a
- *                      time in past to current time, with
- *                      a desired frequency
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _query_cbk_args         : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are
- *                                changed/accessed
- *      write_freq_thresold     : Desired Write Frequency lower limit
- *      read_freq_thresold      : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int find_recently_changed_files_freq(gfdb_conn_node_t *_conn,
-                                gf_query_callback_t query_callback,
-                                void *_query_cbk_args,
-                                gfdb_time_t *from_time,
-                                int write_freq_thresold,
-                                int read_freq_thresold,
-                                gf_boolean_t _clear_counters);
-
-typedef int (*find_recently_changed_files_freq_t) (gfdb_conn_node_t *_conn_node,
-                                                   gf_query_callback_t query_callback,
-                                                   void *_query_cbk_args,
-                                                   gfdb_time_t *from_time,
-                                                   int write_freq_thresold,
-                                                   int read_freq_thresold,
-                                                   gf_boolean_t _clear_counters);
-
-typedef const
-char *(*get_db_path_key_t)();
-
-/*Libgfdb API Function: Clear the heat for all the files
- *
- * Arguments:
- *      _conn_node              : GFDB Connection node
- *
- * Returns : if successful return 0 or
- *          -ve value in case of failure
- **/
-int
-clear_files_heat (gfdb_conn_node_t *_conn_node);
-
-typedef int (*clear_files_heat_t) (gfdb_conn_node_t *_conn_node);
-
-
-
-/* Libgfdb API Function: Function to extract version of the db
- *  Arguments:
- *      gfdb_conn_node_t *_conn_node        : GFDB Connection node
- *      char **version  : the version is extracted as a string
- *                   and will be stored in this variable.
- *                   The freeing of the memory should be done by the caller.
- * Return:
- *      On success return the length of the version string that is
- *      extracted.
- *      On failure return -1
- * */
-int
-get_db_version (gfdb_conn_node_t *_conn_node, char **version);
-
-typedef int (*get_db_version_t)(gfdb_conn_node_t *_conn_node,
-                                        char **version);
-
-
-/* Libgfdb API Function: Function to extract param from the db
- *  Arguments:
- *      gfdb_conn_node_t *_conn_node        : GFDB Connection node
- *      char *param_key     : param to be extracted
- *      char **param_value  : the value of the param that is
- *                       extracted. This function will allocate memory
- *                       to pragma_value. The caller should free the memory.
- * Return:
- *      On success return the lenght of the param value that is
- *      extracted.
- *      On failure return -1
- * */
-int
-get_db_params (gfdb_conn_node_t *_conn_node,
-                char *param_key,
-                char **param_value);
-
-typedef int (*get_db_params_t)(gfdb_conn_node_t *db_conn,
-                                     char *param_key,
-                                     char **param_value);
-
-
-/* Libgfdb API Function: Function to set db params
- * Arguments:
- *      gfdb_conn_node_t *_conn_node        : GFDB Connection node
- *      char *param_key     : param to be set
- * char *param_value  : param value
- * Return:
- *      On success return 0
- *      On failure return -1
- * */
-int
-set_db_params (gfdb_conn_node_t *_conn_node,
-                char *param_key,
-                char *param_value);
-
-typedef int (*set_db_params_t)(gfdb_conn_node_t *db_conn,
-                                     char *param_key,
-                                     char *param_value);
-
-/*Libgfdb API Function: Compact the database.
- *
- * Arguments:
- *      _conn_node                      :  GFDB Connection node
- *      _compact_active                 :  Is compaction currently on?
- *      _compact_mode_switched          :  Was the compaction switch flipped?
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-int
-compact_db (gfdb_conn_node_t *_conn_node, gf_boolean_t _compact_active,
-            gf_boolean_t _compact_mode_switched);
-
-typedef int (*compact_db_t)(gfdb_conn_node_t *db_conn,
-                            gf_boolean_t compact_active,
-                            gf_boolean_t compact_mode_switched);
-
-
-typedef struct gfdb_methods_s {
-        init_db_t                       init_db;
-        fini_db_t                       fini_db;
-        find_unchanged_for_time_t       find_unchanged_for_time;
-        find_recently_changed_files_t   find_recently_changed_files;
-        find_unchanged_for_time_freq_t  find_unchanged_for_time_freq;
-        find_recently_changed_files_freq_t find_recently_changed_files_freq;
-        clear_files_heat_t              clear_files_heat;
-        get_db_version_t                get_db_version;
-        get_db_params_t                 get_db_params;
-        set_db_params_t                 set_db_params;
-        /* Do not expose dbpath directly. Expose it via an */
-        /* access function: get_db_path_key(). */
-        char                            *dbpath;
-        get_db_path_key_t               get_db_path_key;
-
-        /* Query Record related functions */
-        gfdb_query_record_new_t         gfdb_query_record_new;
-        gfdb_query_record_free_t        gfdb_query_record_free;
-        gfdb_add_link_to_query_record_t gfdb_add_link_to_query_record;
-        gfdb_write_query_record_t       gfdb_write_query_record;
-        gfdb_read_query_record_t        gfdb_read_query_record;
-
-        /* Link info related functions */
-        gfdb_link_info_new_t            gfdb_link_info_new;
-        gfdb_link_info_free_t           gfdb_link_info_free;
-
-        /* Compaction related functions */
-        compact_db_t                    compact_db;
-} gfdb_methods_t;
-
-void get_gfdb_methods (gfdb_methods_t *methods);
-
-typedef void (*get_gfdb_methods_t) (gfdb_methods_t *methods);
-
-#endif
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_helper.c b/libglusterfs/src/gfdb/gfdb_data_store_helper.c
deleted file mode 100644
index fba5ec5a252..00000000000
--- a/libglusterfs/src/gfdb/gfdb_data_store_helper.c
+++ /dev/null
@@ -1,612 +0,0 @@
-
-#include "gfdb_data_store_helper.h"
-#include "syscall.h"
-
-/******************************************************************************
- *
- *                       Query record related functions
- *
- * ****************************************************************************/
-
-/*Create a single link info structure*/
-gfdb_link_info_t*
-gfdb_link_info_new ()
-{
-        gfdb_link_info_t *link_info = NULL;
-
-        link_info = GF_CALLOC (1, sizeof(gfdb_link_info_t),
-                                        gf_mt_gfdb_link_info_t);
-        if (!link_info) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
-                        LG_MSG_NO_MEMORY, "Memory allocation failed for "
-                        "link_info ");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&link_info->list);
-
-out:
-
-        return link_info;
-}
-
-/*Destroy a link info structure*/
-void
-gfdb_link_info_free(gfdb_link_info_t *link_info)
-{
-        GF_FREE (link_info);
-}
-
-
-/*Function to create the query_record*/
-gfdb_query_record_t *
-gfdb_query_record_new()
-{
-        int ret = -1;
-        gfdb_query_record_t *query_record = NULL;
-
-        query_record = GF_CALLOC (1, sizeof(gfdb_query_record_t),
-                                        gf_mt_gfdb_query_record_t);
-        if (!query_record) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
-                        LG_MSG_NO_MEMORY, "Memory allocation failed for "
-                        "query_record ");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&query_record->link_list);
-
-        ret = 0;
-out:
-        if (ret == -1) {
-                GF_FREE (query_record);
-        }
-        return query_record;
-}
-
-
-/*Function to delete a single linkinfo from list*/
-static void
-gfdb_delete_linkinfo_from_list (gfdb_link_info_t **link_info)
-{
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, link_info, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, *link_info, out);
-
-        /*Remove hard link from list*/
-        list_del(&(*link_info)->list);
-        gfdb_link_info_free (*link_info);
-        link_info = NULL;
-out:
-        return;
-}
-
-
-/*Function to destroy link_info list*/
-void
-gfdb_free_link_info_list (gfdb_query_record_t *query_record)
-{
-        gfdb_link_info_t        *link_info = NULL;
-        gfdb_link_info_t        *temp = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-
-        list_for_each_entry_safe(link_info, temp,
-                        &query_record->link_list, list)
-        {
-                gfdb_delete_linkinfo_from_list (&link_info);
-                link_info = NULL;
-        }
-
-out:
-        return;
-}
-
-
-
-/* Function to add linkinfo to the query record */
-int
-gfdb_add_link_to_query_record (gfdb_query_record_t      *query_record,
-                           uuid_t                   pgfid,
-                           char               *base_name)
-{
-        int ret                                 = -1;
-        gfdb_link_info_t *link_info             = NULL;
-        int base_name_len                       = 0;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, pgfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, base_name, out);
-
-        link_info = gfdb_link_info_new ();
-        if (!link_info) {
-                goto out;
-        }
-
-        gf_uuid_copy (link_info->pargfid, pgfid);
-        base_name_len = strlen (base_name);
-        memcpy (link_info->file_name, base_name, base_name_len);
-        link_info->file_name[base_name_len] = '\0';
-
-        list_add_tail (&link_info->list,
-                        &query_record->link_list);
-
-        query_record->link_count++;
-
-        ret = 0;
-out:
-        if (ret) {
-                gfdb_link_info_free (link_info);
-                link_info = NULL;
-        }
-        return ret;
-}
-
-
-
-/*Function to destroy query record*/
-void
-gfdb_query_record_free(gfdb_query_record_t *query_record)
-{
-        if (query_record) {
-                gfdb_free_link_info_list (query_record);
-                GF_FREE (query_record);
-        }
-}
-
-
-/******************************************************************************
-                SERIALIZATION/DE-SERIALIZATION OF QUERY RECORD
-*******************************************************************************/
-/******************************************************************************
- The on disk format of query record is as follows,
-
-+---------------------------------------------------------------------------+
-| Length of serialized query record |       Serialized Query Record         |
-+---------------------------------------------------------------------------+
-             4 bytes                     Length of serialized query record
-                                                      |
-                                                      |
-     -------------------------------------------------|
-     |
-     |
-     V
-   Serialized Query Record Format:
-   +---------------------------------------------------------------------------+
-   | GFID |  Link count   |  <LINK INFO>  |.....                      | FOOTER |
-   +---------------------------------------------------------------------------+
-     16 B        4 B         Link Length                                  4 B
-                                |                                          |
-                                |                                          |
-   -----------------------------|                                          |
-   |                                                                       |
-   |                                                                       |
-   V                                                                       |
-   Each <Link Info> will be serialized as                                  |
-   +-----------------------------------------------+                       |
-   | PGID | BASE_NAME_LENGTH |      BASE_NAME      |                       |
-   +-----------------------------------------------+                       |
-     16 B       4 B             BASE_NAME_LENGTH                           |
-                                                                           |
-                                                                           |
-   ------------------------------------------------------------------------|
-   |
-   |
-   V
-   FOOTER is a magic number 0xBAADF00D indicating the end of the record.
-   This also serves as a serialized schema validator.
- * ****************************************************************************/
-
-#define GFDB_QUERY_RECORD_FOOTER 0xBAADF00D
-#define UUID_LEN                 16
-
-/*Function to get the potential length of the serialized buffer*/
-static int32_t
-gfdb_query_record_serialized_length (gfdb_query_record_t *query_record)
-{
-        int32_t len                         = -1;
-        gfdb_link_info_t *link_info     = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-
-        /* Length of GFID */
-        len = UUID_LEN;
-
-        /* length of number of links*/
-        len += sizeof (int32_t);
-
-        list_for_each_entry (link_info, &query_record->link_list, list) {
-
-                /* length of PFID */
-                len += UUID_LEN;
-
-                /* Add size of base name length*/
-                len += sizeof (int32_t);
-
-                /* Length of base_name */
-                len += strlen (link_info->file_name);
-
-        }
-
-        /* length of footer */
-        len += sizeof (int32_t);
-out:
-        return len;
-}
-
-/* Function for serializing query record.
- *
- * Query Record Serialization Format
- * +---------------------------------------------------------------------------+
- * | GFID |  Link count   |  <LINK INFO>  |.....                      | FOOTER |
- * +---------------------------------------------------------------------------+
- *   16 B        4 B         Link Length                                  4 B
- *
- *
- * Each <Link Info> will be serialized as
- * +-----------------------------------------------+
- * | PGID | BASE_NAME_LENGTH |      BASE_NAME      |
- * +-----------------------------------------------+
- *   16 B       4 B             BASE_NAME_LENGTH
- *
- *
- * FOOTER is a magic number 0xBAADF00D indicating the end of the record.
- * This also serves as a serialized schema validator.
- *
- * The function will allocate memory to the serialized buffer,
- * the caller needs to free it.
- * Returns the length of the serialized buffer on success
- * or -1 on failure.
- *
- * */
-static int
-gfdb_query_record_serialize (gfdb_query_record_t *query_record,
-                             char **in_buffer)
-{
-        gfdb_link_info_t *link_info     = NULL;
-        int              count          = -1;
-        int              base_name_len  = 0;
-        int              buffer_length  = 0;
-        int              footer         = GFDB_QUERY_RECORD_FOOTER;
-        char             *buffer        = NULL;
-        char             *ret_buffer    = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE,
-                             (query_record->link_count > 0), out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, in_buffer, out);
-
-
-        /* Calculate the total length of the serialized buffer */
-        buffer_length = gfdb_query_record_serialized_length (query_record);
-        if (buffer_length <= 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Failed to calculate the length of "
-                        "serialized buffer");
-                goto out;
-        }
-
-        /* Allocate memory to the serialized buffer */
-        ret_buffer = GF_CALLOC (1, buffer_length,  gf_common_mt_char);
-        if (!ret_buffer) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Memory allocation failed for "
-                        "serialized buffer.");
-                goto out;
-        }
-
-        buffer = ret_buffer;
-
-        count = 0;
-
-        /* Copying the GFID */
-        memcpy (buffer, query_record->gfid, UUID_LEN);
-        buffer += UUID_LEN;
-        count += UUID_LEN;
-
-        /* Copying the number of links */
-        memcpy (buffer, &query_record->link_count, sizeof (int32_t));
-        buffer += sizeof (int32_t);
-        count += sizeof (int32_t);
-
-        list_for_each_entry (link_info, &query_record->link_list, list) {
-
-                /* Copying the PFID */
-                memcpy(buffer, link_info->pargfid, UUID_LEN);
-                buffer += UUID_LEN;
-                count += UUID_LEN;
-
-                /* Copying base name length*/
-                base_name_len = strlen (link_info->file_name);
-                memcpy (buffer, &base_name_len, sizeof (int32_t));
-                buffer += sizeof (int32_t);
-                count += sizeof (int32_t);
-
-                /* Length of base_name */
-                memcpy(buffer, link_info->file_name, base_name_len);
-                buffer += base_name_len;
-                count += base_name_len;
-
-        }
-
-        /* Copying the Footer of the record */
-        memcpy (buffer, &footer, sizeof (int32_t));
-        buffer += sizeof (int32_t);
-        count += sizeof (int32_t);
-
-out:
-        if (count < 0) {
-                GF_FREE (ret_buffer);
-                ret_buffer = NULL;
-        }
-        *in_buffer = ret_buffer;
-        return count;
-}
-
-static gf_boolean_t
-is_serialized_buffer_valid (char *in_buffer, int buffer_length) {
-        gf_boolean_t    ret        = _gf_false;
-        int             footer     = 0;
-
-        /* Read the footer */
-        in_buffer += (buffer_length - sizeof (int32_t));
-        memcpy (&footer, in_buffer, sizeof (int32_t));
-
-        /*
-         * if the footer is not GFDB_QUERY_RECORD_FOOTER
-         * then the serialized record is invalid
-         *
-         * */
-        if (footer != GFDB_QUERY_RECORD_FOOTER) {
-                goto out;
-        }
-
-        ret = _gf_true;
-out:
-        return ret;
-}
-
-
-static int
-gfdb_query_record_deserialize (char *in_buffer,
-                               int buffer_length,
-                               gfdb_query_record_t **query_record)
-{
-        int ret                                 = -1;
-        char *buffer                            = NULL;
-        int i                                   = 0;
-        gfdb_link_info_t *link_info             = NULL;
-        int count                               = 0;
-        int base_name_len                       = 0;
-        gfdb_query_record_t *ret_qrecord        = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, in_buffer, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (buffer_length > 0), out);
-
-        if (!is_serialized_buffer_valid (in_buffer, buffer_length)) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Invalid serialized query record");
-                goto out;
-        }
-
-        buffer = in_buffer;
-
-        ret_qrecord = gfdb_query_record_new ();
-        if (!ret_qrecord) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Failed to allocate space to "
-                        "gfdb_query_record_t");
-                goto out;
-        }
-
-        /* READ GFID */
-        memcpy ((ret_qrecord)->gfid, buffer, UUID_LEN);
-        buffer += UUID_LEN;
-        count += UUID_LEN;
-
-        /* Read the number of link */
-        memcpy (&(ret_qrecord->link_count), buffer, sizeof (int32_t));
-        buffer += sizeof (int32_t);
-        count += sizeof (int32_t);
-
-        /* Read all the links */
-        for (i = 0; i < ret_qrecord->link_count; i++) {
-                if (count >= buffer_length) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DB_ERROR, "Invalid serialized "
-                                "query record");
-                        ret = -1;
-                        goto out;
-                }
-
-                link_info = gfdb_link_info_new ();
-                if (!link_info) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DB_ERROR, "Failed to create link_info");
-                        goto out;
-                }
-
-                /* READ PGFID */
-                memcpy (link_info->pargfid, buffer, UUID_LEN);
-                buffer += UUID_LEN;
-                count += UUID_LEN;
-
-                /* Read base name length */
-                memcpy (&base_name_len, buffer, sizeof (int32_t));
-                buffer += sizeof (int32_t);
-                count += sizeof (int32_t);
-
-                /* READ basename */
-                memcpy (link_info->file_name, buffer, base_name_len);
-                buffer += base_name_len;
-                count += base_name_len;
-                link_info->file_name[base_name_len] = '\0';
-
-                /* Add link_info to the list */
-                list_add_tail (&link_info->list,
-                               &(ret_qrecord->link_list));
-
-                /* Reseting link_info */
-                link_info = NULL;
-        }
-
-        ret = 0;
-out:
-        if (ret) {
-                gfdb_query_record_free (ret_qrecord);
-                ret_qrecord = NULL;
-        }
-        *query_record = ret_qrecord;
-        return ret;
-}
-
-
-
-
-
-/* Function to write query record to file
- *
- * Disk format
- * +---------------------------------------------------------------------------+
- * | Length of serialized query record |       Serialized Query Record         |
- * +---------------------------------------------------------------------------+
- *            4 bytes                     Length of serialized query record
- *
- * Please refer gfdb_query_record_serialize () for format of
- * Serialized Query Record
- *
- * */
-int
-gfdb_write_query_record (int fd,
-                        gfdb_query_record_t *query_record)
-{
-        int ret                 = -1;
-        int buffer_len          = 0;
-        char *buffer            = NULL;
-        int write_len           = 0;
-        char *write_buffer      = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (fd >= 0), out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-
-        buffer_len = gfdb_query_record_serialize (query_record, &buffer);
-        if (buffer_len < 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Failed to serialize query record");
-                goto out;
-        }
-
-        /* Serialize the buffer length and write to file */
-        ret = write (fd, &buffer_len, sizeof (int32_t));
-        if (ret < 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DB_ERROR, "Failed to write buffer length"
-                                " to file");
-                goto out;
-        }
-
-        /* Write the serialized query record to file */
-        write_len = buffer_len;
-        write_buffer = buffer;
-        while ((ret = write (fd, write_buffer, write_len)) <  write_len) {
-                if (ret < 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, errno,
-                                LG_MSG_DB_ERROR, "Failed to write serialized "
-                                "query record to file");
-                        goto out;
-                }
-
-                write_buffer += ret;
-                write_len -= ret;
-        }
-
-        ret = 0;
-out:
-        GF_FREE (buffer);
-        return ret;
-}
-
-
-
-/* Function to read query record from file.
- * Allocates memory to query record and
- * returns length of serialized query record when successful
- * Return -1 when failed.
- * Return 0 when reached EOF.
- * */
-int
-gfdb_read_query_record (int fd,
-                        gfdb_query_record_t **query_record)
-{
-        int ret                 = -1;
-        int buffer_len          = 0;
-        int read_len            = 0;
-        char *buffer            = NULL;
-        char *read_buffer       = NULL;
-
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (fd >= 0), out);
-        GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
-
-
-        /* Read serialized query record length from the file*/
-        ret = sys_read (fd, &buffer_len, sizeof (int32_t));
-        if (ret < 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DB_ERROR, "Failed reading buffer length"
-                                " from file");
-                goto out;
-        }
-        /* EOF */
-        else if (ret == 0) {
-                ret = 0;
-                goto out;
-        }
-
-        /* Allocating memory to the serialization buffer */
-        buffer = GF_CALLOC (1, buffer_len,  gf_common_mt_char);
-        if (!buffer) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Failed to allocate space to "
-                        "serialized buffer");
-                goto out;
-        }
-
-
-        /* Read the serialized query record from file */
-        read_len = buffer_len;
-        read_buffer = buffer;
-        while ((ret = sys_read (fd, read_buffer, read_len)) < read_len) {
-
-                /*Any error */
-                if (ret < 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, errno,
-                                LG_MSG_DB_ERROR, "Failed to read serialized "
-                                "query record from file");
-                        goto out;
-                }
-                /* EOF */
-                else if (ret == 0) {
-                        gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                                LG_MSG_DB_ERROR, "Invalid query record or "
-                                "corrupted query file");
-                        ret = -1;
-                        goto out;
-                }
-
-                read_buffer += ret;
-                read_len -= ret;
-        }
-
-        ret = gfdb_query_record_deserialize (buffer, buffer_len,
-                                             query_record);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        LG_MSG_DB_ERROR, "Failed to de-serialize query record");
-                goto out;
-        }
-
-        ret = buffer_len;
-out:
-        GF_FREE (buffer);
-        return ret;
-}
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_helper.h b/libglusterfs/src/gfdb/gfdb_data_store_helper.h
deleted file mode 100644
index ce1f1c52281..00000000000
--- a/libglusterfs/src/gfdb/gfdb_data_store_helper.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GFDB_DATA_STORE_HELPER_H
-#define __GFDB_DATA_STORE_HELPER_H
-
-#include <time.h>
-#include <sys/time.h>
-#include <string.h>
-#include <fcntl.h>
-
-#include "common-utils.h"
-#include "compat-uuid.h"
-#include "gfdb_mem-types.h"
-#include "dict.h"
-#include "byte-order.h"
-#include "libglusterfs-messages.h"
-
-
-#define GFDB_DATA_STORE               "gfdbdatastore"
-
-/*******************************************************************************
- *
- *                     Query related data structure and functions
- *
- * ****************************************************************************/
-
-#ifdef NAME_MAX
-#define GF_NAME_MAX NAME_MAX
-#else
-#define GF_NAME_MAX 255
-#endif
-
-/*Structure to hold the link information*/
-typedef struct gfdb_link_info {
-        uuid_t                          pargfid;
-        char                            file_name[GF_NAME_MAX];
-        struct list_head                list;
-} gfdb_link_info_t;
-
-
-/*Structure used for querying purpose*/
-typedef struct gfdb_query_record {
-        uuid_t                          gfid;
-        /*This is the hardlink list*/
-        struct list_head                link_list;
-        int                             link_count;
-} gfdb_query_record_t;
-
-/*Create a single link info structure*/
-gfdb_link_info_t *gfdb_link_info_new ();
-typedef gfdb_link_info_t *(*gfdb_link_info_new_t) ();
-
-/*Destroy a link info structure*/
-void
-gfdb_link_info_free (gfdb_link_info_t *gfdb_link_info);
-typedef void
-(*gfdb_link_info_free_t) (gfdb_link_info_t *gfdb_link_info);
-
-/* Function to create the query_record */
-gfdb_query_record_t *
-gfdb_query_record_new();
-typedef gfdb_query_record_t *
-(*gfdb_query_record_new_t)();
-
-
-
-
-/* Fuction to add linkinfo to query record */
-int
-gfdb_add_link_to_query_record (gfdb_query_record_t      *gfdb_query_record,
-                           uuid_t                   pgfid,
-                           char               *base_name);
-typedef int
-(*gfdb_add_link_to_query_record_t) (gfdb_query_record_t *, uuid_t, char *);
-
-
-
-
-/*Function to destroy query record*/
-void
-gfdb_query_record_free (gfdb_query_record_t *gfdb_query_record);
-typedef void
-(*gfdb_query_record_free_t) (gfdb_query_record_t *);
-
-
-
-
-
-
-/* Function to write query record to file */
-int
-gfdb_write_query_record (int fd,
-                        gfdb_query_record_t *gfdb_query_record);
-typedef int
-(*gfdb_write_query_record_t) (int, gfdb_query_record_t *);
-
-
-
-
-
-/* Function to read query record from file.
- * Allocates memory to query record and return 0 when successful
- * Return -1 when failed.
- * Return 0 when EOF.
- * */
-int
-gfdb_read_query_record (int fd,
-                        gfdb_query_record_t **gfdb_query_record);
-typedef int
-(*gfdb_read_query_record_t) (int, gfdb_query_record_t **);
-
-
-#endif
-\ No newline at end of file
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_types.h b/libglusterfs/src/gfdb/gfdb_data_store_types.h
deleted file mode 100644
index d0c96370eb8..00000000000
--- a/libglusterfs/src/gfdb/gfdb_data_store_types.h
+++ /dev/null
@@ -1,586 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GFDB_DATA_STORE_TYPE_H
-#define __GFDB_DATA_STORE_TYPE_H
-
-#include "gfdb_data_store_helper.h"
-
-/*
- * Helps in dynamically choosing log level
- * */
-static inline gf_loglevel_t
-_gfdb_log_level (gf_loglevel_t given_level,
-                 gf_boolean_t ignore_level)
-{
-        return (ignore_level) ? GF_LOG_DEBUG : given_level;
-}
-
-typedef enum gf_db_operation {
-        GFDB_INVALID_DB_OP = -1,
-        /* Query DB OPS : All the Query DB_OP should be added */
-        /* in between START and END */
-        GFDB_QUERY_DB_OP_START, /* Start of Query DB_OP  */
-        GFDB_QUERY_DB_OP,
-        GF_FTABLE_EXISTS_DB_OP,
-        GFDB_QUERY_DB_OP_END, /* End of Query DB_OP */
-        /* Non-Query DB OPS */
-        GFDB_DB_CREATE_DB_OP,
-        GFDB_GFID_EXIST_DB_OP,
-        GFDB_W_INSERT_DB_OP,
-        GFDB_WU_INSERT_DB_OP,
-        GFDB_W_UPDATE_DB_OP,
-        GFDB_WU_UPDATE_DB_OP,
-        GFDB_W_DELETE_DB_OP,
-        GFDB_UW_DELETE_DB_OP,
-        GFDB_WFC_UPDATE_DB_OP,
-        GFDB_RFC_UPDATE_DB_OP,
-        GFDB_DB_COMPACT_DB_OP /* Added for VACUUM/manual compaction support */
-} gf_db_operation_t;
-
-
-#define GF_COL_MAX_NUM                  2
-#define GF_COL_ALL                      " * "
-
-/* Column/fields names used in the DB.
- * If any new field is added should be updated here*/
-#define GF_COL_GF_ID                    "GF_ID"
-#define GF_COL_GF_PID                   "GF_PID"
-#define GF_COL_FILE_NAME                "FNAME"
-#define GF_COL_WSEC                     "W_SEC"
-#define GF_COL_WMSEC                    "W_MSEC"
-#define GF_COL_UWSEC                    "UW_SEC"
-#define GF_COL_UWMSEC                   "UW_MSEC"
-#define GF_COL_WSEC_READ                "W_READ_SEC"
-#define GF_COL_WMSEC_READ               "W_READ_MSEC"
-#define GF_COL_UWSEC_READ               "UW_READ_SEC"
-#define GF_COL_UWMSEC_READ              "UW_READ_MSEC"
-#define GF_COL_WDEL_FLAG                "W_DEL_FLAG"
-#define GF_COL_WRITE_FREQ_CNTR          "WRITE_FREQ_CNTR"
-#define GF_COL_READ_FREQ_CNTR           "READ_FREQ_CNTR"
-#define GF_COL_LINK_UPDATE              "LINK_UPDATE"
-
-
-/***********************Time related********************************/
-/*1 sec = 1000000 microsec*/
-#define GFDB_MICROSEC         1000000
-
-/*All the gfdb times are represented using this structure*/
-typedef  struct timeval gfdb_time_t;
-
-/*Convert time into  seconds*/
-static inline uint64_t
-gfdb_time_2_usec(gfdb_time_t *gfdb_time)
-{
-        GF_ASSERT(gfdb_time);
-        return ((uint64_t) gfdb_time->tv_sec * GFDB_MICROSEC) + gfdb_time->tv_usec;
-}
-
-/******************************************************************************
- *
- *              Insert/Update Record related data structures/functions
- *
- * ****************************************************************************/
-
-/*Indicated a generic synchronous write to the db
- * This may or may not be implemented*/
-typedef enum gfdb_sync_type {
-        GFDB_INVALID_SYNC = -1,
-        GFDB_DB_ASYNC,
-        GFDB_DB_SYNC
-} gfdb_sync_type_t;
-
-/*Strings related to the abvove sync type*/
-#define GFDB_STR_DB_ASYNC        "async"
-#define GFDB_STR_DB_SYNC         "sync"
-
-/*To convert sync type from string to gfdb_sync_type_t*/
-static inline int
-gf_string2gfdbdbsync (char *sync_option)
-{
-        int ret = -1;
-
-        if (!sync_option)
-                goto out;
-        if (strcmp(sync_option, GFDB_STR_DB_ASYNC) == 0) {
-                ret = GFDB_DB_ASYNC;
-        } else if (strcmp(sync_option, GFDB_STR_DB_SYNC) == 0) {
-                ret = GFDB_DB_SYNC;
-        }
-out:
-        return ret;
-}
-
-/*Indicated different types of db*/
-typedef enum gfdb_db_type {
-        GFDB_INVALID_DB = -1,
-        GFDB_HASH_FILE_STORE,
-        GFDB_ROCKS_DB,
-        GFDB_SQLITE3,
-        GFDB_HYPERDEX,
-        GFDB_DB_END /*Add DB type Entries above this only*/
-} gfdb_db_type_t;
-
-/*String related to the db types*/
-#define GFDB_STR_HASH_FILE_STORE      "hashfile"
-#define GFDB_STR_ROCKS_DB             "rocksdb"
-#define GFDB_STR_SQLITE3              "sqlite3"
-#define GFDB_STR_HYPERDEX             "hyperdex"
-
-/*Convert db type in string to gfdb_db_type_t*/
-static inline int
-gf_string2gfdbdbtype (char *db_option)
-{
-        int ret = -1;
-
-        if (!db_option)
-                goto out;
-        if (strcmp(db_option, GFDB_STR_HASH_FILE_STORE) == 0) {
-                ret = GFDB_HASH_FILE_STORE;
-        } else if (strcmp(db_option, GFDB_STR_ROCKS_DB) == 0) {
-                ret = GFDB_ROCKS_DB;
-        } else if (strcmp(db_option, GFDB_STR_SQLITE3) == 0) {
-                ret = GFDB_SQLITE3;
-        } else if (strcmp(db_option, GFDB_STR_HYPERDEX) == 0) {
-                ret = GFDB_HYPERDEX;
-        }
-out:
-        return ret;
-}
-
-/*Tells the path of the fop*/
-typedef enum gfdb_fop_path {
-        GFDB_FOP_INVALID = -1,
-        /*Filler value for zero*/
-        GFDB_FOP_PATH_ZERO = 0,
-        /*have wind path below this*/
-        GFDB_FOP_WIND = 1,
-        GFDB_FOP_WDEL = 2,
-        /*have unwind path below this*/
-        GFDB_FOP_UNWIND = 4,
-        /*Delete unwind path*/
-        GFDB_FOP_UNDEL = 8,
-        GFDB_FOP_UNDEL_ALL = 16
-} gfdb_fop_path_t;
-/*Strings related to the above fop path*/
-#define GFDB_STR_FOP_INVALID "INVALID"
-#define GFDB_STR_FOP_WIND "ENTRY"
-#define GFDB_STR_FOP_UNWIND "EXIT"
-#define GFDB_STR_FOP_WDEL "WDEL"
-#define GFDB_STR_FOP_UNDEL "UNDEL"
-
-static inline gf_boolean_t
-iswindpath(gfdb_fop_path_t gfdb_fop_path)
-{
-        return ((gfdb_fop_path == GFDB_FOP_WIND) ||
-                        (gfdb_fop_path == GFDB_FOP_WDEL)) ?
-                                                _gf_true : _gf_false;
-}
-
-static inline gf_boolean_t
-isunwindpath(gfdb_fop_path_t gfdb_fop_path)
-{
-        return (gfdb_fop_path >= GFDB_FOP_UNWIND) ? _gf_true : _gf_false;
-}
-
-/*Tell what type of fop it was
- * Like whether a dentry fop or a inode fop
- * Read fop or a write fop etc*/
-typedef enum gfdb_fop_type {
-        GFDB_FOP_INVALID_OP = -1,
-        /*Filler value for zero*/
-        GFDB_FOP_TYPE_ZERO = 0,
-        GFDB_FOP_DENTRY_OP = 1,
-        GFDB_FOP_DENTRY_CREATE_OP = 2,
-        GFDB_FOP_INODE_OP = 4,
-        GFDB_FOP_WRITE_OP = 8,
-        GFDB_FOP_READ_OP = 16
-} gfdb_fop_type_t;
-
-#define GFDB_FOP_INODE_WRITE\
-        (GFDB_FOP_INODE_OP | GFDB_FOP_WRITE_OP)
-
-#define GFDB_FOP_DENTRY_WRITE\
-        (GFDB_FOP_DENTRY_OP | GFDB_FOP_WRITE_OP)
-
-#define GFDB_FOP_CREATE_WRITE\
-        (GFDB_FOP_DENTRY_CREATE_OP | GFDB_FOP_WRITE_OP)
-
-#define GFDB_FOP_INODE_READ\
-        (GFDB_FOP_INODE_OP | GFDB_FOP_READ_OP)
-
-static inline gf_boolean_t
-isreadfop(gfdb_fop_type_t fop_type)
-{
-        return (fop_type & GFDB_FOP_READ_OP) ? _gf_true : _gf_false;
-}
-
-static inline gf_boolean_t
-isdentryfop(gfdb_fop_type_t fop_type)
-{
-        return ((fop_type & GFDB_FOP_DENTRY_OP) ||
-                (fop_type & GFDB_FOP_DENTRY_CREATE_OP)) ? _gf_true : _gf_false;
-}
-
-static inline gf_boolean_t
-isdentrycreatefop(gfdb_fop_type_t fop_type)
-{
-        return (fop_type & GFDB_FOP_DENTRY_CREATE_OP) ?
-                        _gf_true : _gf_false;
-}
-
-/*The structure that is used to send insert/update the databases
- * using insert_db api*/
-typedef struct gfdb_db_record {
-        /* GFID */
-        uuid_t                          gfid;
-        /* Used during a rename refer ctr_rename() in changetimerecorder
-         * xlator*/
-        uuid_t                          old_gfid;
-        /* Parent GFID */
-        uuid_t                          pargfid;
-        uuid_t                          old_pargfid;
-        /* File names */
-        char                            file_name[GF_NAME_MAX + 1];
-        char                            old_file_name[GF_NAME_MAX + 1];
-        /* FOP type and FOP path*/
-        gfdb_fop_type_t                 gfdb_fop_type;
-        gfdb_fop_path_t                 gfdb_fop_path;
-        /*Time of change or access*/
-        gfdb_time_t                     gfdb_wind_change_time;
-        gfdb_time_t                     gfdb_unwind_change_time;
-        /* For crash consistancy while inserting/updating hard links */
-        gf_boolean_t                    islinkupdate;
-        /* For link consistency we do a double update i.e mark the link
-         * during the wind and during the unwind we update/delete the link.
-         * This has a performance hit. We give a choice here whether we need
-         * link consistency to be spoton or not using link_consistency flag.
-         * This will have only one link update */
-         gf_boolean_t                   link_consistency;
-        /* For dentry fops we can choose to ignore recording of unwind time */
-        /* For inode fops "record_exit" volume option does the trick,       */
-        /* but for dentry fops we update the LINK_UPDATE, so an extra       */
-        /* flag is provided to ignore the recording of the unwind time.     */
-        gf_boolean_t                    do_record_uwind_time;
-        /* Global flag to record or not record counters */
-        gf_boolean_t                    do_record_counters;
-        /* Global flag to Record/Not Record wind or wind time.
-         * This flag will overrule do_record_uwind_time*/
-        gf_boolean_t                    do_record_times;
-        /* Ignoring errors while inserting.
-         * */
-        gf_boolean_t                    ignore_errors;
-} gfdb_db_record_t;
-
-
-/*******************************************************************************
- *
- *                           Signatures for the plugin functions
- *                           i.e Any plugin should implementment
- *                           these functions to integrate with
- *                           libgfdb.
- *
- * ****************************************************************************/
-
-/*Call back function for querying the database*/
-typedef int
-(*gf_query_callback_t)(gfdb_query_record_t *, void *);
-
-/* Used to initialize db connection
- * Arguments:
- *      args         : Dictionary containing database specific parameters
- *      db_conn      : pointer to plugin specific data base connection
- *                     that will be created. If the call is successful
- *                     db_conn will contain the plugin specific connection
- *                     If call is unsuccessful will have NULL.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_init_db_t)(dict_t *args, void **db_conn);
-
-
-
-
-/* Used to terminate/de-initialize db connection
- *                      (Destructor function for db connection object)
- * Arguments:
- *      db_conn        : plugin specific data base connection
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_fini_db_t)(void **db_conn);
-
-
-
-
-/*Used to insert/updated records in the database
- * Arguments:
- *      db_conn        : plugin specific data base connection
- *      gfdb_db_record :  Record to be inserted/updated
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_insert_record_t)(void *db_conn,
-                        gfdb_db_record_t *db_record);
-
-
-
-
-/*Used to delete record from the database
- * Arguments:
- *      db_conn        : plugin specific data base connection
- *      gfdb_db_record :  Record to be deleted
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_delete_record_t)(void *db_conn,
-                        gfdb_db_record_t *db_record);
-
-
-
-
-/*Used to compact the database
- * Arguments:
- *      db_conn                        :  GFDB Connection node
- *      compact_active                 :  Is compaction currently on?
- *      compact_mode_switched          :  Was the compaction switch flipped?
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_compact_db_t)(void *db_conn, gf_boolean_t compact_active,
-                     gf_boolean_t compact_mode_switched);
-
-
-
-
-/* Query all the records from the database
- * Arguments:
- *      db_conn        : plugin specific data base connection
- *      query_callback  : Call back function that will be called
- *                        for every record found
- *      _query_cbk_args : Custom argument passed for the call back
- *                        function query_callback
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_find_all_t)(void *db_conn,
-                               gf_query_callback_t query_callback,
-                               void *_cbk_args);
-
-
-
-
-/* Query records/files that have not changed/accessed
- *                      from a time in past to current time
- * Arguments:
- *      db_conn                 : plugin specific data base connection
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _cbk_args               : Custom argument passed for the call back
- *                                function query_callback
- *      for_time                : Time from where the file/s are not
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_find_unchanged_for_time_t)(void *db_conn,
-                                  gf_query_callback_t query_callback,
-                                  void *_cbk_args,
-                                  gfdb_time_t *_time);
-
-
-
-/* Query records/files that have changed/accessed from a
- *                      time in past to current time
- * Arguments:
- *      db_conn                 : plugin specific data base connection
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _cbk_args               : Custom argument passed for the call back
- *                                function query_callback
- *      _time                   : Time from where the file/s are
- *                                changed/accessed
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_find_recently_changed_files_t)(void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_cbk_args, gfdb_time_t *_time);
-
-/* Query records/files that have not changed/accessed
- * from a time in past to current time, with
- * a desired frequency
- *
- * Arguments:
- *      db_conn                 : plugin specific data base connection
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _cbk_args               : Custom argument passed for the call back
- *                                function query_callback
- *      _time                   : Time from where the file/s are not
- *                                changed/accessed
- *      _write_freq             : Desired Write Frequency lower limit
- *      _read_freq              : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_find_unchanged_for_time_freq_t)
-                                        (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_cbk_args, gfdb_time_t *_time,
-                                        int _write_freq, int _read_freq,
-                                        gf_boolean_t _clear_counters);
-
-
-
-
-/* Query records/files that have changed/accessed from a
- * time in past to current time, with a desired frequency
- * Arguments:
- *      db_conn                 : plugin specific data base connection
- *      query_callback          : Call back function that will be called
- *                                for every record found
- *      _cbk_args               : Custom argument passed for the call back
- *                                function query_callback
- *      _time                   : Time from where the file/s are
- *                                changed/accessed
- *      _write_freq             : Desired Write Frequency lower limit
- *      _read_freq              : Desired Read Frequency lower limit
- *      _clear_counters         : If true, Clears all the frequency counters of
- *                                all files.
- * Returns : if successful return 0 or
- *          -ve value in case of failure*/
-typedef int
-(*gfdb_find_recently_changed_files_freq_t)(void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_cbk_args, gfdb_time_t *_time,
-                                        int _write_freq, int _read_freq,
-                                        gf_boolean_t _clear_counters);
-
-
-typedef int (*gfdb_clear_files_heat_t)(void *db_conn);
-
-typedef int (*gfdb_get_db_version_t)(void *db_conn,
-                                        char **version);
-
-typedef int (*gfdb_get_db_params_t)(void *db_conn,
-                                char *param_key,
-                                char **param_value);
-
-typedef int (*gfdb_set_db_params_t)(void *db_conn,
-                                char *param_key,
-                                char *param_value);
-
-
-
-/*Data structure holding all the above plugin function pointers*/
-typedef struct gfdb_db_operations {
-        gfdb_init_db_t                        init_db_op;
-        gfdb_fini_db_t                        fini_db_op;
-        gfdb_insert_record_t                  insert_record_op;
-        gfdb_delete_record_t                  delete_record_op;
-        gfdb_compact_db_t                     compact_db_op;
-        gfdb_find_all_t                       find_all_op;
-        gfdb_find_unchanged_for_time_t        find_unchanged_for_time_op;
-        gfdb_find_recently_changed_files_t    find_recently_changed_files_op;
-        gfdb_find_unchanged_for_time_freq_t
-                                        find_unchanged_for_time_freq_op;
-        gfdb_find_recently_changed_files_freq_t
-                                        find_recently_changed_files_freq_op;
-        gfdb_clear_files_heat_t clear_files_heat_op;
-        gfdb_get_db_version_t           get_db_version;
-        gfdb_get_db_params_t           get_db_params;
-        gfdb_set_db_params_t           set_db_params;
-} gfdb_db_operations_t;
-
-/*******************************************************************************
- *
- * Database connection object: This objected is maitained by libgfdb for each
- * database connection created.
- * gf_db_connection     : DB connection specific to the plugin
- * gfdb_db_operations   : Contains all the libgfdb API implementation
- *                        from the plugin.
- * gfdb_db_type         : Type of database
- *
- * ****************************************************************************/
-
-
-typedef struct gfdb_connection {
-        void *gf_db_connection;
-        gfdb_db_operations_t gfdb_db_operations;
-        gfdb_db_type_t gfdb_db_type;
-} gfdb_connection_t;
-
-
-
-
-/*******************************************************************************
- *
- *                    Macros for get and set db options
- *
- * ****************************************************************************/
-
-
-/*Set param_key : str_value into param_dict*/
-#define SET_DB_PARAM_TO_DICT(comp_name, params_dict, param_key,\
-                                                        str_value, ret, error)\
-        do {\
-                data_t *data     = NULL;\
-                data            = str_to_data (str_value);\
-                if (!data)\
-                        goto error;\
-                ret = dict_add (params_dict, param_key, data);\
-                if (ret) {\
-                        gf_msg (comp_name, GF_LOG_ERROR, 0,\
-                                LG_MSG_SET_PARAM_FAILED, "Failed setting %s "\
-                                "to params dictionary", param_key);\
-                        data_destroy (data);\
-                        goto error;\
-                };\
-        } while (0)
-
-/*get str_value of param_key from param_dict*/
-#define GET_DB_PARAM_FROM_DICT(comp_name, params_dict, param_key, str_value,\
-                                error)\
-        do {\
-                data_t *data    = NULL;\
-                data = dict_get (params_dict, param_key);\
-                if (!data) {\
-                        gf_msg (comp_name, GF_LOG_ERROR, 0,\
-                                LG_MSG_GET_PARAM_FAILED, "Failed to retrieve "\
-                                "%s from params", param_key);\
-                        goto error;\
-                } else {\
-                        str_value = data->data;\
-                };\
-        } while (0)
-
-
-/*get str_value of param_key from param_dict. if param_key is not present
- * set _default_v to str_value */
-#define GET_DB_PARAM_FROM_DICT_DEFAULT(comp_name, params_dict, param_key,\
-                                str_value, _default_v)\
-        do {\
-                data_t *data    = NULL;\
-                data = dict_get (params_dict, param_key);\
-                if (!data) {\
-                        str_value = _default_v;\
-                        gf_msg (comp_name, GF_LOG_WARNING, 0,\
-                                LG_MSG_GET_PARAM_FAILED, "Failed to retrieve "\
-                                "%s from params.Assigning default value: %s",\
-                                param_key, _default_v);\
-                } else {\
-                        str_value = data->data;\
-                };\
-        } while (0)
-
-
-#endif
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3.c b/libglusterfs/src/gfdb/gfdb_sqlite3.c
deleted file mode 100644
index 094028361c5..00000000000
--- a/libglusterfs/src/gfdb/gfdb_sqlite3.c
+++ /dev/null
@@ -1,1563 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3.h"
-#include "gfdb_sqlite3_helper.h"
-#include "libglusterfs-messages.h"
-#include "syscall.h"
-
-/******************************************************************************
- *
- *                      Util functions
- *
- * ***************************************************************************/
-gf_sql_connection_t *
-gf_sql_connection_init ()
-{
-        gf_sql_connection_t *gf_sql_conn = NULL;
-
-        gf_sql_conn = GF_CALLOC (1, sizeof(gf_sql_connection_t),
-                        gf_mt_sql_connection_t);
-        if (gf_sql_conn == NULL) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, ENOMEM,
-                        LG_MSG_NO_MEMORY, "Error allocating memory to "
-                        "gf_sql_connection_t ");
-        }
-
-        return gf_sql_conn;
-}
-
-void
-gf_sql_connection_fini (gf_sql_connection_t **sql_connection)
-{
-        if (!sql_connection)
-                return;
-        GF_FREE (*sql_connection);
-        *sql_connection = NULL;
-}
-
-const char *
-gf_sql_jm2str (gf_sql_journal_mode_t jm)
-{
-        switch (jm) {
-        case gf_sql_jm_delete:
-                return GF_SQL_JM_DELETE;
-        case gf_sql_jm_truncate:
-                return GF_SQL_JM_TRUNCATE;
-        case gf_sql_jm_persist:
-                return GF_SQL_JM_PERSIST;
-        case gf_sql_jm_memory:
-                return GF_SQL_JM_MEMORY;
-        case gf_sql_jm_wal:
-                return GF_SQL_JM_WAL;
-        case gf_sql_jm_off:
-                return GF_SQL_JM_OFF;
-        case gf_sql_jm_invalid:
-                break;
-        }
-        return NULL;
-}
-
-gf_sql_journal_mode_t
-gf_sql_str2jm (const char *jm_str)
-{
-        if (!jm_str) {
-                return gf_sql_jm_invalid;
-        } else if (strcmp (jm_str, GF_SQL_JM_DELETE) == 0) {
-                return gf_sql_jm_delete;
-        } else if (strcmp (jm_str, GF_SQL_JM_TRUNCATE) == 0) {
-                return gf_sql_jm_truncate;
-        } else if (strcmp (jm_str, GF_SQL_JM_PERSIST) == 0) {
-                return gf_sql_jm_persist;
-        } else if (strcmp (jm_str, GF_SQL_JM_MEMORY) == 0) {
-                return gf_sql_jm_memory;
-        } else if (strcmp (jm_str, GF_SQL_JM_WAL) == 0) {
-                return gf_sql_jm_wal;
-        } else if (strcmp (jm_str, GF_SQL_JM_OFF) == 0) {
-                return gf_sql_jm_off;
-        }
-        return gf_sql_jm_invalid;
-}
-
-const char *
-gf_sql_av_t2str (gf_sql_auto_vacuum_t sql_av)
-{
-        switch (sql_av) {
-        case gf_sql_av_none:
-                return GF_SQL_AV_NONE;
-        case gf_sql_av_full:
-                return GF_SQL_AV_FULL;
-        case gf_sql_av_incr:
-                return GF_SQL_AV_INCR;
-        case gf_sql_av_invalid:
-                break;
-        }
-        return NULL;
-}
-
-gf_sql_auto_vacuum_t
-gf_sql_str2av_t (const char *av_str)
-{
-        if (!av_str) {
-                return gf_sql_av_invalid;
-        } else if (strcmp (av_str, GF_SQL_AV_NONE) == 0) {
-                return gf_sql_av_none;
-        } else if (strcmp (av_str, GF_SQL_AV_FULL) == 0) {
-                return gf_sql_av_full;
-        } else if (strcmp (av_str, GF_SQL_AV_INCR) == 0) {
-                return gf_sql_av_incr;
-        }
-        return gf_sql_av_invalid;
-}
-
-const char *
-gf_sync_t2str (gf_sql_sync_t sql_sync)
-{
-        switch (sql_sync) {
-        case gf_sql_sync_off:
-                return GF_SQL_SYNC_OFF;
-        case gf_sql_sync_normal:
-                return GF_SQL_SYNC_NORMAL;
-        case gf_sql_sync_full:
-                return GF_SQL_SYNC_FULL;
-        case gf_sql_sync_invalid:
-                break;
-        }
-        return NULL;
-}
-
-gf_sql_sync_t
-gf_sql_str2sync_t (const char *sync_str)
-{
-        if (!sync_str) {
-                return gf_sql_sync_invalid;
-        } else if (strcmp (sync_str, GF_SQL_SYNC_OFF) == 0) {
-                return gf_sql_sync_off;
-        } else if (strcmp (sync_str, GF_SQL_SYNC_NORMAL) == 0) {
-                return gf_sql_sync_normal;
-        } else if (strcmp (sync_str, GF_SQL_SYNC_FULL) == 0) {
-                return gf_sql_sync_full;
-        }
-        return gf_sql_sync_invalid;
-}
-
-
-/*TODO replace GF_CALLOC by mem_pool or iobuff if required for performace */
-static char *
-sql_stmt_init ()
-{
-        char *sql_stmt = NULL;
-
-        sql_stmt = GF_CALLOC (GF_STMT_SIZE_MAX, sizeof(char),
-                        gf_common_mt_char);
-
-        if (!sql_stmt) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, ENOMEM,
-                        LG_MSG_NO_MEMORY, "Error allocating memory to SQL "
-                        "Statement ");
-                goto out;
-        }
-out:
-        return sql_stmt;
-}
-
-/*TODO replace GF_FREE by mem_pool or iobuff if required for performace */
-static void
-sql_stmt_fini (char **sql_stmt)
-{
-        GF_FREE (*sql_stmt);
-}
-
-/******************************************************************************
- *                      DB Essential functions used by
- *                      > gf_open_sqlite3_conn ()
- *                      > gf_close_sqlite3_conn ()
- * ***************************************************************************/
-static sqlite3 *
-gf_open_sqlite3_conn(char *sqlite3_db_path, int flags)
-{
-        sqlite3 *sqlite3_db_conn = NULL;
-        int ret = -1;
-
-        GF_ASSERT (sqlite3_db_path);
-
-        /*Creates DB if not created*/
-        ret = sqlite3_open_v2 (sqlite3_db_path, &sqlite3_db_conn, flags, NULL);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_DB_ERROR,
-                        "FATAL: Could open %s : %s",
-                        sqlite3_db_path, sqlite3_errmsg (sqlite3_db_conn));
-        }
-        return sqlite3_db_conn;
-}
-
-static int
-gf_close_sqlite3_conn(sqlite3 *sqlite3_db_conn)
-{
-        int ret = 0;
-
-        GF_ASSERT (sqlite3_db_conn);
-
-        if (sqlite3_db_conn) {
-                ret = sqlite3_close (sqlite3_db_conn);
-                if (ret != SQLITE_OK) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_CONNECTION_ERROR, "FATAL: sqlite3 close"
-                                " connection failed %s",
-                                sqlite3_errmsg (sqlite3_db_conn));
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************************************************************
-*
-*                      Database init / fini / create table
-*
-* ***************************************************************************/
-
-
-/*Function to fill db operations*/
-void
-gf_sqlite3_fill_db_operations(gfdb_db_operations_t  *gfdb_db_ops)
-{
-        GF_ASSERT (gfdb_db_ops);
-
-        gfdb_db_ops->init_db_op = gf_sqlite3_init;
-        gfdb_db_ops->fini_db_op = gf_sqlite3_fini;
-
-        gfdb_db_ops->insert_record_op = gf_sqlite3_insert;
-        gfdb_db_ops->delete_record_op = gf_sqlite3_delete;
-        gfdb_db_ops->compact_db_op = gf_sqlite3_vacuum;
-
-        gfdb_db_ops->find_all_op = gf_sqlite3_find_all;
-        gfdb_db_ops->find_unchanged_for_time_op =
-                        gf_sqlite3_find_unchanged_for_time;
-        gfdb_db_ops->find_recently_changed_files_op =
-                        gf_sqlite3_find_recently_changed_files;
-        gfdb_db_ops->find_unchanged_for_time_freq_op =
-                        gf_sqlite3_find_unchanged_for_time_freq;
-        gfdb_db_ops->find_recently_changed_files_freq_op =
-                        gf_sqlite3_find_recently_changed_files_freq;
-
-        gfdb_db_ops->clear_files_heat_op = gf_sqlite3_clear_files_heat;
-
-        gfdb_db_ops->get_db_version = gf_sqlite3_version;
-
-        gfdb_db_ops->get_db_params = gf_sqlite3_pragma;
-
-        gfdb_db_ops->set_db_params = gf_sqlite3_set_pragma;
-}
-
-
-static int
-create_filetable (sqlite3 *sqlite3_db_conn)
-{
-        int ret                         =       -1;
-        char *sql_stmt                  =       NULL;
-        char *sql_strerror              =       NULL;
-
-        GF_ASSERT(sqlite3_db_conn);
-
-        sql_stmt = sql_stmt_init ();
-        if (!sql_stmt) {
-                ret = ENOMEM;
-                goto out;
-        }
-
-        GF_CREATE_STMT(sql_stmt);
-
-        ret = sqlite3_exec (sqlite3_db_conn, sql_stmt, NULL, NULL,
-                                &sql_strerror);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_EXEC_FAILED,
-                        "Failed executing: %s : %s", sql_stmt, sql_strerror);
-                sqlite3_free (sql_strerror);
-                        ret = -1;
-                        goto out;
-        }
-
-
-        ret = 0;
-out:
-        sql_stmt_fini (&sql_stmt);
-        return ret;
-}
-
-
-
-
-static int
-apply_sql_params_db(gf_sql_connection_t *sql_conn, dict_t *param_dict)
-{
-        int ret = -1;
-        char *temp_str = NULL;
-        char sqlite3_config_str[GF_NAME_MAX] = "";
-
-        GF_ASSERT(sql_conn);
-        GF_ASSERT(param_dict);
-
-        /*Extract sql page_size from param_dict,
-         * if not specified default value will be GF_SQL_DEFAULT_PAGE_SIZE*/
-        temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_PAGE_SIZE, temp_str,
-                        GF_SQL_DEFAULT_PAGE_SIZE);
-        sql_conn->page_size = atoi (temp_str);
-        /*Apply page_size on the sqlite db*/
-        GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "page_size", "%zd",
-                        sql_conn->page_size, ret, out);
-
-
-
-        /*Extract sql cache size from param_dict,
-         * if not specified default value will be
-         * GF_SQL_DEFAULT_CACHE_SIZE pages*/
-        temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_CACHE_SIZE, temp_str,
-                        GF_SQL_DEFAULT_CACHE_SIZE);
-        sql_conn->cache_size = atoi (temp_str);
-        /*Apply cache size on the sqlite db*/
-        GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "cache_size", "%zd",
-                        sql_conn->cache_size, ret, out);
-
-
-
-
-        /*Extract sql journal mode from param_dict,
-         * if not specified default value will be
-         * GF_SQL_DEFAULT_JOURNAL_MODE i.e "wal"*/
-        temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_JOURNAL_MODE, temp_str,
-                        GF_SQL_DEFAULT_JOURNAL_MODE);
-        sql_conn->journal_mode = gf_sql_str2jm (temp_str);
-        /*Apply journal mode to the sqlite db*/
-        GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "journal_mode", "%s",
-                        temp_str, ret, out);
-
-
-
-        /*Only when the journal mode is WAL, wal_autocheckpoint makes sense*/
-        if (sql_conn->journal_mode == gf_sql_jm_wal) {
-                /*Extract sql wal auto check point from param_dict
-                * if not specified default value will be
-                * GF_SQL_DEFAULT_WAL_AUTOCHECKPOINT pages*/
-                temp_str = NULL;
-                GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_WAL_AUTOCHECK, temp_str,
-                        GF_SQL_DEFAULT_WAL_AUTOCHECKPOINT);
-                sql_conn->wal_autocheckpoint = atoi(temp_str);
-                /*Apply wal auto check point to the sqlite db*/
-                GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "wal_autocheckpoint",
-                        "%zd", sql_conn->wal_autocheckpoint, ret, out);
-        }
-
-
-
-        /*Extract sql synchronous from param_dict
-         * if not specified default value will be GF_SQL_DEFAULT_SYNC*/
-         temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_SYNC, temp_str, GF_SQL_DEFAULT_SYNC);
-        sql_conn->synchronous = gf_sql_str2sync_t (temp_str);
-        /*Apply synchronous to the sqlite db*/
-        GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "synchronous", "%d",
-                        sql_conn->synchronous, ret, out);
-
-
-
-        /*Extract sql auto_vacuum from param_dict
-         * if not specified default value will be GF_SQL_DEFAULT_AUTO_VACUUM*/
-         temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(GFDB_STR_SQLITE3, param_dict,
-                        GFDB_SQL_PARAM_AUTO_VACUUM, temp_str,
-                        GF_SQL_DEFAULT_AUTO_VACUUM);
-        sql_conn->auto_vacuum = gf_sql_str2av_t (temp_str);
-        /*Apply auto_vacuum to the sqlite db*/
-        GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, "auto_vacuum", "%d",
-                        sql_conn->auto_vacuum, ret, out);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-int
-gf_sqlite3_init (dict_t *args, void **db_conn) {
-        int ret                         = -1;
-        gf_sql_connection_t *sql_conn   = NULL;
-        struct stat stbuf               = {0,};
-        gf_boolean_t    is_dbfile_exist = _gf_false;
-        char *temp_str                  = NULL;
-
-        GF_ASSERT (args);
-        GF_ASSERT (db_conn);
-
-        if (*db_conn != NULL) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_CONNECTION_ERROR, "DB Connection is not "
-                        "empty!");
-                return 0;
-        }
-
-        if (!sqlite3_threadsafe ()) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_NOT_MULTITHREAD_MODE,
-                        "sqlite3 is not in multithreaded mode");
-                goto out;
-        }
-
-        sql_conn = gf_sql_connection_init ();
-        if (!sql_conn) {
-                goto out;
-        }
-
-        /*Extract sql db path from args*/
-        temp_str = NULL;
-        GET_DB_PARAM_FROM_DICT(GFDB_STR_SQLITE3, args,
-                        GFDB_SQL_PARAM_DBPATH, temp_str, out);
-        strncpy(sql_conn->sqlite3_db_path, temp_str, PATH_MAX-1);
-        sql_conn->sqlite3_db_path[PATH_MAX-1] = 0;
-
-        is_dbfile_exist = (sys_stat (sql_conn->sqlite3_db_path, &stbuf) == 0) ?
-                                                _gf_true : _gf_false;
-
-        /*Creates DB if not created*/
-        sql_conn->sqlite3_db_conn = gf_open_sqlite3_conn (
-                                        sql_conn->sqlite3_db_path,
-                                        SQLITE_OPEN_READWRITE |
-                                        SQLITE_OPEN_CREATE);
-        if (!sql_conn->sqlite3_db_conn) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_CONNECTION_ERROR,
-                        "Failed creating db connection");
-                goto out;
-        }
-
-        /* If the file exist we skip the config part
-         * and creation of the schema */
-        if (is_dbfile_exist)
-                goto db_exists;
-
-
-        /*Apply sqlite3 params to database*/
-        ret = apply_sql_params_db (sql_conn, args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_SET_PARAM_FAILED, "Failed applying sql params"
-                        " to %s", sql_conn->sqlite3_db_path);
-                goto out;
-        }
-
-        /*Create the schema if NOT present*/
-        ret = create_filetable (sql_conn->sqlite3_db_conn);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_CREATE_FAILED, "Failed Creating %s Table",
-                        GF_FILE_TABLE);
-               goto out;
-        }
-
-db_exists:
-        ret = 0;
-out:
-        if (ret) {
-                gf_sqlite3_fini ((void **)&sql_conn);
-        }
-
-        *db_conn = sql_conn;
-
-        return ret;
-}
-
-
-int
-gf_sqlite3_fini (void **db_conn)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn = NULL;
-
-        GF_ASSERT (db_conn);
-        sql_conn = *db_conn;
-
-        if (sql_conn) {
-                if (sql_conn->sqlite3_db_conn) {
-                        ret = gf_close_sqlite3_conn (sql_conn->sqlite3_db_conn);
-                        if (ret) {
-                                /*Logging of error done in
-                                 * gf_close_sqlite3_conn()*/
-                                goto out;
-                        }
-                        sql_conn->sqlite3_db_conn = NULL;
-                }
-                gf_sql_connection_fini (&sql_conn);
-        }
-        *db_conn = sql_conn;
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************************************************************
- *
- *                      INSERT/UPDATE/DELETE Operations
- *
- *
- * ***************************************************************************/
-
-int gf_sqlite3_insert(void *db_conn, gfdb_db_record_t *gfdb_db_record)
-{
-        int ret                         =       -1;
-        gf_sql_connection_t *sql_conn   =       db_conn;
-
-        CHECK_SQL_CONN(sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-
-        switch (gfdb_db_record->gfdb_fop_path) {
-        case GFDB_FOP_WIND:
-                ret = gf_sql_insert_wind (sql_conn, gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_INSERT_FAILED, "Failed wind insert");
-                        goto out;
-                }
-                break;
-        case GFDB_FOP_UNWIND:
-                ret = gf_sql_insert_unwind (sql_conn, gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_INSERT_FAILED, "Failed unwind insert");
-                        goto out;
-                }
-                break;
-
-        case GFDB_FOP_WDEL:
-                ret = gf_sql_update_delete_wind (sql_conn, gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_UPDATE_FAILED, "Failed updating delete "
-                                "during wind");
-                        goto out;
-                }
-                break;
-        case GFDB_FOP_UNDEL:
-        case GFDB_FOP_UNDEL_ALL:
-                ret = gf_sql_delete_unwind (sql_conn, gfdb_db_record);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_DELETE_FAILED, "Failed deleting");
-                        goto out;
-                }
-                break;
-        case GFDB_FOP_INVALID:
-        default:
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_INVALID_FOP,
-                        "Cannot record to DB: Invalid FOP");
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-gf_sqlite3_delete(void *db_conn, gfdb_db_record_t *gfdb_db_record)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn = db_conn;
-
-        CHECK_SQL_CONN(sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************************************************************
- *
- *                      SELECT QUERY FUNCTIONS
- *
- *
- * ***************************************************************************/
-
-static int
-gf_get_basic_query_stmt (char **out_stmt)
-{
-        int ret = -1;
-        ret = gf_asprintf (out_stmt, "select GF_FILE_TB.GF_ID,"
-                                  "GF_FLINK_TB.GF_PID ,"
-                                  "GF_FLINK_TB.FNAME "
-                                  "from GF_FLINK_TB, GF_FILE_TB "
-                                  "where "
-                                  "GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID ");
-        if (ret <= 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed to create base query statement");
-                *out_stmt = NULL;
-        }
-        return ret;
-}
-
-
-
-
-
-/*
- * Find All files recorded in the DB
- * Input:
- *      query_callback  :       query callback fuction to handle
- *                              result records from the query
- * */
-int
-gf_sqlite3_find_all (void *db_conn, gf_query_callback_t query_callback,
-                        void *query_cbk_args)
-{
-        int ret                                 =       -1;
-        char *query_str                         =       NULL;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *prep_stmt                 =       NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
-
-        ret = gf_get_basic_query_stmt (&query_str);
-        if (ret <= 0) {
-                goto out;
-        }
-
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
-                                &prep_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed to prepare statement %s :"
-                        "%s", query_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = gf_sql_query_function (prep_stmt, query_callback, query_cbk_args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed Query %s", query_str);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        sqlite3_finalize (prep_stmt);
-        GF_FREE (query_str);
-        return ret;
-}
-
-
-/*
- * Find recently changed files from the DB
- * Input:
- *      query_callback  :       query callback fuction to handle
- *                              result records from the query
- *      from_time       :       Time to define what is recent
- * */
-int
-gf_sqlite3_find_recently_changed_files(void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *query_cbk_args,
-                                        gfdb_time_t *from_time)
-{
-        int ret                                 =       -1;
-        char *query_str                         =       NULL;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *prep_stmt                 =       NULL;
-        uint64_t  from_time_usec                =       0;
-        char *base_query_str                    =       NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
-
-        ret = gf_get_basic_query_stmt (&base_query_str);
-        if (ret <= 0) {
-                goto out;
-        }
-
-        ret = gf_asprintf (&query_str, "%s AND"
-                /*First condition: For writes*/
-                "( ((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_WMSEC ") >= ? )"
-                " OR "
-                /*Second condition: For reads*/
-                "((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_RWMSEC ") >= ?) )"
-                /* Order by write wind time in a descending order
-                 * i.e most hot files w.r.t to write */
-                " ORDER BY GF_FILE_TB.W_SEC DESC",
-                base_query_str);
-
-        if (ret < 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed creating query statement");
-                query_str = NULL;
-                goto out;
-        }
-
-        from_time_usec = gfdb_time_2_usec (from_time);
-
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
-                               &prep_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed to prepare statement %s :"
-                        " %s", query_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 1, from_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
-                        "%"PRIu64" : %s", from_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind read wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 2, from_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
-                        "%"PRIu64" : %s ", from_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the query*/
-        ret = gf_sql_query_function (prep_stmt, query_callback, query_cbk_args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed Query %s", query_str);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        sqlite3_finalize (prep_stmt);
-        GF_FREE (base_query_str);
-        GF_FREE (query_str);
-        return ret;
-}
-
-
-/*
- * Find unchanged files from a specified time from the DB
- * Input:
- *      query_callback  :       query callback fuction to handle
- *                              result records from the query
- *      for_time        :        Time from where the file/s are not changed
- * */
-int
-gf_sqlite3_find_unchanged_for_time (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *query_cbk_args,
-                                        gfdb_time_t *for_time)
-{
-        int ret                                 =       -1;
-        char *query_str                         =       NULL;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *prep_stmt                 =       NULL;
-        uint64_t  for_time_usec                 =       0;
-        char *base_query_str                    =       NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
-
-        ret = gf_get_basic_query_stmt (&base_query_str);
-        if (ret <= 0) {
-                goto out;
-        }
-
-        ret = gf_asprintf (&query_str, "%s AND "
-                /*First condition: For writes*/
-                "( ((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_WMSEC ") <= ? )"
-                " AND "
-                /*Second condition: For reads*/
-                "((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_RWMSEC ") <= ?) )"
-                /* Order by write wind time in a ascending order
-                 * i.e most cold files w.r.t to write */
-                " ORDER BY GF_FILE_TB.W_SEC ASC",
-                base_query_str);
-
-        if (ret < 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed to create query statement");
-                query_str = NULL;
-                goto out;
-        }
-
-        for_time_usec = gfdb_time_2_usec (for_time);
-
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
-                               &prep_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed to prepare statement %s :"
-                        " %s", query_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 1, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind read wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 2, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the query*/
-        ret = gf_sql_query_function (prep_stmt, query_callback, query_cbk_args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed Query %s", query_str);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        sqlite3_finalize (prep_stmt);
-        GF_FREE (base_query_str);
-        GF_FREE (query_str);
-        return ret;
-}
-
-
-
-
-
-/*
- * Find recently changed files with a specific frequency from the DB
- * Input:
- *      db_conn         :       db connection object
- *      query_callback  :       query callback fuction to handle
- *                              result records from the query
- *      from_time       :       Time to define what is recent
- *      freq_write_cnt  :       Frequency thresold for write
- *      freq_read_cnt   :       Frequency thresold for read
- *      clear_counters  :       Clear counters (r/w) for all inodes in DB
- * */
-int
-gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *query_cbk_args,
-                                        gfdb_time_t *from_time,
-                                        int freq_write_cnt,
-                                        int freq_read_cnt,
-                                        gf_boolean_t clear_counters)
-{
-        int ret                                 =       -1;
-        char *query_str                         =       NULL;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *prep_stmt                 =       NULL;
-        uint64_t  from_time_usec                =       0;
-        char *base_query_str                    =       NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
-
-        ret = gf_get_basic_query_stmt (&base_query_str);
-        if (ret <= 0) {
-                goto out;
-        }
-        ret = gf_asprintf (&query_str, "%s AND "
-                /*First condition: For Writes*/
-                "( ( ((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_WMSEC ") >= ? )"
-                " AND "" (" GF_COL_TB_WFC " >= ? ) )"
-                " OR "
-                /*Second condition: For Reads */
-                "( ((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_RWMSEC ") >= ?)"
-                " AND "" (" GF_COL_TB_RFC " >= ? ) ) )"
-                /* Order by write wind time and write freq in a descending order
-                 * i.e most hot files w.r.t to write */
-                " ORDER BY GF_FILE_TB.W_SEC DESC, "
-                "GF_FILE_TB.WRITE_FREQ_CNTR DESC",
-                base_query_str);
-
-        if (ret < 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed to create query statement");
-                query_str = NULL;
-                goto out;
-        }
-
-        from_time_usec = gfdb_time_2_usec (from_time);
-
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
-                                &prep_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed to prepare statement %s :"
-                        " %s", query_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 1, from_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
-                        "%"PRIu64" : %s", from_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write frequency thresold*/
-        ret = sqlite3_bind_int (prep_stmt, 2, freq_write_cnt);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind freq_write_cnt "
-                        "%d : %s", freq_write_cnt,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        /*Bind read wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 3, from_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
-                        "%"PRIu64" : %s", from_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind read frequency thresold*/
-        ret = sqlite3_bind_int (prep_stmt, 4, freq_read_cnt);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind freq_read_cnt "
-                        "%d : %s", freq_read_cnt,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the query*/
-        ret = gf_sql_query_function (prep_stmt, query_callback, query_cbk_args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed Query %s", query_str);
-                goto out;
-        }
-
-
-
-        /*Clear counters*/
-        if (clear_counters) {
-                ret = gf_sql_clear_counters (sql_conn);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear"
-                                " counters!");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        sqlite3_finalize (prep_stmt);
-        GF_FREE (base_query_str);
-        GF_FREE (query_str);
-        return ret;
-}
-
-
-
-
-/*
- * Find unchanged files from a specified time, w.r.t to frequency, from the DB
- * Input:
- *      query_callback  :       query callback fuction to handle
- *                              result records from the query
- *      for_time       :        Time from where the file/s are not changed
- *      freq_write_cnt  :       Frequency thresold for write
- *      freq_read_cnt   :       Frequency thresold for read
- *      clear_counters  :       Clear counters (r/w) for all inodes in DB
- * */
-int
-gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *query_cbk_args,
-                                        gfdb_time_t *for_time,
-                                        int freq_write_cnt,
-                                        int freq_read_cnt,
-                                        gf_boolean_t clear_counters)
-{
-        int ret                                 =       -1;
-        char *query_str                         =       NULL;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *prep_stmt                 =       NULL;
-        uint64_t  for_time_usec                 =       0;
-        char *base_query_str                    =       NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
-
-        ret = gf_get_basic_query_stmt (&base_query_str);
-        if (ret <= 0) {
-                goto out;
-        }
-
-        ret = gf_asprintf (&query_str, "%s AND "
-                /*First condition: For Writes
-                 * Files that have write wind time smaller than for_time
-                 * OR
-                 * File that have write wind time greater than for_time,
-                 * but write_frequency less than freq_write_cnt*/
-                "( ( ((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_WMSEC ") < ? )"
-                " OR "
-                "( (" GF_COL_TB_WFC " < ? ) AND"
-                "((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_WMSEC ") >= ? ) ) )"
-                " AND "
-                /*Second condition: For Reads
-                 * Files that have reaASCd wind time smaller than for_time
-                 * OR
-                 * File that have read wind time greater than for_time,
-                 * but write_frequency less than freq_write_cnt*/
-                "( ((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_RWMSEC ") < ? )"
-                " OR "
-                "( (" GF_COL_TB_RFC " < ? ) AND"
-                "((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
-                GF_COL_TB_RWMSEC ") >= ? ) ) ) )"
-                /* Order by write wind time and write freq in ascending order
-                 * i.e most cold files w.r.t to write */
-                " ORDER BY GF_FILE_TB.W_SEC ASC, "
-                "GF_FILE_TB.WRITE_FREQ_CNTR ASC",
-                base_query_str);
-
-        if (ret < 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed to create query statement");
-                query_str = NULL;
-                goto out;
-        }
-
-        for_time_usec = gfdb_time_2_usec (for_time);
-
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
-                                &prep_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed to prepare delete "
-                        "statement %s : %s", query_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 1, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write frequency thresold*/
-        ret = sqlite3_bind_int (prep_stmt, 2, freq_write_cnt);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind freq_write_cnt"
-                        " %d : %s", freq_write_cnt,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind write wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 3, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-
-        /*Bind read wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 4, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind read frequency thresold*/
-        ret = sqlite3_bind_int (prep_stmt, 5, freq_read_cnt);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind freq_read_cnt "
-                        "%d : %s", freq_read_cnt,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind read wind time*/
-        ret = sqlite3_bind_int64 (prep_stmt, 6, for_time_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
-                        "%"PRIu64" : %s", for_time_usec,
-                        sqlite3_errmsg(sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the query*/
-        ret = gf_sql_query_function (prep_stmt, query_callback, query_cbk_args);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed Query %s", query_str);
-                goto out;
-        }
-
-
-        /*Clear counters*/
-        if (clear_counters) {
-                ret = gf_sql_clear_counters (sql_conn);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear "
-                                "counters!");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        sqlite3_finalize (prep_stmt);
-        GF_FREE (base_query_str);
-        GF_FREE (query_str);
-        return ret;
-}
-
-
-int
-gf_sqlite3_clear_files_heat (void *db_conn)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-
-        CHECK_SQL_CONN (sql_conn, out);
-
-        ret = gf_sql_clear_counters (sql_conn);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear "
-                        "files heat");
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/* Function to extract version of sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char **version  : the version is extracted as a string and will be stored in
- *                   this variable. The freeing of the memory should be done by
- *                   the caller.
- * Return:
- *      On success return the lenght of the version string that is
- *      extracted.
- *      On failure return -1
- * */
-int
-gf_sqlite3_version (void *db_conn, char **version)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        sqlite3_stmt *pre_stmt = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-
-        ret = sqlite3_prepare_v2 (sql_conn->sqlite3_db_conn,
-                                "SELECT SQLITE_VERSION()",
-                                -1, &pre_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed init prepare stmt %s",
-                        sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = sqlite3_step(pre_stmt);
-        if (ret != SQLITE_ROW) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_GET_RECORD_FAILED, "Failed to get records "
-                        "from db : %s", sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = gf_asprintf (version, "%s", sqlite3_column_text (pre_stmt, 0));
-        if (ret <= 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed extracting version");
-        }
-
-out:
-        sqlite3_finalize (pre_stmt);
-
-        return ret;
-}
-
-
-
-/* Function to extract PRAGMA from sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char *pragma_key     : PRAGMA or setting to be extracted
- * char **pragma_value  : the value of the PRAGMA or setting that is
- *                        extracted. This function will allocate memory
- *                        to pragma_value. The caller should free the memory
- * Return:
- *      On success return the lenght of the pragma/setting value that is
- *      extracted.
- *      On failure return -1
- * */
-int
-gf_sqlite3_pragma (void *db_conn, char *pragma_key, char **pragma_value)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn = db_conn;
-        sqlite3_stmt *pre_stmt = NULL;
-        char *sqlstring = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pragma_key, out);
-
-        ret = gf_asprintf (&sqlstring, "PRAGMA %s;", pragma_key);
-        if (ret <= 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed allocating memory");
-                goto out;
-        }
-
-        ret = sqlite3_prepare_v2 (sql_conn->sqlite3_db_conn,
-                                  sqlstring, -1, &pre_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed init prepare stmt %s",
-                        sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = sqlite3_step (pre_stmt);
-        if (ret != SQLITE_ROW) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_GET_RECORD_FAILED, "Failed to get records "
-                        "from db : %s", sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        if (pragma_value) {
-                ret = gf_asprintf (pragma_value, "%s",
-                                   sqlite3_column_text (pre_stmt, 0));
-                if (ret <= 0) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_QUERY_FAILED, "Failed to get %s from db",
-                                pragma_key);
-                }
-        }
-
-        ret = 0;
-out:
-        GF_FREE (sqlstring);
-
-        sqlite3_finalize (pre_stmt);
-
-        return ret;
-}
-
-/* Function to set PRAGMA to sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char *pragma_key     : PRAGMA to be set
- * char *pragma_value   : the value of the PRAGMA
- * Return:
- *      On success return 0
- *      On failure return -1
- * */
-int
-gf_sqlite3_set_pragma (void *db_conn, char *pragma_key, char *pragma_value)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn = db_conn;
-        char sqlstring[GF_NAME_MAX] = "";
-        char *db_pragma_value = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pragma_key, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pragma_value, out);
-
-        GF_SQLITE3_SET_PRAGMA(sqlstring, pragma_key, "%s",
-                              pragma_value, ret, out);
-
-        ret = gf_sqlite3_pragma (db_conn, pragma_key, &db_pragma_value);
-        if (ret < 0) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
-                        "Failed to get %s pragma", pragma_key);
-        } else {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_INFO, 0, 0,
-                        "Value set on DB %s : %s", pragma_key, db_pragma_value);
-        }
-        GF_FREE (db_pragma_value);
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-/* Function to vacuum of sqlite db
- * Input:
- * void *db_conn                      : Sqlite connection
- * gf_boolean_t compact_active        : Is compaction on?
- * gf_boolean_t compact_mode_switched : Did we just flip the compaction swtich?
- * Return:
- *      On success return 0
- *      On failure return -1
- * */
-int
-gf_sqlite3_vacuum (void *db_conn, gf_boolean_t compact_active,
-                   gf_boolean_t compact_mode_switched)
-{
-        int ret = -1;
-        gf_sql_connection_t *sql_conn           =       db_conn;
-        char *sqlstring = NULL;
-        char *sql_strerror = NULL;
-        gf_boolean_t changing_pragma = _gf_true;
-
-        CHECK_SQL_CONN (sql_conn, out);
-
-        if (GF_SQL_COMPACT_DEF == GF_SQL_COMPACT_NONE) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_INFO, 0,
-                        LG_MSG_COMPACT_STATUS,
-                        "VACUUM type is off: no VACUUM to do");
-                goto out;
-        }
-
-        if (compact_mode_switched) {
-                if (compact_active) { /* Then it was OFF before.
-                                        So turn everything on */
-                        ret = 0;
-                        switch (GF_SQL_COMPACT_DEF) {
-                        case GF_SQL_COMPACT_FULL:
-                                ret = gf_sqlite3_set_pragma (db_conn,
-                                                             "auto_vacuum",
-                                                             GF_SQL_AV_FULL);
-                                break;
-                        case GF_SQL_COMPACT_INCR:
-                                ret = gf_sqlite3_set_pragma (db_conn,
-                                                             "auto_vacuum",
-                                                             GF_SQL_AV_INCR);
-                                break;
-                        case GF_SQL_COMPACT_MANUAL:
-                                changing_pragma = _gf_false;
-                        default:
-                                ret = -1;
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_COMPACT_FAILED,
-                                        "VACUUM type undefined");
-                                goto out;
-                                break;
-                        }
-
-                } else { /* Then it was ON before, so turn it all off */
-                        if (GF_SQL_COMPACT_DEF == GF_SQL_COMPACT_FULL ||
-                           GF_SQL_COMPACT_DEF == GF_SQL_COMPACT_INCR) {
-                                ret = gf_sqlite3_set_pragma (db_conn,
-                                                             "auto_vacuum",
-                                                             GF_SQL_AV_NONE);
-                        } else {
-                                changing_pragma = _gf_false;
-                        }
-                }
-
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_TRACE, 0,
-                                LG_MSG_PREPARE_FAILED,
-                                "Failed to set the pragma");
-                        goto out;
-                }
-
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_INFO, 0,
-                        LG_MSG_COMPACT_STATUS, "Turning compaction %i",
-                        GF_SQL_COMPACT_DEF);
-
-                /* If we move from an auto_vacuum scheme to off, */
-                /* or vice-versa, we must VACUUM to save the change. */
-                /* In the case of a manual VACUUM scheme, we might as well */
-                /* run a manual VACUUM now if we */
-                if (changing_pragma || compact_active) {
-                        ret = gf_asprintf (&sqlstring, "VACUUM;");
-                        if (ret <= 0) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_PREPARE_FAILED,
-                                        "Failed allocating memory");
-                                goto out;
-                        }
-                        gf_msg(GFDB_STR_SQLITE3, GF_LOG_INFO, 0,
-                               LG_MSG_COMPACT_STATUS, "Sealed with a VACUUM");
-                }
-        } else { /* We are active, so it's time to VACUUM */
-                if (!compact_active) { /* Did we somehow enter an inconsistent
-                                          state? */
-                        ret = -1;
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_PREPARE_FAILED,
-                                "Tried to VACUUM when compaction inactive");
-                        goto out;
-                }
-
-                gf_msg(GFDB_STR_SQLITE3, GF_LOG_TRACE, 0,
-                       LG_MSG_COMPACT_STATUS,
-                       "Doing regular vacuum of type %i", GF_SQL_COMPACT_DEF);
-
-                switch (GF_SQL_COMPACT_DEF) {
-                case GF_SQL_COMPACT_INCR: /* INCR auto_vacuum */
-                        ret = gf_asprintf(&sqlstring,
-                                          "PRAGMA incremental_vacuum;");
-                        if (ret <= 0) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_PREPARE_FAILED,
-                                        "Failed allocating memory");
-                                goto out;
-                        }
-                        gf_msg(GFDB_STR_SQLITE3, GF_LOG_INFO, 0,
-                               LG_MSG_COMPACT_STATUS,
-                               "Will commence an incremental VACUUM");
-                        break;
-                /* (MANUAL) Invoke the VACUUM command */
-                case GF_SQL_COMPACT_MANUAL:
-                        ret = gf_asprintf(&sqlstring, "VACUUM;");
-                        if (ret <= 0) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_PREPARE_FAILED,
-                                        "Failed allocating memory");
-                                goto out;
-                        }
-                        gf_msg(GFDB_STR_SQLITE3, GF_LOG_INFO, 0,
-                               LG_MSG_COMPACT_STATUS,
-                               "Will commence a VACUUM");
-                        break;
-                /* (FULL) The database does the compaction itself. */
-                /* We cannot do anything else, so we can leave */
-                /* without sending anything to the database */
-                case GF_SQL_COMPACT_FULL:
-                        ret = 0;
-                        goto success;
-                /* Any other state must be an error. Note that OFF */
-                /* cannot hit this statement since we immediately leave */
-                /* in that case */
-                default:
-                        ret = -1;
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_COMPACT_FAILED,
-                                "VACUUM type undefined");
-                        goto out;
-                        break;
-                }
-        }
-
-        gf_msg(GFDB_STR_SQLITE3, GF_LOG_TRACE, 0, LG_MSG_COMPACT_STATUS,
-               "SQLString == %s", sqlstring);
-
-        ret = sqlite3_exec(sql_conn->sqlite3_db_conn, sqlstring, NULL, NULL,
-                           &sql_strerror);
-
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_GET_RECORD_FAILED, "Failed to vacuum "
-                        "the db : %s", sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-success:
-        gf_msg(GFDB_STR_SQLITE3, GF_LOG_INFO, 0, LG_MSG_COMPACT_STATUS,
-               compact_mode_switched ? "Successfully changed VACUUM on/off"
-               : "DB successfully VACUUM");
-out:
-        GF_FREE(sqlstring);
-
-        return ret;
-}
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3.h b/libglusterfs/src/gfdb/gfdb_sqlite3.h
deleted file mode 100644
index 52b84da251c..00000000000
--- a/libglusterfs/src/gfdb/gfdb_sqlite3.h
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GFDB_SQLITE3_H
-#define __GFDB_SQLITE3_H
-
-
-/*Sqlite3 header file*/
-#include <sqlite3.h>
-
-#include "logging.h"
-#include "gfdb_data_store_types.h"
-#include "gfdb_mem-types.h"
-#include "libglusterfs-messages.h"
-
-#define GF_STMT_SIZE_MAX        2048
-
-#define GF_DB_NAME              "gfdb.db"
-#define GF_FILE_TABLE           "GF_FILE_TB"
-#define GF_FILE_LINK_TABLE      "GF_FLINK_TB"
-#define GF_MASTER_TABLE         "sqlite_master"
-
-/*Since we have multiple tables to be created we put it in a transaction*/
-#define GF_CREATE_STMT(out_str)\
-do {\
-        sprintf (out_str , "BEGIN; CREATE TABLE IF NOT EXISTS "\
-                GF_FILE_TABLE\
-                "(GF_ID TEXT PRIMARY KEY NOT NULL, "\
-                "W_SEC INTEGER NOT NULL DEFAULT 0, "\
-                "W_MSEC INTEGER NOT NULL DEFAULT 0, "\
-                "UW_SEC INTEGER NOT NULL DEFAULT 0, "\
-                "UW_MSEC INTEGER NOT NULL DEFAULT 0, "\
-                "W_READ_SEC INTEGER NOT NULL DEFAULT 0, "\
-                "W_READ_MSEC INTEGER NOT NULL DEFAULT 0, "\
-                "UW_READ_SEC INTEGER NOT NULL DEFAULT 0, "\
-                "UW_READ_MSEC INTEGER NOT NULL DEFAULT 0, "\
-                "WRITE_FREQ_CNTR INTEGER NOT NULL DEFAULT 1, "\
-                "READ_FREQ_CNTR INTEGER NOT NULL DEFAULT 1); "\
-                "CREATE TABLE IF NOT EXISTS "\
-                GF_FILE_LINK_TABLE\
-                "(GF_ID TEXT NOT NULL, "\
-                "GF_PID TEXT NOT NULL, "\
-                "FNAME TEXT NOT NULL, "\
-                "W_DEL_FLAG INTEGER NOT NULL DEFAULT 0, "\
-                "LINK_UPDATE INTEGER NOT NULL DEFAULT 0, "\
-                "PRIMARY KEY ( GF_ID, GF_PID, FNAME) "\
-                ");"\
-                "COMMIT;"\
-                );;\
-} while (0)
-
-#define GF_COL_TB_WSEC          GF_FILE_TABLE "." GF_COL_WSEC
-#define GF_COL_TB_WMSEC         GF_FILE_TABLE "." GF_COL_WMSEC
-#define GF_COL_TB_UWSEC         GF_FILE_TABLE "." GF_COL_UWSEC
-#define GF_COL_TB_UWMSEC        GF_FILE_TABLE "." GF_COL_UWMSEC
-#define GF_COL_TB_RWSEC         GF_FILE_TABLE "." GF_COL_WSEC_READ
-#define GF_COL_TB_RWMSEC        GF_FILE_TABLE "." GF_COL_WMSEC_READ
-#define GF_COL_TB_RUWSEC        GF_FILE_TABLE "." GF_COL_UWSEC_READ
-#define GF_COL_TB_RUWMSEC       GF_FILE_TABLE "." GF_COL_UWMSEC_READ
-#define GF_COL_TB_WFC           GF_FILE_TABLE "." GF_COL_WRITE_FREQ_CNTR
-#define GF_COL_TB_RFC           GF_FILE_TABLE "." GF_COL_READ_FREQ_CNTR
-
-
-/*******************************************************************************
-*                      SQLITE3 Connection details and PRAGMA
-* ****************************************************************************/
-
-#define GF_SQL_AV_NONE  "none"
-#define GF_SQL_AV_FULL  "full"
-#define GF_SQL_AV_INCR  "incremental"
-
-#define GF_SQL_SYNC_OFF         "off"
-#define GF_SQL_SYNC_NORMAL      "normal"
-#define GF_SQL_SYNC_FULL        "full"
-
-#define GF_SQL_JM_DELETE        "delete"
-#define GF_SQL_JM_TRUNCATE      "truncate"
-#define GF_SQL_JM_PERSIST       "persist"
-#define GF_SQL_JM_MEMORY        "memory"
-#define GF_SQL_JM_WAL           "wal"
-#define GF_SQL_JM_OFF           "off"
-
-#define GF_SQL_COMPACT_NONE   0
-#define GF_SQL_COMPACT_FULL   1
-#define GF_SQL_COMPACT_INCR   2
-#define GF_SQL_COMPACT_MANUAL 3
-
-#define GF_SQL_COMPACT_DEF    GF_SQL_COMPACT_INCR
-typedef enum gf_sql_auto_vacuum {
-        gf_sql_av_none = 0,
-        gf_sql_av_full,
-        gf_sql_av_incr,
-        gf_sql_av_invalid
-} gf_sql_auto_vacuum_t;
-
-typedef enum gf_sql_sync {
-        gf_sql_sync_off = 0,
-        gf_sql_sync_normal,
-        gf_sql_sync_full,
-        gf_sql_sync_invalid
-} gf_sql_sync_t;
-
-
-typedef enum gf_sql_journal_mode {
-        gf_sql_jm_wal = 0,
-        gf_sql_jm_delete,
-        gf_sql_jm_truncate,
-        gf_sql_jm_persist,
-        gf_sql_jm_memory,
-        gf_sql_jm_off,
-        gf_sql_jm_invalid
-} gf_sql_journal_mode_t;
-
-
-typedef struct gf_sql_connection {
-        char                    sqlite3_db_path[PATH_MAX];
-        sqlite3                 *sqlite3_db_conn;
-        ssize_t                 cache_size;
-        ssize_t                 page_size;
-        ssize_t                 wal_autocheckpoint;
-        gf_sql_journal_mode_t   journal_mode;
-        gf_sql_sync_t           synchronous;
-        gf_sql_auto_vacuum_t    auto_vacuum;
-} gf_sql_connection_t;
-
-
-
-#define CHECK_SQL_CONN(sql_conn, out)\
-do {\
-        GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, sql_conn, out);\
-        if (!sql_conn->sqlite3_db_conn) {\
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,\
-                        LG_MSG_CONNECTION_INIT_FAILED,\
-                        "sqlite3 connection not initialized");\
-                goto out;\
-        };\
-} while (0)
-
-#define GF_SQLITE3_SET_PRAGMA(sqlite3_config_str, param_key, format, value,\
-                        ret, error)\
-do {\
-        sprintf (sqlite3_config_str, "PRAGMA %s = " format ,  param_key,\
-                value);\
-        ret = sqlite3_exec (sql_conn->sqlite3_db_conn, sqlite3_config_str,\
-                NULL, NULL, NULL);\
-        if (ret != SQLITE_OK) {\
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_EXEC_FAILED,\
-                        "Failed executing: %s : %s",\
-                        sqlite3_config_str, sqlite3_errmsg\
-                                               (sql_conn->sqlite3_db_conn));\
-                        ret = -1;\
-                        goto error;\
-        };\
-} while (0)
-
-/************************SQLITE3 PARAMS KEYS***********************************/
-#define GFDB_SQL_PARAM_DBPATH           "sql-db-path"
-#define GFDB_SQL_PARAM_CACHE_SIZE       "sql-db-cachesize"
-#define GFDB_SQL_PARAM_PAGE_SIZE        "sql-db-pagesize"
-#define GFDB_SQL_PARAM_JOURNAL_MODE     "sql-db-journalmode"
-#define GFDB_SQL_PARAM_WAL_AUTOCHECK    "sql-db-wal-autocheckpoint"
-#define GFDB_SQL_PARAM_SYNC             "sql-db-sync"
-#define GFDB_SQL_PARAM_AUTO_VACUUM      "sql-db-autovacuum"
-
-#define GF_SQL_DEFAULT_DBPATH                   ""
-#define GF_SQL_DEFAULT_PAGE_SIZE                "4096"
-#define GF_SQL_DEFAULT_CACHE_SIZE               "12500"
-#define GF_SQL_DEFAULT_WAL_AUTOCHECKPOINT       "25000"
-#define GF_SQL_DEFAULT_JOURNAL_MODE             GF_SQL_JM_WAL
-#define GF_SQL_DEFAULT_SYNC                     GF_SQL_SYNC_OFF
-#define GF_SQL_DEFAULT_AUTO_VACUUM              GF_SQL_AV_NONE
-
-
-/* Defines the indexs for sqlite params
- * The order should be maintained*/
-typedef enum sqlite_param_index {
-        sql_dbpath_ix = 0,
-        sql_pagesize_ix,
-        sql_cachesize_ix,
-        sql_journalmode_ix,
-        sql_walautocheck_ix,
-        sql_dbsync_ix,
-        sql_autovacuum_ix,
-        /*This should be in the end*/
-        sql_index_max
-} sqlite_param_index_t;
-
-/* Array to hold the sqlite param keys
- * The order should be maintained as sqlite_param_index_t*/
-static char *sqlite_params_keys[] = {
-        GFDB_SQL_PARAM_DBPATH,
-        GFDB_SQL_PARAM_PAGE_SIZE,
-        GFDB_SQL_PARAM_CACHE_SIZE,
-        GFDB_SQL_PARAM_JOURNAL_MODE,
-        GFDB_SQL_PARAM_WAL_AUTOCHECK,
-        GFDB_SQL_PARAM_SYNC,
-        GFDB_SQL_PARAM_AUTO_VACUUM
-};
-
-
-/* Array of default values for sqlite params
- * The order should be maintained as sqlite_param_index_t*/
-static char *sqlite_params_default_value[] = {
-        GF_SQL_DEFAULT_DBPATH,
-        GF_SQL_DEFAULT_PAGE_SIZE,
-        GF_SQL_DEFAULT_CACHE_SIZE,
-        GF_SQL_DEFAULT_JOURNAL_MODE,
-        GF_SQL_DEFAULT_WAL_AUTOCHECKPOINT,
-        GF_SQL_DEFAULT_SYNC,
-        GF_SQL_DEFAULT_AUTO_VACUUM
-};
-
-/*Extract sql params from page_size to auto_vacumm
- * The dbpath is extracted in a different way*/
-static inline int
-gfdb_set_sql_params(char *comp_name, dict_t *from_dict, dict_t *to_dict)
-{
-        sqlite_param_index_t sql_index  = sql_pagesize_ix;
-        char *_val_str                  = NULL;
-        int ret                      = -1;
-
-        GF_ASSERT (comp_name);
-        GF_ASSERT (from_dict);
-        GF_ASSERT (to_dict);
-
-        /*Extact and Set of the sql params from page_size*/
-        for (sql_index = sql_pagesize_ix; sql_index < sql_index_max;
-                sql_index++) {
-                _val_str = NULL;
-                GET_DB_PARAM_FROM_DICT_DEFAULT (comp_name, from_dict,
-                        sqlite_params_keys[sql_index], _val_str,
-                        sqlite_params_default_value[sql_index]);
-                SET_DB_PARAM_TO_DICT (comp_name, to_dict,
-                        sqlite_params_keys[sql_index], _val_str, ret, out);
-        }
-out:
-        return ret;
-}
-
-
-
-
-/*************************SQLITE3 GFDB PLUGINS*********************************/
-
-/*Db init and fini modules*/
-int gf_sqlite3_fini (void **db_conn);
-int gf_sqlite3_init (dict_t *args, void **db_conn);
-
-/*insert/update/delete modules*/
-int gf_sqlite3_insert (void *db_conn, gfdb_db_record_t *);
-int gf_sqlite3_delete (void *db_conn, gfdb_db_record_t *);
-
-/*querying modules*/
-int gf_sqlite3_find_all (void *db_conn, gf_query_callback_t,
-                        void *_query_cbk_args);
-int gf_sqlite3_find_unchanged_for_time (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *for_time);
-int gf_sqlite3_find_recently_changed_files (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *from_time);
-int gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *for_time,
-                                        int write_freq_cnt,
-                                        int read_freq_cnt,
-                                        gf_boolean_t clear_counters);
-int gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
-                                        gf_query_callback_t query_callback,
-                                        void *_query_cbk_args,
-                                        gfdb_time_t *from_time,
-                                        int write_freq_cnt,
-                                        int read_freq_cnt,
-                                        gf_boolean_t clear_counters);
-
-int gf_sqlite3_clear_files_heat (void *db_conn);
-
-/* Function to extract version of sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char **version  : the version is extracted as a string and will be stored in
- *                   this variable. The freeing of the memory should be done by
- *                   the caller.
- * Return:
- *      On success return the lenght of the version string that is
- *      extracted.
- *      On failure return -1
- * */
-int gf_sqlite3_version (void *db_conn, char **version);
-
-/* Function to extract PRAGMA or setting from sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char *pragma_key     : PRAGMA or setting to be extracted
- * char **pragma_value  : the value of the PRAGMA or setting that is
- *                        extracted. This function will allocate memory
- *                        to pragma_value. The caller should free the memory
- * Return:
- *      On success return the lenght of the pragma/setting value that is
- *      extracted.
- *      On failure return -1
- * */
-int gf_sqlite3_pragma (void *db_conn, char *pragma_key, char **pragma_value);
-
-/* Function to set PRAGMA to sqlite db
- * Input:
- * void *db_conn        : Sqlite connection
- * char *pragma_key     : PRAGMA to be set
- * char *pragma_value   : the value of the PRAGMA
- * Return:
- *      On success return 0
- *      On failure return -1
- * */
-int
-gf_sqlite3_set_pragma (void *db_conn, char *pragma_key, char *pragma_value);
-
-/* Function to vacuum of sqlite db
- * Input:
- * void *db_conn                      : Sqlite connection
- * gf_boolean_t compact_active        : Is compaction on?
- * gf_boolean_t compact_mode_switched : Did we just flip the compaction swtich?
- * Return:
- *      On success return 0
- *      On failure return -1
- * */
-int
-gf_sqlite3_vacuum (void *db_conn, gf_boolean_t compact_active,
-                   gf_boolean_t compact_mode_switched);
-
-void gf_sqlite3_fill_db_operations (gfdb_db_operations_t  *gfdb_db_ops);
-
-
-#endif
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c b/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c
deleted file mode 100644
index 8e1e27ff082..00000000000
--- a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c
+++ /dev/null
@@ -1,1371 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3_helper.h"
-
-
-#define GFDB_SQL_STMT_SIZE 256
-
-/*****************************************************************************
- *
- *                 Helper function to execute actual sql queries
- *
- *
- * ****************************************************************************/
-
-static int
-gf_sql_delete_all (gf_sql_connection_t  *sql_conn,
-                  char                  *gfid,
-                  gf_boolean_t          ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *delete_file_stmt = NULL;
-        sqlite3_stmt *delete_link_stmt = NULL;
-        char *delete_link_str = "DELETE FROM "
-                           GF_FILE_LINK_TABLE
-                           " WHERE GF_ID = ? ;";
-        char *delete_file_str = "DELETE FROM "
-                           GF_FILE_TABLE
-                           " WHERE GF_ID = ? ;";
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-
-        /*
-         * Delete all links associated with this GFID
-         *
-         * */
-        /*Prepare statement for delete all links*/
-        ret = sqlite3_prepare(sql_conn->sqlite3_db_conn, delete_link_str, -1,
-                                &delete_link_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
-                        "statement %s : %s", delete_link_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (delete_link_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (delete_link_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        delete_link_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        /*
-         * Delete entry from file table associated with this GFID
-         *
-         * */
-        /*Prepare statement for delete all links*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, delete_file_str, -1,
-                                &delete_file_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
-                        "statement %s : %s", delete_file_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (delete_file_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (delete_file_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        delete_file_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (delete_file_stmt);
-        sqlite3_finalize (delete_link_stmt);
-        return ret;
-}
-
-static int
-gf_sql_delete_link (gf_sql_connection_t  *sql_conn,
-                   char                 *gfid,
-                   char                 *pargfid,
-                   char                 *basename,
-                   gf_boolean_t         ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *delete_stmt = NULL;
-        char *delete_str = "DELETE FROM "
-                           GF_FILE_LINK_TABLE
-                           " WHERE GF_ID = ? AND GF_PID = ?"
-                           " AND FNAME = ?;";
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
-
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, delete_str, -1,
-                                &delete_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
-                        "statement %s : %s", delete_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (delete_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED,
-                        "Failed binding gfid %s : %s", gfid,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind pargfid*/
-        ret = sqlite3_bind_text (delete_stmt, 2, pargfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
-                        ": %s", pargfid,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind basename*/
-        ret = sqlite3_bind_text (delete_stmt, 3, basename, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
-                        "%s", basename,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step(delete_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        delete_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (delete_stmt);
-        return ret;
-}
-
-
-
-static int
-gf_sql_update_link_flags (gf_sql_connection_t  *sql_conn,
-                   char                 *gfid,
-                   char                 *pargfid,
-                   char                 *basename,
-                   int                  update_flag,
-                   gf_boolean_t         is_update_or_delete,
-                   gf_boolean_t         ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *update_stmt = NULL;
-        char *update_column = NULL;
-        char update_str[1024] = "";
-
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
-
-        update_column = (is_update_or_delete) ? "LINK_UPDATE" : "W_DEL_FLAG";
-
-        sprintf (update_str, "UPDATE "
-                            GF_FILE_LINK_TABLE
-                            " SET %s = ?"
-                            " WHERE GF_ID = ? AND GF_PID = ? AND FNAME = ?;",
-                            update_column);
-
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, update_str, -1,
-                                &update_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing update "
-                        "statement %s : %s", update_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        /*Bind link_update*/
-        ret = sqlite3_bind_int (update_stmt, 1, update_flag);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding update_flag %d "
-                        ": %s", update_flag,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (update_stmt, 2, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind pargfid*/
-        ret = sqlite3_bind_text (update_stmt, 3, pargfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
-                        ": %s", pargfid,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind basename*/
-        ret = sqlite3_bind_text (update_stmt, 4, basename, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
-                        "%s", basename,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step(update_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        update_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (update_stmt);
-        return ret;
-}
-
-
-static int
-gf_sql_insert_link (gf_sql_connection_t  *sql_conn,
-                   char                 *gfid,
-                   char                 *pargfid,
-                   char                 *basename,
-                   gf_boolean_t         link_consistency,
-                   gf_boolean_t         ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *insert_stmt = NULL;
-        char insert_str[GFDB_SQL_STMT_SIZE] = "";
-
-        sprintf (insert_str, "INSERT INTO "
-                           GF_FILE_LINK_TABLE
-                           " (GF_ID, GF_PID, FNAME,"
-                           " W_DEL_FLAG, LINK_UPDATE) "
-                           " VALUES (?, ?, ?, 0, %d);",
-                           link_consistency);
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
-
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
-                                &insert_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED,
-                        "Failed preparing insert "
-                        "statement %s : %s", insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED,
-                        "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind pargfid*/
-        ret = sqlite3_bind_text (insert_stmt, 2, pargfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                        0, LG_MSG_BINDING_FAILED,
-                        "Failed binding parent gfid %s "
-                        ": %s", pargfid,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind basename*/
-        ret = sqlite3_bind_text (insert_stmt, 3, basename, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                        0, LG_MSG_BINDING_FAILED,
-                        "Failed binding basename %s : %s", basename,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors),
-                        0, LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared "
-                        "stmt %s %s %s %s : %s",
-                        gfid, pargfid, basename, insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (insert_stmt);
-        return ret;
-}
-
-
-static int
-gf_sql_update_link (gf_sql_connection_t  *sql_conn,
-                   char                 *gfid,
-                   char                 *pargfid,
-                   char                 *basename,
-                   char                 *old_pargfid,
-                   char                 *old_basename,
-                   gf_boolean_t         link_consistency,
-                   gf_boolean_t         ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *insert_stmt = NULL;
-        char insert_str[GFDB_SQL_STMT_SIZE] = "";
-
-        sprintf (insert_str, "INSERT INTO "
-                            GF_FILE_LINK_TABLE
-                            " (GF_ID, GF_PID, FNAME,"
-                            " W_DEL_FLAG, LINK_UPDATE) "
-                            " VALUES (? , ?, ?, 0, %d);",
-                            link_consistency);
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, old_pargfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, old_basename, out);
-
-        /*
-         *
-         * Delete the old link
-         *
-         * */
-         ret = gf_sql_delete_link (sql_conn, gfid, old_pargfid,
-                                old_basename, ignore_errors);
-        if (ret) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_DELETE_FAILED, "Failed deleting old link");
-                goto out;
-        }
-
-        /*
-         *
-         * insert new link
-         *
-         * */
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
-                                &insert_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
-                        "statement %s : %s", insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind new pargfid*/
-        ret = sqlite3_bind_text (insert_stmt, 2, pargfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
-                        ": %s", pargfid,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind new basename*/
-        ret = sqlite3_bind_text (insert_stmt, 3, basename, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
-                        "%s", basename,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (insert_stmt);
-        return ret;
-}
-
-static int
-gf_sql_insert_write_wind_time (gf_sql_connection_t      *sql_conn,
-                                char                    *gfid,
-                                gfdb_time_t             *wind_time,
-                                gf_boolean_t            ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *insert_stmt = NULL;
-        char *insert_str = "INSERT INTO "
-                           GF_FILE_TABLE
-                           "(GF_ID, W_SEC, W_MSEC, UW_SEC, UW_MSEC)"
-                           " VALUES (?, ?, ?, 0, 0);";
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, wind_time, out);
-
-
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
-                        &insert_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
-                        "statement %s : %s", insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind wind secs*/
-        ret = sqlite3_bind_int (insert_stmt, 2, wind_time->tv_sec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
-                        "secs %ld : %s", wind_time->tv_sec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind wind msecs*/
-        ret = sqlite3_bind_int (insert_stmt, 3, wind_time->tv_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
-                        "msecs %ld : %s", wind_time->tv_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt GFID:%s %s : %s",
-                        gfid, insert_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (insert_stmt);
-        return ret;
-}
-
-
-
-/*Update write/read times for both wind and unwind*/
-static int
-gf_update_time (gf_sql_connection_t    *sql_conn,
-                char                    *gfid,
-                gfdb_time_t             *update_time,
-                gf_boolean_t            record_counter,
-                gf_boolean_t            is_wind,
-                gf_boolean_t            is_read,
-                gf_boolean_t            ignore_errors)
-{
-        int ret = -1;
-        sqlite3_stmt *update_stmt = NULL;
-        char update_str[1024] = "";
-        char *freq_cntr_str = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, update_time, out);
-
-        /*
-         * Constructing the prepare statement string.
-         *
-         * */
-        /*For write time*/
-        if (!is_read) {
-                if (is_wind) {
-                        /*if record counter is on*/
-                        freq_cntr_str = (record_counter) ?
-                        ", WRITE_FREQ_CNTR = WRITE_FREQ_CNTR + 1" : "";
-
-                        /*Prefectly safe as we will not go array of bound*/
-                        sprintf (update_str, "UPDATE "
-                                GF_FILE_TABLE
-                                " SET W_SEC = ?, W_MSEC = ? "
-                                " %s"/*place for read freq counters*/
-                                " WHERE GF_ID = ? ;", freq_cntr_str);
-                } else {
-                        /*Prefectly safe as we will not go array of bound*/
-                        sprintf (update_str, "UPDATE "
-                                GF_FILE_TABLE
-                                " SET UW_SEC = ?, UW_MSEC = ? ;");
-                }
-        }
-        /*For Read Time update*/
-        else {
-                if (is_wind) {
-                        /*if record counter is on*/
-                        freq_cntr_str = (record_counter) ?
-                        ", READ_FREQ_CNTR = READ_FREQ_CNTR + 1" : "";
-
-                        /*Prefectly safe as we will not go array of bound*/
-                        sprintf (update_str, "UPDATE "
-                                GF_FILE_TABLE
-                                " SET W_READ_SEC = ?, W_READ_MSEC = ? "
-                                " %s"/*place for read freq counters*/
-                                " WHERE GF_ID = ? ;", freq_cntr_str);
-                } else {
-                        /*Prefectly safe as we will not go array of bound*/
-                        sprintf (update_str, "UPDATE "
-                                GF_FILE_TABLE
-                                " SET UW_READ_SEC = ?, UW_READ_MSEC = ? ;");
-                }
-        }
-
-        /*Prepare statement*/
-        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, update_str, -1,
-                                &update_stmt, 0);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
-                        "statement %s : %s", update_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind time secs*/
-        ret = sqlite3_bind_int (update_stmt, 1, update_time->tv_sec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
-                        "secs %ld : %s", update_time->tv_sec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind time msecs*/
-        ret = sqlite3_bind_int (update_stmt, 2, update_time->tv_usec);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
-                        "msecs %ld : %s", update_time->tv_usec,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Bind gfid*/
-        ret = sqlite3_bind_text (update_stmt, 3, gfid, -1, NULL);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
-                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        /*Execute the prepare statement*/
-        if (sqlite3_step (update_stmt) != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3,
-                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
-                        LG_MSG_EXEC_FAILED,
-                        "Failed executing the prepared stmt %s : %s",
-                        update_str,
-                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        /*Free prepared statement*/
-        sqlite3_finalize (update_stmt);
-        return ret;
-}
-
-/******************************************************************************
- *
- *                      Helper functions for gf_sqlite3_insert()
- *
- *
- * ****************************************************************************/
-
-int
-gf_sql_insert_wind (gf_sql_connection_t  *sql_conn,
-                   gfdb_db_record_t     *gfdb_db_record)
-{
-        int ret                 = -1;
-        gfdb_time_t *modtime    = NULL;
-        char *pargfid_str       = NULL;
-        char *gfid_str          = NULL;
-        char *old_pargfid_str   = NULL;
-        gf_boolean_t its_wind   = _gf_true;/*remains true for this function*/
-
-
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-
-        gfid_str = gf_strdup (uuid_utoa (gfdb_db_record->gfid));
-        if (!gfid_str) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
-                        "Creating gfid string failed.");
-                goto out;
-        }
-
-        modtime = &gfdb_db_record->gfdb_wind_change_time;
-
-        /* handle all dentry based operations */
-        if (isdentryfop (gfdb_db_record->gfdb_fop_type)) {
-                /*Parent GFID is always set*/
-                pargfid_str = gf_strdup (uuid_utoa (gfdb_db_record->pargfid));
-                if (!pargfid_str) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                0, LG_MSG_CREATE_FAILED, "Creating gfid string "
-                                "failed.");
-                        goto out;
-                }
-
-                /* handle create, mknod */
-                if (isdentrycreatefop (gfdb_db_record->gfdb_fop_type)) {
-                        /*insert link*/
-                        ret = gf_sql_insert_link(sql_conn,
-                                        gfid_str, pargfid_str,
-                                        gfdb_db_record->file_name,
-                                        gfdb_db_record->link_consistency,
-                                        _gf_true);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3,
-                                        _gfdb_log_level (GF_LOG_WARNING,
-                                                gfdb_db_record->ignore_errors),
-                                        0,
-                                        LG_MSG_INSERT_FAILED, "Failed "
-                                        "inserting link in DB");
-                                /* Even if link creation is failed we
-                                 * continue with the creation of file record.
-                                 * This covers to cases
-                                 * 1) Lookup heal: If the file record from
-                                 * gf_file_tb is deleted but the link record
-                                 * still exist. Lookup heal will attempt a heal
-                                 * with create_wind set. The link heal will fail
-                                 * as there is already a record and if we dont
-                                 * ignore the error we will not heal the
-                                 * gf_file_tb.
-                                 * 2) Rename file in cold tier: During a rename
-                                 * of a file that is there in cold tier. We get
-                                 * an link record created in hot tier for the
-                                 * linkto file. When the file gets heated and
-                                 * moves to hot tier there will be attempt from
-                                 * ctr lookup heal to create link and file
-                                 * record and If we dont ignore the error we
-                                 * will not heal the gf_file_tb.
-                                 * */
-                        }
-                        gfdb_db_record->islinkupdate = gfdb_db_record->
-                                                        link_consistency;
-
-                        /*
-                         * Only for create/mknod insert wind time
-                         * for the first time
-                         * */
-                        ret = gf_sql_insert_write_wind_time (sql_conn, gfid_str,
-                                        modtime, gfdb_db_record->ignore_errors);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3,
-                                        _gfdb_log_level (GF_LOG_ERROR,
-                                                gfdb_db_record->ignore_errors),
-                                        0, LG_MSG_INSERT_FAILED,
-                                        "Failed inserting wind time in DB");
-                                goto out;
-                        }
-                        goto out;
-                }
-                /*handle rename, link */
-                else {
-                        /*rename*/
-                        if (strlen (gfdb_db_record->old_file_name) != 0) {
-                                old_pargfid_str = gf_strdup (uuid_utoa (
-                                                gfdb_db_record->old_pargfid));
-                                if (!old_pargfid_str) {
-                                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                                0, LG_MSG_CREATE_FAILED,
-                                                "Creating gfid string failed.");
-                                        goto out;
-                                }
-                                ret = gf_sql_update_link (sql_conn, gfid_str,
-                                                pargfid_str,
-                                                gfdb_db_record->file_name,
-                                                old_pargfid_str,
-                                                gfdb_db_record->old_file_name,
-                                                gfdb_db_record->
-                                                        link_consistency,
-                                                gfdb_db_record->ignore_errors);
-                                if (ret) {
-                                        gf_msg (GFDB_STR_SQLITE3,
-                                                _gfdb_log_level (GF_LOG_ERROR,
-                                                gfdb_db_record->ignore_errors),
-                                                0, LG_MSG_UPDATE_FAILED,
-                                                "Failed updating link");
-                                        goto out;
-                                }
-                                gfdb_db_record->islinkupdate = gfdb_db_record->
-                                                        link_consistency;
-                        }
-                        /*link*/
-                        else {
-                                ret = gf_sql_insert_link (sql_conn,
-                                                gfid_str, pargfid_str,
-                                                gfdb_db_record->file_name,
-                                                gfdb_db_record->
-                                                        link_consistency,
-                                                gfdb_db_record->ignore_errors);
-                                if (ret) {
-                                        gf_msg (GFDB_STR_SQLITE3,
-                                                _gfdb_log_level (GF_LOG_ERROR,
-                                                gfdb_db_record->ignore_errors),
-                                                0, LG_MSG_INSERT_FAILED,
-                                                "Failed inserting link in DB");
-                                        goto out;
-                                }
-                                gfdb_db_record->islinkupdate = gfdb_db_record->
-                                                        link_consistency;
-                        }
-                }
-        }
-
-        /* update times only when said!*/
-        if (gfdb_db_record->do_record_times) {
-                /*All fops update times read or write*/
-                ret = gf_update_time (sql_conn, gfid_str, modtime,
-                                gfdb_db_record->do_record_counters,
-                                its_wind,
-                                isreadfop (gfdb_db_record->gfdb_fop_type),
-                                gfdb_db_record->ignore_errors);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                gfdb_db_record->ignore_errors), 0,
-                                LG_MSG_UPDATE_FAILED, "Failed update wind time"
-                                " in DB");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        GF_FREE (gfid_str);
-        GF_FREE (pargfid_str);
-        GF_FREE (old_pargfid_str);
-        return ret;
-}
-
-
-
-
-int
-gf_sql_insert_unwind (gf_sql_connection_t  *sql_conn,
-                     gfdb_db_record_t     *gfdb_db_record)
-{
-
-        int ret = -1;
-        gfdb_time_t *modtime    = NULL;
-        gf_boolean_t its_wind   = _gf_true;/*remains true for this function*/
-        char *gfid_str = NULL;
-        char *pargfid_str = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
-        if (!gfid_str) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_CREATE_FAILED, "Creating gfid string failed.");
-                goto out;
-        }
-
-        /*Only update if recording unwind is set*/
-        if (gfdb_db_record->do_record_times &&
-                gfdb_db_record->do_record_uwind_time) {
-                modtime = &gfdb_db_record->gfdb_unwind_change_time;
-                ret = gf_update_time (sql_conn, gfid_str, modtime,
-                        gfdb_db_record->do_record_counters,
-                        (!its_wind),
-                        isreadfop (gfdb_db_record->gfdb_fop_type),
-                        gfdb_db_record->ignore_errors);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                        gfdb_db_record->ignore_errors),
-                                0, LG_MSG_UPDATE_FAILED, "Failed update unwind "
-                                "time in DB");
-                        goto out;
-                }
-        }
-
-        /*For link creation and changes we use link updated*/
-        if (gfdb_db_record->islinkupdate &&
-                isdentryfop(gfdb_db_record->gfdb_fop_type)) {
-
-                pargfid_str = gf_strdup(uuid_utoa(gfdb_db_record->pargfid));
-                if (!pargfid_str) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                0, LG_MSG_CREATE_FAILED,
-                                "Creating pargfid_str string failed.");
-                        goto out;
-                }
-
-                ret = gf_sql_update_link_flags (sql_conn, gfid_str, pargfid_str,
-                                        gfdb_db_record->file_name, 0, _gf_true,
-                                        gfdb_db_record->ignore_errors);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                        gfdb_db_record->ignore_errors),
-                                0, LG_MSG_UPDATE_FAILED,
-                                "Failed updating link flags in unwind");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        GF_FREE (gfid_str);
-        GF_FREE (pargfid_str);
-        return ret;
-}
-
-
-int
-gf_sql_update_delete_wind (gf_sql_connection_t  *sql_conn,
-                          gfdb_db_record_t     *gfdb_db_record)
-{
-        int ret = -1;
-        char *gfid_str          = NULL;
-        char *pargfid_str       = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
-        if (!gfid_str) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
-                        "Creating gfid string failed.");
-                goto out;
-        }
-
-        pargfid_str = gf_strdup (uuid_utoa(gfdb_db_record->pargfid));
-        if (!pargfid_str) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                0, LG_MSG_CREATE_FAILED, "Creating pargfid_str "
-                                "string failed.");
-                        goto out;
-        }
-
-        if (gfdb_db_record->link_consistency) {
-                ret = gf_sql_update_link_flags (sql_conn, gfid_str, pargfid_str,
-                                        gfdb_db_record->file_name, 1,
-                                        _gf_false,
-                                        gfdb_db_record->ignore_errors);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                        gfdb_db_record->ignore_errors),
-                                0, LG_MSG_UPDATE_FAILED,
-                                "Failed updating link flags in wind");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        GF_FREE (gfid_str);
-        GF_FREE (pargfid_str);
-        return ret;
-}
-
-int
-gf_sql_delete_unwind (gf_sql_connection_t  *sql_conn,
-                          gfdb_db_record_t     *gfdb_db_record)
-{
-        int ret = -1;
-        char *gfid_str = NULL;
-        char *pargfid_str = NULL;
-        gfdb_time_t *modtime    = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);
-
-        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
-        if (!gfid_str) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
-                        "Creating gfid string failed.");
-                goto out;
-        }
-
-        /*Nuke all the entries for this GFID from DB*/
-        if (gfdb_db_record->gfdb_fop_path == GFDB_FOP_UNDEL_ALL) {
-                gf_sql_delete_all (sql_conn, gfid_str,
-                                gfdb_db_record->ignore_errors);
-        }
-        /*Remove link entries only*/
-        else if (gfdb_db_record->gfdb_fop_path == GFDB_FOP_UNDEL) {
-
-                pargfid_str = gf_strdup(uuid_utoa(gfdb_db_record->pargfid));
-                if (!pargfid_str) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                0, LG_MSG_CREATE_FAILED, "Creating pargfid_str "
-                                "string failed.");
-                        goto out;
-                }
-
-                /* Special performance case:
-                 * Updating wind time in unwind for delete. This is done here
-                 * as in the wind path we will not know whether its the last
-                 * link or not. For a last link there is not use to update any
-                 * wind or unwind time!*/
-                if (gfdb_db_record->do_record_times) {
-                        /*Update the wind write times*/
-                        modtime = &gfdb_db_record->gfdb_wind_change_time;
-                        ret = gf_update_time (sql_conn, gfid_str, modtime,
-                                gfdb_db_record->do_record_counters,
-                                _gf_true,
-                                isreadfop (gfdb_db_record->gfdb_fop_type),
-                                gfdb_db_record->ignore_errors);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3,
-                                        _gfdb_log_level (GF_LOG_ERROR,
-                                                gfdb_db_record->ignore_errors),
-                                        0, LG_MSG_UPDATE_FAILED,
-                                        "Failed update wind time in DB");
-                                goto out;
-                        }
-                }
-
-                modtime = &gfdb_db_record->gfdb_unwind_change_time;
-
-                ret = gf_sql_delete_link(sql_conn, gfid_str, pargfid_str,
-                                        gfdb_db_record->file_name,
-                                        gfdb_db_record->ignore_errors);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_DELETE_FAILED, "Failed deleting link");
-                        goto out;
-                }
-
-                if (gfdb_db_record->do_record_times &&
-                        gfdb_db_record->do_record_uwind_time) {
-                        ret = gf_update_time (sql_conn, gfid_str, modtime,
-                                gfdb_db_record->do_record_counters,
-                                _gf_false,
-                                isreadfop(gfdb_db_record->gfdb_fop_type),
-                                gfdb_db_record->ignore_errors);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3,
-                                        _gfdb_log_level (GF_LOG_ERROR,
-                                                gfdb_db_record->ignore_errors),
-                                        0, LG_MSG_UPDATE_FAILED,
-                                        "Failed update unwind time in DB");
-                                goto out;
-                        }
-                }
-        } else {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                        0, LG_MSG_INVALID_UPLINK, "Invalid unlink option");
-                goto out;
-        }
-        ret = 0;
-out:
-        GF_FREE (gfid_str);
-        GF_FREE (pargfid_str);
-        return ret;
-}
-
-/******************************************************************************
- *
- *                      Find/Query helper functions
- *
- * ****************************************************************************/
-int
-gf_sql_query_function (sqlite3_stmt              *prep_stmt,
-                      gf_query_callback_t       query_callback,
-                      void                      *_query_cbk_args)
-{
-        int ret                                         = -1;
-        gfdb_query_record_t *query_record               = NULL;
-        char *text_column                               = NULL;
-        sqlite3 *db_conn                                = NULL;
-        uuid_t  prev_gfid                               = {0};
-        uuid_t  curr_gfid                               = {0};
-        uuid_t  pgfid                                   = {0};
-        char *base_name                                 = NULL;
-        gf_boolean_t is_first_record                    = _gf_true;
-        gf_boolean_t is_query_empty                     = _gf_true;
-
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, prep_stmt, out);
-        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, query_callback, out);
-
-        db_conn = sqlite3_db_handle(prep_stmt);
-
-        /*
-         * Loop to access queried rows
-         * Each db record will have 3 columns
-         * GFID, PGFID, FILE_NAME
-         *
-         * For file with multiple hard links we will get multiple query rows
-         * with the same GFID, but different PGID and FILE_NAME Combination
-         * For Example if a file with
-         *         GFID = 00000000-0000-0000-0000-000000000006
-         * has 3 hardlinks file1, file2 and file3 in 3 different folder
-         * with GFID's
-         * 00000000-0000-0000-0000-0000EFC00001,
-         * 00000000-0000-0000-0000-00000ABC0001 and
-         * 00000000-0000-0000-0000-00000ABC00CD
-         * Then there will be 3 records
-         *         GFID         : 00000000-0000-0000-0000-000000000006
-         *         PGFID        : 00000000-0000-0000-0000-0000EFC00001
-         *         FILE_NAME    : file1
-         *
-         *         GFID         : 00000000-0000-0000-0000-000000000006
-         *         PGFID        : 00000000-0000-0000-0000-00000ABC0001
-         *         FILE_NAME    : file2
-         *
-         *         GFID         : 00000000-0000-0000-0000-000000000006
-         *         PGFID        : 00000000-0000-0000-0000-00000ABC00CD
-         *         FILE_NAME    : file3
-         *
-         * This is retrieved and added to a single query_record
-         *
-         * query_record->gfid = 00000000-0000-0000-0000-000000000006
-         *                  ->link_info = {00000000-0000-0000-0000-0000EFC00001,
-         *                                 "file1"}
-         *                                  |
-         *                                  V
-         *             link_info = {00000000-0000-0000-0000-00000ABC0001,
-         *                                 "file2"}
-         *                                  |
-         *                                  V
-         *             link_info = {00000000-0000-0000-0000-00000ABC0001,
-         *                                 "file3",
-         *                                 list}
-         *
-         * This query record is sent to the registered query_callback()
-         *
-         * */
-        while ((ret = sqlite3_step (prep_stmt)) == SQLITE_ROW) {
-
-                if (sqlite3_column_count(prep_stmt) > 0) {
-
-                        is_query_empty = _gf_false;
-
-                        /*Retrieving GFID - column index is 0*/
-                        text_column = (char *)sqlite3_column_text
-                                                        (prep_stmt, 0);
-                        if (!text_column) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_GET_ID_FAILED, "Failed to"
-                                        "retrieve GFID");
-                                goto out;
-                        }
-                        ret = gf_uuid_parse (text_column, curr_gfid);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_PARSE_FAILED, "Failed to parse "
-                                        "GFID");
-                                goto out;
-                        }
-
-                        /*
-                         * if the previous record was not of the current gfid
-                         * call the call_back function and send the
-                         * query record, which will have all the link_info
-                         * objects associated with this gfid
-                         *
-                         * */
-                        if (gf_uuid_compare (curr_gfid, prev_gfid) != 0) {
-
-                                /* If this is not the first record */
-                                if (!is_first_record) {
-                                        /*Call the call_back function provided*/
-                                        ret = query_callback (query_record,
-                                                        _query_cbk_args);
-                                        if (ret) {
-                                                gf_msg (GFDB_STR_SQLITE3,
-                                                        GF_LOG_ERROR, 0,
-                                                LG_MSG_QUERY_CALL_BACK_FAILED,
-                                                        "Query call back "
-                                                        "failed");
-                                                goto out;
-                                        }
-
-                                }
-
-                                /*Clear the query record*/
-                                gfdb_query_record_free (query_record);
-                                query_record = NULL;
-                                query_record = gfdb_query_record_new ();
-                                if (!query_record) {
-                                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
-                                                0, LG_MSG_CREATE_FAILED,
-                                                "Failed to create "
-                                                "query_record");
-                                        goto out;
-                                }
-
-                                gf_uuid_copy(query_record->gfid,
-                                                                curr_gfid);
-                                gf_uuid_copy(prev_gfid, curr_gfid);
-
-                        }
-
-                        /* Get PGFID */
-                        text_column = (char *)sqlite3_column_text
-                                                        (prep_stmt, 1);
-                        if (!text_column) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_GET_ID_FAILED, "Failed to"
-                                        " retrieve GF_ID");
-                                goto out;
-                        }
-                        ret = gf_uuid_parse (text_column, pgfid);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_PARSE_FAILED, "Failed to parse "
-                                        "GF_ID");
-                                goto out;
-                        }
-
-                        /* Get Base name */
-                        text_column = (char *)sqlite3_column_text
-                                                        (prep_stmt, 2);
-                        if (!text_column) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_GET_ID_FAILED, "Failed to"
-                                        " retrieve GF_ID");
-                                goto out;
-                        }
-                        base_name = text_column;
-
-
-                        /* Add link info to the list */
-                        ret = gfdb_add_link_to_query_record (query_record,
-                                                         pgfid, base_name);
-                        if (ret) {
-                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                        LG_MSG_GET_ID_FAILED, "Failed to"
-                                        " add link info to query record");
-                                goto out;
-                        }
-
-                        is_first_record = _gf_false;
-
-                }
-
-        }
-
-        if (ret != SQLITE_DONE) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                        LG_MSG_GET_RECORD_FAILED, "Failed to retrieve records "
-                        "from db : %s", sqlite3_errmsg (db_conn));
-                ret = -1;
-                goto out;
-        }
-
-
-        if (!is_query_empty) {
-                /*
-                 * Call the call_back function for the last record from the
-                 * Database
-                 * */
-                ret = query_callback (query_record, _query_cbk_args);
-                if (ret) {
-                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
-                                LG_MSG_QUERY_CALL_BACK_FAILED,
-                                "Query call back failed");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        gfdb_query_record_free (query_record);
-        query_record = NULL;
-        return ret;
-}
-
-
-
-int
-gf_sql_clear_counters (gf_sql_connection_t *sql_conn)
-{
-        int ret                 = -1;
-        char *sql_strerror      = NULL;
-        char *query_str         = NULL;
-
-        CHECK_SQL_CONN (sql_conn, out);
-
-        query_str = "UPDATE "
-                    GF_FILE_TABLE
-                    " SET " GF_COL_READ_FREQ_CNTR " = 0 , "
-                    GF_COL_WRITE_FREQ_CNTR " = 0 ;";
-
-        ret = sqlite3_exec (sql_conn->sqlite3_db_conn, query_str, NULL, NULL,
-                                &sql_strerror);
-        if (ret != SQLITE_OK) {
-                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_EXEC_FAILED,
-                        "Failed to execute: %s : %s",
-                        query_str, sql_strerror);
-                sqlite3_free (sql_strerror);
-                        ret = -1;
-                        goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.h b/libglusterfs/src/gfdb/gfdb_sqlite3_helper.h
deleted file mode 100644
index 0d222305d01..00000000000
--- a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GFDB_SQLITE3_HELPER_H
-#define __GFDB_SQLITE3_HELPER_H
-
-
-#include "gfdb_sqlite3.h"
-
-/******************************************************************************
- *
- *                      Helper functions for gf_sqlite3_insert()
- *
- * ****************************************************************************/
-
-
-int
-gf_sql_insert_wind (gf_sql_connection_t  *sql_conn,
-                   gfdb_db_record_t     *gfdb_db_record);
-
-int
-gf_sql_insert_unwind (gf_sql_connection_t  *sql_conn,
-                     gfdb_db_record_t     *gfdb_db_record);
-
-
-int
-gf_sql_update_delete_wind (gf_sql_connection_t  *sql_conn,
-                          gfdb_db_record_t     *gfdb_db_record);
-
-int
-gf_sql_delete_unwind (gf_sql_connection_t  *sql_conn,
-                          gfdb_db_record_t     *gfdb_db_record);
-
-
-
-
-
-/******************************************************************************
- *
- *                      Find/Query helper functions
- *
- * ****************************************************************************/
-
-
-int
-gf_sql_query_function (sqlite3_stmt              *prep_stmt,
-                      gf_query_callback_t       query_callback,
-                      void                      *_query_cbk_args);
-
-int
-gf_sql_clear_counters (gf_sql_connection_t *sql_conn);
-
-#endif
diff --git a/libglusterfs/src/gidcache.c b/libglusterfs/src/gidcache.c
index f2d98abefc3..64a93802f76 100644
--- a/libglusterfs/src/gidcache.c
+++ b/libglusterfs/src/gidcache.c
@@ -8,45 +8,48 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "gidcache.h"
-#include "mem-pool.h"
+#include "glusterfs/gidcache.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/common-utils.h"
 
 /*
  * We treat this as a very simple set-associative LRU cache, with entries aged
  * out after a configurable interval.  Hardly rocket science, but lots of
  * details to worry about.
  */
-#define BUCKET_START(p,n)       ((p) + ((n) * AUX_GID_CACHE_ASSOC))
+#define BUCKET_START(p, n) ((p) + ((n)*AUX_GID_CACHE_ASSOC))
 
 /*
  * Initialize the cache.
  */
-int gid_cache_init(gid_cache_t *cache, uint32_t timeout)
+int
+gid_cache_init(gid_cache_t *cache, uint32_t timeout)
 {
-	if (!cache)
-		return -1;
+    if (!cache)
+        return -1;
 
-	LOCK_INIT(&cache->gc_lock);
-	cache->gc_max_age = timeout;
-	cache->gc_nbuckets = AUX_GID_CACHE_BUCKETS;
-	memset(cache->gc_cache, 0, sizeof(gid_list_t) * AUX_GID_CACHE_SIZE);
+    LOCK_INIT(&cache->gc_lock);
+    cache->gc_max_age = timeout;
+    cache->gc_nbuckets = AUX_GID_CACHE_BUCKETS;
+    memset(cache->gc_cache, 0, sizeof(gid_list_t) * AUX_GID_CACHE_SIZE);
 
-	return 0;
+    return 0;
 }
 
 /*
  * Reconfigure the cache timeout.
  */
-int gid_cache_reconf(gid_cache_t *cache, uint32_t timeout)
+int
+gid_cache_reconf(gid_cache_t *cache, uint32_t timeout)
 {
-        if (!cache)
-                return -1;
+    if (!cache)
+        return -1;
 
-        LOCK(&cache->gc_lock);
-        cache->gc_max_age = timeout;
-        UNLOCK(&cache->gc_lock);
+    LOCK(&cache->gc_lock);
+    cache->gc_max_age = timeout;
+    UNLOCK(&cache->gc_lock);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -54,153 +57,155 @@ int gid_cache_reconf(gid_cache_t *cache, uint32_t timeout)
  * an additional allocation and memory copy. The caller should copy the data and
  * release (unlock) the cache as soon as possible.
  */
-const gid_list_t *gid_cache_lookup(gid_cache_t *cache, uint64_t id,
-				   uint64_t uid, uint64_t gid)
+const gid_list_t *
+gid_cache_lookup(gid_cache_t *cache, uint64_t id, uint64_t uid, uint64_t gid)
 {
-	int bucket;
-	int i;
-	time_t now;
-	const gid_list_t *agl;
-
-	LOCK(&cache->gc_lock);
-	now = time(NULL);
-	bucket = id % cache->gc_nbuckets;
-	agl = BUCKET_START(cache->gc_cache, bucket);
-	for (i = 0; i < AUX_GID_CACHE_ASSOC; i++, agl++) {
-		if (!agl->gl_list)
-			continue;
-		if (agl->gl_id != id)
-			continue;
-
-		/*
-		  @uid and @gid reflect the latest UID/GID of the
-		   process performing the syscall (taken from frame->root).
-
-		   If the UID and GID has changed for the PID since the
-		   time we cached it, we should treat the cache as having
-		   stale values and query them freshly.
-		*/
-		if (agl->gl_uid != uid || agl->gl_gid != gid)
-			break;
-
-		/*
-		 * We don't put new entries in the cache when expiration=0, but
-		 * there might be entries still in there if expiration was
-		 * changed very recently.  Writing the check this way ensures
-		 * that they're not used.
-		 */
-		if (now < agl->gl_deadline) {
-			return agl;
-		}
-
-		/*
-		 * We're not going to find any more UID matches, and reaping
-		 * is handled further down to maintain LRU order.
-		 */
-		break;
-	}
-	UNLOCK(&cache->gc_lock);
-	return NULL;
+    int bucket;
+    int i;
+    time_t now;
+    const gid_list_t *agl;
+
+    now = gf_time();
+    LOCK(&cache->gc_lock);
+    bucket = id % cache->gc_nbuckets;
+    agl = BUCKET_START(cache->gc_cache, bucket);
+    for (i = 0; i < AUX_GID_CACHE_ASSOC; i++, agl++) {
+        if (!agl->gl_list)
+            continue;
+        if (agl->gl_id != id)
+            continue;
+
+        /*
+          @uid and @gid reflect the latest UID/GID of the
+           process performing the syscall (taken from frame->root).
+
+           If the UID and GID has changed for the PID since the
+           time we cached it, we should treat the cache as having
+           stale values and query them freshly.
+        */
+        if (agl->gl_uid != uid || agl->gl_gid != gid)
+            break;
+
+        /*
+         * We don't put new entries in the cache when expiration=0, but
+         * there might be entries still in there if expiration was
+         * changed very recently.  Writing the check this way ensures
+         * that they're not used.
+         */
+        if (now < agl->gl_deadline) {
+            return agl;
+        }
+
+        /*
+         * We're not going to find any more UID matches, and reaping
+         * is handled further down to maintain LRU order.
+         */
+        break;
+    }
+    UNLOCK(&cache->gc_lock);
+    return NULL;
 }
 
 /*
  * Release an entry found via lookup.
  */
-void gid_cache_release(gid_cache_t *cache, const gid_list_t *agl)
+void
+gid_cache_release(gid_cache_t *cache, const gid_list_t *agl)
 {
-	UNLOCK(&cache->gc_lock);
+    UNLOCK(&cache->gc_lock);
 }
 
 /*
  * Add a new list entry to the cache. If an entry for this ID already exists,
  * update it.
  */
-int gid_cache_add(gid_cache_t *cache, gid_list_t *gl)
+int
+gid_cache_add(gid_cache_t *cache, gid_list_t *gl)
 {
-	gid_list_t *agl;
-	int bucket;
-	int i;
-	time_t now;
-
-	if (!gl || !gl->gl_list)
-		return -1;
-
-	if (!cache->gc_max_age)
-		return 0;
-
-	LOCK(&cache->gc_lock);
-	now = time(NULL);
-
-	/*
-	 * Scan for the first free entry or one that matches this id. The id
-	 * check is added to address a bug where the cache might contain an
-	 * expired entry for this id. Since lookup occurs in LRU order and
-	 * does not reclaim entries, it will always return failure on discovery
-	 * of an expired entry. This leads to duplicate entries being added,
-	 * which still do not satisfy lookups until the expired entry (and
-	 * everything before it) is reclaimed.
-	 *
-	 * We address this through reuse of an entry already allocated to this
-	 * id, whether expired or not, since we have obviously already received
-	 * more recent data. The entry is repopulated with the new data and a new
-	 * deadline and is pushed forward to reside as the last populated entry in
-	 * the bucket.
-	 */
-	bucket = gl->gl_id % cache->gc_nbuckets;
-	agl = BUCKET_START(cache->gc_cache, bucket);
-	for (i = 0; i < AUX_GID_CACHE_ASSOC; ++i, ++agl) {
-		if (agl->gl_id == gl->gl_id)
-			break;
-		if (!agl->gl_list)
-			break;
-	}
-
-	/*
-	 * The way we allocate free entries naturally places the newest
-	 * ones at the highest indices, so evicting the lowest makes
-	 * sense, but that also means we can't just replace it with the
-	 * one that caused the eviction.  That would cause us to thrash
-	 * the first entry while others remain idle.  Therefore, we
-	 * need to slide the other entries down and add the new one at
-	 * the end just as if the *last* slot had been free.
-	 *
-	 * Deadline expiration is also handled here, since the oldest
-	 * expired entry will be in the first position.  This does mean
-	 * the bucket can stay full of expired entries if we're idle
-	 * but, if the small amount of extra memory or scan time before
-	 * we decide to evict someone ever become issues, we could
-	 * easily add a reaper thread.
-	 */
-
-	if (i >= AUX_GID_CACHE_ASSOC) {
-		/* cache full, evict the first (LRU) entry */
-		i = 0;
-		agl = BUCKET_START(cache->gc_cache, bucket);
-		GF_FREE(agl->gl_list);
-	} else if (agl->gl_list) {
-		/* evict the old entry we plan to reuse */
-		GF_FREE(agl->gl_list);
-	}
-
-	/*
-	 * If we have evicted an entry, slide the subsequent populated entries
-	 * back and populate the last entry.
-	 */
-	for (; i < AUX_GID_CACHE_ASSOC - 1; i++) {
-		if (!agl[1].gl_list)
-			break;
-		agl[0] = agl[1];
-		agl++;
-	}
-
-	agl->gl_id = gl->gl_id;
-	agl->gl_uid = gl->gl_uid;
-	agl->gl_gid = gl->gl_gid;
-	agl->gl_count = gl->gl_count;
-	agl->gl_list = gl->gl_list;
-	agl->gl_deadline = now + cache->gc_max_age;
-
-	UNLOCK(&cache->gc_lock);
-
-	return 1;
+    gid_list_t *agl;
+    int bucket;
+    int i;
+    time_t now;
+
+    if (!gl || !gl->gl_list)
+        return -1;
+
+    if (!cache->gc_max_age)
+        return 0;
+
+    now = gf_time();
+    LOCK(&cache->gc_lock);
+
+    /*
+     * Scan for the first free entry or one that matches this id. The id
+     * check is added to address a bug where the cache might contain an
+     * expired entry for this id. Since lookup occurs in LRU order and
+     * does not reclaim entries, it will always return failure on discovery
+     * of an expired entry. This leads to duplicate entries being added,
+     * which still do not satisfy lookups until the expired entry (and
+     * everything before it) is reclaimed.
+     *
+     * We address this through reuse of an entry already allocated to this
+     * id, whether expired or not, since we have obviously already received
+     * more recent data. The entry is repopulated with the new data and a new
+     * deadline and is pushed forward to reside as the last populated entry in
+     * the bucket.
+     */
+    bucket = gl->gl_id % cache->gc_nbuckets;
+    agl = BUCKET_START(cache->gc_cache, bucket);
+    for (i = 0; i < AUX_GID_CACHE_ASSOC; ++i, ++agl) {
+        if (agl->gl_id == gl->gl_id)
+            break;
+        if (!agl->gl_list)
+            break;
+    }
+
+    /*
+     * The way we allocate free entries naturally places the newest
+     * ones at the highest indices, so evicting the lowest makes
+     * sense, but that also means we can't just replace it with the
+     * one that caused the eviction.  That would cause us to thrash
+     * the first entry while others remain idle.  Therefore, we
+     * need to slide the other entries down and add the new one at
+     * the end just as if the *last* slot had been free.
+     *
+     * Deadline expiration is also handled here, since the oldest
+     * expired entry will be in the first position.  This does mean
+     * the bucket can stay full of expired entries if we're idle
+     * but, if the small amount of extra memory or scan time before
+     * we decide to evict someone ever become issues, we could
+     * easily add a reaper thread.
+     */
+
+    if (i >= AUX_GID_CACHE_ASSOC) {
+        /* cache full, evict the first (LRU) entry */
+        i = 0;
+        agl = BUCKET_START(cache->gc_cache, bucket);
+        GF_FREE(agl->gl_list);
+    } else if (agl->gl_list) {
+        /* evict the old entry we plan to reuse */
+        GF_FREE(agl->gl_list);
+    }
+
+    /*
+     * If we have evicted an entry, slide the subsequent populated entries
+     * back and populate the last entry.
+     */
+    for (; i < AUX_GID_CACHE_ASSOC - 1; i++) {
+        if (!agl[1].gl_list)
+            break;
+        agl[0] = agl[1];
+        agl++;
+    }
+
+    agl->gl_id = gl->gl_id;
+    agl->gl_uid = gl->gl_uid;
+    agl->gl_gid = gl->gl_gid;
+    agl->gl_count = gl->gl_count;
+    agl->gl_list = gl->gl_list;
+    agl->gl_deadline = now + cache->gc_max_age;
+
+    UNLOCK(&cache->gc_lock);
+
+    return 1;
 }
diff --git a/libglusterfs/src/glfs-message-id.h b/libglusterfs/src/glfs-message-id.h
deleted file mode 100644
index 8601c5cc5ab..00000000000
--- a/libglusterfs/src/glfs-message-id.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
-  Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _GLFS_MESSAGE_ID_H_
-#define _GLFS_MESSAGE_ID_H_
-
-/* Base of all message IDs, all message IDs would be
- * greater than this */
-#define GLFS_MSGID_BASE         100000
-
-/* Segment size of allocated range. Any component needing more than this
- * segment size should take multiple segments (at times non contiguous,
- * if extensions are being made post the next segment already allocated) */
-#define GLFS_MSGID_SEGMENT      1000
-
-/* Per module message segments allocated */
-/* NOTE: For any new module add to the end the modules */
-#define GLFS_MSGID_COMP_GLUSTERFSD         GLFS_MSGID_BASE
-#define GLFS_MSGID_COMP_GLUSTERFSD_END     GLFS_MSGID_COMP_GLUSTERFSD + \
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_LIBGLUSTERFS       GLFS_MSGID_COMP_GLUSTERFSD_END
-#define GLFS_MSGID_COMP_LIBGLUSTERFS_END   GLFS_MSGID_COMP_LIBGLUSTERFS + \
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_RPC_LIB            GLFS_MSGID_COMP_LIBGLUSTERFS_END
-#define GLFS_MSGID_COMP_RPC_LIB_END        GLFS_MSGID_COMP_RPC_LIB + \
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_RPC_TRANS_RDMA     GLFS_MSGID_COMP_RPC_LIB_END
-#define GLFS_MSGID_COMP_RPC_TRANS_RDMA_END (GLFS_MSGID_COMP_RPC_TRANS_RDMA + \
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_API                GLFS_MSGID_COMP_RPC_TRANS_RDMA_END
-#define GLFS_MSGID_COMP_API_END            GLFS_MSGID_COMP_API + \
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_CLI                GLFS_MSGID_COMP_API_END
-#define GLFS_MSGID_COMP_CLI_END            GLFS_MSGID_COMP_CLI + \
-                                           GLFS_MSGID_SEGMENT
-
-/* glusterd has a lot of messages, taking 2 segments for the same */
-#define GLFS_MSGID_GLUSTERD                GLFS_MSGID_COMP_CLI_END
-#define GLFS_MSGID_GLUSTERD_END            GLFS_MSGID_GLUSTERD + \
-                                           GLFS_MSGID_SEGMENT + \
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_AFR                GLFS_MSGID_GLUSTERD_END
-#define GLFS_MSGID_COMP_AFR_END            GLFS_MSGID_COMP_AFR +\
-                                           GLFS_MSGID_SEGMENT
-
-#define GLFS_MSGID_COMP_DHT                GLFS_MSGID_COMP_AFR_END
-#define GLFS_MSGID_COMP_DHT_END            GLFS_MSGID_COMP_DHT +\
-                                           GLFS_MSGID_SEGMENT
-
-
-/* there is no component called 'common', however reserving this segment
- * for common actions/errors like dict_{get/set}, memory accounting*/
-
-#define GLFS_MSGID_COMP_COMMON             GLFS_MSGID_COMP_DHT_END
-#define GLFS_MSGID_COMP_COMMON_END         (GLFS_MSGID_COMP_COMMON +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_UPCALL             GLFS_MSGID_COMP_COMMON_END
-#define GLFS_MSGID_COMP_UPCALL_END         (GLFS_MSGID_COMP_UPCALL +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_NFS                GLFS_MSGID_COMP_UPCALL_END
-#define GLFS_MSGID_COMP_NFS_END            (GLFS_MSGID_COMP_NFS +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_POSIX              GLFS_MSGID_COMP_NFS_END
-#define GLFS_MSGID_COMP_POSIX_END          (GLFS_MSGID_COMP_POSIX +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_PC                 GLFS_MSGID_COMP_POSIX_END
-#define GLFS_MSGID_COMP_PC_END             (GLFS_MSGID_COMP_PC +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_PS                 GLFS_MSGID_COMP_PC_END
-#define GLFS_MSGID_COMP_PS_END             (GLFS_MSGID_COMP_PS +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_BITROT_STUB        GLFS_MSGID_COMP_PS_END
-#define GLFS_MSGID_COMP_BITROT_STUB_END    (GLFS_MSGID_COMP_BITROT_STUB +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_CHANGELOG          GLFS_MSGID_COMP_BITROT_STUB_END
-#define GLFS_MSGID_COMP_CHANGELOG_END      (GLFS_MSGID_COMP_CHANGELOG +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_BITROT_BITD        GLFS_MSGID_COMP_CHANGELOG_END
-#define GLFS_MSGID_COMP_BITROT_BITD_END    (GLFS_MSGID_COMP_BITROT_BITD +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_RPC_TRANS_SOCKET        GLFS_MSGID_COMP_BITROT_BITD_END
-#define GLFS_MSGID_COMP_RPC_TRANS_SOCKET_END    (GLFS_MSGID_COMP_RPC_TRANS_SOCKET + \
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_QUOTA              GLFS_MSGID_COMP_RPC_TRANS_SOCKET_END
-#define GLFS_MSGID_COMP_QUOTA_END          (GLFS_MSGID_COMP_QUOTA +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_CTR                GLFS_MSGID_COMP_QUOTA_END
-#define GLFS_MSGID_COMP_CTR_END            (GLFS_MSGID_COMP_CTR+\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_EC               GLFS_MSGID_COMP_CTR_END
-#define GLFS_MSGID_COMP_EC_END           (GLFS_MSGID_COMP_EC +\
-                                         GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_IO_CACHE                GLFS_MSGID_COMP_EC_END
-#define GLFS_MSGID_COMP_IO_CACHE_END            (GLFS_MSGID_COMP_IO_CACHE+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_IO_THREADS              GLFS_MSGID_COMP_IO_CACHE_END
-#define GLFS_MSGID_COMP_IO_THREADS_END          (GLFS_MSGID_COMP_IO_THREADS+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_MD_CACHE                GLFS_MSGID_COMP_IO_THREADS_END
-#define GLFS_MSGID_COMP_MD_CACHE_END            (GLFS_MSGID_COMP_MD_CACHE+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_OPEN_BEHIND             GLFS_MSGID_COMP_MD_CACHE_END
-#define GLFS_MSGID_COMP_OPEN_BEHIND_END         (GLFS_MSGID_COMP_OPEN_BEHIND+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_QUICK_READ              GLFS_MSGID_COMP_OPEN_BEHIND_END
-#define GLFS_MSGID_COMP_QUICK_READ_END          (GLFS_MSGID_COMP_QUICK_READ+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_READ_AHEAD              GLFS_MSGID_COMP_QUICK_READ_END
-#define GLFS_MSGID_COMP_READ_AHEAD_END          (GLFS_MSGID_COMP_READ_AHEAD+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_READDIR_AHEAD           GLFS_MSGID_COMP_READ_AHEAD_END
-#define GLFS_MSGID_COMP_READDIR_AHEAD_END       (GLFS_MSGID_COMP_READDIR_AHEAD+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_SYMLINK_CACHE           \
-GLFS_MSGID_COMP_READDIR_AHEAD_END
-#define GLFS_MSGID_COMP_SYMLINK_CACHE_END \
-(GLFS_MSGID_COMP_SYMLINK_CACHE+ \
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_WRITE_BEHIND             \
-GLFS_MSGID_COMP_SYMLINK_CACHE_END
-#define GLFS_MSGID_COMP_WRITE_BEHIND_END        (GLFS_MSGID_COMP_WRITE_BEHIND+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_CHANGELOG_LIB           GLFS_MSGID_COMP_WRITE_BEHIND_END
-#define GLFS_MSGID_COMP_CHANGELOG_LIB_END       (GLFS_MSGID_COMP_CHANGELOG_LIB+\
-                                                GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_SHARD               GLFS_MSGID_COMP_CHANGELOG_LIB_END
-#define GLFS_MSGID_COMP_SHARD_END           (GLFS_MSGID_COMP_SHARD +\
-                                             GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_JBR                     GLFS_MSGID_COMP_SHARD_END
-#define GLFS_MSGID_COMP_JBR_END                 (GLFS_MSGID_COMP_JBR +\
-                                                 GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_PL               GLFS_MSGID_COMP_JBR_END
-#define GLFS_MSGID_COMP_PL_END           (GLFS_MSGID_COMP_PL +\
-                                         GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_DC                     GLFS_MSGID_COMP_PL_END
-#define GLFS_MSGID_COMP_DC_END                 (GLFS_MSGID_COMP_PL_END+\
-                                                 GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_LEASES             GLFS_MSGID_COMP_DC_END
-#define GLFS_MSGID_COMP_LEASES_END         (GLFS_MSGID_COMP_LEASES +\
-                                           GLFS_MSGID_SEGMENT)
-
-#define GLFS_MSGID_COMP_INDEX              GLFS_MSGID_COMP_LEASES_END
-#define GLFS_MSGID_COMP_INDEX_END          (GLFS_MSGID_COMP_INDEX +\
-                                            GLFS_MSGID_SEGMENT)
-/* --- new segments for messages goes above this line --- */
-
-#endif /* !_GLFS_MESSAGE_ID_H_ */
diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c
index c537971cbb4..ae06f8be386 100644
--- a/libglusterfs/src/globals.c
+++ b/libglusterfs/src/globals.c
@@ -10,81 +10,80 @@
 
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "globals.h"
-#include "xlator.h"
-#include "mem-pool.h"
-#include "syncop.h"
-#include "libglusterfs-messages.h"
-#include "upcall-utils.h"
+#include "glusterfs/syncop.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 const char *gf_fop_list[GF_FOP_MAXVALUE] = {
-        [GF_FOP_NULL]        = "NULL",
-        [GF_FOP_STAT]        = "STAT",
-        [GF_FOP_READLINK]    = "READLINK",
-        [GF_FOP_MKNOD]       = "MKNOD",
-        [GF_FOP_MKDIR]       = "MKDIR",
-        [GF_FOP_UNLINK]      = "UNLINK",
-        [GF_FOP_RMDIR]       = "RMDIR",
-        [GF_FOP_SYMLINK]     = "SYMLINK",
-        [GF_FOP_RENAME]      = "RENAME",
-        [GF_FOP_LINK]        = "LINK",
-        [GF_FOP_TRUNCATE]    = "TRUNCATE",
-        [GF_FOP_OPEN]        = "OPEN",
-        [GF_FOP_READ]        = "READ",
-        [GF_FOP_WRITE]       = "WRITE",
-        [GF_FOP_STATFS]      = "STATFS",
-        [GF_FOP_FLUSH]       = "FLUSH",
-        [GF_FOP_FSYNC]       = "FSYNC",
-        [GF_FOP_SETXATTR]    = "SETXATTR",
-        [GF_FOP_GETXATTR]    = "GETXATTR",
-        [GF_FOP_REMOVEXATTR] = "REMOVEXATTR",
-        [GF_FOP_OPENDIR]     = "OPENDIR",
-        [GF_FOP_FSYNCDIR]    = "FSYNCDIR",
-        [GF_FOP_ACCESS]      = "ACCESS",
-        [GF_FOP_CREATE]      = "CREATE",
-        [GF_FOP_FTRUNCATE]   = "FTRUNCATE",
-        [GF_FOP_FSTAT]       = "FSTAT",
-        [GF_FOP_LK]          = "LK",
-        [GF_FOP_LOOKUP]      = "LOOKUP",
-        [GF_FOP_READDIR]     = "READDIR",
-        [GF_FOP_INODELK]     = "INODELK",
-        [GF_FOP_FINODELK]    = "FINODELK",
-        [GF_FOP_ENTRYLK]     = "ENTRYLK",
-        [GF_FOP_FENTRYLK]    = "FENTRYLK",
-        [GF_FOP_XATTROP]     = "XATTROP",
-        [GF_FOP_FXATTROP]    = "FXATTROP",
-        [GF_FOP_FSETXATTR]   = "FSETXATTR",
-        [GF_FOP_FGETXATTR]   = "FGETXATTR",
-        [GF_FOP_RCHECKSUM]   = "RCHECKSUM",
-        [GF_FOP_SETATTR]     = "SETATTR",
-        [GF_FOP_FSETATTR]    = "FSETATTR",
-        [GF_FOP_READDIRP]    = "READDIRP",
-        [GF_FOP_GETSPEC]     = "GETSPEC",
-        [GF_FOP_FORGET]      = "FORGET",
-        [GF_FOP_RELEASE]     = "RELEASE",
-        [GF_FOP_RELEASEDIR]  = "RELEASEDIR",
-        [GF_FOP_FREMOVEXATTR]= "FREMOVEXATTR",
-	[GF_FOP_FALLOCATE]   = "FALLOCATE",
-	[GF_FOP_DISCARD]     = "DISCARD",
-        [GF_FOP_ZEROFILL]    = "ZEROFILL",
-        [GF_FOP_IPC]         = "IPC",
-        [GF_FOP_SEEK]        = "SEEK",
-        [GF_FOP_LEASE]       = "LEASE",
-        [GF_FOP_COMPOUND]    = "COMPOUND",
-        [GF_FOP_GETACTIVELK] = "GETACTIVELK",
-        [GF_FOP_SETACTIVELK] = "SETACTIVELK",
+    [GF_FOP_NULL] = "NULL",
+    [GF_FOP_STAT] = "STAT",
+    [GF_FOP_READLINK] = "READLINK",
+    [GF_FOP_MKNOD] = "MKNOD",
+    [GF_FOP_MKDIR] = "MKDIR",
+    [GF_FOP_UNLINK] = "UNLINK",
+    [GF_FOP_RMDIR] = "RMDIR",
+    [GF_FOP_SYMLINK] = "SYMLINK",
+    [GF_FOP_RENAME] = "RENAME",
+    [GF_FOP_LINK] = "LINK",
+    [GF_FOP_TRUNCATE] = "TRUNCATE",
+    [GF_FOP_OPEN] = "OPEN",
+    [GF_FOP_READ] = "READ",
+    [GF_FOP_WRITE] = "WRITE",
+    [GF_FOP_STATFS] = "STATFS",
+    [GF_FOP_FLUSH] = "FLUSH",
+    [GF_FOP_FSYNC] = "FSYNC",
+    [GF_FOP_SETXATTR] = "SETXATTR",
+    [GF_FOP_GETXATTR] = "GETXATTR",
+    [GF_FOP_REMOVEXATTR] = "REMOVEXATTR",
+    [GF_FOP_OPENDIR] = "OPENDIR",
+    [GF_FOP_FSYNCDIR] = "FSYNCDIR",
+    [GF_FOP_ACCESS] = "ACCESS",
+    [GF_FOP_CREATE] = "CREATE",
+    [GF_FOP_FTRUNCATE] = "FTRUNCATE",
+    [GF_FOP_FSTAT] = "FSTAT",
+    [GF_FOP_LK] = "LK",
+    [GF_FOP_LOOKUP] = "LOOKUP",
+    [GF_FOP_READDIR] = "READDIR",
+    [GF_FOP_INODELK] = "INODELK",
+    [GF_FOP_FINODELK] = "FINODELK",
+    [GF_FOP_ENTRYLK] = "ENTRYLK",
+    [GF_FOP_FENTRYLK] = "FENTRYLK",
+    [GF_FOP_XATTROP] = "XATTROP",
+    [GF_FOP_FXATTROP] = "FXATTROP",
+    [GF_FOP_FSETXATTR] = "FSETXATTR",
+    [GF_FOP_FGETXATTR] = "FGETXATTR",
+    [GF_FOP_RCHECKSUM] = "RCHECKSUM",
+    [GF_FOP_SETATTR] = "SETATTR",
+    [GF_FOP_FSETATTR] = "FSETATTR",
+    [GF_FOP_READDIRP] = "READDIRP",
+    [GF_FOP_GETSPEC] = "GETSPEC",
+    [GF_FOP_FORGET] = "FORGET",
+    [GF_FOP_RELEASE] = "RELEASE",
+    [GF_FOP_RELEASEDIR] = "RELEASEDIR",
+    [GF_FOP_FREMOVEXATTR] = "FREMOVEXATTR",
+    [GF_FOP_FALLOCATE] = "FALLOCATE",
+    [GF_FOP_DISCARD] = "DISCARD",
+    [GF_FOP_ZEROFILL] = "ZEROFILL",
+    [GF_FOP_IPC] = "IPC",
+    [GF_FOP_SEEK] = "SEEK",
+    [GF_FOP_LEASE] = "LEASE",
+    [GF_FOP_COMPOUND] = "COMPOUND",
+    [GF_FOP_GETACTIVELK] = "GETACTIVELK",
+    [GF_FOP_SETACTIVELK] = "SETACTIVELK",
+    [GF_FOP_PUT] = "PUT",
+    [GF_FOP_ICREATE] = "ICREATE",
+    [GF_FOP_NAMELINK] = "NAMELINK",
+    [GF_FOP_COPY_FILE_RANGE] = "COPY_FILE_RANGE",
 };
 
 const char *gf_upcall_list[GF_UPCALL_FLAGS_MAXVALUE] = {
-        [GF_UPCALL_NULL] = "NULL",
-        [GF_UPCALL] = "UPCALL",
-        [GF_UPCALL_CI_STAT] = "CI_IATT",
-        [GF_UPCALL_CI_XATTR] = "CI_XATTR",
-        [GF_UPCALL_CI_RENAME] = "CI_RENAME",
-        [GF_UPCALL_CI_NLINK] = "CI_UNLINK",
-        [GF_UPCALL_CI_FORGET] = "CI_FORGET",
-        [GF_UPCALL_LEASE_RECALL] = "LEASE_RECALL",
+    [GF_UPCALL_NULL] = "NULL",
+    [GF_UPCALL] = "UPCALL",
+    [GF_UPCALL_CI_STAT] = "CI_IATT",
+    [GF_UPCALL_CI_XATTR] = "CI_XATTR",
+    [GF_UPCALL_CI_RENAME] = "CI_RENAME",
+    [GF_UPCALL_CI_NLINK] = "CI_UNLINK",
+    [GF_UPCALL_CI_FORGET] = "CI_FORGET",
+    [GF_UPCALL_LEASE_RECALL] = "LEASE_RECALL",
 };
 
 /* THIS */
@@ -95,366 +94,273 @@ const char *gf_upcall_list[GF_UPCALL_FLAGS_MAXVALUE] = {
 glusterfs_ctx_t *global_ctx = NULL;
 pthread_mutex_t global_ctx_mutex = PTHREAD_MUTEX_INITIALIZER;
 xlator_t global_xlator;
-static pthread_key_t this_xlator_key;
-static pthread_key_t synctask_key;
-static pthread_key_t uuid_buf_key;
-static char          global_uuid_buf[GF_UUID_BUF_SIZE];
-static pthread_key_t lkowner_buf_key;
-static char          global_lkowner_buf[GF_LKOWNER_BUF_SIZE];
-static pthread_key_t leaseid_buf_key;
 static int gf_global_mem_acct_enable = 1;
 static pthread_once_t globals_inited = PTHREAD_ONCE_INIT;
 
+static pthread_key_t free_key;
+
+static __thread xlator_t *thread_xlator = NULL;
+static __thread void *thread_synctask = NULL;
+static __thread void *thread_leaseid = NULL;
+static __thread struct syncopctx thread_syncopctx = {};
+static __thread char thread_uuid_buf[GF_UUID_BUF_SIZE] = {};
+static __thread char thread_lkowner_buf[GF_LKOWNER_BUF_SIZE] = {};
+static __thread char thread_leaseid_buf[GF_LEASE_ID_BUF_SIZE] = {};
 
 int
-gf_global_mem_acct_enable_get (void)
+gf_global_mem_acct_enable_get(void)
 {
-	return gf_global_mem_acct_enable;
+    return gf_global_mem_acct_enable;
 }
 
 int
-gf_global_mem_acct_enable_set (int val)
+gf_global_mem_acct_enable_set(int val)
 {
-	gf_global_mem_acct_enable = val;
-	return 0;
+    gf_global_mem_acct_enable = val;
+    return 0;
 }
 
-void
-glusterfs_this_destroy (void *ptr)
-{
-        FREE (ptr);
-}
+static struct xlator_cbks global_cbks = {
+    .forget = NULL,
+    .release = NULL,
+    .releasedir = NULL,
+    .invalidate = NULL,
+    .client_destroy = NULL,
+    .client_disconnect = NULL,
+    .ictxmerge = NULL,
+    .ictxsize = NULL,
+    .fdctxsize = NULL,
+};
 
+/* This is required to get through the check in graph.c */
+static struct xlator_fops global_fops = {};
 
-int
-glusterfs_this_init ()
+static int
+global_xl_reconfigure(xlator_t *this, dict_t *options)
 {
-        int  ret = 0;
-
-        ret = pthread_key_create (&this_xlator_key, glusterfs_this_destroy);
-        if (ret != 0) {
-                gf_msg ("", GF_LOG_WARNING, ret,
-                        LG_MSG_PTHREAD_KEY_CREATE_FAILED, "failed to create "
-                        "the pthread key");
-                return ret;
-        }
+    int ret = -1;
+    gf_boolean_t bool_opt = _gf_false;
 
-        global_xlator.name = "glusterfs";
-        global_xlator.type = "global";
+    /* This is not added in volume dump, hence adding the options in log
+       would be helpful for debugging later */
+    dict_dump_to_log(options);
 
-        INIT_LIST_HEAD (&global_xlator.volume_options);
-
-        return ret;
-}
+    GF_OPTION_RECONF("measure-latency", bool_opt, options, bool, out);
+    this->ctx->measure_latency = bool_opt;
 
+    GF_OPTION_RECONF("metrics-dump-path", this->ctx->config.metrics_dumppath,
+                     options, str, out);
 
-xlator_t **
-__glusterfs_this_location ()
-{
-        xlator_t **this_location = NULL;
-        int        ret = 0;
-
-        this_location = pthread_getspecific (this_xlator_key);
-
-        if (!this_location) {
-                this_location = CALLOC (1, sizeof (*this_location));
-                if (!this_location)
-                        goto out;
-
-                ret = pthread_setspecific (this_xlator_key, this_location);
-                if (ret != 0) {
-                        FREE (this_location);
-                        this_location = NULL;
-                        goto out;
-                }
-        }
+    /* TODO: add more things here */
+    ret = 0;
 out:
-        if (this_location) {
-                if (!*this_location)
-                        *this_location = &global_xlator;
-        }
-        return this_location;
+    return ret;
 }
 
-
-xlator_t *
-glusterfs_this_get ()
+static int
+global_xl_init(xlator_t *this)
 {
-        xlator_t **this_location = NULL;
+    int ret = -1;
+    gf_boolean_t bool_opt = false;
 
-        this_location = __glusterfs_this_location ();
-        if (!this_location)
-                return &global_xlator;
+    GF_OPTION_INIT("measure-latency", bool_opt, bool, out);
+    this->ctx->measure_latency = bool_opt;
 
-        return *this_location;
-}
-
-
-int
-glusterfs_this_set (xlator_t *this)
-{
-        xlator_t **this_location = NULL;
+    GF_OPTION_INIT("metrics-dump-path", this->ctx->config.metrics_dumppath, str,
+                   out);
 
-        this_location = __glusterfs_this_location ();
-        if (!this_location)
-                return -ENOMEM;
+    ret = 0;
 
-        *this_location = this;
-
-        return 0;
+out:
+    return ret;
 }
 
-/* SYNCOPCTX */
-static pthread_key_t syncopctx_key;
-
 static void
-syncopctx_key_destroy (void *ptr)
+global_xl_fini(xlator_t *this)
 {
-	struct syncopctx *opctx = ptr;
-
-	if (opctx) {
-		if (opctx->groups)
-			GF_FREE (opctx->groups);
-
-		GF_FREE (opctx);
-	}
-
-	return;
+    return;
 }
 
-void *
-syncopctx_getctx ()
-{
-	void *opctx = NULL;
+struct volume_options global_xl_options[] = {
+    {.key = {"measure-latency"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"global", "context"},
+     .description = "Use this option to toggle measuring latency"},
+    {.key = {"metrics-dump-path"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "{{gluster_workdir}}/metrics",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"global", "context"},
+     .description = "Use this option to set the metrics dump path"},
+
+    {
+        .key = {NULL},
+    },
+};
 
-	opctx = pthread_getspecific (syncopctx_key);
+static volume_opt_list_t global_xl_opt_list;
 
-	return opctx;
+void
+glusterfs_this_init()
+{
+    global_xlator.name = "glusterfs";
+    global_xlator.type = GF_GLOBAL_XLATOR_NAME;
+    global_xlator.cbks = &global_cbks;
+    global_xlator.fops = &global_fops;
+    global_xlator.reconfigure = global_xl_reconfigure;
+    global_xlator.init = global_xl_init;
+    global_xlator.fini = global_xl_fini;
+
+    INIT_LIST_HEAD(&global_xlator.volume_options);
+    INIT_LIST_HEAD(&global_xl_opt_list.list);
+    global_xl_opt_list.given_opt = global_xl_options;
+
+    list_add_tail(&global_xl_opt_list.list, &global_xlator.volume_options);
 }
 
-int
-syncopctx_setctx (void *ctx)
+xlator_t **
+__glusterfs_this_location()
 {
-	int ret = 0;
+    xlator_t **this_location;
 
-	ret = pthread_setspecific (syncopctx_key, ctx);
+    this_location = &thread_xlator;
+    if (*this_location == NULL) {
+        thread_xlator = &global_xlator;
+    }
 
-	return ret;
+    return this_location;
 }
 
-static int
-syncopctx_init (void)
+xlator_t *
+glusterfs_this_get()
 {
-	int ret;
-
-	ret = pthread_key_create (&syncopctx_key, syncopctx_key_destroy);
-
-	return ret;
+    return *__glusterfs_this_location();
 }
 
-/* SYNCTASK */
-
-int
-synctask_init ()
+void
+glusterfs_this_set(xlator_t *this)
 {
-        int  ret = 0;
-
-        ret = pthread_key_create (&synctask_key, NULL);
-
-        return ret;
+    thread_xlator = this;
 }
 
+/* SYNCOPCTX */
+
 void *
-synctask_get ()
+syncopctx_getctx()
 {
-        void   *synctask = NULL;
-
-        synctask = pthread_getspecific (synctask_key);
-
-        return synctask;
+    return &thread_syncopctx;
 }
 
+/* SYNCTASK */
 
-int
-synctask_set (void *synctask)
+void *
+synctask_get()
 {
-        int     ret = 0;
-
-        pthread_setspecific (synctask_key, synctask);
-
-        return ret;
+    return thread_synctask;
 }
 
-//UUID_BUFFER
-
 void
-glusterfs_uuid_buf_destroy (void *ptr)
+synctask_set(void *synctask)
 {
-        FREE (ptr);
+    thread_synctask = synctask;
 }
 
-int
-glusterfs_uuid_buf_init ()
-{
-        int ret = 0;
-
-        ret = pthread_key_create (&uuid_buf_key,
-                                  glusterfs_uuid_buf_destroy);
-        return ret;
-}
+// UUID_BUFFER
 
 char *
-glusterfs_uuid_buf_get ()
+glusterfs_uuid_buf_get()
 {
-        char *buf;
-        int ret = 0;
-
-        buf = pthread_getspecific (uuid_buf_key);
-        if(!buf) {
-                buf = MALLOC (GF_UUID_BUF_SIZE);
-                ret = pthread_setspecific (uuid_buf_key, (void *) buf);
-                if (ret)
-                        buf = global_uuid_buf;
-        }
-        return buf;
+    return thread_uuid_buf;
 }
 
 /* LKOWNER_BUFFER */
 
-void
-glusterfs_lkowner_buf_destroy (void *ptr)
+char *
+glusterfs_lkowner_buf_get()
 {
-        FREE (ptr);
+    return thread_lkowner_buf;
 }
 
-int
-glusterfs_lkowner_buf_init ()
-{
-        int ret = 0;
-
-        ret = pthread_key_create (&lkowner_buf_key,
-                                  glusterfs_lkowner_buf_destroy);
-        return ret;
-}
+/* Leaseid buffer */
 
 char *
-glusterfs_lkowner_buf_get ()
+glusterfs_leaseid_buf_get()
 {
-        char *buf;
-        int ret = 0;
-
-        buf = pthread_getspecific (lkowner_buf_key);
-        if(!buf) {
-                buf = MALLOC (GF_LKOWNER_BUF_SIZE);
-                ret = pthread_setspecific (lkowner_buf_key, (void *) buf);
-                if (ret)
-                        buf = global_lkowner_buf;
-        }
-        return buf;
+    char *buf = NULL;
+
+    buf = thread_leaseid;
+    if (buf == NULL) {
+        buf = thread_leaseid_buf;
+        thread_leaseid = buf;
+    }
+
+    return buf;
 }
 
-/* Leaseid buffer */
-void
-glusterfs_leaseid_buf_destroy (void *ptr)
+char *
+glusterfs_leaseid_exist()
 {
-        FREE (ptr);
+    return thread_leaseid;
 }
 
-int
-glusterfs_leaseid_buf_init ()
+static void
+glusterfs_cleanup(void *ptr)
 {
-        int ret = 0;
+    if (thread_syncopctx.groups != NULL) {
+        GF_FREE(thread_syncopctx.groups);
+    }
 
-        ret = pthread_key_create (&leaseid_buf_key,
-                                  glusterfs_leaseid_buf_destroy);
-        return ret;
+    mem_pool_thread_destructor(NULL);
 }
 
-char *
-glusterfs_leaseid_buf_get ()
+void
+gf_thread_needs_cleanup(void)
 {
-        char *buf = NULL;
-        int   ret = 0;
-
-        buf = pthread_getspecific (leaseid_buf_key);
-        if (!buf) {
-                buf = CALLOC (1, GF_LEASE_ID_BUF_SIZE);
-                ret = pthread_setspecific (leaseid_buf_key, (void *) buf);
-                if (ret) {
-                        FREE (buf);
-                        buf = NULL;
-                }
-        }
-        return buf;
+    /* The value stored in free_key TLS is not really used for anything, but
+     * pthread implementation doesn't call the TLS destruction function unless
+     * it's != NULL. This function must be called whenever something is
+     * allocated for this thread so that glusterfs_cleanup() will be called
+     * and resources can be released. */
+    (void)pthread_setspecific(free_key, (void *)1);
 }
 
 static void
-gf_globals_init_once ()
+gf_globals_init_once()
 {
-        int ret = 0;
-
-        ret = glusterfs_this_init ();
-        if (ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_TRANSLATOR_INIT_FAILED,
-                        "ERROR: glusterfs-translator init failed");
-                goto out;
-        }
-
-        ret = glusterfs_uuid_buf_init ();
-        if(ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_UUID_BUF_INIT_FAILED,
-                        "ERROR: glusterfs uuid buffer init failed");
-                goto out;
-        }
-
-        ret = glusterfs_lkowner_buf_init ();
-        if(ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_LKOWNER_BUF_INIT_FAILED,
-                        "ERROR: glusterfs lkowner buffer init failed");
-                goto out;
-        }
-
-        ret = glusterfs_leaseid_buf_init ();
-        if (ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_LEASEID_BUF_INIT_FAILED,
-                        "ERROR: glusterfs leaseid buffer init failed");
-                goto out;
-        }
-
-        ret = synctask_init ();
-        if (ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_SYNCTASK_INIT_FAILED,
-                        "ERROR: glusterfs synctask init failed");
-                goto out;
-        }
-
-        ret = syncopctx_init ();
-        if (ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_SYNCOPCTX_INIT_FAILED,
-                        "ERROR: glusterfs syncopctx init failed");
-                goto out;
-        }
-out:
+    int ret = 0;
+
+    glusterfs_this_init();
+
+    /* This is needed only to cleanup the potential allocation of
+     * thread_syncopctx.groups. */
+    ret = pthread_key_create(&free_key, glusterfs_cleanup);
+    if (ret != 0) {
+        gf_msg("", GF_LOG_ERROR, ret, LG_MSG_PTHREAD_KEY_CREATE_FAILED,
+               "failed to create the pthread key");
+
+        gf_msg("", GF_LOG_CRITICAL, 0, LG_MSG_GLOBAL_INIT_FAILED,
+               "Exiting as global initialization failed");
 
-        if (ret) {
-                gf_msg ("", GF_LOG_CRITICAL, 0, LG_MSG_GLOBAL_INIT_FAILED,
-                        "Exiting as global initialization failed");
-                exit (ret);
-        }
+        exit(ret);
+    }
 }
 
 int
-glusterfs_globals_init (glusterfs_ctx_t *ctx)
+glusterfs_globals_init(glusterfs_ctx_t *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_log_globals_init (ctx, GF_LOG_INFO);
+    gf_log_globals_init(ctx, GF_LOG_INFO);
 
-        ret =  pthread_once (&globals_inited, gf_globals_init_once);
+    ret = pthread_once(&globals_inited, gf_globals_init_once);
 
-        if (ret)
-                gf_msg ("", GF_LOG_CRITICAL, ret, LG_MSG_PTHREAD_FAILED,
-                        "pthread_once failed");
+    if (ret)
+        gf_msg("", GF_LOG_CRITICAL, ret, LG_MSG_PTHREAD_FAILED,
+               "pthread_once failed");
 
-        return ret;
+    return ret;
 }
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
deleted file mode 100644
index e186d58cea6..00000000000
--- a/libglusterfs/src/globals.h
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _GLOBALS_H
-#define _GLOBALS_H
-
-#define GF_DEFAULT_BASE_PORT 24007
-#define GF_DEFAULT_VOLFILE_TRANSPORT "tcp"
-
-#define GD_OP_VERSION_KEY     "operating-version"
-#define GD_MIN_OP_VERSION_KEY "minimum-operating-version"
-#define GD_MAX_OP_VERSION_KEY "maximum-operating-version"
-
-/* Gluster versions - OP-VERSION mapping
- *
- * 3.3.x                - 1
- * 3.4.x                - 2
- * 3.5.0                - 3
- * 3.5.1                - 30501
- * 3.6.0                - 30600
- * 3.7.0                - 30700
- * 3.7.1                - 30701
- * 3.7.2                - 30702
- *
- * Starting with Gluster v3.6, the op-version will be multi-digit integer values
- * based on the Glusterfs version, instead of a simply incrementing integer
- * value. The op-version for a given X.Y.Z release will be an integer XYZ, with
- * Y and Z 2 digit always 2 digits wide and padded with 0 when needed. This
- * should allow for some gaps between two Y releases for backports of features
- * in Z releases.
- */
-#define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly
-                                should not change */
-#define GD_OP_VERSION_MAX  GD_OP_VERSION_4_0_0 /* MAX VERSION is the maximum
-                                                  count in VME table, should
-                                                  keep changing with
-                                                  introduction of newer
-                                                  versions */
-
-#define GD_OP_VERSION_3_6_0    30600 /* Op-Version for GlusterFS 3.6.0 */
-
-#define GD_OP_VERSION_3_7_0    30700 /* Op-version for GlusterFS 3.7.0 */
-
-#define GD_OP_VERSION_3_7_1    30701 /* Op-version for GlusterFS 3.7.1 */
-
-#define GD_OP_VERSION_3_7_2    30702 /* Op-version for GlusterFS 3.7.2 */
-
-#define GD_OP_VERSION_3_7_3    30703 /* Op-version for GlusterFS 3.7.3 */
-
-#define GD_OP_VERSION_3_7_4    30704 /* Op-version for GlusterFS 3.7.4 */
-
-#define GD_OP_VERSION_3_7_5    30705 /* Op-version for GlusterFS 3.7.5 */
-
-#define GD_OP_VERSION_3_7_6    30706 /* Op-version for GlusterFS 3.7.6 */
-
-#define GD_OP_VERSION_3_7_7    30707 /* Op-version for GlusterFS 3.7.7 */
-
-#define GD_OP_VERSION_3_7_10   30710 /* Op-version for GlusterFS 3.7.10 */
-
-#define GD_OP_VERSION_3_7_12   30712 /* Op-version for GlusterFS 3.7.12 */
-
-#define GD_OP_VERSION_3_8_0    30800 /* Op-version for GlusterFS 3.8.0 */
-
-#define GD_OP_VERSION_3_8_3    30803 /* Op-version for GlusterFS 3.8.3 */
-
-#define GD_OP_VERSION_3_8_4    30804 /* Op-version for GlusterFS 3.8.4 */
-
-#define GD_OP_VERSION_3_9_0    30900 /* Op-version for GlusterFS 3.9.0 */
-
-#define GD_OP_VERSION_4_0_0    40000 /* Op-version for GlusterFS 4.0.0 */
-
-#define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0
-
-#include "xlator.h"
-
-/* THIS */
-#define THIS (*__glusterfs_this_location())
-#define DECLARE_OLD_THIS        xlator_t *old_THIS = THIS
-
-/*
- * a more comprehensive feature test is shown at
- * http://lists.iptel.org/pipermail/semsdev/2010-October/005075.html
- * this is sufficient for RHEL5 i386 builds
- */
-#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__i386__)
-# define INCREMENT_ATOMIC(lk, op) __sync_add_and_fetch(&op, 1)
-# define DECREMENT_ATOMIC(lk, op) __sync_sub_and_fetch(&op, 1)
-#else
-/* These are only here for old gcc, e.g. on RHEL5 i386.
- * We're not ever going to use this in an if stmt,
- * but let's be pedantically correct for style points */
-# define INCREMENT_ATOMIC(lk, op) do { LOCK (&lk); ++op; UNLOCK (&lk); } while (0)
-/* this is a gcc 'statement expression', it works with llvm/clang too */
-# define DECREMENT_ATOMIC(lk, op) ({ LOCK (&lk); --op; UNLOCK (&lk); op; })
-#endif
-
-xlator_t **__glusterfs_this_location (void);
-xlator_t *glusterfs_this_get (void);
-int glusterfs_this_set (xlator_t *);
-
-/* syncopctx */
-void *syncopctx_getctx (void);
-int syncopctx_setctx (void *ctx);
-
-/* task */
-void *synctask_get (void);
-int synctask_set (void *);
-
-/* uuid_buf */
-char *glusterfs_uuid_buf_get (void);
-/* lkowner_buf */
-char *glusterfs_lkowner_buf_get (void);
-/* leaseid buf */
-char *glusterfs_leaseid_buf_get (void);
-
-/* init */
-int glusterfs_globals_init (glusterfs_ctx_t *ctx);
-
-extern const char *gf_fop_list[];
-extern const char *gf_upcall_list[];
-
-/* mem acct enable/disable */
-int gf_global_mem_acct_enable_get (void);
-int gf_global_mem_acct_enable_set (int val);
-#endif /* !_GLOBALS_H */
diff --git a/libglusterfs/src/glusterfs-acl.h b/libglusterfs/src/glusterfs-acl.h
deleted file mode 100644
index 55f94ff0509..00000000000
--- a/libglusterfs/src/glusterfs-acl.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _GLUSTERFS_ACL_H
-#define _GLUSTERFS_ACL_H
-
-
-/* WARNING: Much if this code is restricted to Linux usage.
- *
- * It would be much cleaner to replace the code with something that is based on
- * libacl (or its libc implementation on *BSD).
- *
- * Initial work for replacing this Linux specific implementation has been
- * started as part of the "Improve POSIX ACLs" feature. Functionality for this
- * feature has been added to the end of this file.
- */
-
-#include <stdint.h>
-#include <sys/types.h> /* For uid_t */
-
-#include "locking.h" /* For gf_lock_t in struct posix_acl_conf */
-
-#define ACL_PROGRAM    100227
-#define ACLV3_VERSION  3
-
-#define POSIX_ACL_MINIMAL_ACE_COUNT   3
-
-#define POSIX_ACL_READ                (0x04)
-#define POSIX_ACL_WRITE               (0x02)
-#define POSIX_ACL_EXECUTE             (0x01)
-
-#define POSIX_ACL_UNDEFINED_TAG       (0x00)
-#define POSIX_ACL_USER_OBJ            (0x01)
-#define POSIX_ACL_USER                (0x02)
-#define POSIX_ACL_GROUP_OBJ           (0x04)
-#define POSIX_ACL_GROUP               (0x08)
-#define POSIX_ACL_MASK                (0x10)
-#define POSIX_ACL_OTHER               (0x20)
-
-#define POSIX_ACL_UNDEFINED_ID        (-1)
-
-#define POSIX_ACL_XATTR_VERSION       (0x02)
-
-#define POSIX_ACL_ACCESS_XATTR        "system.posix_acl_access"
-#define POSIX_ACL_DEFAULT_XATTR       "system.posix_acl_default"
-
-struct posix_acl_xattr_entry {
-        uint16_t            tag;
-        uint16_t            perm;
-        uint32_t            id;
-};
-
-struct posix_acl_xattr_header {
-        uint32_t                        version;
-        struct posix_acl_xattr_entry    entries[];
-};
-
-typedef struct posix_acl_xattr_entry  posix_acl_xattr_entry;
-typedef struct posix_acl_xattr_header posix_acl_xattr_header;
-
-static inline size_t
-posix_acl_xattr_size (unsigned int count)
-{
-        return (sizeof(posix_acl_xattr_header) +
-                       (count * sizeof(posix_acl_xattr_entry)));
-}
-
-static inline ssize_t
-posix_acl_xattr_count (size_t size)
-{
-        if (size < sizeof(posix_acl_xattr_header))
-                return (-1);
-        size -= sizeof(posix_acl_xattr_header);
-        if (size % sizeof(posix_acl_xattr_entry))
-                return (-1);
-        return (size / sizeof(posix_acl_xattr_entry));
-}
-
-struct posix_ace {
-        uint16_t     tag;
-        uint16_t     perm;
-        uint32_t     id;
-};
-
-
-struct posix_acl {
-        int               refcnt;
-        int               count;
-        struct posix_ace  entries[];
-};
-
-struct posix_acl_ctx {
-        uid_t             uid;
-        gid_t             gid;
-        mode_t            perm;
-        struct posix_acl *acl_access;
-        struct posix_acl *acl_default;
-};
-
-struct posix_acl_conf {
-        gf_lock_t         acl_lock;
-        uid_t             super_uid;
-        struct posix_acl *minimal_acl;
-};
-
-
-/* Above this comment, the legacy POSIX ACL support is kept until it is not
- * used anymore. Below you will find the more portable version to support POSIX
- * ACls based on the implementation of libacl (see sys/acl.h). */
-
-/* virtual xattrs passed over RPC, not stored on disk */
-#define GF_POSIX_ACL_ACCESS       "glusterfs.posix.acl"
-#define GF_POSIX_ACL_DEFAULT      "glusterfs.posix.default_acl"
-#define GF_POSIX_ACL_REQUEST(key) \
-        (!strncmp(key, GF_POSIX_ACL_ACCESS, strlen(GF_POSIX_ACL_ACCESS)) || \
-         !strncmp(key, GF_POSIX_ACL_DEFAULT, strlen(GF_POSIX_ACL_DEFAULT)))
-
-#ifdef HAVE_SYS_ACL_H /* only NetBSD does not support POSIX ACLs */
-
-#include <sys/acl.h>
-
-static inline const char*
-gf_posix_acl_get_key (const acl_type_t type)
-{
-        char *acl_key = NULL;
-
-        switch (type) {
-        case ACL_TYPE_ACCESS:
-                acl_key = GF_POSIX_ACL_ACCESS;
-                break;
-        case ACL_TYPE_DEFAULT:
-                acl_key = GF_POSIX_ACL_DEFAULT;
-                break;
-        default:
-                errno = EINVAL;
-        }
-
-        return acl_key;
-}
-
-static inline const acl_type_t
-gf_posix_acl_get_type (const char *key)
-{
-        acl_type_t type = 0;
-
-        if (!strncmp (key, GF_POSIX_ACL_ACCESS, strlen (GF_POSIX_ACL_ACCESS)))
-                type = ACL_TYPE_ACCESS;
-        else if (!strncmp (key, GF_POSIX_ACL_DEFAULT,
-                           strlen (GF_POSIX_ACL_DEFAULT)))
-                type = ACL_TYPE_DEFAULT;
-        else
-                errno = EINVAL;
-
-        return type;
-}
-
-#endif /* HAVE_SYS_ACL_H */
-#endif /* _GLUSTERFS_ACL_H */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
deleted file mode 100644
index 24b8065b66d..00000000000
--- a/libglusterfs/src/glusterfs.h
+++ /dev/null
@@ -1,573 +0,0 @@
-/*
-  Copyright (c) 2008-2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _GLUSTERFS_H
-#define _GLUSTERFS_H
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <netdb.h>
-#include <errno.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <arpa/inet.h>
-#include <sys/poll.h>
-#include <pthread.h>
-#include <limits.h> /* For PATH_MAX */
-
-#include "glusterfs-fops.h" /* generated XDR values for FOPs */
-
-#include "list.h"
-#include "locking.h"
-#include "logging.h"
-#include "lkowner.h"
-#include "compat-uuid.h"
-
-#define GF_YES 1
-#define GF_NO  0
-
-#ifndef O_LARGEFILE
-/* savannah bug #20053, patch for compiling on darwin */
-#define O_LARGEFILE 0100000 /* from bits/fcntl.h */
-#endif
-
-#ifndef O_FMODE_EXEC
-/* redhat bug 843080, added from linux/fs.h */
-#define O_FMODE_EXEC 040 //0x20
-#endif
-
-#ifndef O_DIRECT
-/* savannah bug #20050, #20052 */
-#define O_DIRECT 0 /* From asm/fcntl.h */
-#endif
-
-#ifndef O_DIRECTORY
-/* FreeBSD does not need O_DIRECTORY */
-#define O_DIRECTORY 0
-#endif
-
-#ifndef EBADFD
-/* Mac OS X does not have EBADFD */
-#define EBADFD EBADF
-#endif
-
-#ifndef FNM_EXTMATCH
-#define FNM_EXTMATCH 0
-#endif
-
-#define GLUSTERD_MAX_SNAP_NAME  255
-#define ZR_MOUNTPOINT_OPT       "mountpoint"
-#define ZR_ATTR_TIMEOUT_OPT     "attribute-timeout"
-#define ZR_ENTRY_TIMEOUT_OPT    "entry-timeout"
-#define ZR_NEGATIVE_TIMEOUT_OPT "negative-timeout"
-#define ZR_DIRECT_IO_OPT        "direct-io-mode"
-#define ZR_STRICT_VOLFILE_CHECK "strict-volfile-check"
-#define ZR_DUMP_FUSE            "dump-fuse"
-#define ZR_FUSE_MOUNTOPTS       "fuse-mountopts"
-
-#define GF_XATTR_CLRLK_CMD      "glusterfs.clrlk"
-#define GF_XATTR_PATHINFO_KEY   "trusted.glusterfs.pathinfo"
-#define GF_XATTR_NODE_UUID_KEY  "trusted.glusterfs.node-uuid"
-#define GF_REBAL_FIND_LOCAL_SUBVOL "glusterfs.find-local-subvol"
-#define GF_XATTR_VOL_ID_KEY   "trusted.glusterfs.volume-id"
-#define GF_XATTR_LOCKINFO_KEY   "trusted.glusterfs.lockinfo"
-#define GF_META_LOCK_KEY        "glusterfs.lock-migration-meta-lock"
-#define GF_META_UNLOCK_KEY      "glusterfs.lock-migration-meta-unlock"
-#define GF_XATTR_GET_REAL_FILENAME_KEY "glusterfs.get_real_filename:"
-#define GF_XATTR_USER_PATHINFO_KEY   "glusterfs.pathinfo"
-#define GF_INTERNAL_IGNORE_DEEM_STATFS "ignore-deem-statfs"
-#define GF_XATTR_IOSTATS_DUMP_KEY "trusted.io-stats-dump"
-
-#define GF_READDIR_SKIP_DIRS       "readdir-filter-directories"
-#define GF_MDC_LOADED_KEY_NAMES     "glusterfs.mdc.loaded.key.names"
-
-#define BD_XATTR_KEY             "user.glusterfs"
-#define GF_PREOP_PARENT_KEY      "glusterfs.preop.parent.key"
-#define GF_PREOP_CHECK_FAILED    "glusterfs.preop.check.failed"
-
-#define XATTR_IS_PATHINFO(x)  ((strncmp (x, GF_XATTR_PATHINFO_KEY,       \
-                                        strlen (x)) == 0) ||             \
-                              (strncmp (x, GF_XATTR_USER_PATHINFO_KEY,   \
-                                         strlen (x)) == 0))
-#define XATTR_IS_NODE_UUID(x) (strncmp (x, GF_XATTR_NODE_UUID_KEY,      \
-                                        strlen (GF_XATTR_NODE_UUID_KEY)) == 0)
-#define XATTR_IS_LOCKINFO(x) (strncmp (x, GF_XATTR_LOCKINFO_KEY,        \
-                                       strlen (GF_XATTR_LOCKINFO_KEY)) == 0)
-
-#define XATTR_IS_BD(x) (strncmp (x, BD_XATTR_KEY, strlen (BD_XATTR_KEY)) == 0)
-
-#define GF_XATTR_LINKINFO_KEY   "trusted.distribute.linkinfo"
-#define GFID_XATTR_KEY          "trusted.gfid"
-#define PGFID_XATTR_KEY_PREFIX  "trusted.pgfid."
-#define VIRTUAL_GFID_XATTR_KEY_STR  "glusterfs.gfid.string"
-#define VIRTUAL_GFID_XATTR_KEY      "glusterfs.gfid"
-#define UUID_CANONICAL_FORM_LEN 36
-
-#define GET_ANCESTRY_PATH_KEY "glusterfs.ancestry.path"
-#define GET_ANCESTRY_DENTRY_KEY "glusterfs.ancestry.dentry"
-
-#define BITROT_DEFAULT_CURRENT_VERSION  (unsigned long)1
-#define BITROT_DEFAULT_SIGNING_VERSION  (unsigned long)0
-
-/* on-disk object signature keys */
-#define BITROT_OBJECT_BAD_KEY       "trusted.bit-rot.bad-file"
-#define BITROT_CURRENT_VERSION_KEY  "trusted.bit-rot.version"
-#define BITROT_SIGNING_VERSION_KEY  "trusted.bit-rot.signature"
-
-/* globally usable bad file marker */
-#define GLUSTERFS_BAD_INODE         "glusterfs.bad-inode"
-
-/* on-disk size of signing xattr (not the signature itself) */
-#define BITROT_SIGNING_XATTR_SIZE_KEY  "trusted.glusterfs.bit-rot.size"
-
-/* GET/SET object signature */
-#define GLUSTERFS_GET_OBJECT_SIGNATURE "trusted.glusterfs.get-signature"
-#define GLUSTERFS_SET_OBJECT_SIGNATURE "trusted.glusterfs.set-signature"
-
-/* operation needs to be durable on-disk */
-#define GLUSTERFS_DURABLE_OP           "trusted.glusterfs.durable-op"
-
-/* key for version exchange b/w bitrot stub and changelog */
-#define GLUSTERFS_VERSION_XCHG_KEY     "glusterfs.version.xchg"
-
-#define GLUSTERFS_INTERNAL_FOP_KEY  "glusterfs-internal-fop"
-#define DHT_CHANGELOG_RENAME_OP_KEY   "changelog.rename-op"
-
-#define ZR_FILE_CONTENT_STR     "glusterfs.file."
-#define ZR_FILE_CONTENT_STRLEN 15
-
-#define GLUSTERFS_WRITE_IS_APPEND "glusterfs.write-is-append"
-#define GLUSTERFS_WRITE_UPDATE_ATOMIC "glusterfs.write-update-atomic"
-#define GLUSTERFS_OPEN_FD_COUNT "glusterfs.open-fd-count"
-#define GLUSTERFS_INODELK_COUNT "glusterfs.inodelk-count"
-#define GLUSTERFS_ENTRYLK_COUNT "glusterfs.entrylk-count"
-#define GLUSTERFS_POSIXLK_COUNT "glusterfs.posixlk-count"
-#define GLUSTERFS_PARENT_ENTRYLK "glusterfs.parent-entrylk"
-#define GLUSTERFS_INODELK_DOM_COUNT "glusterfs.inodelk-dom-count"
-#define GFID_TO_PATH_KEY "glusterfs.gfid2path"
-#define GF_XATTR_STIME_PATTERN "trusted.glusterfs.*.stime"
-#define GF_XATTR_XTIME_PATTERN "trusted.glusterfs.*.xtime"
-#define GF_XATTR_TRIGGER_SYNC "glusterfs.geo-rep.trigger-sync"
-
-/* quota xattrs */
-#define QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
-#define QUOTA_LIMIT_KEY "trusted.glusterfs.quota.limit-set"
-#define QUOTA_LIMIT_OBJECTS_KEY "trusted.glusterfs.quota.limit-objects"
-#define VIRTUAL_QUOTA_XATTR_CLEANUP_KEY "glusterfs.quota-xattr-cleanup"
-#define QUOTA_READ_ONLY_KEY "trusted.glusterfs.quota.read-only"
-
-/* Index xlator related */
-#define GF_XATTROP_INDEX_GFID "glusterfs.xattrop_index_gfid"
-#define GF_XATTROP_ENTRY_CHANGES_GFID "glusterfs.xattrop_entry_changes_gfid"
-#define GF_XATTROP_INDEX_COUNT "glusterfs.xattrop_index_count"
-#define GF_XATTROP_DIRTY_GFID "glusterfs.xattrop_dirty_gfid"
-#define GF_XATTROP_DIRTY_COUNT "glusterfs.xattrop_dirty_count"
-#define GF_XATTROP_ENTRY_IN_KEY "glusterfs.xattrop-entry-create"
-#define GF_XATTROP_ENTRY_OUT_KEY "glusterfs.xattrop-entry-delete"
-#define GF_INDEX_IA_TYPE_GET_REQ "glusterfs.index-ia-type-get-req"
-#define GF_INDEX_IA_TYPE_GET_RSP "glusterfs.index-ia-type-get-rsp"
-
-#define GF_HEAL_INFO "glusterfs.heal-info"
-#define GF_AFR_HEAL_SBRAIN "glusterfs.heal-sbrain"
-#define GF_AFR_SBRAIN_STATUS "replica.split-brain-status"
-#define GF_AFR_SBRAIN_CHOICE "replica.split-brain-choice"
-#define GF_AFR_SPB_CHOICE_TIMEOUT "replica.split-brain-choice-timeout"
-#define GF_AFR_SBRAIN_RESOLVE "replica.split-brain-heal-finalize"
-#define GF_AFR_ADD_BRICK "trusted.add-brick"
-#define GF_AFR_REPLACE_BRICK "trusted.replace-brick"
-#define GF_AFR_DIRTY "trusted.afr.dirty"
-#define GF_XATTROP_ENTRY_OUT "glusterfs.xattrop-entry-delete"
-#define GF_XATTROP_PURGE_INDEX "glusterfs.xattrop-purge-index"
-
-#define GF_GFIDLESS_LOOKUP "gfidless-lookup"
-/* replace-brick and pump related internal xattrs */
-#define RB_PUMP_CMD_START       "glusterfs.pump.start"
-#define RB_PUMP_CMD_PAUSE       "glusterfs.pump.pause"
-#define RB_PUMP_CMD_COMMIT      "glusterfs.pump.commit"
-#define RB_PUMP_CMD_ABORT       "glusterfs.pump.abort"
-#define RB_PUMP_CMD_STATUS      "glusterfs.pump.status"
-
-#define GLUSTERFS_MARKER_DONT_ACCOUNT_KEY "glusters.marker.dont-account"
-#define GLUSTERFS_RDMA_INLINE_THRESHOLD       (2048)
-#define GLUSTERFS_RDMA_MAX_HEADER_SIZE        (228) /* (sizeof (rdma_header_t)                 \
-                                                       + RDMA_MAX_SEGMENTS \
-                                                       * sizeof (rdma_read_chunk_t))
-                                                       */
-
-#define GLUSTERFS_RPC_REPLY_SIZE               24
-
-#define STARTING_EVENT_THREADS                 1
-
-#define ZR_FILE_CONTENT_REQUEST(key) (!strncmp(key, ZR_FILE_CONTENT_STR, \
-                                               ZR_FILE_CONTENT_STRLEN))
-
-#define DEFAULT_VAR_RUN_DIRECTORY        DATADIR "/run/gluster"
-#define DEFAULT_GLUSTERFSD_MISC_DIRETORY DATADIR "/lib/misc/glusterfsd"
-#ifdef GF_LINUX_HOST_OS
-#define GLUSTERD_DEFAULT_WORKDIR DATADIR "/lib/glusterd"
-#else
-#define GLUSTERD_DEFAULT_WORKDIR DATADIR "/db/glusterd"
-#endif
-#define GF_REPLICATE_TRASH_DIR           ".landfill"
-
-/* GlusterFS's maximum supported Auxiliary GIDs */
-/* TODO: Keeping it to 200, so that we can fit in 2KB buffer for auth data
- * in RPC server code, if there is ever need for having more aux-gids, then
- * we have to add aux-gid in payload of actors */
-#define GF_MAX_AUX_GROUPS   65535
-
-#define GF_UUID_BUF_SIZE 50
-
-#define GF_REBALANCE_TID_KEY     "rebalance-id"
-#define GF_REMOVE_BRICK_TID_KEY  "remove-brick-id"
-
-#define UUID_CANONICAL_FORM_LEN  36
-
-/* Adding this here instead of any glusterd*.h files as it is also required by
- * cli
- */
-#define DEFAULT_GLUSTERD_SOCKFILE             DATADIR "/run/glusterd.socket"
-
-/* features/marker-quota also needs to have knowledge of link-files so as to
- * exclude them from accounting.
- */
-#define DHT_LINKFILE_MODE        (S_ISVTX)
-
-#define IS_DHT_LINKFILE_MODE(iabuf) ((st_mode_from_ia ((iabuf)->ia_prot, \
-                                                       (iabuf)->ia_type) & ~S_IFMT)\
-                                     == DHT_LINKFILE_MODE)
-#define DHT_LINKFILE_STR "linkto"
-#define DHT_COMMITHASH_STR "commithash"
-
-#define DHT_SKIP_NON_LINKTO_UNLINK  "unlink-only-if-dht-linkto-file"
-#define TIER_SKIP_NON_LINKTO_UNLINK  "unlink-only-if-tier-linkto-file"
-#define TIER_LINKFILE_GFID           "tier-linkfile-gfid"
-#define DHT_SKIP_OPEN_FD_UNLINK     "dont-unlink-for-open-fd"
-#define DHT_IATT_IN_XDATA_KEY       "dht-get-iatt-in-xattr"
-#define GET_LINK_COUNT              "get-link-count"
-
-/*CTR and Marker requires inode dentry link count from posix*/
-#define GF_RESPONSE_LINK_COUNT_XDATA "gf_response_link_count"
-#define GF_REQUEST_LINK_COUNT_XDATA  "gf_request_link_count"
-
-#define CTR_ATTACH_TIER_LOOKUP    "ctr_attach_tier_lookup"
-
-#define GF_LOG_LRU_BUFSIZE_DEFAULT 5
-#define GF_LOG_LRU_BUFSIZE_MIN 0
-#define GF_LOG_LRU_BUFSIZE_MAX 20
-#define GF_LOG_LRU_BUFSIZE_MIN_STR "0"
-#define GF_LOG_LRU_BUFSIZE_MAX_STR "20"
-
-#define GF_LOG_FLUSH_TIMEOUT_DEFAULT 120
-#define GF_LOG_FLUSH_TIMEOUT_MIN 30
-#define GF_LOG_FLUSH_TIMEOUT_MAX 300
-#define GF_LOG_FLUSH_TIMEOUT_MIN_STR "30"
-#define GF_LOG_FLUSH_TIMEOUT_MAX_STR "300"
-
-#define GF_BACKTRACE_LEN        4096
-#define GF_BACKTRACE_FRAME_COUNT 7
-
-#define GF_LK_ADVISORY 0
-#define GF_LK_MANDATORY 1
-
-/* md-cache keys */
-#define MDC_INVALIDATE_IATT "mdc.invalidate.iatt"
-#define MDC_INVALIDATE_XATT "mdc.invalidate.xatt"
-
-const char *fop_enum_to_pri_string (glusterfs_fop_t fop);
-const char *fop_enum_to_string (glusterfs_fop_t fop);
-
-#define GF_SET_IF_NOT_PRESENT 0x1 /* default behaviour */
-#define GF_SET_OVERWRITE      0x2 /* Overwrite with the buf given */
-#define GF_SET_DIR_ONLY       0x4
-#define GF_SET_EPOCH_TIME     0x8 /* used by afr dir lookup selfheal */
-
-/* key value which quick read uses to get small files in lookup cbk */
-#define GF_CONTENT_KEY "glusterfs.content"
-
-struct _xlator_cmdline_option {
-        struct list_head    cmd_args;
-        char               *volume;
-        char               *key;
-        char               *value;
-};
-typedef struct _xlator_cmdline_option xlator_cmdline_option_t;
-
-struct _server_cmdline {
-        struct list_head  list;
-        char              *volfile_server;
-        char              *transport;
-        int               port;
-};
-typedef struct _server_cmdline server_cmdline_t;
-
-#define GF_OPTION_ENABLE   _gf_true
-#define GF_OPTION_DISABLE  _gf_false
-#define GF_OPTION_DEFERRED 2
-
-struct _cmd_args {
-        /* basic options */
-        char             *volfile_server;
-        server_cmdline_t *curr_server;
-        /* List of backup volfile servers, including original */
-        struct list_head volfile_servers;
-        char             *volfile;
-        char             *log_server;
-        gf_loglevel_t    log_level;
-        char            *log_file;
-        char            *log_ident;
-        gf_log_logger_t  logger;
-        gf_log_format_t  log_format;
-        uint32_t         log_buf_size;
-        uint32_t         log_flush_timeout;
-        int32_t          max_connect_attempts;
-        char            *print_exports;
-        char            *print_netgroups;
-        /* advanced options */
-        uint32_t         volfile_server_port;
-        char            *volfile_server_transport;
-        uint32_t         log_server_port;
-        char            *pid_file;
-        char            *sock_file;
-        int              no_daemon_mode;
-        char            *run_id;
-        int              debug_mode;
-        int              read_only;
-        int              acl;
-        int              selinux;
-        int              capability;
-        int              enable_ino32;
-        int              worm;
-        int              mac_compat;
-        int              fopen_keep_cache;
-        int              gid_timeout;
-        char             gid_timeout_set;
-        int              aux_gfid_mount;
-
-        /* need a process wide timer-wheel? */
-        int              global_timer_wheel;
-
-        struct list_head xlator_options;  /* list of xlator_option_t */
-
-	/* fuse options */
-	int              fuse_direct_io_mode;
-	char             *use_readdirp;
-        int              no_root_squash;
-        int              volfile_check;
-        double           fuse_entry_timeout;
-        double           fuse_negative_timeout;
-        double           fuse_attribute_timeout;
-        char            *volume_name;
-        int              fuse_nodev;
-        int              fuse_nosuid;
-        char            *dump_fuse;
-        pid_t            client_pid;
-        int              client_pid_set;
-        unsigned         uid_map_root;
-        int              background_qlen;
-        int              congestion_threshold;
-        char             *fuse_mountopts;
-        int              mem_acct;
-        int              resolve_gids;
-
-        /* key args */
-        char            *mount_point;
-        char            *volfile_id;
-
-        /* required for portmap */
-        int             brick_port;
-        char           *brick_name;
-        int             brick_port2;
-
-        /* Should management connections use SSL? */
-        int             secure_mgmt;
-
-        /* Linux-only OOM killer adjustment */
-#ifdef GF_LINUX_HOST_OS
-        char            *oom_score_adj;
-#endif
-};
-typedef struct _cmd_args cmd_args_t;
-
-
-struct _glusterfs_graph {
-        struct list_head          list;
-        char                      graph_uuid[128];
-        struct timeval            dob;
-        void                     *first;
-        void                     *top;   /* selected by -n */
-        uint32_t                  leaf_count;
-        int                       xl_count;
-        int                       id;    /* Used in logging */
-        int                       used;  /* Should be set when fuse gets
-                                            first CHILD_UP */
-        uint32_t                  volfile_checksum;
-};
-typedef struct _glusterfs_graph glusterfs_graph_t;
-
-
-typedef int32_t (*glusterfsd_mgmt_event_notify_fn_t) (int32_t event, void *data,
-                                                      ...);
-
-typedef enum {
-        MGMT_SSL_NEVER = 0,
-        MGMT_SSL_COPY_IO,
-        MGMT_SSL_ALWAYS
-} mgmt_ssl_t;
-
-struct tvec_base;
-
-struct _glusterfs_ctx {
-        cmd_args_t          cmd_args;
-        char               *process_uuid;
-        FILE               *pidfp;
-        char                fin;
-        void               *timer;
-        void               *ib;
-        struct call_pool   *pool;
-        void               *event_pool;
-        void               *iobuf_pool;
-        void               *logbuf_pool;
-        gf_lock_t           lock;
-        size_t              page_size;
-        struct list_head    graphs; /* double linked list of graphs - one per volfile parse */
-        glusterfs_graph_t  *active; /* the latest graph in use */
-        void               *master; /* fuse, or libglusterfsclient (however, not protocol/server) */
-        void               *mgmt;   /* xlator implementing MOPs for centralized logging, volfile server */
-        void               *listener; /* listener of the commands from glusterd */
-        unsigned char       measure_latency; /* toggle switch for latency measurement */
-        pthread_t           sigwaiter;
-	char               *cmdlinestr;
-        struct mem_pool    *stub_mem_pool;
-        unsigned char       cleanup_started;
-        int                 graph_id; /* Incremented per graph, value should
-                                         indicate how many times the graph has
-                                         got changed */
-        pid_t               mnt_pid; /* pid of the mount agent */
-        int                 process_mode; /*mode in which process is runninng*/
-        struct syncenv     *env;          /* The env pointer to the synctasks */
-
-        struct list_head    mempool_list; /* used to keep a global list of
-                                             mempools, used to log details of
-                                             mempool in statedump */
-        char               *statedump_path;
-
-        struct mem_pool    *dict_pool;
-        struct mem_pool    *dict_pair_pool;
-        struct mem_pool    *dict_data_pool;
-
-        glusterfsd_mgmt_event_notify_fn_t notify; /* Used for xlators to make
-                                                     call to fsd-mgmt */
-        gf_log_handle_t     log; /* all logging related variables */
-
-        int                 mem_acct_enable;
-
-        int                 daemon_pipe[2];
-
-        struct clienttable *clienttable;
-
-        /*
-         * Should management connections use SSL?  This is the only place we
-         * can put it where both daemon-startup and socket code will see it.
-         *
-         * Why is it an int?  Because we're included before common-utils.h,
-         * which defines gf_boolean_t (what we really want).  It doesn't make
-         * any sense, but it's not worth turning the codebase upside-down to
-         * fix it.  Thus, an int.
-         */
-        int                 secure_mgmt;
-
-        /*
-         * Should *our* server/inbound connections use SSL?  This is only true
-         * if we're glusterd and secure_mgmt is set, or if we're glusterfsd
-         * and SSL is set on the I/O path.  It should never be set e.g. for
-         * NFS.
-         */
-        mgmt_ssl_t          secure_srvr;
-        /* Buffer to 'save' backtrace even under OOM-kill like situations*/
-        char btbuf[GF_BACKTRACE_LEN];
-
-        pthread_mutex_t notify_lock;
-        pthread_cond_t notify_cond;
-        int notifying;
-
-        struct tvec_base *timer_wheel; /* global timer-wheel instance */
-
-};
-typedef struct _glusterfs_ctx glusterfs_ctx_t;
-
-glusterfs_ctx_t *glusterfs_ctx_new (void);
-
-struct gf_flock {
-        short        l_type;
-        short        l_whence;
-        off_t        l_start;
-        off_t        l_len;
-        pid_t        l_pid;
-        gf_lkowner_t l_owner;
-};
-
-typedef struct lock_migration_info {
-        struct list_head        list;
-        struct gf_flock         flock;
-        char                   *client_uid;
-        uint32_t                lk_flags;
-} lock_migration_info_t;
-
-#define GF_MUST_CHECK __attribute__((warn_unused_result))
-/*
- * Some macros (e.g. ALLOC_OR_GOTO) set variables in function scope, but the
- * calling function might not only declare the variable to keep the macro happy
- * and not use it otherwise.  In such cases, the following can be used to
- * suppress the "set but not used" warning that would otherwise occur.
- */
-#define GF_UNUSED __attribute__((unused))
-
-/*
- * If present, this has the following effects:
- *
- *      glusterd enables privileged commands over TCP
- *
- *      all code enables SSL for outbound connections to management port
- *
- *      glusterd enables SSL for inbound connections
- *
- * Servers and clients enable/disable SSL among themselves by other means.
- * Making secure management connections conditional on a file is a bit of a
- * hack, but we don't have any other place for such global settings across
- * all of the affected components.  Making it a compile-time option would
- * reduce functionality, both for users and for testing (which can now be
- * done using secure connections f
author	Meghana Madhusudhan <mmadhusu@redhat.com>	2015-05-06 09:58:11 +0530
committer	Kaleb KEITHLEY <kkeithle@redhat.com>	2015-05-06 05:34:56 -0700
commit	e79edf35178f2558d6afb262444294380b76d740 (patch)
tree	22a5e069388e141f8e0a64fc5e3497c5ef1169ed /tests/basic
parent	403a74da5d3287489829c629d42643b8b320ee62 (diff)