diff options
410 files changed, 18680 insertions, 8126 deletions
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index 306b4eebc15..386ed2d8dd5 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -1,13 +1,30 @@ -This repository (glusterfs) uses github issues to track feature requests *only*. +<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! --> -For assistance with bugs, please file a bug in our bugzilla instance [1] +**Description of problem:** -For other queries regarding glusterfs, please post to the users [2] or devel [3] lists as appropriate. -You may further want to subscribe to these lists or view the archives for these lists, see [4] [5]. +**The exact command to reproduce the issue**: + + +**The full output of the command that failed**: +<details> + + + +</details> + +**Expected results:** + + +**Additional info:** + + +**- The output of the `gluster volume info` command**: +<details> + + + +</details> + +**- The operating system / glusterfs version**: -[1] glusterfs Bugzilla: https://bugzilla.redhat.com/enter_bug.cgi?product=GlusterFS -[2] Gluster users list ID: gluster-users@gluster.org -[3] Gluster devel list ID: gluster-devel@gluster.org -[4] Gluster users list subscribe/view archives: https://lists.gluster.org/mailman/listinfo/gluster-users -[5] Gluster devel list subscribe/view archives: https://lists.gluster.org/mailman/listinfo/gluster-devel diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 0ec6eb319ee..e69de29bb2d 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -1,33 +0,0 @@ -Many thanks for your interest in improving GlusterFS! - -GlusterFS does not use GitHub Pull-Requests. Instead, changes are reviewed -on the Gerrit instance of the Gluster Community at https://review.gluster.org - -In order to send your changes for review, follow these steps: - -1. login on https://review.gluster.org with your GitHub account -2. add a public ssh-key to your profile on https://review.gluster.org/#/settings/ssh-keys -3. add the Gerrit remote to your locally cloned git repository - - $ git remote add gerrit ssh://$USER@review.gluster.org/glusterfs.git - -4. configure the commit hooks - - $ git review --setup - -5. post your changes to Gerrit - - $ git review - - -You may need to install the 'git-review' package if 'git review' is not -available. Note that the hooks for the repository make sure to add a ChangeId -label in the commit messages. Gerrit uses the ChangeId to track single patches -and its updated versions. - -For more details, see the documented development workflow at - http://gluster.readthedocs.io/en/latest/Developer-guide/Simplified-Development-Workflow/ - -If there are any troubles or difficulties with these instructions, please -contact us on gluster-devel@gluster.org or on Freenode IRC in the #gluster-dev -channel. diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE new file mode 100644 index 00000000000..502bbd5556c --- /dev/null +++ b/.github/RELEASE_TRACKER_TEMPLATE @@ -0,0 +1,12 @@ +<!-- Please use this template while creating a tracker issue --> + +**Description of problem:** +A tracker issue to track the issues that will be fixed as a part of this release + + +**Major or minor release**: + + +**Release version**: + + diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 00000000000..460e327c6ea --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,25 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 210 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 15 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security +# Label to use when marking an issue as stale +staleLabel: wontfix + +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + Thank you for your contributions. + + Noticed that this issue is not having any activity in last ~6 months! We + are marking this issue as stale because it has not had recent activity. + + It will be closed in 2 weeks if no one responds with a comment here. + + +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: > + Closing this issue as there was no update since my last update on issue. + If this is an issue which is still valid, feel free to open it. diff --git a/.gitignore b/.gitignore index 849a0f1a8be..fc5ba586f8e 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,5 @@ xlators/features/cloudsync/src/cloudsync-autogen-fops.c xlators/features/cloudsync/src/cloudsync-autogen-fops.h xlators/features/utime/src/utime-autogen-fops.c xlators/features/utime/src/utime-autogen-fops.h +tests/basic/metadisp/ftruncate +xlators/features/metadisp/src/fops.c diff --git a/.testignore b/.testignore index 190573191e1..fe8f838bf2b 100644 --- a/.testignore +++ b/.testignore @@ -1,4 +1,6 @@ .github/ISSUE_TEMPLATE +.github/PULL_REQUEST_TEMPLATE +.github/stale.yml .gitignore .mailmap .testignore @@ -6,7 +8,7 @@ rfc.sh submit-for-review.sh AUTHORS -CONTRIBUTING +CONTRIBUTING.md COPYING-GPLV2 COPYING-LGPLV3 ChangeLog diff --git a/CONTRIBUTING b/CONTRIBUTING deleted file mode 100644 index 7bccd88d7e5..00000000000 --- a/CONTRIBUTING +++ /dev/null @@ -1,25 +0,0 @@ - Developer's Certificate of Origin 1.1 - - By making a contribution to this project, I certify that: - - (a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - - (b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - - (c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - - (d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..65fc3497104 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,114 @@ +# GlusterFS project Contribution guidelines + +## Development Workflow + +We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here. + + +#### Get the Repository setup + +0. Fork Repository + - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork). + +1. Clone Repository + - Clone the glusterfs repo freshly from github using below steps. + +``` + git clone git@github.com:${username}/glusterfs.git + cd glusterfs/ + git remote add upstream git@github.com:gluster/glusterfs.git +``` + +About two tasks are one time for the life time. You can continue to use the same repository for all the work in future. + +#### Development & Other flows + +0. Issue: + - Make sure there is an issue filed for the task you are working on. + - If it is not filed, open the issue with all the description. + - If it is a bug fix, add label "Type:Bug". + - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label. + +1. Code: + - Start coding + - Build and test locally + - Make sure clang-format is installed and is run on the patch. + +2. Keep up-to-date + - GlusterFS is a large project with many developers, so there would be one or the other patch everyday. + - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened. + - Git provides many options to keep up-to-date, below is one of them +``` + git fetch upstream + git rebase upstream/devel +``` + - It is recommended you keep pushing to your repo every day, so you don't loose any work. + - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`) + +2. Commit Message / PR description: + - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice. + - PRs continue to have the title of format "component: \<title\>", like it is practiced now. + - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS. + - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message. + - Here, NNNN is the Issue ID in glusterfs repository. + - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line. + - Can do this by `-s` option for `git commit`. + - If the PR is not ready for review, apply the label `work-in-progress`. + - Check the availability of "Draft PR" is present for you, if yes, use that instead. + +3. Tests: + - All the required smoke tests would be auto-triggered. + - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment. + - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group. + - Ask for help as comment in PR if you have any questions about the process! + +4. Review Process: + - `+2` : is equivalent to "Approve" from the people in the maintainer's group. + - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment. + - `-1` : provide details on required changes and pick "Request Changes" while submitting your review. + - `-2` : done by adding the `DO-NOT-MERGE` label. + + - Any further discussions can happen as comments in the PR. + +5. Making changes: + - There are 2 approaches to submit changes done after addressing review comments. + - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN) + - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option. + +6. Merging: + - GlusterFS project follows 'Squash and Merge' method + - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link. + - This also makes every merge a complete patch, which has passed all tests. + - The merging of the patch is expected to be done by the maintainers. + - It can be done when all the tests (smoke and regression) pass. + - When the PR has 'Approved' flag from corresponding maintainer. + - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email. + +## By contributing to this project, the contributor would need to agree to below. + +### Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + diff --git a/MAINTAINERS b/MAINTAINERS index b4d8195e0e9..953e8755fd9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -53,8 +53,7 @@ General Project Architects -------------------------- M: Amar Tumballi <amarts@gmail.com> M: Xavier Hernandez <xhernandez@redhat.com> -P: Pranith Karampuri <pkarampu@redhat.com> -P: Raghavendra Gowdappa <rgowdapp@redhat.com> +P: Pranith Karampuri <pranith.karampuri@phonepe.com> P: Atin Mukherjee <amukherj@redhat.com> xlators: @@ -67,13 +66,14 @@ F: xlators/system/posix-acl/ Arbiter M: Ravishankar N <ravishankar@redhat.com> -P: Pranith Karampuri <pkarampu@redhat.com> +P: Pranith Karampuri <pranith.karampuri@phonepe.com> S: Maintained F: xlators/features/arbiter/ Automatic File Replication (AFR) -M: Pranith Karampuri <pkarampu@redhat.com> -P: Ravishankar N <ravishankar@redhat.com> +M: Pranith Karampuri <pranith.karampuri@phonepe.com> +M: Ravishankar N <ravishankar@redhat.com> +P: Karthik US <ksubrahm@redhat.com> S: Maintained F: xlators/cluster/afr/ @@ -96,13 +96,12 @@ S: Maintained F: xlators/features/changelog/ Distributed Hashing Table (DHT) -M: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Susant Palai <spalai@redhat.com> S: Maintained F: xlators/cluster/dht/ Erasure Coding -M: Pranith Karampuri <pkarampu@redhat.com> +M: Pranith Karampuri <pranith.karampuri@phonepe.com> M: Xavier Hernandez <xhernandez@redhat.com> P: Ashish Pandey <aspandey@redhat.com> S: Maintained @@ -115,20 +114,18 @@ F: xlators/debug/error-gen/ FUSE Bridge M: Csaba Henk <chenk@redhat.com> -P: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Niels de Vos <ndevos@redhat.com> S: Maintained F: xlators/mount/ Index -M: Pranith Karampuri <pkarampu@redhat.com> +M: Pranith Karampuri <pranith.karampuri@phonepe.com> P: Ravishankar N <ravishankar@redhat.com> S: Maintained F: xlators/features/index/ IO Cache -M: Raghavendra Gowdappa <rgowdapp@redhat.com> -P: Mohammed Rafi KC <rkavunga@redhat.com> +P: Mohammed Rafi KC <rafi.kavungal@iternity.com> S: Maintained F: xlators/performance/io-cache/ @@ -139,7 +136,7 @@ S: Maintained F: xlators/debug/io-stats/ IO threads -M: Pranith Karampuri <pkarampu@redhat.com> +M: Pranith Karampuri <pranith.karampuri@phonepe.com> P: Ravishankar N <ravishankar@redhat.com> S: Maintained F: xlators/performance/io-threads/ @@ -158,13 +155,12 @@ S: Maintained F: xlators/features/locks/ Marker -M: Raghavendra Gowdappa <rgowdapp@redhat.com> M: Kotresh HR <khiremat@redhat.com> S: Maintained F: xlators/features/marker/ Meta -M: Mohammed Rafi KC <rkavunga@redhat.com> +M: Mohammed Rafi KC <rafi.kavungal@iternity.com> S: Maintained F: xlators/features/meta/ @@ -176,7 +172,7 @@ F: xlators/performance/md-cache/ Negative-lookup Cache M: Poornima G <pgurusid@redhat.com> -P: Pranith Karampuri <pkarampu@redhat.com> +P: Pranith Karampuri <pranith.karampuri@phonepe.com> S: Maintained F: xlators/performance/nl-cache/ @@ -188,7 +184,6 @@ S: Odd Fixes F: xlators/nfs/server/ Open-behind -M: Raghavendra Gowdappa <rgowdapp@redhat.com> S: Maintained F: xlators/performance/open-behind/ @@ -200,25 +195,21 @@ S: Maintained F: xlators/storage/posix/ Quick-read -M: Raghavendra Gowdappa <rgowdapp@redhat.com> S: Maintained F: xlators/performance/quick-read/ Quota -M: Raghavendra Gowdappa <rgowdapp@redhat.com> M: Shyamsundar Ranganathan <srangana@redhat.com> P: Hari Gowtham <hgowtham@redhat.com> S: Maintained F: xlators/features/quota/ Read-ahead -M: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Csaba Henk <chenk@redhat.com> S: Maintained F: xlators/performance/read-ahead/ Readdir-ahead -M: Raghavendra Gowdappa <rgowdapp@redhat.com> S: Maintained F: xlators/performance/readdir-ahead/ @@ -242,7 +233,6 @@ S: Maintained F: xlators/features/upcall/ Write-behind -M: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Csaba Henk <chenk@redhat.com> S: Maintained F: xlators/performance/write-behind/ @@ -292,8 +282,7 @@ M: Xavier Hernandez <xhernandez@redhat.com> M: Jeff Darcy <jeff@pl.atyp.us> P: Kaleb Keithley <kkeithle@redhat.com> P: Niels de Vos <ndevos@redhat.com> -P: Pranith Karampuri <pkarampu@redhat.com> -P: Raghavendra Gowdappa <rgowdapp@redhat.com> +P: Pranith Karampuri <pranith.karampuri@phonepe.com> P: Shyamsundar Ranganathan <srangana@redhat.com> S: Maintained F: libglusterfs/ @@ -316,13 +305,11 @@ F: xlators/mgmt/glusterd/ Protocol M: Niels de Vos <ndevos@redhat.com> -M: Raghavendra Gowdappa <rgowdapp@redhat.com> -P: Mohammed Rafi KC <rkavunga@redhat.com> +P: Mohammed Rafi KC <rafi.kavungal@iternity.com> S: Maintained F: xlators/protocol/ Remote Procedure Call subsystem -M: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Mohit Agrawal <moagrawa@redhat.com> S: Maintained F: rpc/rpc-lib/ @@ -330,17 +317,16 @@ F: rpc/xdr/ Snapshot M: Raghavendra Bhat <raghavendra@redhat.com> -P: Mohammed Rafi KC <rkavunga@redhat.com> +P: Mohammed Rafi KC <rafi.kavungal@iternity.com> P: Sunny Kumar <sunkumar@redhat.com> S: Maintained F: xlators/mgmt/glusterd/src/glusterd-snap* F: extras/snap-scheduler.py Socket subsystem -M: Raghavendra Gowdappa <rgowdapp@redhat.com> P: Krutika Dhananjay <kdhananj@redhat.com> P: Milind Changire <mchangir@redhat.com> -P: Mohammed Rafi KC <rkavunga@redhat.com> +P: Mohammed Rafi KC <rafi.kavungal@iternity.com> P: Mohit Agrawal <moagrawa@redhat.com> S: Maintained F: rpc/rpc-transport/socket/ @@ -521,3 +507,4 @@ M: Luis Pabon M: Ira Cooper M: Shwetha Panduranga M: Nithya Balachandran +M: Raghavendra Gowdappa diff --git a/Makefile.am b/Makefile.am index aa0ca5c2a04..98ea5c1038d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,9 +19,13 @@ pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile +clean-local: + find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f + gitclean: distclean find . -name Makefile.in -exec rm -f {} \; find . -name mount.glusterfs -exec rm -f {} \; + find . -name .deps -o -name .libs | xargs rm -rf rm -fr autom4te.cache rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp @@ -49,4 +53,3 @@ gen-VERSION: ./build-aux/pkg-version --full \ > $(abs_top_builddir)/$(distdir)/VERSION; \ fi - diff --git a/README.md b/README.md index 92f829e431e..9d68e033782 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,7 @@ petabytes. It provides interfaces for object, block and file storage. ## Development - Contributions to gluster in the form of patches and new feature additions can - be made by following steps outlined at [Developers Guide](http://docs.gluster.org/en/latest/Developer-guide/Developers-Index/#contributing-to-the-gluster-community). + The development workflow is documented in [Contributors guide](CONTRIBUTING.md) ## Documentation The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org). diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c index 8d6d11423e8..6aa3c5602d1 100644 --- a/api/src/glfs-fops.c +++ b/api/src/glfs-fops.c @@ -292,6 +292,7 @@ glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt, statx->glfs_st_attributes_mask = 0; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0) void priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx) { @@ -371,7 +372,6 @@ priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx) iatt->ia_attributes = statx->glfs_st_attributes; iatt->ia_attributes_mask = statx->glfs_st_attributes_mask; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0); void glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid) @@ -415,6 +415,7 @@ glfs_loc_unlink(loc_t *loc) return 0; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0) struct glfs_fd * pub_glfs_open(struct glfs *fs, const char *path, int flags) { @@ -509,8 +510,7 @@ invalid_fs: return glfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0) int pub_glfs_close(struct glfs_fd *glfd) { @@ -565,8 +565,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0) int pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat) { @@ -607,8 +606,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0) int pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat) { @@ -649,8 +647,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0) int priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask, struct glfs_stat *statxbuf) @@ -704,8 +701,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0) int pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat) { @@ -754,8 +750,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0) struct glfs_fd * pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode) { @@ -902,8 +897,6 @@ invalid_fs: return glfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0); - #ifdef HAVE_SEEK_HOLE static int glfs_seek(struct glfs_fd *glfd, off_t offset, int whence) @@ -957,6 +950,7 @@ out: } #endif +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0) off_t pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence) { @@ -1012,8 +1006,6 @@ invalid_fs: return -1; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0); - static ssize_t glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, off_t offset, int flags, struct glfs_stat *poststat) @@ -1091,6 +1083,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0) ssize_t pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, off_t offset, int flags) @@ -1098,8 +1091,7 @@ pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0) ssize_t pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags) { @@ -1108,6 +1100,11 @@ pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags) }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = buf; iov.iov_len = count; @@ -1116,8 +1113,7 @@ pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0); - +GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0) ssize_t pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, int flags) @@ -1135,8 +1131,7 @@ pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0) ssize_t pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, int flags, struct glfs_stat *poststat) @@ -1154,21 +1149,23 @@ pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0) ssize_t pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags); return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0); - struct glfs_io { struct glfs_fd *glfd; int op; @@ -1370,6 +1367,7 @@ invalid_fs: return -1; } +GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0) int pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec, int count, off_t offset, int flags, glfs_io_cbk34 fn, @@ -1379,8 +1377,7 @@ pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec, (void *)fn, data); } -GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0) int pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec, int count, off_t offset, int flags, glfs_io_cbk fn, @@ -1390,8 +1387,7 @@ pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec, _gf_false, fn, data); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0) int pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags, glfs_io_cbk34 fn, void *data) @@ -1401,6 +1397,11 @@ pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags, }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = buf; iov.iov_len = count; @@ -1410,8 +1411,7 @@ pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0) int pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags, glfs_io_cbk fn, void *data) @@ -1421,6 +1421,11 @@ pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags, }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = buf; iov.iov_len = count; @@ -1430,8 +1435,7 @@ pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0) int pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, int flags, glfs_io_cbk34 fn, void *data) @@ -1450,8 +1454,7 @@ pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0) int pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count, off_t offset, int flags, glfs_io_cbk fn, void *data) @@ -1470,34 +1473,40 @@ pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0) int pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags, glfs_io_cbk34 fn, void *data) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags, _gf_true, (void *)fn, data); return ret; } -GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0) int pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags, glfs_io_cbk fn, void *data) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags, _gf_false, fn, data); return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0); - static ssize_t glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, off_t offset, int flags, struct glfs_stat *prestat, @@ -1583,6 +1592,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0) ssize_t pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in, struct glfs_fd *glfd_out, off64_t *off_out, size_t len, @@ -1736,8 +1746,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0) ssize_t pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, off_t offset, int flags) @@ -1745,8 +1754,7 @@ pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt, return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0) ssize_t pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags) { @@ -1755,6 +1763,11 @@ pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags) }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = (void *)buf; iov.iov_len = count; @@ -1763,21 +1776,24 @@ pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0) ssize_t pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags); return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0); - +GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0) ssize_t pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count, off_t offset, int flags) @@ -1795,8 +1811,7 @@ pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0) ssize_t pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count, off_t offset, int flags, struct glfs_stat *prestat, @@ -1815,8 +1830,6 @@ pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0); - extern glfs_t * pub_glfs_from_glfd(glfs_fd_t *); @@ -1935,6 +1948,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0) int pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec, int count, off_t offset, int flags, glfs_io_cbk34 fn, @@ -1944,8 +1958,7 @@ pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec, _gf_true, (void *)fn, data); } -GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0) int pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec, int count, off_t offset, int flags, glfs_io_cbk fn, @@ -1955,8 +1968,7 @@ pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec, _gf_false, fn, data); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0) int pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count, int flags, glfs_io_cbk34 fn, void *data) @@ -1966,6 +1978,11 @@ pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count, }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = (void *)buf; iov.iov_len = count; @@ -1975,8 +1992,7 @@ pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0) int pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count, int flags, glfs_io_cbk fn, void *data) @@ -1986,6 +2002,11 @@ pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count, }; ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + iov.iov_base = (void *)buf; iov.iov_len = count; @@ -1995,8 +2016,7 @@ pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0) int pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count, off_t offset, int flags, glfs_io_cbk34 fn, void *data) @@ -2015,8 +2035,7 @@ pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count, return ret; } -GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0) int pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count, off_t offset, int flags, glfs_io_cbk fn, void *data) @@ -2035,34 +2054,40 @@ pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0) int pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags, glfs_io_cbk34 fn, void *data) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags, _gf_true, (void *)fn, data); return ret; } -GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0) int pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count, int flags, glfs_io_cbk fn, void *data) { ssize_t ret = 0; + if (glfd == NULL) { + errno = EBADF; + return -1; + } + ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags, _gf_false, fn, data); return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0); - static int glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2127,14 +2152,14 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0) int pub_glfs_fsync34(struct glfs_fd *glfd) { return glfs_fsync_common(glfd, NULL, NULL); } -GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0) int pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2142,8 +2167,6 @@ pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat, return glfs_fsync_common(glfd, prestat, poststat); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0); - static int glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *prebuf, @@ -2224,6 +2247,7 @@ out: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0) int pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data) { @@ -2240,8 +2264,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0) int pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data) { @@ -2258,8 +2281,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0); - static int glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2324,14 +2345,14 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0) int pub_glfs_fdatasync34(struct glfs_fd *glfd) { return glfs_fdatasync_common(glfd, NULL, NULL); } -GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0) int pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2339,8 +2360,7 @@ pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat, return glfs_fdatasync_common(glfd, prestat, poststat); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0); - +GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0) int pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data) { @@ -2357,8 +2377,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0) int pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data) { @@ -2375,8 +2394,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0); - static int glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2442,14 +2459,14 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0) int pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset) { return glfs_ftruncate_common(glfd, offset, NULL, NULL); } -GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0) int pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset, struct glfs_stat *prestat, struct glfs_stat *poststat) @@ -2457,8 +2474,7 @@ pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset, return glfs_ftruncate_common(glfd, offset, prestat, poststat); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15) int pub_glfs_truncate(struct glfs *fs, const char *path, off_t length) { @@ -2504,8 +2520,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15); - static int glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *prebuf, @@ -2598,6 +2612,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0) int pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn, void *data) @@ -2606,8 +2621,7 @@ pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn, data); } -GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0) int pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn, void *data) @@ -2615,8 +2629,7 @@ pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn, return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0) int pub_glfs_access(struct glfs *fs, const char *path, int mode) { @@ -2662,8 +2675,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0) int pub_glfs_symlink(struct glfs *fs, const char *data, const char *path) { @@ -2753,8 +2765,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0) int pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz) { @@ -2811,8 +2822,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0) int pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev) { @@ -2902,8 +2912,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0) int pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode) { @@ -2993,8 +3002,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0) int pub_glfs_unlink(struct glfs *fs, const char *path) { @@ -3050,8 +3058,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0) int pub_glfs_rmdir(struct glfs *fs, const char *path) { @@ -3106,8 +3113,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0) int pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath) { @@ -3196,8 +3202,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0) int pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath) { @@ -3283,8 +3288,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0) struct glfs_fd * pub_glfs_opendir(struct glfs *fs, const char *path) { @@ -3365,8 +3369,7 @@ invalid_fs: return glfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0) int pub_glfs_closedir(struct glfs_fd *glfd) { @@ -3387,22 +3390,30 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0) long pub_glfs_telldir(struct glfs_fd *fd) { + if (fd == NULL) { + errno = EBADF; + return -1; + } + return fd->offset; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0) void pub_glfs_seekdir(struct glfs_fd *fd, long offset) { gf_dirent_t *entry = NULL; gf_dirent_t *tmp = NULL; + if (fd == NULL) { + errno = EBADF; + return; + } + if (fd->offset == offset) return; @@ -3425,8 +3436,6 @@ pub_glfs_seekdir(struct glfs_fd *fd, long offset) */ } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0); - static int glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, @@ -3517,6 +3526,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0) int pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len, glfs_io_cbk34 fn, void *data) @@ -3525,8 +3535,7 @@ pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len, data); } -GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0) int pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len, glfs_io_cbk fn, void *data) @@ -3534,8 +3543,6 @@ pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len, return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0); - static int glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, @@ -3628,6 +3635,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0) int pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len, glfs_io_cbk34 fn, void *data) @@ -3636,8 +3644,7 @@ pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len, data); } -GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0) int pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len, glfs_io_cbk fn, void *data) @@ -3645,8 +3652,6 @@ pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len, return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0); - void gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent) { @@ -3806,6 +3811,7 @@ unlock: return buf; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0) int pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat, struct dirent *ext, struct dirent **res) @@ -3861,8 +3867,7 @@ invalid_fs: return -1; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0) int pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf, struct dirent **res) @@ -3870,8 +3875,7 @@ pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf, return pub_glfs_readdirplus_r(glfd, 0, buf, res); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0) struct dirent * pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat) { @@ -3885,16 +3889,14 @@ pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat) return res; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0) struct dirent * pub_glfs_readdir(struct glfs_fd *glfd) { return pub_glfs_readdirplus(glfd, NULL); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0) int pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf) { @@ -3940,8 +3942,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0) int pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat, int follow) @@ -4001,8 +4002,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0) int pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) { @@ -4055,8 +4055,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0) int pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode) { @@ -4073,8 +4072,7 @@ pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0) int pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode) { @@ -4091,8 +4089,7 @@ pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0) int pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid) { @@ -4117,8 +4114,7 @@ pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0) int pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid) { @@ -4143,8 +4139,7 @@ pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0) int pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid) { @@ -4169,8 +4164,7 @@ pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0) int pub_glfs_utimens(struct glfs *fs, const char *path, const struct timespec times[2]) @@ -4190,8 +4184,7 @@ pub_glfs_utimens(struct glfs *fs, const char *path, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0) int pub_glfs_lutimens(struct glfs *fs, const char *path, const struct timespec times[2]) @@ -4211,8 +4204,7 @@ pub_glfs_lutimens(struct glfs *fs, const char *path, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0) int pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2]) { @@ -4231,8 +4223,6 @@ pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2]) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0); - int glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name) { @@ -4332,6 +4322,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0) ssize_t pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name, void *value, size_t size) @@ -4339,8 +4330,7 @@ pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name, return glfs_getxattr_common(fs, path, name, value, size, 1); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0) ssize_t pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name, void *value, size_t size) @@ -4348,8 +4338,7 @@ pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name, return glfs_getxattr_common(fs, path, name, value, size, 0); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0) ssize_t pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value, size_t size) @@ -4412,8 +4401,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0); - int glfs_listxattr_process(void *value, size_t size, dict_t *xattr) { @@ -4497,22 +4484,21 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0) ssize_t pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size) { return glfs_listxattr_common(fs, path, value, size, 1); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0) ssize_t pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size) { return glfs_listxattr_common(fs, path, value, size, 0); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0) ssize_t pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size) { @@ -4562,8 +4548,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0); - int glfs_setxattr_common(struct glfs *fs, const char *path, const char *name, const void *value, size_t size, int flags, int follow) @@ -4643,6 +4627,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0) int pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name, const void *value, size_t size, int flags) @@ -4650,8 +4635,7 @@ pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name, return glfs_setxattr_common(fs, path, name, value, size, flags, 1); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0) int pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name, const void *value, size_t size, int flags) @@ -4659,8 +4643,7 @@ pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name, return glfs_setxattr_common(fs, path, name, value, size, flags, 0); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0) int pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value, size_t size, int flags) @@ -4735,8 +4718,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0); - int glfs_removexattr_common(struct glfs *fs, const char *path, const char *name, int follow) @@ -4787,22 +4768,21 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0) int pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name) { return glfs_removexattr_common(fs, path, name, 1); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0) int pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name) { return glfs_removexattr_common(fs, path, name, 0); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0) int pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name) { @@ -4845,8 +4825,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0) int pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset, size_t len) @@ -4897,8 +4876,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0) int pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len) { @@ -4948,8 +4926,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0) int pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len) { @@ -4997,8 +4974,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0) int pub_glfs_chdir(struct glfs *fs, const char *path) { @@ -5048,8 +5024,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0) int pub_glfs_fchdir(struct glfs_fd *glfd) { @@ -5101,8 +5076,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0); - static gf_boolean_t warn_realpath = _gf_true; /* log once */ static char * @@ -5185,22 +5158,21 @@ invalid_fs: return retpath; } +GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0) char * pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path) { return glfs_realpath_common(fs, path, resolved_path, _gf_true); } -GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17) char * pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path) { return glfs_realpath_common(fs, path, resolved_path, _gf_false); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0) char * pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n) { @@ -5249,8 +5221,6 @@ invalid_fs: return buf; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0); - static void gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock) { @@ -5357,6 +5327,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0) int pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock, glfs_lock_mode_t lk_mode) @@ -5393,16 +5364,14 @@ out: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0) int pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock) { return glfs_lock_common(glfd, cmd, flock, NULL); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7) int pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len) { @@ -5438,8 +5407,8 @@ out: invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0) struct glfs_fd * pub_glfs_dup(struct glfs_fd *glfd) { @@ -5490,8 +5459,6 @@ invalid_fs: return dupfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0); - static void glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data) { @@ -5976,6 +5943,7 @@ out: * Otherwise all the upcall events are queued up in a list * to be read/polled by the applications. */ +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0) void priv_glfs_process_upcall_event(struct glfs *fs, void *data) { @@ -6043,7 +6011,6 @@ out: err: return; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0); ssize_t glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object, @@ -6229,6 +6196,7 @@ glfs_release_xreaddirp_stat(void *ptr) * Given glfd of a directory, this function does readdirp and returns * xstat along with dirents. */ +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0) int pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags, struct glfs_xreaddirp_stat **xstat_p, @@ -6337,8 +6305,8 @@ out: invalid_fs: return -1; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0) struct stat * pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat) { @@ -6354,7 +6322,6 @@ pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat) out: return NULL; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0); void gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease) @@ -6374,6 +6341,7 @@ glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease) memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE); } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0) int pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease, glfs_recall_cbk fn, void *data) @@ -6475,5 +6443,3 @@ out: invalid_fs: return ret; } - -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0); diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c index cdac07124a5..53c2ee896f9 100644 --- a/api/src/glfs-handleops.c +++ b/api/src/glfs-handleops.c @@ -60,6 +60,7 @@ glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt, return; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4) struct glfs_object * pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent, const char *path, struct stat *stat, int follow) @@ -126,8 +127,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4); - +GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2) struct glfs_object * pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent, const char *path, struct stat *stat) @@ -135,8 +135,7 @@ pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent, return pub_glfs_h_lookupat(fs, parent, path, stat, 0); } -GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0) int pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object, struct statvfs *statvfs) @@ -194,8 +193,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2) int pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat) { @@ -259,8 +257,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2) int pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object, struct stat *stat) @@ -317,8 +314,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2); - int glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object, dict_t **xattr, const char *name, @@ -380,6 +375,7 @@ out: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1) int pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object, const char *name, void *value, size_t size) @@ -416,8 +412,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2) int pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object, struct stat *stat, int valid) @@ -480,8 +475,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0) int pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object, const char *name, const void *value, size_t size, @@ -568,8 +562,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1) int pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object, const char *name) @@ -626,8 +619,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2) struct glfs_fd * pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags) { @@ -727,8 +719,7 @@ invalid_fs: return glfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2) struct glfs_object * pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path, int flags, mode_t mode, struct stat *stat) @@ -840,8 +831,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6) struct glfs_object * pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path, int flags, mode_t mode, @@ -975,8 +965,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2) struct glfs_object * pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path, mode_t mode, struct stat *stat) @@ -1074,8 +1063,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2) struct glfs_object * pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path, mode_t mode, dev_t dev, struct stat *stat) @@ -1172,8 +1160,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2) int pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path) { @@ -1244,8 +1231,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2) struct glfs_fd * pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object) { @@ -1327,8 +1313,7 @@ invalid_fs: return glfd; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0) int pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask) { @@ -1385,8 +1370,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2) ssize_t pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle, int len) @@ -1417,8 +1401,7 @@ out: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2) struct glfs_object * pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len, struct stat *stat) @@ -1541,8 +1524,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2) int pub_glfs_h_close(struct glfs_object *object) { @@ -1555,8 +1537,7 @@ pub_glfs_h_close(struct glfs_object *object) return 0; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2) int pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset) { @@ -1616,8 +1597,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2) struct glfs_object * pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent, const char *name, const char *data, struct stat *stat) @@ -1716,8 +1696,7 @@ invalid_fs: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2) int pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf, size_t bufsiz) @@ -1782,8 +1761,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2) int pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc, struct glfs_object *parent, const char *name) @@ -1880,8 +1858,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2) int pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir, const char *oldname, struct glfs_object *newdir, @@ -1991,8 +1968,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2); - /* * Given a handle/gfid, find if the corresponding inode is present in * the inode table. If yes create and return the corresponding glfs_object. @@ -2200,6 +2175,7 @@ glfs_release_upcall(void *ptr) * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures * cannot be freed while in this routine. */ +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16) int pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg) { @@ -2317,8 +2293,6 @@ err: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16); - static gf_boolean_t log_upcall370 = _gf_true; /* log once */ /* The old glfs_h_poll_upcall interface requires intimate knowledge of the @@ -2332,6 +2306,7 @@ static gf_boolean_t log_upcall370 = _gf_true; /* log once */ * * WARNING: this function will be removed in the future. */ +GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0) int pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg) { @@ -2399,12 +2374,11 @@ out: return ret; } -GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0); - #ifdef HAVE_ACL_LIBACL_H #include <glusterfs/glusterfs-acl.h> #include <acl/libacl.h> +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0) int pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object, const acl_type_t type, const acl_t acl) @@ -2453,6 +2427,7 @@ invalid_fs: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0) acl_t pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object, const acl_type_t type) @@ -2507,6 +2482,7 @@ invalid_fs: return acl; } #else /* !HAVE_ACL_LIBACL_H */ +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0) acl_t pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object, const acl_type_t type) @@ -2515,6 +2491,7 @@ pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object, return NULL; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0) int pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object, const acl_type_t type, const acl_t acl) @@ -2523,10 +2500,9 @@ pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object, return -1; } #endif -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0); -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0); /* The API to perform read using anonymous fd */ +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0) ssize_t pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object, const void *buf, size_t count, off_t offset) @@ -2550,9 +2526,8 @@ pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0); - /* The API to perform write using anonymous fd */ +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0) ssize_t pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object, const void *buf, size_t count, off_t offset) @@ -2576,8 +2551,7 @@ pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object, return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0) struct glfs_object * pub_glfs_object_copy(struct glfs_object *src) { @@ -2599,8 +2573,8 @@ pub_glfs_object_copy(struct glfs_object *src) out: return object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0) struct glfs_object * pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) { @@ -2616,8 +2590,8 @@ pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) out: return NULL; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0) int pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object, struct glfs_lease *lease) @@ -2679,5 +2653,3 @@ out: invalid_fs: return ret; } - -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0); diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h index 55401b2910e..7cc3b18a104 100644 --- a/api/src/glfs-internal.h +++ b/api/src/glfs-internal.h @@ -16,6 +16,7 @@ #include <glusterfs/upcall-utils.h> #include "glfs-handles.h" #include <glusterfs/refcount.h> +#include <glusterfs/syncop.h> #define GLFS_SYMLINK_MAX_FOLLOW 2048 @@ -80,25 +81,40 @@ #ifndef GFAPI_PRIVATE #define GFAPI_PRIVATE(sym, ver) /**/ #endif +#if __GNUC__ >= 10 #define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver) \ - asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver)) + __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver)))) #define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver) \ - asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver)) + __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver)))) #define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver) \ - asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver)) + __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver)))) #define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver) \ - asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver)) + __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver)))) + +#else +#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver) \ + asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver)); + +#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver) \ + asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver)); + +#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver) \ + asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver)); + +#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver) \ + asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver)); +#endif #define STR(str) #str #else #ifndef GFAPI_PUBLIC -#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver)) +#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver)); #endif #ifndef GFAPI_PRIVATE #define GFAPI_PRIVATE(sym, ver) \ - __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver)) + __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver)); #endif #define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver) /**/ #define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/ @@ -207,6 +223,7 @@ struct glfs { glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */ void *up_data; /* Opaque data provided by application * during upcall registration */ + struct list_head waitq; /* waiting synctasks */ }; /* This enum is used to maintain the state of glfd. In case of async fops @@ -442,6 +459,34 @@ glfs_process_upcall_event(struct glfs *fs, void *data) THIS = glfd->fd->inode->table->xl->ctx->master; \ } while (0) +#define __GLFS_LOCK_WAIT(fs) \ + do { \ + struct synctask *task = NULL; \ + \ + task = synctask_get(); \ + \ + if (task) { \ + list_add_tail(&task->waitq, &fs->waitq); \ + pthread_mutex_unlock(&fs->mutex); \ + synctask_yield(task, NULL); \ + pthread_mutex_lock(&fs->mutex); \ + } else { \ + /* non-synctask */ \ + pthread_cond_wait(&fs->cond, &fs->mutex); \ + } \ + } while (0) + +#define __GLFS_SYNCTASK_WAKE(fs) \ + do { \ + struct synctask *waittask = NULL; \ + \ + while (!list_empty(&fs->waitq)) { \ + waittask = list_entry(fs->waitq.next, struct synctask, waitq); \ + list_del_init(&waittask->waitq); \ + synctask_wake(waittask); \ + } \ + } while (0) + /* By default all lock attempts from user context must use glfs_lock() and glfs_unlock(). This allows @@ -466,10 +511,10 @@ glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration) pthread_mutex_lock(&fs->mutex); while (!fs->init) - pthread_cond_wait(&fs->cond, &fs->mutex); + __GLFS_LOCK_WAIT(fs); while (wait_for_migration && fs->migration_in_progress) - pthread_cond_wait(&fs->cond, &fs->mutex); + __GLFS_LOCK_WAIT(fs); return 0; } diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c index 7271e35ec3f..7c82b8cd162 100644 --- a/api/src/glfs-mgmt.c +++ b/api/src/glfs-mgmt.c @@ -364,6 +364,7 @@ out: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0) int pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size) { @@ -418,8 +419,6 @@ invalid_fs: return -1; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0); - int glfs_get_volume_info(struct glfs *fs) { @@ -1014,11 +1013,6 @@ glfs_mgmt_init(struct glfs *fs) if (ret) goto out; - if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) { - ctx->secure_mgmt = 1; - ctx->ssl_cert_depth = glusterfs_read_secure_access_file(); - } - rpc = rpc_clnt_new(options, THIS, THIS->name, 8); if (!rpc) { ret = -1; diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c index 4b6f76961ee..8a393ecb464 100644 --- a/api/src/glfs-resolve.c +++ b/api/src/glfs-resolve.c @@ -65,6 +65,9 @@ __glfs_first_lookup(struct glfs *fs, xlator_t *subvol) fs->migration_in_progress = 0; pthread_cond_broadcast(&fs->cond); + /* wake up other waiting tasks */ + __GLFS_SYNCTASK_WAKE(fs); + return ret; } @@ -153,9 +156,13 @@ __glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode, fs->migration_in_progress = 0; pthread_cond_broadcast(&fs->cond); + /* wake up other waiting tasks */ + __GLFS_SYNCTASK_WAKE(fs); + return newinode; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0) int priv_glfs_loc_touchup(loc_t *loc) { @@ -170,8 +177,6 @@ priv_glfs_loc_touchup(loc_t *loc) return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0); - int glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode, char **lpath) @@ -461,6 +466,7 @@ out: return inode; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0) int priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at, const char *origpath, loc_t *loc, struct iatt *iatt, @@ -611,8 +617,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0); - int glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath, loc_t *loc, struct iatt *iatt, int follow, int reval) @@ -641,6 +645,7 @@ out: return ret; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0) int priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath, loc_t *loc, struct iatt *iatt, int reval) @@ -651,7 +656,6 @@ priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath, return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0); int glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath, @@ -722,6 +726,7 @@ glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd) 0, }; char uuid1[64]; + dict_t *xdata = NULL; oldinode = oldfd->inode; oldsubvol = oldinode->table->xl; @@ -730,7 +735,16 @@ glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd) return fd_ref(oldfd); if (!oldsubvol->switched) { - ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, NULL, NULL); + xdata = dict_new(); + if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) { + gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED, + "err=%s", "last-fsync set failed", "gfid=%s", + uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s", + graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, + NULL); + } + + ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL); DECODE_SYNCOP_ERR(ret); if (ret) { gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED, @@ -810,6 +824,9 @@ out: newfd = NULL; } + if (xdata) + dict_unref(xdata); + return newfd; } @@ -830,6 +847,9 @@ __glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd) fs->migration_in_progress = 0; pthread_cond_broadcast(&fs->cond); + /* wake up other waiting tasks */ + __GLFS_SYNCTASK_WAKE(fs); + return newfd; } @@ -956,6 +976,7 @@ __glfs_active_subvol(struct glfs *fs) return new_subvol; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0) void priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol) { @@ -983,8 +1004,7 @@ priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol) } } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0) xlator_t * priv_glfs_active_subvol(struct glfs *fs) { @@ -1012,8 +1032,6 @@ priv_glfs_active_subvol(struct glfs *fs) return subvol; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0); - int __glfs_cwd_set(struct glfs *fs, inode_t *inode) { diff --git a/api/src/glfs.c b/api/src/glfs.c index 5f683451c21..b4bf1423f6d 100644 --- a/api/src/glfs.c +++ b/api/src/glfs.c @@ -251,6 +251,11 @@ glfs_volumes_init(struct glfs *fs) if (!vol_assigned(cmd_args)) return -1; + if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) { + fs->ctx->secure_mgmt = 1; + fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file(); + } + if (cmd_args->volfile_server) { ret = glfs_mgmt_init(fs); goto out; @@ -275,6 +280,7 @@ out: /////////////////////////////////////////////////////////////////////////////// +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0) int pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key, const char *value) @@ -324,8 +330,7 @@ invalid_fs: return -1; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1) int pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport, const char *host, const int port) @@ -385,8 +390,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0) int pub_glfs_set_volfile_server(struct glfs *fs, const char *transport, const char *host, int port) @@ -465,8 +469,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0); - /* * * Used to free the arguments allocated by glfs_set_volfile_server() */ @@ -509,6 +511,7 @@ glfs_free_xlator_options(cmd_args_t *cmd_args) } } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2) int pub_glfs_setfsuid(uid_t fsuid) { @@ -518,8 +521,7 @@ pub_glfs_setfsuid(uid_t fsuid) return syncopctx_setfsuid(&fsuid); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2) int pub_glfs_setfsgid(gid_t fsgid) { @@ -529,8 +531,7 @@ pub_glfs_setfsgid(gid_t fsgid) return syncopctx_setfsgid(&fsgid); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2) int pub_glfs_setfsgroups(size_t size, const gid_t *list) { @@ -540,8 +541,7 @@ pub_glfs_setfsgroups(size_t size, const gid_t *list) return syncopctx_setfsgroups(size, list); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0) int pub_glfs_setfsleaseid(glfs_leaseid_t leaseid) { @@ -563,8 +563,6 @@ pub_glfs_setfsleaseid(glfs_leaseid_t leaseid) return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0); - int get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd) { @@ -652,14 +650,19 @@ unset_fop_attr(dict_t **fop_attr) *fop_attr = NULL; } } + +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0) struct glfs * pub_glfs_from_glfd(struct glfs_fd *glfd) { + if (glfd == NULL) { + errno = EBADF; + return NULL; + } + return glfd->fs; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0); - static void glfs_fd_destroy(struct glfs_fd *glfd) { @@ -737,6 +740,7 @@ glfs_new_fs(const char *volname) INIT_LIST_HEAD(&fs->openfds); INIT_LIST_HEAD(&fs->upcall_list); + INIT_LIST_HEAD(&fs->waitq); PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX, err); @@ -807,6 +811,7 @@ unlock: return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0) struct glfs * pub_glfs_new(const char *volname) { @@ -913,8 +918,7 @@ out: return fs; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0) struct glfs * priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx) { @@ -933,8 +937,7 @@ out: return fs; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0) void priv_glfs_free_from_ctx(struct glfs *fs) { @@ -970,8 +973,7 @@ priv_glfs_free_from_ctx(struct glfs *fs) FREE(fs); } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0) int pub_glfs_set_volfile(struct glfs *fs, const char *volfile) { @@ -988,8 +990,7 @@ pub_glfs_set_volfile(struct glfs *fs, const char *volfile) return 0; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0) int pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel) { @@ -1027,8 +1028,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0); - int glfs_init_wait(struct glfs *fs) { @@ -1047,6 +1046,7 @@ glfs_init_wait(struct glfs *fs) return ret; } +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0) void priv_glfs_init_done(struct glfs *fs, int ret) { @@ -1077,8 +1077,6 @@ out: return; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0); - int glfs_init_common(struct glfs *fs) { @@ -1118,6 +1116,7 @@ glfs_init_async(struct glfs *fs, glfs_init_cbk cbk) return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0) int pub_glfs_init(struct glfs *fs) { @@ -1150,8 +1149,6 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0); - static int glusterfs_ctx_destroy(glusterfs_ctx_t *ctx) { @@ -1229,6 +1226,7 @@ glusterfs_ctx_destroy(glusterfs_ctx_t *ctx) return ret; } +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0) int pub_glfs_fini(struct glfs *fs) { @@ -1240,6 +1238,7 @@ pub_glfs_fini(struct glfs *fs) call_pool_t *call_pool = NULL; int fs_init = 0; int err = -1; + struct synctask *waittask = NULL; DECLARE_OLD_THIS; @@ -1261,6 +1260,13 @@ pub_glfs_fini(struct glfs *fs) call_pool = fs->ctx->pool; + /* Wake up any suspended synctasks */ + while (!list_empty(&fs->waitq)) { + waittask = list_entry(fs->waitq.next, struct synctask, waitq); + list_del_init(&waittask->waitq); + synctask_wake(waittask); + } + while (countdown--) { /* give some time for background frames to finish */ pthread_mutex_lock(&fs->mutex); @@ -1411,8 +1417,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0) ssize_t pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len) { @@ -1438,8 +1443,7 @@ invalid_fs: return res; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0) int priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out) { @@ -1467,8 +1471,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0); - +GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1) int priv_glfs_setfspid(struct glfs *fs, pid_t pid) { @@ -1482,107 +1485,104 @@ priv_glfs_setfspid(struct glfs *fs, pid_t pid) return ret; } -GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16) void pub_glfs_free(void *ptr) { GLFS_FREE(ptr); } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16) struct glfs * pub_glfs_upcall_get_fs(struct glfs_upcall *arg) { return arg->fs; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16) enum glfs_upcall_reason pub_glfs_upcall_get_reason(struct glfs_upcall *arg) { return arg->reason; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16) void * pub_glfs_upcall_get_event(struct glfs_upcall *arg) { return arg->event; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16) struct glfs_object * pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg) { return arg->object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16) uint64_t pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg) { return arg->flags; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16) struct stat * pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg) { return &arg->buf; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16) uint64_t pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg) { return arg->expire_time_attr; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16) struct glfs_object * pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg) { return arg->p_object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16) struct stat * pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg) { return &arg->p_buf; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16) struct glfs_object * pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg) { return arg->oldp_object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16) struct stat * pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg) { return &arg->oldp_buf; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16); +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6) struct glfs_object * pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg) { return arg->object; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6) uint32_t pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg) { return arg->lease_type; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6); /* definitions of the GLFS_SYSRQ_* chars are in glfs.h */ static struct glfs_sysrq_help { @@ -1592,6 +1592,7 @@ static struct glfs_sysrq_help { {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"}, {0, NULL}}; +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0) int pub_glfs_sysrq(struct glfs *fs, char sysrq) { @@ -1640,8 +1641,7 @@ out: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0) int pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list, glfs_upcall_cbk cbk, void *data) @@ -1697,8 +1697,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0) int pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list) { @@ -1745,8 +1744,7 @@ invalid_fs: return ret; } -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0); - +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0) int pub_glfs_set_statedump_path(struct glfs *fs, const char *path) { @@ -1806,5 +1804,3 @@ err: invalid_fs: return -1; } - -GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0); diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c index ba3c1ea6446..2c9a5f01bb1 100644 --- a/cli/src/cli-cmd-global.c +++ b/cli/src/cli-cmd-global.c @@ -27,8 +27,6 @@ #include <glusterfs/syscall.h> #include <glusterfs/common-utils.h> -extern rpc_clnt_prog_t *cli_rpc_prog; - int cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word, const char **words, int wordcount); diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c index 658677bbca3..e961d88da86 100644 --- a/cli/src/cli-cmd-misc.c +++ b/cli/src/cli-cmd-misc.c @@ -18,10 +18,6 @@ #include "cli-mem-types.h" #include "protocol-common.h" -extern struct rpc_clnt *global_rpc; - -extern rpc_clnt_prog_t *cli_rpc_prog; - extern struct cli_cmd volume_cmds[]; extern struct cli_cmd bitrot_cmds[]; extern struct cli_cmd quota_cmds[]; diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c index 875f1c3fbe3..34620b4a31b 100644 --- a/cli/src/cli-cmd-parser.c +++ b/cli/src/cli-cmd-parser.c @@ -3906,8 +3906,6 @@ heal_command_type_get(const char *command) [GF_SHD_OP_HEAL_INDEX] = NULL, [GF_SHD_OP_HEAL_FULL] = "full", [GF_SHD_OP_INDEX_SUMMARY] = "info", - [GF_SHD_OP_HEALED_FILES] = NULL, - [GF_SHD_OP_HEAL_FAILED_FILES] = NULL, [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL, [GF_SHD_OP_STATISTICS] = "statistics", [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL, diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c index 342b1d67aaa..084998701d8 100644 --- a/cli/src/cli-cmd-peer.c +++ b/cli/src/cli-cmd-peer.c @@ -20,10 +20,6 @@ #include "protocol-common.h" #include <glusterfs/events.h> -extern struct rpc_clnt *global_rpc; - -extern rpc_clnt_prog_t *cli_rpc_prog; - int cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word, const char **words, int wordcount); diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c index 176c6d745b5..859d6b2e40d 100644 --- a/cli/src/cli-cmd-snapshot.c +++ b/cli/src/cli-cmd-snapshot.c @@ -17,8 +17,6 @@ #include "cli-cmd.h" #include "cli-mem-types.h" -extern rpc_clnt_prog_t *cli_rpc_prog; - int cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word, const char **words, int wordcount); diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c index d5732630582..801e8f4efed 100644 --- a/cli/src/cli-cmd-system.c +++ b/cli/src/cli-cmd-system.c @@ -18,10 +18,6 @@ #include "cli-mem-types.h" #include "protocol-common.h" -extern struct rpc_clnt *global_rpc; - -extern rpc_clnt_prog_t *cli_rpc_prog; - int cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word, const char **words, int wordcount); diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c index 82b4a7da225..f238851586e 100644 --- a/cli/src/cli-cmd-volume.c +++ b/cli/src/cli-cmd-volume.c @@ -28,10 +28,6 @@ #include <glusterfs/common-utils.h> #include <glusterfs/events.h> -extern struct rpc_clnt *global_rpc; -extern struct rpc_clnt *global_quotad_rpc; - -extern rpc_clnt_prog_t *cli_rpc_prog; extern rpc_clnt_prog_t cli_quotad_clnt; static int diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c index af009757e93..2d458b16a56 100644 --- a/cli/src/cli-cmd.c +++ b/cli/src/cli-cmd.c @@ -28,7 +28,7 @@ static pthread_cond_t conn = PTHREAD_COND_INITIALIZER; static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER; int cli_op_ret = 0; -int connected = 0; +static gf_boolean_t connected = _gf_false; static unsigned cli_cmd_needs_connection(struct cli_cmd_word *word) @@ -328,19 +328,32 @@ cli_cmd_await_connected(unsigned conn_timo) } int32_t -cli_cmd_broadcast_connected() +cli_cmd_broadcast_connected(gf_boolean_t status) { pthread_mutex_lock(&conn_mutex); { - connected = 1; + connected = status; pthread_cond_broadcast(&conn); } - pthread_mutex_unlock(&conn_mutex); return 0; } +gf_boolean_t +cli_cmd_connected(void) +{ + gf_boolean_t status; + + pthread_mutex_lock(&conn_mutex); + { + status = connected; + } + pthread_mutex_unlock(&conn_mutex); + + return status; +} + int cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame, rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref, diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c index 52ab97ee815..772b8f75bd9 100644 --- a/cli/src/cli-quotad-client.c +++ b/cli/src/cli-quotad-client.c @@ -10,9 +10,6 @@ #include "cli-quotad-client.h" -extern struct rpc_clnt global_quotad_rpc; -extern struct rpc_clnt_program cli_quotad_clnt; - int cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref, xlator_t *this, @@ -60,7 +57,7 @@ cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog, } /* Send the msg */ - ret = rpc_clnt_submit(&global_quotad_rpc, prog, procnum, cbkfn, &iov, count, + ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count, NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL); ret = 0; diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c index 0f57d94b506..9b6b0c7fa50 100644 --- a/cli/src/cli-rpc-ops.c +++ b/cli/src/cli-rpc-ops.c @@ -47,10 +47,7 @@ enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK }; -extern struct rpc_clnt *global_quotad_rpc; rpc_clnt_prog_t cli_quotad_clnt; -extern rpc_clnt_prog_t *cli_rpc_prog; -extern int connected; static int32_t gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data); @@ -158,9 +155,9 @@ gf_cli_probe_cbk(struct rpc_req *req, struct iovec *iov, int count, gf_log("cli", GF_LOG_INFO, "Received resp to probe"); - if (rsp.op_ret) { - if (rsp.op_errstr && rsp.op_errstr[0] != '\0') { - snprintf(msg, sizeof(msg), "%s", rsp.op_errstr); + if (rsp.op_errstr && rsp.op_errstr[0] != '\0') { + snprintf(msg, sizeof(msg), "%s", rsp.op_errstr); + if (rsp.op_ret) { gf_log("cli", GF_LOG_ERROR, "%s", msg); } } @@ -3406,7 +3403,7 @@ gf_cli_quota_list(cli_local_t *local, char *volname, dict_t *dict, char *default_sl, int count, int op_ret, int op_errno, char *op_errstr) { - if (!connected) + if (!cli_cmd_connected()) goto out; if (count > 0) { @@ -4875,14 +4872,6 @@ gf_cli_gsync_status_output(dict_t *dict, gf_boolean_t is_detail) ret = -1; goto out; } - for (i = 0; i < gsync_count; i++) { - sts_vals[i] = GF_CALLOC(1, sizeof(gf_gsync_status_t), - gf_common_mt_char); - if (!sts_vals[i]) { - ret = -1; - goto out; - } - } ret = gf_cli_read_status_data(dict, sts_vals, spacing, gsync_count, num_of_fields); @@ -5352,7 +5341,8 @@ cmd_profile_volume_brick_out(dict_t *dict, int count, int interval) double total_percentage_latency = 0; for (i = 0; i < 32; i++) { - snprintf(key, sizeof(key), "%d-%d-read-%d", count, interval, (1 << i)); + snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, count, interval, + (1U << i)); ret = dict_get_uint64(dict, key, &rb_counts[i]); if (ret) { gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key); @@ -5360,7 +5350,8 @@ cmd_profile_volume_brick_out(dict_t *dict, int count, int interval) } for (i = 0; i < 32; i++) { - snprintf(key, sizeof(key), "%d-%d-write-%d", count, interval, (1 << i)); + snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, count, interval, + (1U << i)); ret = dict_get_uint64(dict, key, &wb_counts[i]); if (ret) { gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key); @@ -5446,7 +5437,8 @@ cmd_profile_volume_brick_out(dict_t *dict, int count, int interval) if ((rb_counts[i] == 0) && (wb_counts[i] == 0)) continue; per_line++; - snprintf(output + index, sizeof(output) - index, "%19db+ ", (1 << i)); + snprintf(output + index, sizeof(output) - index, "%19" PRIu32 "b+ ", + (1U << i)); if (rb_counts[i]) { snprintf(read_blocks + index, sizeof(read_blocks) - index, "%21" PRId64 " ", rb_counts[i]); @@ -5733,7 +5725,7 @@ gf_cli_top_volume_cbk(struct rpc_req *req, struct iovec *iov, int count, double time = 0; int32_t time_sec = 0; long int time_usec = 0; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char *openfd_str = NULL; @@ -7823,7 +7815,7 @@ cmd_heal_volume_brick_out(dict_t *dict, int brick) char *status = NULL; uint64_t i = 0; uint32_t time = 0; - char timestr[32] = {0}; + char timestr[GF_TIMESTR_SIZE] = {0}; char *shd_status = NULL; snprintf(key, sizeof key, "%d-hostname", brick); @@ -7926,8 +7918,6 @@ gf_is_cli_heal_get_command(gf_xl_afr_op_t heal_op) [GF_SHD_OP_HEAL_INDEX] = 0, [GF_SHD_OP_HEAL_FULL] = 0, [GF_SHD_OP_INDEX_SUMMARY] = 1, - [GF_SHD_OP_HEALED_FILES] = 1, - [GF_SHD_OP_HEAL_FAILED_FILES] = 1, [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1, [GF_SHD_OP_STATISTICS] = 1, [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1, @@ -8014,12 +8004,6 @@ gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count, case GF_SHD_OP_INDEX_SUMMARY: heal_op_str = "list of entries to be healed"; break; - case GF_SHD_OP_HEALED_FILES: - heal_op_str = "list of healed entries"; - break; - case GF_SHD_OP_HEAL_FAILED_FILES: - heal_op_str = "list of heal failed entries"; - break; case GF_SHD_OP_SPLIT_BRAIN_FILES: heal_op_str = "list of split brain entries"; break; @@ -8032,12 +8016,14 @@ gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count, case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA: heal_op_str = "count of entries to be healed per replica"; break; - /* The below 4 cases are never hit; they're coded only to make - * compiler warnings go away.*/ case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE: case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME: case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK: case GF_SHD_OP_HEAL_SUMMARY: + case GF_SHD_OP_HEALED_FILES: + case GF_SHD_OP_HEAL_FAILED_FILES: + /* These cases are never hit; they're coded just to silence the + * compiler warnings.*/ break; case GF_SHD_OP_INVALID: @@ -8112,8 +8098,6 @@ gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count, cmd_heal_volume_statistics_heal_count_out(dict, i); break; case GF_SHD_OP_INDEX_SUMMARY: - case GF_SHD_OP_HEALED_FILES: - case GF_SHD_OP_HEAL_FAILED_FILES: case GF_SHD_OP_SPLIT_BRAIN_FILES: for (i = 0; i < brick_count; i++) cmd_heal_volume_brick_out(dict, i); diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c index 5f33aa9e0a3..069de75801c 100644 --- a/cli/src/cli-xml-output.c +++ b/cli/src/cli-xml-output.c @@ -1661,15 +1661,15 @@ cli_xml_output_vol_top_rw_perf(xmlTextWriterPtr writer, dict_t *dict, int ret = -1; char *filename = NULL; uint64_t throughput = 0; - long int time_sec = 0; - long int time_usec = 0; - char timestr[256] = { + struct timeval tv = { + 0, + }; + char timestr[GF_TIMESTR_SIZE] = { 0, }; char key[1024] = { 0, }; - int len; /* <file> */ ret = xmlTextWriterStartElement(writer, (xmlChar *)"file"); @@ -1692,19 +1692,16 @@ cli_xml_output_vol_top_rw_perf(xmlTextWriterPtr writer, dict_t *dict, XML_RET_CHECK_AND_GOTO(ret, out); snprintf(key, sizeof(key), "%d-time-sec-%d", brick_index, member_index); - ret = dict_get_int32(dict, key, (int32_t *)&time_sec); + ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_sec); if (ret) goto out; snprintf(key, sizeof(key), "%d-time-usec-%d", brick_index, member_index); - ret = dict_get_int32(dict, key, (int32_t *)&time_usec); + ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_usec); if (ret) goto out; - gf_time_fmt(timestr, sizeof timestr, time_sec, gf_timefmt_FT); - len = strlen(timestr); - snprintf(timestr + len, sizeof(timestr) - len, ".%" GF_PRI_SUSECONDS, - time_usec); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"time", "%s", timestr); XML_RET_CHECK_AND_GOTO(ret, out); @@ -1953,11 +1950,11 @@ cli_xml_output_vol_profile_stats(xmlTextWriterPtr writer, dict_t *dict, XML_RET_CHECK_AND_GOTO(ret, out); ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size", - "%" PRIu32, (1 << i)); + "%" PRIu32, (1U << i)); XML_RET_CHECK_AND_GOTO(ret, out); - snprintf(key, sizeof(key), "%d-%d-read-%d", brick_index, interval, - (1 << i)); + snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, brick_index, interval, + (1U << i)); ret = dict_get_uint64(dict, key, &read_count); if (ret) read_count = 0; @@ -1965,8 +1962,8 @@ cli_xml_output_vol_profile_stats(xmlTextWriterPtr writer, dict_t *dict, "%" PRIu64, read_count); XML_RET_CHECK_AND_GOTO(ret, out); - snprintf(key, sizeof(key), "%d-%d-write-%d", brick_index, interval, - (1 << i)); + snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, brick_index, + interval, (1U << i)); ret = dict_get_uint64(dict, key, &write_count); if (ret) write_count = 0; diff --git a/cli/src/cli.c b/cli/src/cli.c index 2c96b7ae6e6..a52b39c5fb8 100644 --- a/cli/src/cli.c +++ b/cli/src/cli.c @@ -33,12 +33,6 @@ #include <malloc.h> #endif -#ifdef HAVE_MALLOC_STATS -#ifdef DEBUG -#include <mcheck.h> -#endif -#endif - #include "cli.h" #include "cli-quotad-client.h" #include "cli-cmd.h" @@ -61,7 +55,6 @@ #include "xdr-generic.h" -extern int connected; /* using argp for command line parsing */ const char *argp_program_version = @@ -78,6 +71,7 @@ const char *argp_program_version = const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">"; struct rpc_clnt *global_quotad_rpc; + struct rpc_clnt *global_rpc; rpc_clnt_prog_t *cli_rpc_prog; @@ -309,14 +303,14 @@ cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, switch (event) { case RPC_CLNT_CONNECT: { - cli_cmd_broadcast_connected(); + cli_cmd_broadcast_connected(_gf_true); gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT"); break; } case RPC_CLNT_DISCONNECT: { + cli_cmd_broadcast_connected(_gf_false); gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT"); - connected = 0; if (!global_state->prompt && global_state->await_connected) { ret = 1; cli_out( diff --git a/cli/src/cli.h b/cli/src/cli.h index cd4db3dac2c..c0d933e8f8a 100644 --- a/cli/src/cli.h +++ b/cli/src/cli.h @@ -189,6 +189,12 @@ typedef ssize_t (*cli_serialize_t)(struct iovec outmsg, void *args); extern struct cli_state *global_state; /* use only in readline callback */ +extern struct rpc_clnt *global_quotad_rpc; + +extern struct rpc_clnt *global_rpc; + +extern rpc_clnt_prog_t *cli_rpc_prog; + typedef const char *(*cli_selector_t)(void *wcon); char * @@ -329,11 +335,14 @@ cli_local_get(); void cli_local_wipe(cli_local_t *local); +gf_boolean_t +cli_cmd_connected(); + int32_t -cli_cmd_await_connected(); +cli_cmd_await_connected(unsigned timeout); int32_t -cli_cmd_broadcast_connected(); +cli_cmd_broadcast_connected(gf_boolean_t status); int cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event, diff --git a/configure.ac b/configure.ac index 0d236f6c359..e2d6fd66cec 100644 --- a/configure.ac +++ b/configure.ac @@ -164,6 +164,8 @@ AC_CONFIG_FILES([Makefile xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile + xlators/features/metadisp/Makefile + xlators/features/metadisp/src/Makefile xlators/playground/Makefile xlators/playground/template/Makefile xlators/playground/template/src/Makefile @@ -273,36 +275,65 @@ AC_ARG_ENABLE([debug], [Enable debug build options.])) if test "x$enable_debug" = "xyes"; then BUILD_DEBUG=yes - GF_CFLAGS="${GF_CFLAGS} -g -rdynamic -O0 -DDEBUG" + GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG" else BUILD_DEBUG=no fi +SANITIZER=none + AC_ARG_ENABLE([asan], AC_HELP_STRING([--enable-asan], [Enable Address Sanitizer support])) if test "x$enable_asan" = "xyes"; then - BUILD_ASAN=yes - AC_CHECK_LIB([asan], [__asan_report_error], , - [AC_MSG_ERROR([libasan.so not found, this is required for --enable-asan])]) - GF_CFLAGS="${GF_CFLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer" - dnl -lasan always need to be the first library, otherwise libxml complains - GF_LDFLAGS="-lasan ${GF_LDFLAGS}" -else - BUILD_ASAN=no + SANITIZER=asan + AC_CHECK_LIB([asan], [__asan_init], , + [AC_MSG_ERROR([--enable-asan requires libasan.so, exiting])]) + GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=address -fno-omit-frame-pointer" + GF_LDFLAGS="${GF_LDFLAGS} -lasan" fi -AC_ARG_ENABLE([atan], +AC_ARG_ENABLE([tsan], AC_HELP_STRING([--enable-tsan], - [Enable ThreadSanitizer support])) + [Enable Thread Sanitizer support])) if test "x$enable_tsan" = "xyes"; then - BUILD_TSAN=yes + if test "x$SANITIZER" != "xnone"; then + AC_MSG_ERROR([only one sanitizer can be enabled at once]) + fi + SANITIZER=tsan AC_CHECK_LIB([tsan], [__tsan_init], , - [AC_MSG_ERROR([libtsan.so not found, this is required for --enable-tsan])]) - GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=thread" + [AC_MSG_ERROR([--enable-tsan requires libtsan.so, exiting])]) + if test "x$ac_cv_lib_tsan___tsan_init" = xyes; then + AC_MSG_CHECKING([whether tsan API can be used]) + saved_CFLAGS=${CFLAGS} + CFLAGS="${CFLAGS} -fsanitize=thread" + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([ + [#include <sanitizer/tsan_interface.h>]], + [[__tsan_create_fiber(0)]])], + [TSAN_API=yes], [TSAN_API=no]) + AC_MSG_RESULT([$TSAN_API]) + if test x$TSAN_API = "xyes"; then + AC_DEFINE(HAVE_TSAN_API, 1, [Define if tsan API can be used.]) + fi + CFLAGS=${saved_CFLAGS} + fi + GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=thread -fno-omit-frame-pointer" GF_LDFLAGS="${GF_LDFLAGS} -ltsan" -else - BUILD_TSAN=no +fi + +AC_ARG_ENABLE([ubsan], + AC_HELP_STRING([--enable-ubsan], + [Enable Undefined Behavior Sanitizer support])) +if test "x$enable_ubsan" = "xyes"; then + if test "x$SANITIZER" != "xnone"; then + AC_MSG_ERROR([only one sanitizer can be enabled at once]) + fi + SANITIZER=ubsan + AC_CHECK_LIB([ubsan], [__ubsan_default_options], , + [AC_MSG_ERROR([--enable-ubsan requires libubsan.so, exiting])]) + GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=undefined -fno-omit-frame-pointer" + GF_LDFLAGS="${GF_LDFLAGS} -lubsan" fi # Initialize CFLAGS before usage @@ -378,12 +409,27 @@ esac # --enable-valgrind prevents calling dlclose(), this leaks memory AC_ARG_ENABLE([valgrind], - AC_HELP_STRING([--enable-valgrind], - [Enable valgrind for resource leak debugging.])) -if test "x$enable_valgrind" = "xyes"; then - AC_DEFINE(RUN_WITH_VALGRIND, 1, [define if all processes should run under valgrind]) -fi - + AC_HELP_STRING([--enable-valgrind@<:@=memcheck,drd@:>@], + [Enable valgrind for resource leak (memcheck, which is + the default) or thread synchronization (drd) debugging.])) +case x$enable_valgrind in + xmemcheck|xyes) + AC_DEFINE(RUN_WITH_MEMCHECK, 1, + [Define if all processes should run under 'valgrind --tool=memcheck'.]) + VALGRIND_TOOL=memcheck + ;; + xdrd) + AC_DEFINE(RUN_WITH_DRD, 1, + [Define if all processes should run under 'valgrind --tool=drd'.]) + VALGRIND_TOOL=drd + ;; + x|xno) + VALGRIND_TOOL=no + ;; + *) + AC_MSG_ERROR([Please specify --enable-valgrind@<:@=memcheck,drd@:>@]) + ;; +esac AC_ARG_WITH([previous-options], [AS_HELP_STRING([--with-previous-options], @@ -684,6 +730,14 @@ if test "x$enable_fuse_client" != "xno"; then fi AC_SUBST(FUSE_CLIENT_SUBDIR) + +AC_ARG_ENABLE([fuse-notifications], + AS_HELP_STRING([--disable-fuse-notifications], [Disable FUSE notifications])) + +AS_IF([test "x$enable_fuse_notifications" != "xno"], [ + AC_DEFINE([HAVE_FUSE_NOTIFICATIONS], [1], [Use FUSE notifications]) +]) + # end FUSE section @@ -731,6 +785,9 @@ AC_ARG_ENABLE([georeplication], BUILD_SYNCDAEMON=no case $host_os in + freebsd*) +#do nothing + ;; linux*) #do nothing ;; @@ -770,6 +827,17 @@ fi AC_SUBST(GEOREP_EXTRAS_SUBDIR) AM_CONDITIONAL(USE_GEOREP, test "x$enable_georeplication" != "xno") +# METADISP section +AC_ARG_ENABLE([metadisp], + AC_HELP_STRING([--enable-metadisp], + [Enable the metadata dispersal xlator])) +BUILD_METADISP=no +if test "x${enable_metadisp}" = "xyes"; then + BUILD_METADISP=yes +fi +AM_CONDITIONAL([BUILD_METADISP], [test "x$BUILD_METADISP" = "xyes"]) +# end METADISP section + # Events section AC_ARG_ENABLE([events], AC_HELP_STRING([--disable-events], @@ -908,6 +976,25 @@ if test "x${have_backtrace}" = "xyes"; then fi AC_SUBST(HAVE_BACKTRACE) +dnl Old (before C11) compiler can compile (but not link) this: +dnl +dnl int main () { +dnl _Static_assert(1, "True"); +dnl return 0; +dnl } +dnl +dnl assuming that _Static_assert is an implicitly declared function. So +dnl we're trying to link just to make sure that this is not the case. + +AC_MSG_CHECKING([whether $CC supports C11 _Static_assert]) +AC_TRY_LINK([], [_Static_assert(1, "True");], + [STATIC_ASSERT=yes], [STATIC_ASSERT=no]) + +AC_MSG_RESULT([$STATIC_ASSERT]) +if test x$STATIC_ASSERT = "xyes"; then + AC_DEFINE(HAVE_STATIC_ASSERT, 1, [Define if C11 _Static_assert is supported.]) +fi + if test "x${have_backtrace}" != "xyes"; then AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);], [], @@ -915,11 +1002,11 @@ AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);], fi dnl glusterfs prints memory usage to stderr by sending it SIGUSR1 -AC_CHECK_FUNC([malloc_stats], [have_malloc_stats=yes]) -if test "x${have_malloc_stats}" = "xyes"; then - AC_DEFINE(HAVE_MALLOC_STATS, 1, [define if found malloc_stats]) +AC_CHECK_FUNC([mallinfo], [have_mallinfo=yes]) +if test "x${have_mallinfo}" = "xyes"; then + AC_DEFINE(HAVE_MALLINFO, 1, [define if found mallinfo]) fi -AC_SUBST(HAVE_MALLOC_STATS) +AC_SUBST(HAVE_MALLINFO) dnl Linux, Solaris, Cygwin AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec]) @@ -946,6 +1033,9 @@ case $host_os in CFLAGS="${CFLAGS} -isystem /usr/local/include" ARGP_LDADD=-largp ;; + *netbsd*) + ARGP_LDADD=-largp + ;; esac dnl argp-standalone does not provide a pkg-config file AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp])) @@ -1015,6 +1105,19 @@ else CFLAGS=${OLD_CFLAGS} fi +AC_CHECK_FUNC([syncfs], [have_syncfs=yes]) +if test "x${have_syncfs}" = "xyes"; then + AC_DEFINE(HAVE_SYNCFS, 1, [define if syncfs exists]) +else + OLD_CFLAGS=${CFLAGS} + CFLAGS="-D_GNU_SOURCE" + AC_CHECK_DECL([SYS_syncfs], , , [#include <sys/syscall.h>]) + if test "x${ac_cv_have_decl_SYS_syncfs}" = "xyes"; then + AC_DEFINE(HAVE_SYNCFS_SYS, 1, [define if SYS_syncfs is available]) + fi + CFLAGS=${OLD_CFLAGS} +fi + BUILD_NANOSECOND_TIMESTAMPS=no AC_CHECK_FUNC([utimensat], [have_utimensat=yes]) if test "x${have_utimensat}" = "xyes"; then @@ -1207,10 +1310,6 @@ case $host_os in ;; *freebsd*) GF_HOST_OS="GF_BSD_HOST_OS" - GF_CFLAGS="${GF_CFLAGS} -O0" - GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME" - GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME" - GF_CFLAGS="${GF_CFLAGS} -D_LIBGEN_H_" GF_CFLAGS="${GF_CFLAGS} -DO_DSYNC=0" GF_CFLAGS="${GF_CFLAGS} -Dxdr_quad_t=xdr_longlong_t" GF_CFLAGS="${GF_CFLAGS} -Dxdr_u_quad_t=xdr_u_longlong_t" @@ -1491,9 +1590,9 @@ case $host_os in ;; esac dnl GF_XLATOR_DEFAULT_LDFLAGS is for most xlators that expose a common set of symbols -GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym $(UUID_LIBS) $(GF_NO_UNDEFINED)' +GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)' dnl GF_XLATOR_LDFLAGS is for xlators that expose extra symbols, e.g. dht -GF_XLATOR_LDFLAGS='-avoid-version $(UUID_LIBS) $(GF_NO_UNDEFINED)' +GF_XLATOR_LDFLAGS='-avoid-version $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)' AC_SUBST(GF_HOST_OS) AC_SUBST(GF_CFLAGS) @@ -1508,6 +1607,13 @@ AC_SUBST(AM_LIBTOOLFLAGS) AC_SUBST(GF_NO_UNDEFINED) AC_SUBST(GF_XLATOR_DEFAULT_LDFLAGS) AC_SUBST(GF_XLATOR_LDFLAGS) +AC_SUBST(GF_XLATOR_MGNT_LIBADD) + +case $host_os in + *freebsd*) + GF_XLATOR_MGNT_LIBADD="-lutil -lprocstat" + ;; +esac CONTRIBDIR='$(top_srcdir)/contrib' AC_SUBST(CONTRIBDIR) @@ -1579,8 +1685,8 @@ echo "readline : $BUILD_READLINE" echo "georeplication : $BUILD_SYNCDAEMON" echo "Linux-AIO : $BUILD_LIBAIO" echo "Enable Debug : $BUILD_DEBUG" -echo "Enable ASAN : $BUILD_ASAN" -echo "Enable TSAN : $BUILD_TSAN" +echo "Run with Valgrind : $VALGRIND_TOOL" +echo "Sanitizer enabled : $SANITIZER" echo "Use syslog : $USE_SYSLOG" echo "XML output : $BUILD_XML_OUTPUT" echo "Unit Tests : $BUILD_UNITTEST" @@ -1598,17 +1704,14 @@ echo "IPV6 default : $with_ipv6_default" echo "Use TIRPC : $with_libtirpc" echo "With Python : ${PYTHON_VERSION}" echo "Cloudsync : $BUILD_CLOUDSYNC" +echo "Metadata dispersal : $BUILD_METADISP" echo "Link with TCMALLOC : $BUILD_TCMALLOC" echo -if test "x$BUILD_ASAN" = "xyes"; then - echo "### Run below command before executing your tests if your system" - echo "### has 'gcc --version' above 7.x (works on Fedora 27 and Above)" - echo "export ASAN_OPTIONS=log_path=/var/log/glusterfs/asan-output.log" - echo "" - echo "### Above is required to get details of asan run, as glusterfs" - echo "### processes are daemon processes. Further details and more" - echo "### options can be found under 'Run-time flags' at" - echo "### https://github.com/google/sanitizers/wiki/AddressSanitizerFlags" - echo +# dnl Note: ${X^^} capitalization assumes bash >= 4.x +if test "x$SANITIZER" != "xnone"; then + echo "Note: since glusterfs processes are daemon processes, use" + echo "'export ${SANITIZER^^}_OPTIONS=log_path=/path/to/xxx.log' to collect" + echo "sanitizer output. Further details and more options can be" + echo "found at https://github.com/google/sanitizers." fi diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c index ffa0a4b6316..06ff191f542 100644 --- a/contrib/fuse-lib/mount.c +++ b/contrib/fuse-lib/mount.c @@ -52,12 +52,16 @@ gf_fuse_unmount (const char *mountpoint, int fd) if (geteuid () == 0) { fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1); return; + } else { + GFFUSE_LOGERR ("fuse: Effective-uid: %d", geteuid()); } res = umount2 (mountpoint, 2); if (res == 0) return; + GFFUSE_LOGERR ("fuse: failed to unmount %s: %s", + mountpoint, strerror (errno)); pid = fork (); if (pid == -1) return; @@ -67,6 +71,8 @@ gf_fuse_unmount (const char *mountpoint, int fd) "--", mountpoint, NULL }; execvp (FUSERMOUNT_PROG, (char **)argv); + GFFUSE_LOGERR ("fuse: failed to execute fuserumount: %s", + strerror (errno)); _exit (1); } waitpid (pid, NULL, 0); @@ -384,6 +390,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, build_iovec (&iov, &iovlen, "from", "/dev/fuse", -1); build_iovec (&iov, &iovlen, "volname", source, -1); build_iovec (&iov, &iovlen, "fd", fdstr, -1); + build_iovec (&iov, &iovlen, "allow_other", NULL, -1); ret = nmount (iov, iovlen, mountflags); #else ret = mount (source, mountpoint, fstype, mountflags, diff --git a/contrib/sunrpc/xdr_sizeof.c b/contrib/sunrpc/xdr_sizeof.c deleted file mode 100644 index ca1f7bf0a5e..00000000000 --- a/contrib/sunrpc/xdr_sizeof.c +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (c) 1999 Apple Computer, Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * Portions Copyright (c) 1999 Apple Computer, Inc. All Rights - * Reserved. This file contains Original Code and/or Modifications of - * Original Code as defined in and that are subject to the Apple Public - * Source License Version 1.1 (the "License"). You may not use this file - * except in compliance with the License. Please obtain a copy of the - * License at http://www.apple.com/publicsource and read it before using - * this file. - * - * The Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON- INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -/* - * Sun RPC is a product of Sun Microsystems, Inc. and is provided for - * unrestricted use provided that this legend is included on all tape - * media and as a part of the software program in whole or part. Users - * may copy or modify Sun RPC without charge, but are not authorized - * to license or distribute it to anyone else except as part of a product or - * program developed by the user. - * - * SUN RPC IS PROVIDED AS IS WITH NO WARRANTIES OF ANY KIND INCLUDING THE - * WARRANTIES OF DESIGN, MERCHANTIBILITY AND FITNESS FOR A PARTICULAR - * PURPOSE, OR ARISING FROM A COURSE OF DEALING, USAGE OR TRADE PRACTICE. - * - * Sun RPC is provided with no support and without any obligation on the - * part of Sun Microsystems, Inc. to assist in its use, correction, - * modification or enhancement. - * - * SUN MICROSYSTEMS, INC. SHALL HAVE NO LIABILITY WITH RESPECT TO THE - * INFRINGEMENT OF COPYRIGHTS, TRADE SECRETS OR ANY PATENTS BY SUN RPC - * OR ANY PART THEREOF. - * - * In no event will Sun Microsystems, Inc. be liable for any lost revenue - * or profits or other special, indirect and consequential damages, even if - * Sun has been advised of the possibility of such damages. - * - * Sun Microsystems, Inc. - * 2550 Garcia Avenue - * Mountain View, California 94043 - */ - -/* - * xdr_sizeof.c - * - * Copyright 1990 Sun Microsystems, Inc. - * - * General purpose routine to see how much space something will use - * when serialized using XDR. - */ - -#ifdef GF_DARWIN_HOST_OS - -#include <rpc/types.h> -#include <rpc/xdr.h> -#include <sys/types.h> -#include <sys/cdefs.h> - -#include <stdlib.h> - -/* ARGSUSED */ -#ifdef GF_DARWIN_HOST_OS -static bool_t -x_putlong (XDR *xdrs, const int *longp) -{ - xdrs->x_handy += BYTES_PER_XDR_UNIT; - return TRUE; -} - -#else -static bool_t -x_putlong (XDR *xdrs, const long *longp) -{ - xdrs->x_handy += BYTES_PER_XDR_UNIT; - return TRUE; -} -#endif - -/* ARGSUSED */ -static bool_t -x_putbytes (XDR *xdrs, const char *bp, u_int len) -{ - xdrs->x_handy += len; - return TRUE; -} - -#ifdef GF_DARWIN_HOST_OS -static u_int -x_getpostn (XDR *xdrs) -{ - return xdrs->x_handy; -} -#else -static u_int -x_getpostn (const XDR *xdrs) -{ - return xdrs->x_handy; -} -#endif - -/* ARGSUSED */ -static bool_t -x_setpostn (XDR *xdrs, u_int len) -{ - /* This is not allowed */ - return FALSE; -} - -static int32_t * -x_inline (XDR *xdrs, u_int len) -{ - if (len == 0) - return NULL; - if (xdrs->x_op != XDR_ENCODE) - return NULL; - if (len < (u_int) (long int) xdrs->x_base) - { - /* x_private was already allocated */ - xdrs->x_handy += len; - return (int32_t *) xdrs->x_private; - } - else - { - /* Free the earlier space and allocate new area */ - free (xdrs->x_private); - if ((xdrs->x_private = (caddr_t) malloc (len)) == NULL) - { - xdrs->x_base = 0; - return NULL; - } - xdrs->x_base = (void *) (long) len; - xdrs->x_handy += len; - return (int32_t *) xdrs->x_private; - } -} - -static int -harmless (void) -{ - /* Always return FALSE/NULL, as the case may be */ - return 0; -} - -static void -x_destroy (XDR *xdrs) -{ - xdrs->x_handy = 0; - xdrs->x_base = 0; - if (xdrs->x_private) - { - free (xdrs->x_private); - xdrs->x_private = NULL; - } - return; -} - -unsigned long -xdr_sizeof (xdrproc_t func, void *data) -{ - XDR x; - struct xdr_ops ops; - bool_t stat; - -#ifdef GF_DARWIN_HOST_OS - typedef bool_t (*dummyfunc1) (XDR *, int *); -#else - typedef bool_t (*dummyfunc1) (XDR *, long *); -#endif - typedef bool_t (*dummyfunc2) (XDR *, caddr_t, u_int); - - ops.x_putlong = x_putlong; - ops.x_putbytes = x_putbytes; - ops.x_inline = x_inline; - ops.x_getpostn = x_getpostn; - ops.x_setpostn = x_setpostn; - ops.x_destroy = x_destroy; - - /* the other harmless ones */ - ops.x_getlong = (dummyfunc1) harmless; - ops.x_getbytes = (dummyfunc2) harmless; - - x.x_op = XDR_ENCODE; - x.x_ops = &ops; - x.x_handy = 0; - x.x_private = (caddr_t) NULL; - x.x_base = (caddr_t) 0; - - stat = func (&x, data, 0); - if (x.x_private) - free (x.x_private); - return (stat == TRUE ? (unsigned) x.x_handy : 0); -} -#endif /* GF_DARWIN_HOST_OS */ diff --git a/doc/developer-guide/fuse-interrupt.md b/doc/developer-guide/fuse-interrupt.md index f92b5532eaf..ec991b81ec5 100644 --- a/doc/developer-guide/fuse-interrupt.md +++ b/doc/developer-guide/fuse-interrupt.md @@ -23,9 +23,10 @@ not exported to a header file). ``` enum fuse_interrupt_state { - INTERRUPT_NONE, + /* ... */ INTERRUPT_SQUELCHED, INTERRUPT_HANDLED, + /* ... */ }; typedef enum fuse_interrupt_state fuse_interrupt_state_t; struct fuse_interrupt_record; @@ -62,8 +63,58 @@ dummy implementation only for demonstration purposes.) Flush is chosen because a `FLUSH` interrupt is easy to trigger (see *tests/features/interrupt.t*). Interrupt handling for flush is switched on by `--fuse-flush-handle-interrupt` (a hidden glusterfs command line flag). -The flush interrupt handling code is guarded by the -`flush_handle_interrupt` Boolean member of `fuse_private_t`. +The implementation of flush interrupt is contained in the +`fuse_flush_interrupt_handler()` function and blocks guarded by the + +``` +if (priv->flush_handle_interrupt) { ... +``` + +conditional (where `priv` is a `*fuse_private_t`). + +### Overview + +"Regular" fuse fops and interrupt handlers interact via a list containing +interrupt records. + +If a fop wishes to have its interrupts handled, it needs to set up an +interrupt record and insert it into the list; also when it's to finish +(ie. in its "cbk" stage) it needs to delete the record from the list. + +If no interrupt happens, basically that's all to it - a list insertion +and deletion. + +However, if an interrupt comes for the fop, the interrupt FUSE request +will carry the data identifying an ongoing fop (that is, its `unique`), +and based on that, the interrupt record will be looked up in the list, and +the specific interrupt handler (a member of the interrupt record) will be +called. + +Usually the fop needs to share some data with the interrupt handler to +enable it to perform its task (also shared via the interrupt record). +The interrupt API offers two approaches to manage shared data: +- _Async or reference-counting strategy_: from the point on when the interrupt + record is inserted to the list, it's owned jointly by the regular fop and + the prospective interrupt handler. Both of them need to check before they + return if the other is still holding a reference; if not, then they are + responsible for reclaiming the shared data. +- _Sync or borrow strategy_: the interrupt handler is considered a borrower + of the shared data. The interrupt handler should not reclaim the shared + data. The fop will wait for the interrupt handler to finish (ie., the borrow + to be returned), then it has to reclaim the shared data. + +The user of the interrupt API need to call the following functions to +instrument this control flow: +- `fuse_interrupt_record_insert()` in the fop to insert the interrupt record to + the list; +- `fuse_interrupt_finish_fop()`in the fop (cbk) and +- `fuse_interrupt_finish_interrupt()`in the interrupt handler + +to perform needed synchronization at the end their tenure. The data management +strategies are implemented by the `fuse_interrupt_finish_*()` functions (which +have an argument to specify which strategy to use); these routines take care +of freeing the interrupt record itself, while the reclamation of the shared data +is left to the API user. ### Usage @@ -75,12 +126,15 @@ steps: call (directly or as async callback) `fuse_interrupt_finish_interrupt()`. The `intstat` argument to `fuse_interrupt_finish_interrupt` should be either `INTERRUPT_SQUELCHED` or `INTERRUPT_HANDLED`. - - `INTERRUPT_SQUELCHED` means that we choose not to handle the interrupt + - `INTERRUPT_SQUELCHED` means that the interrupt could not be delivered and the fop is going on uninterrupted. - `INTERRUPT_HANDLED` means that the interrupt was actually handled. In this case the fop will be answered from interrupt context with errno `EINTR` (that is, the fop should not send a response to the kernel). + (the enum `fuse_interrupt_state` includes further members, which are reserved + for internal use). + We return to the `sync` and `datap` arguments later. - In the `fuse_<FOP>` function create an interrupt record using `fuse_interrupt_record_new()`, passing the incoming `fuse_in_header` and @@ -92,10 +146,10 @@ steps: `fuse_interrupt_record_insert()`. - In `fuse_<FOP>_cbk` call `fuse_interrupt_finish_fop()`. - `fuse_interrupt_finish_fop()` returns a Boolean according to whether the - interrupt was handled. If it was, then the fuse request is already + interrupt was handled. If it was, then the FUSE request is already answered and the stack gets destroyed in `fuse_interrupt_finish_fop` so - `fuse_<FOP>_cbk` can just return (zero). Otherwise follow the standard - cbk logic (answer the fuse request and destroy the stack -- these are + `fuse_<FOP>_cbk()` can just return (zero). Otherwise follow the standard + cbk logic (answer the FUSE request and destroy the stack -- these are typically accomplished by `fuse_err_cbk()`). - The last two argument of `fuse_interrupt_finish_fop()` and `fuse_interrupt_finish_interrupt()` are `gf_boolean_t sync` and @@ -124,7 +178,34 @@ steps: then that pointer will be directed to the `data` member of the interrupt record and it's up to the caller what it's doing with it. - If `sync` is true, interrupt handler can use `datap = NULL`, and - fop handler will have `datap` set. + fop handler will have `datap` point to a valid pointer. - If `sync` is false, and handlers pass a pointer to a pointer for `datap`, they should check if the pointed pointer is NULL before attempting to deal with the data. + +### FUSE answer for the interrupted fop + +The kernel acknowledges a successful interruption for a given FUSE request +if the filesystem daemon answers it with errno EINTR; upon that, the syscall +which induced the request will be abruptly terminated with an interrupt, rather +than returning a value. + +In glusterfs, this can be arranged in two ways. + +- If the interrupt handler wins the race for the interrupt record, ie. + `fuse_interrupt_finish_fop()` returns true to `fuse_<FOP>_cbk()`, then, as + said above, `fuse_<FOP>_cbk()` does not need to answer the FUSE request. + That's because then the interrupt handler will take care about answering + it (with errno EINTR). +- If `fuse_interrupt_finish_fop()` returns false to `fuse_<FOP>_cbk()`, then + this return value does not inform the fop handler whether there was an interrupt + or not. This return value occurs both when fop handler won the race for the + interrupt record against the interrupt handler, and when there was no interrupt + at all. + + However, the internal logic of the fop handler might detect from other + circumstances that an interrupt was delivered. For example, the fop handler + might be sleeping, waiting for some data to arrive, so that a premature + wakeup (with no data present) occurs if the interrupt handler intervenes. In + such cases it's the responsibility of the fop handler to reply the FUSE + request with errro EINTR. diff --git a/doc/developer-guide/identifying-resource-leaks.md b/doc/developer-guide/identifying-resource-leaks.md index 851fc4424bc..950cae79b0a 100644 --- a/doc/developer-guide/identifying-resource-leaks.md +++ b/doc/developer-guide/identifying-resource-leaks.md @@ -174,3 +174,27 @@ In this case, the resource leak can be addressed by adding a single line to the Running the same Valgrind command and comparing the output will show that the memory leak in `xlators/meta/src/meta.c:init` is not reported anymore. + +### Running DRD, the Valgrind thread error detector + +When configuring GlusterFS with: + +```shell +./configure --enable-valgrind +``` + +the default Valgrind tool (Memcheck) is enabled. But it's also possble to select +one of Memcheck or DRD by using: + +```shell +./configure --enable-valgrind=memcheck +``` + +or: + +```shell +./configure --enable-valgrind=drd +``` + +respectively. When using DRD, it's recommended to consult +https://valgrind.org/docs/manual/drd-manual.html before running. diff --git a/doc/developer-guide/thread-naming.md b/doc/developer-guide/thread-naming.md index 74efba28e45..513140d4437 100644 --- a/doc/developer-guide/thread-naming.md +++ b/doc/developer-guide/thread-naming.md @@ -29,10 +29,10 @@ gf_thread_create_detached (pthread_t *thread, As max name length for a thread in POSIX is only 16 characters including the '\0' character, you have to be a little creative with naming. Also, it is important that all Gluster threads have common prefix. Considering these -conditions, we have "gluster" as prefix for all the threads created by these +conditions, we have "glfs_" as prefix for all the threads created by these wrapper functions. It is responsibility of the owner of thread to provide the suffix part of the name. It does not have to be a descriptive name, as it has -only 8 letters to work with. However, it should be unique enough such that it +only 10 letters to work with. However, it should be unique enough such that it can be matched with a table which describes it. If n number of threads are spwaned to perform same function, it is must that the @@ -87,6 +87,7 @@ such that it can be matched with a table below without ambiguity. - posixfsy - posix fsync - posixhc - posix heal - posixjan - posix janitor +- posixrsv - posix reserve - quiesce - quiesce dequeue - rdmaAsyn - rdma async event handler - rdmaehan - rdma completion handler diff --git a/events/src/eventsapiconf.py.in b/events/src/eventsapiconf.py.in index 76b5954d325..700093bee60 100644 --- a/events/src/eventsapiconf.py.in +++ b/events/src/eventsapiconf.py.in @@ -28,6 +28,8 @@ def get_glusterd_workdir(): return glusterd_workdir SERVER_ADDRESS = "0.0.0.0" +SERVER_ADDRESSv4 = "0.0.0.0" +SERVER_ADDRESSv6 = "::1" DEFAULT_CONFIG_FILE = "@SYSCONF_DIR@/glusterfs/eventsconfig.json" CUSTOM_CONFIG_FILE_TO_SYNC = "/events/config.json" CUSTOM_CONFIG_FILE = get_glusterd_workdir() + CUSTOM_CONFIG_FILE_TO_SYNC diff --git a/events/src/glustereventsd.py b/events/src/glustereventsd.py index c4c7b65e332..341a3b60947 100644 --- a/events/src/glustereventsd.py +++ b/events/src/glustereventsd.py @@ -13,6 +13,7 @@ from __future__ import print_function import sys import signal +import threading try: import socketserver except ImportError: @@ -23,10 +24,17 @@ from argparse import ArgumentParser, RawDescriptionHelpFormatter from eventtypes import all_events import handlers import utils -from eventsapiconf import SERVER_ADDRESS, PID_FILE +from eventsapiconf import SERVER_ADDRESSv4, SERVER_ADDRESSv6, PID_FILE from eventsapiconf import AUTO_BOOL_ATTRIBUTES, AUTO_INT_ATTRIBUTES from utils import logger, PidFile, PidFileLockFailed, boolify +# Subclass so that specifically IPv4 packets are captured +class UDPServerv4(socketserver.ThreadingUDPServer): + address_family = socket.AF_INET + +# Subclass so that specifically IPv6 packets are captured +class UDPServerv6(socketserver.ThreadingUDPServer): + address_family = socket.AF_INET6 class GlusterEventsRequestHandler(socketserver.BaseRequestHandler): @@ -89,6 +97,10 @@ def signal_handler_sigusr2(sig, frame): utils.restart_webhook_pool() +def UDP_server_thread(sock): + sock.serve_forever() + + def init_event_server(): utils.setup_logger() utils.load_all() @@ -99,15 +111,26 @@ def init_event_server(): sys.stderr.write("Unable to get Port details from Config\n") sys.exit(1) - # Start the Eventing Server, UDP Server + # Creating the Eventing Server, UDP Server for IPv4 packets + try: + serverv4 = UDPServerv4((SERVER_ADDRESSv4, port), + GlusterEventsRequestHandler) + except socket.error as e: + sys.stderr.write("Failed to start Eventsd for IPv4: {0}\n".format(e)) + sys.exit(1) + # Creating the Eventing Server, UDP Server for IPv6 packets try: - server = socketserver.ThreadingUDPServer( - (SERVER_ADDRESS, port), - GlusterEventsRequestHandler) + serverv6 = UDPServerv6((SERVER_ADDRESSv6, port), + GlusterEventsRequestHandler) except socket.error as e: - sys.stderr.write("Failed to start Eventsd: {0}\n".format(e)) + sys.stderr.write("Failed to start Eventsd for IPv6: {0}\n".format(e)) sys.exit(1) - server.serve_forever() + server_thread1 = threading.Thread(target=UDP_server_thread, + args=(serverv4,)) + server_thread2 = threading.Thread(target=UDP_server_thread, + args=(serverv6,)) + server_thread1.start() + server_thread2.start() def get_args(): diff --git a/events/src/peer_eventsapi.py b/events/src/peer_eventsapi.py index 26b77a09179..4d2e5f35b1c 100644 --- a/events/src/peer_eventsapi.py +++ b/events/src/peer_eventsapi.py @@ -353,8 +353,7 @@ class WebhookModCmd(Cmd): errcode=ERROR_WEBHOOK_NOT_EXISTS, json_output=args.json) - if isinstance(data[args.url], str) or \ - isinstance(data[args.url], unicode): + if isinstance(data[args.url], str): data[args.url]["token"] = data[args.url] if args.bearer_token != "": diff --git a/events/src/utils.py b/events/src/utils.py index 38b707a1b28..6d4e0791a2b 100644 --- a/events/src/utils.py +++ b/events/src/utils.py @@ -13,6 +13,7 @@ import sys import json import os import logging +import logging.handlers import fcntl from errno import EBADF from threading import Thread @@ -98,7 +99,7 @@ def setup_logger(): logger.setLevel(logging.INFO) # create the logging file handler - fh = logging.FileHandler(LOG_FILE) + fh = logging.handlers.WatchedFileHandler(LOG_FILE) formatter = logging.Formatter("[%(asctime)s] %(levelname)s " "[%(module)s - %(lineno)s:%(funcName)s] " diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py index 7bfb6c9652a..5a07e2feab1 100755 --- a/extras/distributed-testing/distributed-test-runner.py +++ b/extras/distributed-testing/distributed-test-runner.py @@ -383,14 +383,17 @@ class Handlers: return self.shell.call("make install") == 0 @synchronized - def prove(self, id, test, timeout, valgrind=False, asan_noleaks=True): + def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True): assert id == self.client_id self.shell.cd(self.gluster_root) env = "DEBUG=1 " - if valgrind: + if valgrind == "memcheck" or valgrind == "yes": cmd = "valgrind" cmd += " --tool=memcheck --leak-check=full --track-origins=yes" cmd += " --show-leak-kinds=all -v prove -v" + elif valgrind == "drd": + cmd = "valgrind" + cmd += " --tool=drd -v prove -v" elif asan_noleaks: cmd = "prove -v" env += "ASAN_OPTIONS=detect_leaks=0 " @@ -827,8 +830,9 @@ parser.add_argument("--port", help="server port to listen", type=int, default=DEFAULT_PORT) # test role parser.add_argument("--tester", help="start tester", action="store_true") -parser.add_argument("--valgrind", help="run tests under valgrind", - action="store_true") +parser.add_argument("--valgrind[=memcheck,drd]", + help="run tests with valgrind tool 'memcheck' or 'drd'", + default="no") parser.add_argument("--asan", help="test with asan enabled", action="store_true") parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks", diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd index 93fc8be5136..f91e8b6b8f7 100644 --- a/extras/ganesha/ocf/ganesha_nfsd +++ b/extras/ganesha/ocf/ganesha_nfsd @@ -36,7 +36,7 @@ else . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs fi -OCF_RESKEY_ha_vol_mnt_default="/var/run/gluster/shared_storage" +OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage" : ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}} ganesha_meta_data() { diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh index 1ffba427457..3040e8138b0 100755 --- a/extras/ganesha/scripts/create-export-ganesha.sh +++ b/extras/ganesha/scripts/create-export-ganesha.sh @@ -64,6 +64,7 @@ echo " Pseudo=\"/$VOL\";" echo ' Protocols = "3", "4" ;' echo ' Transports = "UDP","TCP";' echo ' SecType = "sys";' +echo ' Security_Label = False;' echo " }" } if [ "$OPTION" = "on" ]; diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh index 32af1cab10e..9790a719e10 100644 --- a/extras/ganesha/scripts/ganesha-ha.sh +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -24,11 +24,16 @@ GANESHA_HA_SH=$(realpath $0) HA_NUM_SERVERS=0 HA_SERVERS="" HA_VOL_NAME="gluster_shared_storage" -HA_VOL_MNT="/var/run/gluster/shared_storage" +HA_VOL_MNT="/run/gluster/shared_storage" HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha" SERVICE_MAN="DISTRO_NOT_FOUND" -RHEL6_PCS_CNAME_OPTION="--name" +# rhel, fedora id, version +ID="" +VERSION_ID="" + +PCS9OR10_PCS_CNAME_OPTION="" +PCS9OR10_PCS_CLONE_OPTION="clone" SECRET_PEM="/var/lib/glusterd/nfs/secret.pem" # UNBLOCK RA uses shared_storage which may become unavailable @@ -101,9 +106,9 @@ determine_service_manager () { then SERVICE_MAN="/sbin/service" fi - if [ "${SERVICE_MAN}" == "DISTRO_NOT_FOUND" ] + if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]] then - echo "Service manager not recognized, exiting" + logger "Service manager not recognized, exiting" exit 1 fi } @@ -114,7 +119,7 @@ manage_service () local new_node=${2} local option= - if [ "${action}" == "start" ]; then + if [[ "${action}" == "start" ]]; then option="yes" else option="no" @@ -122,7 +127,7 @@ manage_service () ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ ${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option" - if [ "${SERVICE_MAN}" == "/bin/systemctl" ] + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]] then ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ ${SECRET_PEM} root@${new_node} "${SERVICE_MAN} ${action} nfs-ganesha" @@ -140,7 +145,7 @@ check_cluster_exists() if [ -e /var/run/corosync.pid ]; then cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3) - if [ ${cluster_name} -a ${cluster_name} = ${name} ]; then + if [[ "${cluster_name}X" == "${name}X" ]]; then logger "$name already exists, exiting" exit 0 fi @@ -155,7 +160,7 @@ determine_servers() local tmp_ifs=${IFS} local ha_servers="" - if [ "X${cmd}X" != "XsetupX" -a "X${cmd}X" != "XstatusX" ]; then + if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//') IFS=$' ' for server in ${ha_servers} ; do @@ -193,15 +198,21 @@ setup_cluster() logger "setting up cluster ${name} with the following ${servers}" - pcs cluster auth ${servers} - # pcs cluster setup --name ${name} ${servers} - pcs cluster setup ${RHEL6_PCS_CNAME_OPTION} ${name} --enable --transport udpu ${servers} + # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers} + pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} if [ $? -ne 0 ]; then - logger "pcs cluster setup ${RHEL6_PCS_CNAME_OPTION} ${name} --enable --transport udpu ${servers} failed" + logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out" #set up failed stop all ganesha process and clean up symlinks in cluster stop_ganesha_all "${servers}" exit 1; fi + + # pcs cluster auth ${servers} + pcs cluster auth + if [ $? -ne 0 ]; then + logger "pcs cluster auth failed" + fi + pcs cluster start --all if [ $? -ne 0 ]; then logger "pcs cluster start failed" @@ -217,7 +228,7 @@ setup_cluster() done unclean=$(pcs status | grep -u "UNCLEAN") - while [[ "${unclean}X" = "UNCLEANX" ]]; do + while [[ "${unclean}X" == "UNCLEANX" ]]; do sleep 1 unclean=$(pcs status | grep -u "UNCLEAN") done @@ -244,7 +255,7 @@ setup_finalize_ha() local stopped="" stopped=$(pcs status | grep -u "Stopped") - while [[ "${stopped}X" = "StoppedX" ]]; do + while [[ "${stopped}X" == "StoppedX" ]]; do sleep 1 stopped=$(pcs status | grep -u "Stopped") done @@ -265,7 +276,7 @@ refresh_config () if [ -e ${SECRET_PEM} ]; then while [[ ${3} ]]; do current_host=`echo ${3} | cut -d "." -f 1` - if [ ${short_host} != ${current_host} ]; then + if [[ ${short_host} != ${current_host} ]]; then output=$(ssh -oPasswordAuthentication=no \ -oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \ "dbus-send --print-reply --system --dest=org.ganesha.nfsd \ @@ -398,7 +409,7 @@ wrap_create_virt_ip_constraints() # the result is "node2 node3 node4"; for node2, "node3 node4 node1" # and so on. while [[ ${1} ]]; do - if [ "${1}" = "${primary}" ]; then + if [[ ${1} == ${primary} ]]; then shift while [[ ${1} ]]; do tail=${tail}" "${1} @@ -429,15 +440,15 @@ setup_create_resources() local cibfile=$(mktemp -u) # fixup /var/lib/nfs - logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone" - pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone + logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}" + pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} if [ $? -ne 0 ]; then - logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} --clone failed" + logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed" fi - pcs resource create nfs-mon ocf:heartbeat:ganesha_mon --clone + pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} if [ $? -ne 0 ]; then - logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon --clone failed" + logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed" fi # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace @@ -445,9 +456,9 @@ setup_create_resources() # ganesha-active crm_attribute sleep 5 - pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone notify=true + pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true if [ $? -ne 0 ]; then - logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone failed" + logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed" fi pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1 @@ -616,7 +627,7 @@ addnode_recreate_resources() --after ${add_node}-nfs_block if [ $? -ne 0 ]; then logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ - ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed" + ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed" fi pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 @@ -780,7 +791,7 @@ setup_state_volume() touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state fi for server in ${HA_SERVERS} ; do - if [ ${server} != ${dirname} ]; then + if [[ ${server} != ${dirname} ]]; then ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} fi @@ -794,7 +805,7 @@ setup_state_volume() enable_pacemaker() { while [[ ${1} ]]; do - if [ "${SERVICE_MAN}" == "/usr/bin/systemctl" ]; then + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ ${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker" else @@ -892,7 +903,7 @@ delnode_state_volume() rm -rf ${mnt}/nfs-ganesha/${dirname} for server in ${HA_SERVERS} ; do - if [[ "${server}" != "${dirname}" ]]; then + if [[ ${server} != ${dirname} ]]; then rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} fi @@ -908,8 +919,9 @@ status() local index=1 local nodes - # change tabs to spaces, strip leading spaces - pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*//" > ${scratch} + # change tabs to spaces, strip leading spaces, including any + # new '*' at the beginning of a line introduced in pcs-0.10.x + pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch} nodes[0]=${1}; shift @@ -924,7 +936,7 @@ status() done # print the nodes that are expected to be online - grep -E "^Online:" ${scratch} + grep -E "Online:" ${scratch} echo @@ -963,7 +975,7 @@ status() create_ganesha_conf_file() { - if [ $1 == "yes" ]; + if [[ "$1" == "yes" ]]; then if [ -e $GANESHA_CONF ]; then @@ -1012,6 +1024,13 @@ main() semanage boolean -m gluster_use_execmem --on fi + local osid="" + + osid=$(grep ^ID= /etc/os-release) + eval $(echo ${osid} | grep -F ID=) + osid=$(grep ^VERSION_ID= /etc/os-release) + eval $(echo ${osid} | grep -F VERSION_ID=) + HA_CONFDIR=${1%/}; shift local ha_conf=${HA_CONFDIR}/ganesha-ha.conf local node="" @@ -1032,7 +1051,17 @@ main() determine_servers "setup" - if [ "X${HA_NUM_SERVERS}X" != "X1X" ]; then + # Fedora 29+ and rhel/centos 8 has PCS-0.10.x + # default is pcs-0.10.x options but check for + # rhel/centos 7 (pcs-0.9.x) and adjust accordingly + if [[ ! ${ID} =~ {rhel,centos} ]]; then + if [[ ${VERSION_ID} == 7.* ]]; then + PCS9OR10_PCS_CNAME_OPTION="--name" + PCS9OR10_PCS_CLONE_OPTION="--clone" + fi + fi + + if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then determine_service_manager diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py new file mode 100755 index 00000000000..634576058d6 --- /dev/null +++ b/extras/glusterfs-georep-upgrade.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +""" + +Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> +This file is part of GlusterFS. + +This file is licensed to you under your choice of the GNU Lesser +General Public License, version 3 or any later version (LGPLv3 or +later), or the GNU General Public License, version 2 (GPLv2), in all +cases as published by the Free Software Foundation. + +""" + +import argparse +import errno +import os, sys +import shutil +from datetime import datetime + +def find_htime_path(brick_path): + dirs = [] + htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime') + for file in os.listdir(htime_dir): + if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"): + dirs.append(os.path.join(htime_dir, file)) + else: + raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file))) + return dirs + +def modify_htime_file(brick_path): + htime_file_path_list = find_htime_path(brick_path) + + for htime_file_path in htime_file_path_list: + changelog_path = os.path.join(brick_path, '.glusterfs/changelogs') + temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file') + with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file: + #extract epoch times from htime file + paths = htime_file.read().split("\x00") + + for pth in paths: + epoch_no = pth.split(".")[-1] + changelog = os.path.basename(pth) + #convert epoch time to year, month and day + if epoch_no != '': + date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d")) + #update paths in temp htime file + temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog)) + #create directory in the format year/month/days + path = os.path.join(changelog_path, date) + + if changelog.startswith("CHANGELOG."): + try: + os.makedirs(path, mode = 0o600); + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + #copy existing changelogs to new directory structure, delete old changelog files + shutil.copyfile(pth, os.path.join(path, changelog)) + os.remove(pth) + + #rename temp_htime_file with htime file + os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path)) + os.rename(temp_htime_path, htime_file_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('brick_path', help="This upgrade script, which is to be run on\ + server side, takes brick path as the argument, \ + updates paths inside htime file and alters the directory structure \ + above the changelog files inorder to support new optimised format \ + of the directory structure as per \ + https://review.gluster.org/#/c/glusterfs/+/23733/") + args = parser.parse_args() + modify_htime_file(args.brick_path) diff --git a/extras/group-virt.example b/extras/group-virt.example index 6a66f977b2f..cc37c98a25c 100644 --- a/extras/group-virt.example +++ b/extras/group-virt.example @@ -17,3 +17,8 @@ cluster.choose-local=off client.event-threads=4 server.event-threads=4 performance.client-io-threads=on +network.ping-timeout=20 +server.tcp-user-timeout=20 +server.keepalive-time=10 +server.keepalive-interval=2 +server.keepalive-count=5 diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh index 885ed03ad5b..1f2564b44ff 100755 --- a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh +++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh @@ -79,9 +79,9 @@ done if [ "$option" == "disable" ]; then # Unmount the volume on all the nodes - umount /var/run/gluster/shared_storage - cat /etc/fstab | grep -v "gluster_shared_storage /var/run/gluster/shared_storage/" > /var/run/gluster/fstab.tmp - mv /var/run/gluster/fstab.tmp /etc/fstab + umount /run/gluster/shared_storage + cat /etc/fstab | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab fi if [ "$is_originator" == 1 ]; then @@ -104,8 +104,15 @@ function check_volume_status() echo $status } -mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \ - /var/run/gluster/shared_storage" +key=`echo $5 | cut -d '=' -f 1` +val=`echo $5 | cut -d '=' -f 2` +if [ "$key" == "transport.address-family" ]; then + mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \ + $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage" +else + mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \ + /run/gluster/shared_storage" +fi if [ "$option" == "enable" ]; then retry=0; @@ -120,10 +127,10 @@ if [ "$option" == "enable" ]; then status=$(check_volume_status) done # Mount the volume on all the nodes - umount /var/run/gluster/shared_storage - mkdir -p /var/run/gluster/shared_storage + umount /run/gluster/shared_storage + mkdir -p /run/gluster/shared_storage $mount_cmd - cp /etc/fstab /var/run/gluster/fstab.tmp - echo "$local_node_hostname:/gluster_shared_storage /var/run/gluster/shared_storage/ glusterfs defaults 0 0" >> /var/run/gluster/fstab.tmp - mv /var/run/gluster/fstab.tmp /etc/fstab + cp /etc/fstab /run/gluster/fstab.tmp + echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults 0 0" >> /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab fi diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh index 90ba6bc73a5..7ad6f23ad06 100755 --- a/extras/hook-scripts/start/post/S31ganesha-start.sh +++ b/extras/hook-scripts/start/post/S31ganesha-start.sh @@ -4,7 +4,7 @@ OPTSPEC="volname:,gd-workdir:" VOL= declare -i EXPORT_ID ganesha_key="ganesha.enable" -GANESHA_DIR="/var/run/gluster/shared_storage/nfs-ganesha" +GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha" CONF1="$GANESHA_DIR/ganesha.conf" GLUSTERD_WORKDIR= diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py index 174f2a2a44c..e62f7fc52a3 100755 --- a/extras/quota/quota_fsck.py +++ b/extras/quota/quota_fsck.py @@ -156,12 +156,10 @@ def get_quota_xattr_brick(dpath): xattr_dict = {} xattr_dict['parents'] = {} - for xattr in pairs: + for xattr in pairs[1:]: + xattr = xattr.decode("utf-8") xattr_key = xattr.split("=")[0] - if re.search("# file:", xattr_key): - # skip the file comment - continue - elif xattr_key is "": + if xattr_key == "": # skip any empty lines continue elif not re.search("quota", xattr_key): diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py index 1127be0e976..0e4df77d481 100755 --- a/extras/snap_scheduler/gcron.py +++ b/extras/snap_scheduler/gcron.py @@ -19,10 +19,10 @@ import logging.handlers import fcntl -GCRON_TASKS = "/var/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" +GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag" -LOCK_FILE_DIR = "/var/run/gluster/shared_storage/snaps/lock_files/" +LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/" log = logging.getLogger("gcron-logger") start_time = 0.0 @@ -38,7 +38,8 @@ def initLogger(script_name): sh.setFormatter(formatter) process = subprocess.Popen(["gluster", "--print-logdir"], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() if process.returncode == 0: logfile = os.path.join(out.strip(), script_name[:-3]+".log") diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py index a66c5e3d5ce..e8fcc449a9b 100755 --- a/extras/snap_scheduler/snap_scheduler.py +++ b/extras/snap_scheduler/snap_scheduler.py @@ -67,7 +67,7 @@ except ImportError: SCRIPT_NAME = "snap_scheduler" scheduler_enabled = False log = logging.getLogger(SCRIPT_NAME) -SHARED_STORAGE_DIR="/var/run/gluster/shared_storage" +SHARED_STORAGE_DIR="/run/gluster/shared_storage" GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled" GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled" GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks" @@ -149,7 +149,7 @@ def initLogger(): sh.setFormatter(formatter) process = subprocess.Popen(["gluster", "--print-logdir"], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, universal_newlines=True) logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") fh = logging.FileHandler(logfile) diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb index 872f284fcb8..1aff43377db 100755 --- a/extras/statedumpparse.rb +++ b/extras/statedumpparse.rb @@ -17,25 +17,35 @@ end # client.c:client_fd_lk_ctx_dump uses a six-dash record separator. ARRSEP = /^(-{5,6}=-{5,6})?$/ HEAD = /^\[(.*)\]$/ +INPUT_FORMATS = %w[statedump json] format = 'json' +input_format = 'statedump' tz = '+0000' memstat_select,memstat_reject = //,/\Z./ -human = false OptionParser.new do |op| op.banner << " [<] <STATEDUMP>" - op.on("-f", "--format=F", "json/yaml/memstat") { |s| format = s } + op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s } + op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s } op.on("--timezone=T", "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s } - op.on("--memstat-select=RX", "memstat: select memory types maxtching RX") { |s| + op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s| memstat_select = Regexp.new s } - op.on("--memstat-reject=RX", "memstat: reject memory types maxtching RX") { |s| + op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s| memstat_reject = Regexp.new s } - op.on("--memstat-human", "memstat: human readable") { human = true } end.parse! + +if format =~ /\Amemstat(?:-(.*))?/ + memstat_type = $1 || 'plain' + unless %w[plain human json].include? memstat_type + raise "unknown memstat type #{memstat_type.dump}" + end + format = 'memstat' +end + repr, logsep = case format when 'yaml' require 'yaml' @@ -50,8 +60,11 @@ else end formatter = proc { |e| puts repr.call(e) } -d1 = {} +INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'" + +dumpinfo = {} +# parse a statedump entry elem_cbk = proc { |s,&cbk| arraylike = false s.grep(/\S/).empty? and next @@ -74,15 +87,8 @@ elem_cbk = proc { |s,&cbk| body.reject(&:empty?).map { |e| ea = e.map { |l| k,v = l.split("=",2) - [k, begin - Integer v - rescue ArgumentError - begin - Float v - rescue ArgumentError - v - end - end] + m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v + [k, m ? (m[2] ? Float(v) : Integer(v)) : v] } begin ea.to_h @@ -93,83 +99,110 @@ elem_cbk = proc { |s,&cbk| } if body - cbk.call [head, arraylike ? body : body[0]] + cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])] else STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep) end } +# aggregator routine aggr = case format when 'memstat' - mh = {} + meminfo = {} + # commit memory-related entries to meminfo proc { |k,r| case k when /memusage/ - (mh["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject + (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject when "mempool" r.each {|e| kk = "mempool:#{e['pool-name']}" - (mh["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject + (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject } end } else + # just format data, don't actually aggregate anything proc { |pair| formatter.call pair } end -acc = [] -$<.each { |l| - l = l.strip - if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/ - d1["_meta"]||={} - (d1["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ") - next - end +# processing the data +case input_format +when 'statedump' + acc = [] + $<.each { |l| + l = l.strip + if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/ + dumpinfo["_meta"]||={} + (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ") + next + end - if l =~ HEAD - elem_cbk.call(acc, &aggr) - acc = [l] - next - end + if l =~ HEAD + elem_cbk.call(acc, &aggr) + acc = [l] + next + end - acc << l -} -elem_cbk.call(acc, &aggr) + acc << l + } + elem_cbk.call(acc, &aggr) +when 'json' + $<.each { |l| + r = JSON.load l + case r + when Array + aggr[r] + when Hash + dumpinfo.merge! r + end + } +end +# final actions: output aggregated data case format when 'memstat' - hr = proc { |n| - qa = %w[B kB MB GB] - q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1 - [n*100 / (1 << q*10) / 100.0, qa[q]].join - } - - ma = mh.values.map(&:to_a).inject(:+) - totals = mh.map { |coll,h| [coll, h.values.inject(:+)] }.to_h + ma = meminfo.values.map(&:to_a).inject(:+) + totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h tt = ma.transpose[1].inject(:+) - templ = "%{val} %{key}" - tft = proc { |t| t } - nft = if human - nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max - proc { |n| - hn = hr[n] - " " * (nw - hn.size) + hn + summary_sep,showm = case memstat_type + when 'json' + ["", proc { |k,v| puts({type: k, value: v}.to_json) }] + when 'plain', 'human' + # human-friendly number representation + hr = proc { |n| + qa = %w[B kB MB GB] + q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1 + "%.2f%s" % [n.to_f / (1 << q*10), qa[q]] } + + templ = "%{val} %{key}" + tft = proc { |t| t } + nft = if memstat_type == 'human' + nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max + proc { |n| + hn = hr[n] + " " * (nw - hn.size) + hn + } + else + nw = tt.to_s.size + proc { |n| "%#{nw}d" % n } + end + ## Alternative template, key first: + # templ = "%{key} %{val}" + # tw = ma.transpose[0].map(&:size).max + # tft = proc { |t| t + " " * [tw - t.size, 0].max } + # nft = (memstat_type == 'human') ? hr : proc { |n| n } + ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }] else - nw = tt.to_s.size - proc { |n| "%#{nw}d" % n } + raise 'this should be impossible' end - # templ = "%{key} %{val}" - # tw = ma.transpose[0].map(&:size).max - # tft = proc { |t| t + " " * [tw - t.size, 0].max } - # nft = human ? hr : proc { |n| n } - showm = proc { |k,v| puts templ % {key: tft[k], val: nft[v]} } ma.sort_by { |k,v| v }.each(&showm) - puts + print summary_sep totals.each { |coll,t| showm.call "Total #{coll}", t } showm.call "TOTAL", tt else - formatter.call d1 + formatter.call dumpinfo end diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in index 32542d9f78b..abb0d82911f 100644 --- a/extras/systemd/glusterd.service.in +++ b/extras/systemd/glusterd.service.in @@ -1,6 +1,8 @@ [Unit] Description=GlusterFS, a clustered file-system server Documentation=man:glusterd(8) +StartLimitBurst=6 +StartLimitIntervalSec=3600 Requires=@RPCBIND_SERVICE@ After=network.target @RPCBIND_SERVICE@ Before=network-online.target @@ -10,7 +12,7 @@ Type=forking PIDFile=@localstatedir@/run/glusterd.pid LimitNOFILE=65536 Environment="LOG_LEVEL=INFO" -EnvironmentFile=-@sysconfdir@/sysconfig/glusterd +EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS KillMode=process TimeoutSec=300 @@ -18,7 +20,6 @@ SuccessExitStatus=15 Restart=on-abnormal RestartSec=60 StartLimitBurst=6 -StartLimitIntervalSec=3600 StartLimitInterval=3600 [Install] diff --git a/geo-replication/gsyncd.conf.in b/geo-replication/gsyncd.conf.in index 11e57fdf54f..9688c79fab7 100644 --- a/geo-replication/gsyncd.conf.in +++ b/geo-replication/gsyncd.conf.in @@ -123,7 +123,7 @@ type=bool help=Use this to set Active Passive mode to meta-volume. [meta-volume-mnt] -value=/var/run/gluster/shared_storage +value=/run/gluster/shared_storage help=Meta Volume or Shared Volume mount path [allow-network] diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py index 73a91fea0f8..257ed72c6ae 100644 --- a/geo-replication/syncdaemon/gsyncd.py +++ b/geo-replication/syncdaemon/gsyncd.py @@ -22,8 +22,8 @@ import gsyncdconfig as gconf from rconf import rconf import subcmds from conf import GLUSTERD_WORKDIR, GLUSTERFS_CONFDIR, GCONF_VERSION -from syncdutils import set_term_handler, finalize, lf -from syncdutils import log_raise_exception, FreeObject, escape +from syncdutils import (set_term_handler, finalize, lf, + log_raise_exception, FreeObject, escape) import argsupgrade diff --git a/geo-replication/syncdaemon/gsyncdstatus.py b/geo-replication/syncdaemon/gsyncdstatus.py index 72bcb092f01..1a655ff8887 100644 --- a/geo-replication/syncdaemon/gsyncdstatus.py +++ b/geo-replication/syncdaemon/gsyncdstatus.py @@ -23,8 +23,8 @@ from datetime import datetime from errno import EACCES, EAGAIN, ENOENT import logging -from syncdutils import EVENT_GEOREP_ACTIVE, EVENT_GEOREP_PASSIVE, gf_event -from syncdutils import EVENT_GEOREP_CHECKPOINT_COMPLETED, lf +from syncdutils import (EVENT_GEOREP_ACTIVE, EVENT_GEOREP_PASSIVE, gf_event, + EVENT_GEOREP_CHECKPOINT_COMPLETED, lf) DEFAULT_STATUS = "N/A" MONITOR_STATUS = ("Created", "Started", "Paused", "Stopped") diff --git a/geo-replication/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py index c7d69d7eb2e..e6406c36bd7 100644 --- a/geo-replication/syncdaemon/libcxattr.py +++ b/geo-replication/syncdaemon/libcxattr.py @@ -10,8 +10,8 @@ import os from ctypes import CDLL, get_errno -from py2py3 import bytearray_to_str, gr_create_string_buffer -from py2py3 import gr_query_xattr, gr_lsetxattr, gr_lremovexattr +from py2py3 import (bytearray_to_str, gr_create_string_buffer, + gr_query_xattr, gr_lsetxattr, gr_lremovexattr) class Xattr(object): diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py index 34beadb3552..a3bda7282c0 100644 --- a/geo-replication/syncdaemon/libgfchangelog.py +++ b/geo-replication/syncdaemon/libgfchangelog.py @@ -12,8 +12,9 @@ import os from ctypes import CDLL, RTLD_GLOBAL, get_errno, byref, c_ulong from ctypes.util import find_library from syncdutils import ChangelogException, ChangelogHistoryNotAvailable -from py2py3 import gr_cl_history_changelog, gr_cl_done, gr_create_string_buffer -from py2py3 import gr_cl_register, gr_cl_history_done, bytearray_to_str +from py2py3 import (gr_cl_history_changelog, gr_cl_done, + gr_create_string_buffer, gr_cl_register, + gr_cl_history_done, bytearray_to_str) libgfc = CDLL( diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index 62640d4eaee..9501aeae6b5 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -24,10 +24,11 @@ from datetime import datetime import gsyncdconfig as gconf import libgfchangelog from rconf import rconf -from syncdutils import Thread, GsyncdError, escape_space_newline -from syncdutils import unescape_space_newline, gauxpfx, escape -from syncdutils import lstat, errno_wrap, FreeObject, lf, matching_disk_gfid -from syncdutils import NoStimeAvailable, PartialHistoryAvailable +from syncdutils import (Thread, GsyncdError, escape_space_newline, + unescape_space_newline, gauxpfx, escape, + lstat, errno_wrap, FreeObject, lf, matching_disk_gfid, + NoStimeAvailable, PartialHistoryAvailable, + host_brick_split) URXTIME = (-1, 0) @@ -1466,7 +1467,7 @@ class GMasterChangelogMixin(GMasterCommon): node = rconf.args.resource_remote node_data = node.split("@") node = node_data[-1] - remote_node_ip = node.split(":")[0] + remote_node_ip, _ = host_brick_split(node) self.status.set_slave_node(remote_node_ip) def changelogs_batch_process(self, changes): diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py index 14e77aef27e..6aa7b9dfc99 100644 --- a/geo-replication/syncdaemon/monitor.py +++ b/geo-replication/syncdaemon/monitor.py @@ -22,12 +22,12 @@ from resource import SSH import gsyncdconfig as gconf import libgfchangelog from rconf import rconf -from syncdutils import select, waitpid, errno_wrap, lf, grabpidfile -from syncdutils import set_term_handler, GsyncdError -from syncdutils import Thread, finalize, Volinfo, VolinfoFromGconf -from syncdutils import gf_event, EVENT_GEOREP_FAULTY, get_up_nodes +from syncdutils import (select, waitpid, errno_wrap, lf, grabpidfile, + set_term_handler, GsyncdError, + Thread, finalize, Volinfo, VolinfoFromGconf, + gf_event, EVENT_GEOREP_FAULTY, get_up_nodes, + unshare_propagation_supported) from gsyncdstatus import GeorepStatus, set_monitor_status -from syncdutils import unshare_propagation_supported import py2py3 from py2py3 import pipe diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py index c0ed408cf49..f12c7ceaa36 100644 --- a/geo-replication/syncdaemon/resource.py +++ b/geo-replication/syncdaemon/resource.py @@ -19,8 +19,8 @@ import struct import logging import tempfile import subprocess -from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EACCES -from errno import EISDIR, ENOTEMPTY, ESTALE, EINVAL, EBUSY, EPERM +from errno import (EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EACCES, + EISDIR, ENOTEMPTY, ESTALE, EINVAL, EBUSY, EPERM) import errno from rconf import rconf @@ -31,19 +31,19 @@ import repce from repce import RepceServer, RepceClient from master import gmaster_builder import syncdutils -from syncdutils import GsyncdError, select, privileged, funcode -from syncdutils import entry2pb, gauxpfx, errno_wrap, lstat -from syncdutils import NoStimeAvailable, PartialHistoryAvailable -from syncdutils import ChangelogException, ChangelogHistoryNotAvailable -from syncdutils import get_changelog_log_level, get_rsync_version -from syncdutils import GX_GFID_CANONICAL_LEN -from syncdutils import gf_mount_ready +from syncdutils import (GsyncdError, select, privileged, funcode, + entry2pb, gauxpfx, errno_wrap, lstat, + NoStimeAvailable, PartialHistoryAvailable, + ChangelogException, ChangelogHistoryNotAvailable, + get_changelog_log_level, get_rsync_version, + GX_GFID_CANONICAL_LEN, + gf_mount_ready, lf, Popen, sup, + Xattr, matching_disk_gfid, get_gfid_from_mnt, + unshare_propagation_supported, get_slv_dir_path) from gsyncdstatus import GeorepStatus -from syncdutils import lf, Popen, sup -from syncdutils import Xattr, matching_disk_gfid, get_gfid_from_mnt -from syncdutils import unshare_propagation_supported, get_slv_dir_path -from py2py3 import pipe, str_to_bytearray, entry_pack_reg -from py2py3 import entry_pack_reg_stat, entry_pack_mkdir, entry_pack_symlink +from py2py3 import (pipe, str_to_bytearray, entry_pack_reg, + entry_pack_reg_stat, entry_pack_mkdir, + entry_pack_symlink) ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP') diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py index dc0366266b9..a3df103e76c 100644 --- a/geo-replication/syncdaemon/syncdutils.py +++ b/geo-replication/syncdaemon/syncdutils.py @@ -21,8 +21,8 @@ import subprocess import socket from subprocess import PIPE from threading import Lock, Thread as baseThread -from errno import EACCES, EAGAIN, EPIPE, ENOTCONN, ENOMEM, ECONNABORTED -from errno import EINTR, ENOENT, ESTALE, EBUSY, ENODATA, errorcode +from errno import (EACCES, EAGAIN, EPIPE, ENOTCONN, ENOMEM, ECONNABORTED, + EINTR, ENOENT, ESTALE, EBUSY, ENODATA, errorcode, EIO) from signal import signal, SIGTERM import select as oselect from os import waitpid as owaitpid @@ -344,6 +344,17 @@ def log_raise_exception(excont): ECONNABORTED): logging.error(lf('Gluster Mount process exited', error=errorcode[exc.errno])) + elif isinstance(exc, OSError) and exc.errno == EIO: + logging.error("Getting \"Input/Output error\" " + "is most likely due to " + "a. Brick is down or " + "b. Split brain issue.") + logging.error("This is expected as per design to " + "keep the consistency of the file system. " + "Once the above issue is resolved " + "geo-replication would automatically " + "proceed further.") + logtag = "FAIL" else: logtag = "FAIL" if not logtag and logging.getLogger().isEnabledFor(logging.DEBUG): @@ -714,11 +725,13 @@ def get_slv_dir_path(slv_host, slv_volume, gfid): if not isinstance(realpath, int): basename = os.path.basename(realpath).rstrip('\x00') dirpath = os.path.dirname(realpath) - if dirpath is "/": + if dirpath == "/": pargfid = ROOT_GFID else: dirpath = dirpath.strip("/") pargfid = get_gfid_from_mnt(dirpath) + if isinstance(pargfid, int): + return None dir_entry = os.path.join(pfx, pargfid, basename) return dir_entry @@ -881,6 +894,19 @@ class Popen(subprocess.Popen): self.errfail() +def host_brick_split(value): + """ + IPv6 compatible way to split and get the host + and brick information. Example inputs: + node1.example.com:/exports/bricks/brick1/brick + fe80::af0f:df82:844f:ef66%utun0:/exports/bricks/brick1/brick + """ + parts = value.split(":") + brick = parts[-1] + hostparts = parts[0:-1] + return (":".join(hostparts), brick) + + class Volinfo(object): def __init__(self, vol, host='localhost', prelude=[], master=True): @@ -923,7 +949,7 @@ class Volinfo(object): @memoize def bricks(self): def bparse(b): - host, dirp = b.find("name").text.split(':', 2) + host, dirp = host_brick_split(b.find("name").text) return {'host': host, 'dir': dirp, 'uuid': b.find("hostUuid").text} return [bparse(b) for b in self.get('brick')] @@ -999,6 +1025,16 @@ class VolinfoFromGconf(object): def is_hot(self, brickpath): return False + def is_uuid(self, value): + try: + uuid.UUID(value) + return True + except ValueError: + return False + + def possible_path(self, value): + return "/" in value + @property @memoize def bricks(self): @@ -1012,8 +1048,22 @@ class VolinfoFromGconf(object): out = [] for b in bricks_data: parts = b.split(":") - bpath = parts[2] if len(parts) == 3 else "" - out.append({"host": parts[1], "dir": bpath, "uuid": parts[0]}) + b_uuid = None + if self.is_uuid(parts[0]): + b_uuid = parts[0] + # Set all parts except first + parts = parts[1:] + + if self.possible_path(parts[-1]): + bpath = parts[-1] + # Set all parts except last + parts = parts[0:-1] + + out.append({ + "host": ":".join(parts), # if remaining parts are IPv6 name + "dir": bpath, + "uuid": b_uuid + }) return out diff --git a/geo-replication/tests/unit/test_gsyncdstatus.py b/geo-replication/tests/unit/test_gsyncdstatus.py index 483023dbfe9..9c1aa2ad4ad 100755 --- a/geo-replication/tests/unit/test_gsyncdstatus.py +++ b/geo-replication/tests/unit/test_gsyncdstatus.py @@ -13,11 +13,11 @@ import unittest import os import urllib -from syncdaemon.gstatus import GeorepStatus, set_monitor_status -from syncdaemon.gstatus import get_default_values -from syncdaemon.gstatus import MONITOR_STATUS, DEFAULT_STATUS -from syncdaemon.gstatus import STATUS_VALUES, CRAWL_STATUS_VALUES -from syncdaemon.gstatus import human_time, human_time_utc +from syncdaemon.gstatus import (GeorepStatus, set_monitor_status, + get_default_values, + MONITOR_STATUS, DEFAULT_STATUS, + STATUS_VALUES, CRAWL_STATUS_VALUES, + human_time, human_time_utc) class GeorepStatusTestCase(unittest.TestCase): diff --git a/glusterfs.spec.in b/glusterfs.spec.in index 607fa9bbdcd..b6d63146e14 100644 --- a/glusterfs.spec.in +++ b/glusterfs.spec.in @@ -111,6 +111,12 @@ ## All %%global definitions should be placed here and keep them sorted ## +# selinux booleans whose defalut value needs modification +# these booleans will be consumed by "%%selinux_set_booleans" macro. +%if ( 0%{?rhel} && 0%{?rhel} >= 8 ) +%global selinuxbooleans rsync_full_access=1 rsync_client=1 +%endif + %if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 6 ) %global _with_systemd true %endif @@ -231,7 +237,9 @@ Requires(pre): shadow-utils BuildRequires: systemd %endif -Requires: %{name}-libs%{?_isa} = %{version}-%{release} +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Requires: libgfrpc0%{?_isa} = %{version}-%{release} +Requires: libgfxdr0%{?_isa} = %{version}-%{release} %if ( 0%{?_with_systemd:1} ) %{?systemd_requires} %endif @@ -294,40 +302,10 @@ This package includes the glusterfs binary, the glusterfsd daemon and the libglusterfs and glusterfs translator modules common to both GlusterFS server and client framework. -%package api -Summary: GlusterFS api library -Requires: %{name}%{?_isa} = %{version}-%{release} -Requires: %{name}-client-xlators%{?_isa} = %{version}-%{release} - -%description api -GlusterFS is a distributed file-system capable of scaling to several -petabytes. It aggregates various storage bricks over TCP/IP interconnect -into one large parallel network filesystem. GlusterFS is one of the -most sophisticated file systems in terms of features and extensibility. -It borrows a powerful concept called Translators from GNU Hurd kernel. -Much of the code in GlusterFS is in user space and easily manageable. - -This package provides the glusterfs libgfapi library. - -%package api-devel -Summary: Development Libraries -Requires: %{name}%{?_isa} = %{version}-%{release} -Requires: %{name}-devel%{?_isa} = %{version}-%{release} -Requires: libacl-devel - -%description api-devel -GlusterFS is a distributed file-system capable of scaling to several -petabytes. It aggregates various storage bricks over TCP/IP interconnect -into one large parallel network filesystem. GlusterFS is one of the -most sophisticated file systems in terms of features and extensibility. -It borrows a powerful concept called Translators from GNU Hurd kernel. -Much of the code in GlusterFS is in user space and easily manageable. - -This package provides the api include files. - %package cli Summary: GlusterFS CLI -Requires: %{name}-libs%{?_isa} = %{version}-%{release} +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Requires: libglusterd0%{?_isa} = %{version}-%{release} %description cli GlusterFS is a distributed file-system capable of scaling to several @@ -353,22 +331,6 @@ Much of the code in GlusterFS is in user space and easily manageable. This package provides cloudsync plugins for archival feature. -%package devel -Summary: Development Libraries -Requires: %{name}%{?_isa} = %{version}-%{release} -# Needed for the Glupy examples to work -Requires: %{name}-extra-xlators%{?_isa} = %{version}-%{release} - -%description devel -GlusterFS is a distributed file-system capable of scaling to several -petabytes. It aggregates various storage bricks over TCP/IP interconnect -into one large parallel network filesystem. GlusterFS is one of the -most sophisticated file systems in terms of features and extensibility. -It borrows a powerful concept called Translators from GNU Hurd kernel. -Much of the code in GlusterFS is in user space and easily manageable. - -This package provides the development libraries and include files. - %package extra-xlators Summary: Extra Gluster filesystem Translators # We need python-gluster rpm for gluster module's __init__.py in Python @@ -466,6 +428,14 @@ Requires: python%{_pythonver}-gluster = %{version}-%{release} Requires: rsync Requires: util-linux +# required for setting selinux bools +%if ( 0%{?rhel} && 0%{?rhel} >= 8 ) +Requires(post): policycoreutils-python-utils +Requires(postun): policycoreutils-python-utils +Requires: selinux-policy-targeted +Requires(post): selinux-policy-targeted +BuildRequires: selinux-policy-devel +%endif %description geo-replication GlusterFS is a distributed file-system capable of scaling to several @@ -496,10 +466,155 @@ Much of the code in GlusterFS is in user space and easily manageable. This package provides the glusterfs legacy gNFS server xlator %endif -%package libs -Summary: GlusterFS common libraries +%package -n libglusterfs0 +Summary: GlusterFS libglusterfs library +Requires: libgfrpc0%{?_isa} = %{version}-%{release} +Requires: libgfxdr0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-libs <= %{version}-%{release} +Provides: %{name}-libs = %{version}-%{release} + +%description -n libglusterfs0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the base libglusterfs library + +%package -n libglusterfs-devel +Summary: GlusterFS libglusterfs library +Requires: libgfrpc-devel%{?_isa} = %{version}-%{release} +Requires: libgfxdr-devel%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-devel <= %{version}-%{release} +Provides: %{name}-devel = %{version}-%{release} + +%description -n libglusterfs-devel +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides libglusterfs.so and the gluster C header files. + +%package -n libgfapi0 +Summary: GlusterFS api library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Requires: %{name}-client-xlators%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-api <= %{version}-%{release} +Provides: %{name}-api = %{version}-%{release} + +%description -n libgfapi0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the glusterfs libgfapi library. + +%package -n libgfapi-devel +Summary: Development Libraries +Requires: libglusterfs-devel%{?_isa} = %{version}-%{release} +Requires: libacl-devel +Obsoletes: %{name}-api-devel <= %{version}-%{release} +Provides: %{name}-api-devel = %{version}-%{release} + +%description -n libgfapi-devel +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides libgfapi.so and the api C header files. + +%package -n libgfchangelog0 +Summary: GlusterFS libchangelog library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-libs <= %{version}-%{release} + +%description -n libgfchangelog0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the libgfchangelog library + +%package -n libgfchangelog-devel +Summary: GlusterFS libchangelog library +Requires: libglusterfs-devel%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-devel <= %{version}-%{release} + +%description -n libgfchangelog-devel +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides libgfchangelog.so and changelog C header files. + +%package -n libgfrpc0 +Summary: GlusterFS libgfrpc0 library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-libs <= %{version}-%{release} + +%description -n libgfrpc0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the libgfrpc library + +%package -n libgfrpc-devel +Summary: GlusterFS libgfrpc library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-devel <= %{version}-%{release} + +%description -n libgfrpc-devel +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides libgfrpc.so and rpc C header files. + +%package -n libgfxdr0 +Summary: GlusterFS libgfxdr0 library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-libs <= %{version}-%{release} + +%description -n libgfxdr0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the libgfxdr library + +%package -n libgfxdr-devel +Summary: GlusterFS libgfxdr library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-devel <= %{version}-%{release} -%description libs +%description -n libgfxdr-devel GlusterFS is a distributed file-system capable of scaling to several petabytes. It aggregates various storage bricks over TCP/IP interconnect into one large parallel network filesystem. GlusterFS is one of the @@ -507,7 +622,22 @@ most sophisticated file systems in terms of features and extensibility. It borrows a powerful concept called Translators from GNU Hurd kernel. Much of the code in GlusterFS is in user space and easily manageable. -This package provides the base GlusterFS libraries +This package provides libgfxdr.so. + +%package -n libglusterd0 +Summary: GlusterFS libglusterd library +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Obsoletes: %{name}-libs <= %{version}-%{release} + +%description -n libglusterd0 +GlusterFS is a distributed file-system capable of scaling to several +petabytes. It aggregates various storage bricks over TCP/IP interconnect +into one large parallel network filesystem. GlusterFS is one of the +most sophisticated file systems in terms of features and extensibility. +It borrows a powerful concept called Translators from GNU Hurd kernel. +Much of the code in GlusterFS is in user space and easily manageable. + +This package provides the libglusterd library %package -n python%{_pythonver}-gluster Summary: GlusterFS python library @@ -574,11 +704,15 @@ like Pacemaker. Summary: Clustered file-system server Requires: %{name}%{?_isa} = %{version}-%{release} Requires: %{name}-cli%{?_isa} = %{version}-%{release} -Requires: %{name}-libs%{?_isa} = %{version}-%{release} +Requires: libglusterfs0%{?_isa} = %{version}-%{release} +Requires: libgfchangelog0%{?_isa} = %{version}-%{release} +%if ( 0%{?fedora} && 0%{?fedora} >= 30 || ( 0%{?rhel} && 0%{?rhel} >= 8 ) ) +Requires: glusterfs-selinux >= 0.1.0-2 +%endif # some daemons (like quota) use a fuse-mount, glusterfsd is part of -fuse Requires: %{name}-fuse%{?_isa} = %{version}-%{release} # self-heal daemon, rebalance, nfs-server etc. are actually clients -Requires: %{name}-api%{?_isa} = %{version}-%{release} +Requires: libgfapi0%{?_isa} = %{version}-%{release} Requires: %{name}-client-xlators%{?_isa} = %{version}-%{release} # lvm2 for snapshot, and nfs-utils and rpcbind/portmap for gnfs server Requires: lvm2 @@ -846,9 +980,6 @@ rm -rf %{buildroot} %endif exit 0 -%post api -/sbin/ldconfig - %if ( 0%{!?_without_events:1} ) %post events %systemd_post glustereventsd @@ -864,13 +995,31 @@ exit 0 %if ( 0%{!?_without_georeplication:1} ) %post geo-replication +%if ( 0%{?rhel} && 0%{?rhel} >= 8 ) +%selinux_set_booleans %{selinuxbooleans} +%endif if [ $1 -ge 1 ]; then %systemd_postun_with_restart glusterd fi exit 0 %endif -%post libs +%post -n libglusterfs0 +/sbin/ldconfig + +%post -n libgfapi0 +/sbin/ldconfig + +%post -n libgfchangelog0 +/sbin/ldconfig + +%post -n libgfrpc0 +/sbin/ldconfig + +%post -n libgfxdr0 +/sbin/ldconfig + +%post -n libglusterd0 /sbin/ldconfig %if ( 0%{!?_without_server:1} ) @@ -1116,21 +1265,6 @@ exit 0 %exclude %{_prefix}/lib/ocf/resource.d/heartbeat/* %endif -%files api -%exclude %{_libdir}/*.so -# libgfapi files -%{_libdir}/libgfapi.* -%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator -%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount - %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api.so - -%files api-devel -%{_libdir}/pkgconfig/glusterfs-api.pc -%{_libdir}/libgfapi.so -%dir %{_includedir}/glusterfs -%dir %{_includedir}/glusterfs/api - %{_includedir}/glusterfs/api/* - %files cli %{_sbindir}/gluster %{_mandir}/man8/gluster.8* @@ -1141,14 +1275,33 @@ exit 0 %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsyncs3.so %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsynccvlt.so -%files devel +%files -n libglusterfs-devel %dir %{_includedir}/glusterfs - %{_includedir}/glusterfs/* -%exclude %{_includedir}/glusterfs/api -%exclude %{_libdir}/libgfapi.so -%{_libdir}/*.so + %{_includedir}/glusterfs/*.h + %{_includedir}/glusterfs/server/*.h +%{_libdir}/libglusterfs.so + +%files -n libgfapi-devel +%dir %{_includedir}/glusterfs/api + %{_includedir}/glusterfs/api/*.h +%{_libdir}/libgfapi.so +%{_libdir}/pkgconfig/glusterfs-api.pc + + +%files -n libgfchangelog-devel +%dir %{_includedir}/glusterfs/gfchangelog + %{_includedir}/glusterfs/gfchangelog/*.h +%{_libdir}/libgfchangelog.so %{_libdir}/pkgconfig/libgfchangelog.pc +%files -n libgfrpc-devel +%dir %{_includedir}/glusterfs/rpc + %{_includedir}/glusterfs/rpc/*.h +%{_libdir}/libgfrpc.so + +%files -n libgfxdr-devel +%{_libdir}/libgfxdr.so + %files client-xlators %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator %dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster @@ -1235,9 +1388,26 @@ exit 0 %endif -%files libs -%{_libdir}/*.so.* -%exclude %{_libdir}/libgfapi.* +%files -n libglusterfs0 +%{_libdir}/libglusterfs.so.* + +%files -n libgfapi0 +%{_libdir}/libgfapi.so.* +%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount + %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api.so + +%files -n libgfchangelog0 +%{_libdir}/libgfchangelog.so.* + +%files -n libgfrpc0 +%{_libdir}/libgfrpc.so.* + +%files -n libgfxdr0 +%{_libdir}/libgfxdr.so.* + +%files -n libglusterd0 +%{_libdir}/libglusterd.so.* +%exclude %{_libdir}/libglusterd.so %files -n python%{_pythonver}-gluster # introducing glusterfs module in site packages. @@ -1451,6 +1621,12 @@ exit 0 %endif %changelog +* Thu May 14 2020 Kaleb S. KEITHLEY <kkeithle@redhat.com> +- refactor, common practice, Issue #1126 + +* Mon May 11 2020 Sunny Kumar <sunkumar@redhat.com> +- added requires policycoreutils-python-utils on rhel8 for geo-replication + * Wed Oct 9 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com> - remove leftover bd xlator cruft diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am index 7b8d1dbf1fb..a0a778158d8 100644 --- a/glusterfsd/src/Makefile.am +++ b/glusterfsd/src/Makefile.am @@ -6,14 +6,15 @@ endif glusterfsd_SOURCES = glusterfsd.c glusterfsd-mgmt.c glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ - $(top_builddir)/rpc/xdr/src/libgfxdr.la ${GF_LDADD} -glusterfsd_LDFLAGS = $(GF_LDFLAGS) $(LIB_DL) + $(top_builddir)/rpc/xdr/src/libgfxdr.la $(GF_LDADD) $(LIB_DL) +glusterfsd_LDFLAGS = $(GF_LDFLAGS) gf_attach_SOURCES = gf_attach.c gf_attach_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ $(top_builddir)/api/src/libgfapi.la \ $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ $(top_builddir)/rpc/xdr/src/libgfxdr.la +gf_attach_LDFLAGS = $(GF_LDFLAGS) noinst_HEADERS = glusterfsd.h glusterfsd-mem-types.h glusterfsd-messages.h diff --git a/glusterfsd/src/gf_attach.c b/glusterfsd/src/gf_attach.c index cb6c0f59b0b..c553b0b1f61 100644 --- a/glusterfsd/src/gf_attach.c +++ b/glusterfsd/src/gf_attach.c @@ -19,9 +19,16 @@ #include "xdr-generic.h" #include "glusterd1-xdr.h" +/* In seconds */ +#define CONNECT_TIMEOUT 60 +#define REPLY_TIMEOUT 120 + int done = 0; int rpc_status; +pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + struct rpc_clnt_procedure gf_attach_actors[GLUSTERD_BRICK_MAXVALUE] = { [GLUSTERD_BRICK_NULL] = {"NULL", NULL}, [GLUSTERD_BRICK_OP] = {"BRICK_OP", NULL}, @@ -38,8 +45,12 @@ struct rpc_clnt_program gf_attach_prog = { int32_t my_callback(struct rpc_req *req, struct iovec *iov, int count, void *frame) { + pthread_mutex_lock(&mutex); rpc_status = req->rpc_status; done = 1; + /* Signal main thread which is the only waiter */ + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mutex); return 0; } @@ -48,6 +59,7 @@ int send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op) { int ret = -1; + struct timespec ts; struct iobuf *iobuf = NULL; struct iobref *iobref = NULL; struct iovec iov = { @@ -57,7 +69,6 @@ send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op) call_frame_t *frame = NULL; gd1_mgmt_brick_op_req brick_req; void *req = &brick_req; - int i; brick_req.op = op; brick_req.name = path; @@ -87,12 +98,20 @@ send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op) iov.iov_len = ret; - for (i = 0; i < 60; ++i) { - if (rpc->conn.connected) { - break; - } - sleep(1); + /* Wait for connection */ + timespec_now_realtime(&ts); + ts.tv_sec += CONNECT_TIMEOUT; + pthread_mutex_lock(&rpc->conn.lock); + { + while (!rpc->conn.connected) + if (pthread_cond_timedwait(&rpc->conn.cond, &rpc->conn.lock, &ts) == + ETIMEDOUT) { + fprintf(stderr, "timeout waiting for RPC connection\n"); + pthread_mutex_unlock(&rpc->conn.lock); + return EXIT_FAILURE; + } } + pthread_mutex_unlock(&rpc->conn.lock); frame = create_frame(this, this->ctx->pool); if (!frame) { @@ -104,9 +123,19 @@ send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op) ret = rpc_clnt_submit(rpc, &gf_attach_prog, op, my_callback, &iov, 1, NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL); if (!ret) { - for (i = 0; !done && (i < 120); ++i) { - sleep(1); + /* OK, wait for callback */ + timespec_now_realtime(&ts); + ts.tv_sec += REPLY_TIMEOUT; + pthread_mutex_lock(&mutex); + { + while (!done) + if (pthread_cond_timedwait(&cond, &mutex, &ts) == ETIMEDOUT) { + fprintf(stderr, "timeout waiting for RPC reply\n"); + pthread_mutex_unlock(&mutex); + return EXIT_FAILURE; + } } + pthread_mutex_unlock(&mutex); } out: diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index 45431450185..eaf6796e4c3 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -205,6 +205,7 @@ glusterfs_serialize_reply(rpcsvc_request_t *req, void *arg, retlen = xdr_serialize_generic(*outmsg, arg, xdrproc); if (retlen == -1) { gf_log(THIS->name, GF_LOG_ERROR, "Failed to encode message"); + GF_FREE(iob); goto ret; } @@ -628,7 +629,7 @@ glusterfs_volume_top_perf(const char *brick_path, dict_t *dict, goto out; } - time = (end.tv_sec - begin.tv_sec) * 1e6 + (end.tv_usec - begin.tv_usec); + time = gf_tvdiff(&begin, &end); throughput = total_blks / time; gf_log("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs " @@ -684,7 +685,7 @@ glusterfs_volume_top_perf(const char *brick_path, dict_t *dict, goto out; } - time = (end.tv_sec - begin.tv_sec) * 1e6 + (end.tv_usec - begin.tv_usec); + time = gf_tvdiff(&begin, &end); throughput = total_blks / time; gf_log("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs " diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index e192795baac..dae41f33fef 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -47,12 +47,6 @@ #include <malloc.h> #endif -#ifdef HAVE_MALLOC_STATS -#ifdef DEBUG -#include <mcheck.h> -#endif -#endif - #include <glusterfs/xlator.h> #include <glusterfs/glusterfs.h> #include <glusterfs/compat.h> @@ -1693,6 +1687,10 @@ glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx) INIT_LIST_HEAD(&cmd_args->xlator_options); INIT_LIST_HEAD(&cmd_args->volfile_servers); + ctx->pxl_count = 0; + pthread_mutex_init(&ctx->fd_lock, NULL); + pthread_cond_init(&ctx->fd_cond, NULL); + INIT_LIST_HEAD(&ctx->janitor_fds); lim.rlim_cur = RLIM_INFINITY; lim.rlim_max = RLIM_INFINITY; @@ -1981,7 +1979,7 @@ parse_cmdline(int argc, char *argv[], glusterfs_ctx_t *ctx) struct stat stbuf = { 0, }; - char timestr[32]; + char timestr[GF_TIMESTR_SIZE]; char tmp_logfile[1024] = {0}; char *tmp_logfile_dyn = NULL; char *tmp_logfilebase = NULL; @@ -2085,8 +2083,8 @@ parse_cmdline(int argc, char *argv[], glusterfs_ctx_t *ctx) if (((ret == 0) && (S_ISREG(stbuf.st_mode) || S_ISLNK(stbuf.st_mode))) || (ret == -1)) { - /* Have separate logfile per run */ - gf_time_fmt(timestr, sizeof timestr, time(NULL), gf_timefmt_FT); + /* Have separate logfile per run. */ + gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT); sprintf(tmp_logfile, "%s.%s.%d", cmd_args->log_file, timestr, getpid()); diff --git a/heal/src/glfs-heal.c b/heal/src/glfs-heal.c index ef53dfc0ccb..bf4b47f8760 100644 --- a/heal/src/glfs-heal.c +++ b/heal/src/glfs-heal.c @@ -1055,6 +1055,10 @@ glfsh_set_heal_options(glfs_t *fs, gf_xl_afr_op_t heal_op) if (ret) goto out; + ret = glfs_set_xlator_option(fs, "*-replicate-*", "halo-enabled", "off"); + if (ret) + goto out; + if ((heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) && (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) && (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME)) diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am index cd9d28e3cdc..385e8ef4600 100644 --- a/libglusterfs/src/Makefile.am +++ b/libglusterfs/src/Makefile.am @@ -12,7 +12,8 @@ libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \ -DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel \ -I$(CONTRIBDIR)/xxhash -libglusterfs_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS) +libglusterfs_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS) $(LIB_DL) \ + $(URCU_LIBS) $(URCU_CDS_LIBS) libglusterfs_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) $(GF_LDFLAGS) \ -export-symbols $(top_srcdir)/libglusterfs/src/libglusterfs.sym diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c index 4cd22f4920c..9d377c3c2e1 100644 --- a/libglusterfs/src/client_t.c +++ b/libglusterfs/src/client_t.c @@ -314,8 +314,6 @@ client_destroy(client_t *client) clienttable = client->this->ctx->clienttable; - LOCK_DESTROY(&client->scratch_ctx.lock); - LOCK(&clienttable->lock); { clienttable->cliententries[client->tbl_index].client = NULL; @@ -333,6 +331,8 @@ client_destroy(client_t *client) if (client->subdir_inode) inode_unref(client->subdir_inode); + LOCK_DESTROY(&client->scratch_ctx.lock); + GF_FREE(client->auth.data); GF_FREE(client->auth.username); GF_FREE(client->auth.passwd); diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c index 585952d7641..682cbf28055 100644 --- a/libglusterfs/src/common-utils.c +++ b/libglusterfs/src/common-utils.c @@ -37,6 +37,9 @@ #ifndef GF_LINUX_HOST_OS #include <sys/resource.h> #endif +#ifdef HAVE_SYNCFS_SYS +#include <sys/syscall.h> +#endif #include "glusterfs/compat-errno.h" #include "glusterfs/common-utils.h" @@ -50,6 +53,7 @@ #include "xxhash.h" #include <ifaddrs.h> #include "glusterfs/libglusterfs-messages.h" +#include "glusterfs/glusterfs-acl.h" #ifdef __FreeBSD__ #include <pthread_np.h> #undef BIT_SET @@ -75,6 +79,15 @@ char *vol_type_str[] = { typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size); typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size); +char *xattrs_to_heal[] = {"user.", + POSIX_ACL_ACCESS_XATTR, + POSIX_ACL_DEFAULT_XATTR, + QUOTA_LIMIT_KEY, + QUOTA_LIMIT_OBJECTS_KEY, + GF_SELINUX_XATTR_KEY, + GF_XATTR_MDATA_KEY, + NULL}; + void gf_xxh64_wrapper(const unsigned char *data, size_t const len, unsigned long long const seed, char *xxh64) @@ -425,7 +438,7 @@ gf_resolve_path_parent(const char *path) GF_VALIDATE_OR_GOTO(THIS->name, path, out); - if (strlen(path) <= 0) { + if (0 == strlen(path)) { gf_msg_callingfn(THIS->name, GF_LOG_DEBUG, 0, LG_MSG_INVALID_STRING, "invalid string for 'path'"); goto out; @@ -564,8 +577,14 @@ struct dnscache * gf_dnscache_init(time_t ttl) { struct dnscache *cache = GF_MALLOC(sizeof(*cache), gf_common_mt_dnscache); - if (cache) { - cache->cache_dict = NULL; + if (!cache) + return NULL; + + cache->cache_dict = dict_new(); + if (!cache->cache_dict) { + GF_FREE(cache); + cache = NULL; + } else { cache->ttl = ttl; } @@ -573,6 +592,20 @@ gf_dnscache_init(time_t ttl) } /** + * gf_dnscache_deinit -- cleanup resources used by struct dnscache + */ +void +gf_dnscache_deinit(struct dnscache *cache) +{ + if (!cache) { + gf_msg_plain(GF_LOG_WARNING, "dnscache is NULL"); + return; + } + dict_unref(cache->cache_dict); + GF_FREE(cache); +} + +/** * gf_dnscache_entry_init -- Initialize a dnscache entry * * @return: SUCCESS: Pointer to an allocated dnscache entry struct @@ -620,12 +653,6 @@ gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache) if (!dnscache) goto out; - if (!dnscache->cache_dict) { - dnscache->cache_dict = dict_new(); - if (!dnscache->cache_dict) { - goto out; - } - } cache = dnscache->cache_dict; /* Quick cache lookup to see if we already hold it */ @@ -633,7 +660,7 @@ gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache) if (entrydata) { dnsentry = (struct dnscache_entry *)entrydata->data; /* First check the TTL & timestamp */ - if (time(NULL) - dnsentry->timestamp > dnscache->ttl) { + if (gf_time() - dnsentry->timestamp > dnscache->ttl) { gf_dnscache_entry_deinit(dnsentry); entrydata->data = NULL; /* Mark this as 'null' so * dict_del () doesn't try free @@ -664,23 +691,16 @@ gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache) from_cache = _gf_false; out: /* Insert into the cache */ - if (fqdn && !from_cache) { + if (fqdn && !from_cache && ip) { struct dnscache_entry *entry = gf_dnscache_entry_init(); - if (!entry) { - goto out; - } - entry->fqdn = fqdn; - if (!ip) { - gf_dnscache_entry_deinit(entry); - goto out; + if (entry) { + entry->fqdn = fqdn; + entry->ip = gf_strdup(ip); + entry->timestamp = gf_time(); + entrydata = bin_to_data(entry, sizeof(*entry)); + dict_set(cache, (char *)ip, entrydata); } - - entry->ip = gf_strdup(ip); - entry->timestamp = time(NULL); - - entrydata = bin_to_data(entry, sizeof(*entry)); - dict_set(cache, (char *)ip, entrydata); } return fqdn; } @@ -889,7 +909,7 @@ gf_print_trace(int32_t signum, glusterfs_ctx_t *ctx) char msg[1024] = { 0, }; - char timestr[64] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; call_stack_t *stack = NULL; @@ -929,7 +949,7 @@ gf_print_trace(int32_t signum, glusterfs_ctx_t *ctx) { /* Dump the timestamp of the crash too, so the previous logs can be related */ - gf_time_fmt(timestr, sizeof timestr, time(NULL), gf_timefmt_FT); + gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT); gf_msg_plain_nomem(GF_LOG_ALERT, "time of crash: "); gf_msg_plain_nomem(GF_LOG_ALERT, timestr); } @@ -3113,7 +3133,7 @@ get_mem_size() memsize = page_size * num_pages; #endif -#if defined GF_DARWIN_HOST_OS +#if defined GF_DARWIN_HOST_OS || defined __FreeBSD__ size_t len = sizeof(memsize); int name[] = {CTL_HW, HW_PHYSMEM}; @@ -4127,6 +4147,14 @@ gf_skip_header_section(int fd, int header_len) gf_boolean_t gf_is_pid_running(int pid) { +#ifdef __FreeBSD__ + int ret = -1; + + ret = sys_kill(pid, 0); + if (ret < 0) { + return _gf_false; + } +#else char fname[32] = { 0, }; @@ -4140,6 +4168,7 @@ gf_is_pid_running(int pid) } sys_close(fd); +#endif return _gf_true; } @@ -4381,7 +4410,7 @@ gf_backtrace_end(char *buf, size_t frames) frames = min(frames, GF_BACKTRACE_LEN - pos - 1); - if (frames <= 0) + if (0 == frames) return; memset(buf + pos, ')', frames); @@ -4704,8 +4733,9 @@ recursive_rmdir(const char *delete_path) goto out; } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name); ret = sys_lstat(path, &st); if (ret == -1) { @@ -4731,8 +4761,6 @@ recursive_rmdir(const char *delete_path) gf_msg_debug(this->name, 0, "%s %s", ret ? "Failed to remove" : "Removed", entry->d_name); - - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } ret = sys_closedir(dir); @@ -5412,3 +5440,26 @@ gf_nanosleep(uint64_t nsec) return ret; } + +int +gf_syncfs(int fd) +{ + int ret = 0; +#if defined(HAVE_SYNCFS) + /* Linux with glibc recent enough. */ + ret = syncfs(fd); +#elif defined(HAVE_SYNCFS_SYS) + /* Linux with no library function. */ + ret = syscall(SYS_syncfs, fd); +#else + /* Fallback to generic UNIX stuff. */ + sync(); +#endif + return ret; +} + +char ** +get_xattrs_to_heal() +{ + return xattrs_to_heal; +} diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c index 4a001c29209..3d890b04ec9 100644 --- a/libglusterfs/src/ctx.c +++ b/libglusterfs/src/ctx.c @@ -37,8 +37,12 @@ glusterfs_ctx_new() ctx->log.loglevel = DEFAULT_LOG_LEVEL; -#ifdef RUN_WITH_VALGRIND - ctx->cmd_args.valgrind = _gf_true; +#if defined(RUN_WITH_MEMCHECK) + ctx->cmd_args.vgtool = _gf_memcheck; +#elif defined(RUN_WITH_DRD) + ctx->cmd_args.vgtool = _gf_drd; +#else + ctx->cmd_args.vgtool = _gf_none; #endif /* lock is never destroyed! */ diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c index a1a6ad12527..1d9be9217a6 100644 --- a/libglusterfs/src/dict.c +++ b/libglusterfs/src/dict.c @@ -98,6 +98,7 @@ get_new_dict_full(int size_hint) } dict->free_pair.key = NULL; + dict->totkvlen = 0; LOCK_INIT(&dict->lock); return dict; @@ -410,6 +411,7 @@ dict_set_lk(dict_t *this, char *key, const int key_len, data_t *value, if (pair) { data_t *unref_data = pair->value; pair->value = data_ref(value); + this->totkvlen += (value->len - unref_data->len); data_unref(unref_data); if (key_free) GF_FREE(key); @@ -445,6 +447,7 @@ dict_set_lk(dict_t *this, char *key, const int key_len, data_t *value, } pair->key_hash = key_hash; pair->value = data_ref(value); + this->totkvlen += (keylen + 1 + value->len); /* If the divisor is 1, the modulo is always 0, * in such case avoid hash calculation. @@ -642,6 +645,7 @@ dict_deln(dict_t *this, char *key, const int keylen) else this->members[hashval] = pair->hash_next; + this->totkvlen -= pair->value->len; data_unref(pair->value); if (pair->prev) @@ -652,6 +656,7 @@ dict_deln(dict_t *this, char *key, const int keylen) if (pair->next) pair->next->prev = pair->prev; + this->totkvlen -= (strlen(pair->key) + 1); GF_FREE(pair->key); if (pair == &this->free_pair) { this->free_pair.key = NULL; @@ -701,6 +706,7 @@ dict_destroy(dict_t *this) prev = pair; } + this->totkvlen = 0; if (this->members != &this->members_internal) { mem_put(this->members); } @@ -1105,117 +1111,146 @@ data_to_int64(data_t *data) { VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); - return (int64_t)strtoull(data->data, NULL, 0); + char *endptr = NULL; + int64_t value = 0; + + errno = 0; + value = strtoll(data->data, &endptr, 0); + + if (endptr && *endptr != '\0') + /* Unrecognized characters at the end of string. */ + errno = EINVAL; + if (errno) { + gf_msg_callingfn("dict", GF_LOG_WARNING, errno, + LG_MSG_DATA_CONVERSION_ERROR, + "Error in data conversion: '%s' can't " + "be represented as int64_t", + data->data); + return -1; + } + return value; } +/* Like above but implies signed range check. */ + +#define DATA_TO_RANGED_SIGNED(endptr, value, data, type, min, max) \ + do { \ + errno = 0; \ + value = strtoll(data->data, &endptr, 0); \ + if (endptr && *endptr != '\0') \ + errno = EINVAL; \ + if (errno || value > max || value < min) { \ + gf_msg_callingfn("dict", GF_LOG_WARNING, errno, \ + LG_MSG_DATA_CONVERSION_ERROR, \ + "Error in data conversion: '%s' can't " \ + "be represented as " #type, \ + data->data); \ + return -1; \ + } \ + return (type)value; \ + } while (0) + int32_t data_to_int32(data_t *data) { - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); + char *endptr = NULL; + int64_t value = 0; - return strtoul(data->data, NULL, 0); + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); + DATA_TO_RANGED_SIGNED(endptr, value, data, int32_t, INT_MIN, INT_MAX); } int16_t data_to_int16(data_t *data) { - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); + char *endptr = NULL; + int64_t value = 0; - int16_t value = 0; - - errno = 0; - value = strtol(data->data, NULL, 0); - - if ((value > SHRT_MAX) || (value < SHRT_MIN)) { - errno = ERANGE; - gf_msg_callingfn("dict", GF_LOG_WARNING, errno, - LG_MSG_DATA_CONVERSION_ERROR, - "Error in data" - " conversion: detected overflow"); - return -1; - } - - return (int16_t)value; + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); + DATA_TO_RANGED_SIGNED(endptr, value, data, int16_t, SHRT_MIN, SHRT_MAX); } int8_t data_to_int8(data_t *data) { + char *endptr = NULL; + int64_t value = 0; + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1); + DATA_TO_RANGED_SIGNED(endptr, value, data, int8_t, CHAR_MIN, CHAR_MAX); +} + +uint64_t +data_to_uint64(data_t *data) +{ + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); - int8_t value = 0; + char *endptr = NULL; + uint64_t value = 0; errno = 0; - value = strtol(data->data, NULL, 0); + value = strtoull(data->data, &endptr, 0); - if ((value > SCHAR_MAX) || (value < SCHAR_MIN)) { - errno = ERANGE; + if (endptr && *endptr != '\0') + errno = EINVAL; + if (errno) { gf_msg_callingfn("dict", GF_LOG_WARNING, errno, LG_MSG_DATA_CONVERSION_ERROR, - "Error in data" - " conversion: detected overflow"); + "Error in data conversion: '%s' can't " + "be represented as uint64_t", + data->data); return -1; } - - return (int8_t)value; + return value; } -uint64_t -data_to_uint64(data_t *data) -{ - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); +/* Like above but implies unsigned range check. */ - return strtoll(data->data, NULL, 0); -} +#define DATA_TO_RANGED_UNSIGNED(endptr, value, data, type, max) \ + do { \ + errno = 0; \ + value = strtoull(data->data, &endptr, 0); \ + if (endptr && *endptr != '\0') \ + errno = EINVAL; \ + if (errno || value > max) { \ + gf_msg_callingfn("dict", GF_LOG_WARNING, errno, \ + LG_MSG_DATA_CONVERSION_ERROR, \ + "Error in data conversion: '%s' can't " \ + "be represented as " #type, \ + data->data); \ + return -1; \ + } \ + return (type)value; \ + } while (0) uint32_t data_to_uint32(data_t *data) { - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); + char *endptr = NULL; + uint64_t value = 0; - return strtol(data->data, NULL, 0); + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); + DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint32_t, UINT_MAX); } uint16_t data_to_uint16(data_t *data) { - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); - - uint16_t value = 0; - - errno = 0; - value = strtol(data->data, NULL, 0); + char *endptr = NULL; + uint64_t value = 0; - if ((USHRT_MAX - value) < 0) { - errno = ERANGE; - gf_msg_callingfn("dict", GF_LOG_WARNING, errno, - LG_MSG_DATA_CONVERSION_ERROR, - "Error in data conversion: " - "overflow detected"); - return -1; - } - - return (uint16_t)value; + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); + DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint16_t, USHRT_MAX); } uint8_t data_to_uint8(data_t *data) { - VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); - - errno = 0; - uint32_t value = strtol(data->data, NULL, 0); - - if ((UCHAR_MAX - (uint8_t)value) < 0) { - errno = ERANGE; - gf_msg_callingfn("dict", GF_LOG_WARNING, errno, - LG_MSG_DATA_CONVERSION_ERROR, - "data " - "conversion overflow detected"); - return -1; - } + char *endptr = NULL; + uint64_t value = 0; - return (uint8_t)value; + VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1); + DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint8_t, UCHAR_MAX); } char * @@ -1460,32 +1495,13 @@ fail: * -val error, val = errno */ -int -dict_get_with_ref(dict_t *this, char *key, data_t **data) -{ - if (!this || !key || !data) { - gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG, - "dict OR key (%s) is NULL", key); - return -EINVAL; - } - - return dict_get_with_refn(this, key, strlen(key), data); -} - -int +static int dict_get_with_refn(dict_t *this, char *key, const int keylen, data_t **data) { data_pair_t *pair = NULL; int ret = -ENOENT; uint32_t hash; - if (!this || !key || !data) { - gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG, - "dict OR key (%s) is NULL", key); - ret = -EINVAL; - goto err; - } - hash = (uint32_t)XXH64(key, keylen, 0); LOCK(&this->lock); @@ -1498,10 +1514,22 @@ dict_get_with_refn(dict_t *this, char *key, const int keylen, data_t **data) } } UNLOCK(&this->lock); -err: + return ret; } +int +dict_get_with_ref(dict_t *this, char *key, data_t **data) +{ + if (!this || !key || !data) { + gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG, + "dict OR key (%s) is NULL", key); + return -EINVAL; + } + + return dict_get_with_refn(this, key, strlen(key), data); +} + static int data_to_ptr_common(data_t *data, void **val) { @@ -1675,7 +1703,7 @@ dict_get_int8(dict_t *this, char *key, int8_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1721,7 +1749,7 @@ dict_get_int16(dict_t *this, char *key, int16_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1793,7 +1821,7 @@ dict_get_int32(dict_t *this, char *key, int32_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1858,7 +1886,7 @@ dict_get_int64(dict_t *this, char *key, int64_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1903,7 +1931,7 @@ dict_get_uint16(dict_t *this, char *key, uint16_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1948,7 +1976,7 @@ dict_get_uint32(dict_t *this, char *key, uint32_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -1993,7 +2021,7 @@ dict_get_uint64(dict_t *this, char *key, uint64_t *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -2146,7 +2174,7 @@ _dict_modify_flag(dict_t *this, char *key, int flag, int op) strcpy(pair->key, key); pair->key_hash = hash; pair->value = data_ref(data); - + this->totkvlen += (strlen(key) + 1 + data->len); hashval = hash % this->hash_size; pair->hash_next = this->members[hashval]; this->members[hashval] = pair; @@ -2216,7 +2244,7 @@ dict_get_double(dict_t *this, char *key, double *val) data_t *data = NULL; int ret = 0; - if (!this || !key || !val) { + if (!val) { ret = -EINVAL; goto err; } @@ -2299,7 +2327,7 @@ dict_get_ptr(dict_t *this, char *key, void **ptr) data_t *data = NULL; int ret = 0; - if (!this || !key || !ptr) { + if (!ptr) { ret = -EINVAL; goto err; } @@ -2329,7 +2357,7 @@ dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len) data_t *data = NULL; int ret = 0; - if (!this || !key || !ptr) { + if (!ptr) { ret = -EINVAL; goto err; } @@ -2387,7 +2415,7 @@ dict_get_str(dict_t *this, char *key, char **str) data_t *data = NULL; int ret = -EINVAL; - if (!this || !key || !str) { + if (!str) { goto err; } ret = dict_get_with_ref(this, key, &data); @@ -2561,7 +2589,7 @@ dict_get_bin(dict_t *this, char *key, void **bin) data_t *data = NULL; int ret = -EINVAL; - if (!this || !key || !bin) { + if (!bin) { goto err; } @@ -2664,7 +2692,7 @@ dict_get_gfuuid(dict_t *this, char *key, uuid_t *gfid) data_t *data = NULL; int ret = -EINVAL; - if (!this || !key || !gfid) { + if (!gfid) { goto err; } ret = dict_get_with_ref(this, key, &data); @@ -2697,7 +2725,7 @@ dict_get_mdata(dict_t *this, char *key, struct mdata_iatt *mdata) data_t *data = NULL; int ret = -EINVAL; - if (!this || !key || !mdata) { + if (!mdata) { goto err; } ret = dict_get_with_ref(this, key, &data); @@ -2735,7 +2763,7 @@ dict_get_iatt(dict_t *this, char *key, struct iatt *iatt) data_t *data = NULL; int ret = -EINVAL; - if (!this || !key || !iatt) { + if (!iatt) { goto err; } ret = dict_get_with_ref(this, key, &data); @@ -2872,8 +2900,7 @@ dict_serialized_length_lk(dict_t *this) { int ret = -EINVAL; int count = this->count; - int len = DICT_HDR_LEN; - data_pair_t *pair = this->members_list; + const int keyhdrlen = DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN; if (count < 0) { gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_COUNT_LESS_THAN_ZERO, @@ -2881,41 +2908,7 @@ dict_serialized_length_lk(dict_t *this) goto out; } - while (count) { - if (!pair) { - gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_PAIRS_LESS_THAN_COUNT, - NULL); - goto out; - } - - len += DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN; - - if (!pair->key) { - gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_NULL_PTR, NULL); - goto out; - } - - len += strlen(pair->key) + 1 /* for '\0' */; - - if (!pair->value) { - gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_NULL_PTR, NULL); - goto out; - } - - if (pair->value->len < 0) { - gf_smsg("dict", GF_LOG_ERROR, EINVAL, - LG_MSG_VALUE_LENGTH_LESS_THAN_ZERO, "len=%d", - pair->value->len, NULL); - goto out; - } - - len += pair->value->len; - - pair = pair->next; - count--; - } - - ret = len; + ret = DICT_HDR_LEN + this->totkvlen + (count * keyhdrlen); out: return ret; } diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c index 235128b6044..402c253ca25 100644 --- a/libglusterfs/src/event.c +++ b/libglusterfs/src/event.c @@ -17,6 +17,7 @@ #include <string.h> #include "glusterfs/gf-event.h" +#include "glusterfs/timespec.h" #include "glusterfs/common-utils.h" #include "glusterfs/libglusterfs-messages.h" #include "glusterfs/syscall.h" @@ -266,7 +267,7 @@ gf_event_dispatch_destroy(struct event_pool *event_pool) if (sys_write(fd[1], "dummy", 6) == -1) { break; } - clock_gettime(CLOCK_REALTIME, &sleep_till); + timespec_now_realtime(&sleep_till); sleep_till.tv_sec += 1; ret = pthread_cond_timedwait(&event_pool->cond, &event_pool->mutex, &sleep_till); diff --git a/libglusterfs/src/events.c b/libglusterfs/src/events.c index 6d1e3836477..33157549897 100644 --- a/libglusterfs/src/events.c +++ b/libglusterfs/src/events.c @@ -40,6 +40,7 @@ _gf_event(eventtypes_t event, const char *fmt, ...) char *host = NULL; struct addrinfo hints; struct addrinfo *result = NULL; + struct addrinfo *iter_result_ptr = NULL; xlator_t *this = THIS; char *volfile_server_transport = NULL; @@ -51,13 +52,6 @@ _gf_event(eventtypes_t event, const char *fmt, ...) goto out; } - /* Initialize UDP socket */ - sock = socket(AF_INET, SOCK_DGRAM, 0); - if (sock < 0) { - ret = EVENT_ERROR_SOCKET; - goto out; - } - if (ctx) { volfile_server_transport = ctx->cmd_args.volfile_server_transport; } @@ -66,7 +60,6 @@ _gf_event(eventtypes_t event, const char *fmt, ...) } /* host = NULL returns localhost */ - host = NULL; if (ctx && ctx->cmd_args.volfile_server && (strcmp(volfile_server_transport, "unix"))) { /* If it is client code then volfile_server is set @@ -84,6 +77,24 @@ _gf_event(eventtypes_t event, const char *fmt, ...) goto out; } + // iterate over the result and break when socket creation is success. + for (iter_result_ptr = result; iter_result_ptr != NULL; + iter_result_ptr = iter_result_ptr->ai_next) { + sock = socket(iter_result_ptr->ai_family, iter_result_ptr->ai_socktype, + iter_result_ptr->ai_protocol); + if (sock != -1) { + break; + } + } + /* + * If none of the addrinfo structures lead to a successful socket + * creation, socket creation has failed. + */ + if (sock < 0) { + ret = EVENT_ERROR_SOCKET; + goto out; + } + va_start(arguments, fmt); ret = gf_vasprintf(&msg, fmt, arguments); va_end(arguments); @@ -93,7 +104,7 @@ _gf_event(eventtypes_t event, const char *fmt, ...) goto out; } - ret = gf_asprintf(&eventstr, "%u %d %s", (unsigned)time(NULL), event, msg); + ret = gf_asprintf(&eventstr, "%u %d %s", (unsigned)gf_time(), event, msg); GF_FREE(msg); if (ret <= 0) { ret = EVENT_ERROR_MSG_FORMAT; diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c index e0767a7e61f..62606e91164 100644 --- a/libglusterfs/src/fd.c +++ b/libglusterfs/src/fd.c @@ -502,6 +502,32 @@ out: } void +fd_close(fd_t *fd) +{ + xlator_t *xl, *old_THIS; + + old_THIS = THIS; + + for (xl = fd->inode->table->xl->graph->first; xl != NULL; xl = xl->next) { + if (!xl->call_cleanup) { + THIS = xl; + + if (IA_ISDIR(fd->inode->ia_type)) { + if (xl->cbks->fdclosedir != NULL) { + xl->cbks->fdclosedir(xl, fd); + } + } else { + if (xl->cbks->fdclose != NULL) { + xl->cbks->fdclose(xl, fd); + } + } + } + } + + THIS = old_THIS; +} + +void fd_unref(fd_t *fd) { int32_t refcount = 0; diff --git a/libglusterfs/src/gf-dirent.c b/libglusterfs/src/gf-dirent.c index 9c8c74beb54..a809efc97ef 100644 --- a/libglusterfs/src/gf-dirent.c +++ b/libglusterfs/src/gf-dirent.c @@ -276,7 +276,7 @@ gf_fill_iatt_for_dirent(gf_dirent_t *entry, inode_t *parent, xlator_t *subvol) gf_uuid_copy(loc.pargfid, parent->gfid); loc.name = entry->d_name; loc.parent = inode_ref(parent); - ret = inode_path(loc.inode, entry->d_name, &path); + ret = inode_path(loc.parent, entry->d_name, &path); loc.path = path; if (ret < 0) goto out; diff --git a/libglusterfs/src/gidcache.c b/libglusterfs/src/gidcache.c index 40fcffbb35e..64a93802f76 100644 --- a/libglusterfs/src/gidcache.c +++ b/libglusterfs/src/gidcache.c @@ -10,6 +10,7 @@ #include "glusterfs/gidcache.h" #include "glusterfs/mem-pool.h" +#include "glusterfs/common-utils.h" /* * We treat this as a very simple set-associative LRU cache, with entries aged @@ -64,7 +65,7 @@ gid_cache_lookup(gid_cache_t *cache, uint64_t id, uint64_t uid, uint64_t gid) time_t now; const gid_list_t *agl; - now = time(NULL); + now = gf_time(); LOCK(&cache->gc_lock); bucket = id % cache->gc_nbuckets; agl = BUCKET_START(cache->gc_cache, bucket); @@ -132,7 +133,7 @@ gid_cache_add(gid_cache_t *cache, gid_list_t *gl) if (!cache->gc_max_age) return 0; - now = time(NULL); + now = gf_time(); LOCK(&cache->gc_lock); /* diff --git a/libglusterfs/src/glusterfs/common-utils.h b/libglusterfs/src/glusterfs/common-utils.h index dd73a240c59..f297fdab5c9 100644 --- a/libglusterfs/src/glusterfs/common-utils.h +++ b/libglusterfs/src/glusterfs/common-utils.h @@ -18,6 +18,7 @@ #include <string.h> #include <assert.h> #include <pthread.h> +#include <unistd.h> #include <openssl/md5.h> #ifndef GF_BSD_HOST_OS #include <alloca.h> @@ -26,6 +27,11 @@ #include <fnmatch.h> #include <uuid/uuid.h> +/* FreeBSD, etc. */ +#ifndef __BITS_PER_LONG +#define __BITS_PER_LONG (CHAR_BIT * (sizeof(long))) +#endif + #ifndef ffsll #define ffsll(x) __builtin_ffsll(x) #endif @@ -148,6 +154,9 @@ trap(void); #define GF_THREAD_NAME_LIMIT 16 #define GF_THREAD_NAME_PREFIX "glfs_" +/* Advisory buffer size for formatted timestamps (see gf_time_fmt) */ +#define GF_TIMESTR_SIZE 256 + /* * we could have initialized these as +ve values and treated * them as negative while comparing etc.. (which would have @@ -167,7 +176,8 @@ enum _gf_special_pid { GF_CLIENT_PID_SCRUB = -9, GF_CLIENT_PID_TIER_DEFRAG = -10, GF_SERVER_PID_TRASH = -11, - GF_CLIENT_PID_ADD_REPLICA_MOUNT = -12 + GF_CLIENT_PID_ADD_REPLICA_MOUNT = -12, + GF_CLIENT_PID_SET_UTIME = -13, }; enum _gf_xlator_ipc_targets { @@ -179,6 +189,12 @@ enum _gf_xlator_ipc_targets { typedef enum _gf_special_pid gf_special_pid_t; typedef enum _gf_xlator_ipc_targets _gf_xlator_ipc_targets_t; +/* Array to hold custom xattr keys */ +extern char *xattrs_to_heal[]; + +char ** +get_xattrs_to_heal(); + /* The DHT file rename operation is not a straightforward rename. * It involves creating linkto and linkfiles, and can unlink or rename the * source file depending on the hashed and cached subvols for the source @@ -239,6 +255,8 @@ list_node_del(struct list_node *node); struct dnscache * gf_dnscache_init(time_t ttl); +void +gf_dnscache_deinit(struct dnscache *cache); struct dnscache_entry * gf_dnscache_entry_init(void); void @@ -437,6 +455,15 @@ BIT_VALUE(unsigned char *array, unsigned int index) } while (0) #endif +/* Compile-time assert, borrowed from Linux kernel. */ +#ifdef HAVE_STATIC_ASSERT +#define GF_STATIC_ASSERT(expr, ...) \ + __gf_static_assert(expr, ##__VA_ARGS__, #expr) +#define __gf_static_assert(expr, msg, ...) _Static_assert(expr, msg) +#else +#define GF_STATIC_ASSERT(expr, ...) +#endif + #define GF_ABORT(msg...) \ do { \ gf_msg_callingfn("", GF_LOG_CRITICAL, 0, LG_MSG_ASSERTION_FAILED, \ @@ -467,18 +494,15 @@ union gf_sock_union { #define IOV_MIN(n) min(IOV_MAX, n) -#define GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scr) \ - do { \ - entry = NULL; \ - if (dir) { \ - entry = sys_readdir(dir, scr); \ - while (entry && (!strcmp(entry->d_name, ".") || \ - !fnmatch("*.tmp", entry->d_name, 0) || \ - !strcmp(entry->d_name, ".."))) { \ - entry = sys_readdir(dir, scr); \ - } \ - } \ - } while (0) +static inline gf_boolean_t +gf_irrelevant_entry(struct dirent *entry) +{ + GF_ASSERT(entry); + + return (!strcmp(entry->d_name, ".") || + !fnmatch("*.tmp", entry->d_name, 0) || + !strcmp(entry->d_name, "..")); +} static inline void iov_free(struct iovec *vector, int count) @@ -789,7 +813,7 @@ typedef enum { } gf_timefmts; static inline char * -gf_time_fmt(char *dst, size_t sz_dst, time_t utime, unsigned int fmt) +gf_time_fmt_tv(char *dst, size_t sz_dst, struct timeval *tv, unsigned int fmt) { extern void _gf_timestuff(const char ***, const char ***); static gf_timefmts timefmt_last = (gf_timefmts)-1; @@ -797,6 +821,8 @@ gf_time_fmt(char *dst, size_t sz_dst, time_t utime, unsigned int fmt) static const char **zeros; struct tm tm, *res; int localtime = 0; + int len = 0; + int pos = 0; if (timefmt_last == ((gf_timefmts)-1)) { _gf_timestuff(&fmts, &zeros); @@ -806,15 +832,35 @@ gf_time_fmt(char *dst, size_t sz_dst, time_t utime, unsigned int fmt) fmt = gf_timefmt_default; } localtime = gf_log_get_localtime(); - res = localtime ? localtime_r(&utime, &tm) : gmtime_r(&utime, &tm); - if (utime && (res != NULL)) { - strftime(dst, sz_dst, fmts[fmt], &tm); + res = localtime ? localtime_r(&tv->tv_sec, &tm) + : gmtime_r(&tv->tv_sec, &tm); + if (tv->tv_sec && (res != NULL)) { + len = strftime(dst, sz_dst, fmts[fmt], &tm); + if (len == 0) + return dst; + pos += len; + if (tv->tv_usec >= 0) { + len = snprintf(dst + pos, sz_dst - pos, ".%" GF_PRI_SUSECONDS, + tv->tv_usec); + if (len >= sz_dst - pos) + return dst; + pos += len; + } + strftime(dst + pos, sz_dst - pos, " %z", &tm); } else { strncpy(dst, "N/A", sz_dst); } return dst; } +static inline char * +gf_time_fmt(char *dst, size_t sz_dst, time_t utime, unsigned int fmt) +{ + struct timeval tv = {utime, -1}; + + return gf_time_fmt_tv(dst, sz_dst, &tv, fmt); +} + /* This function helps us use gfid (unique identity) to generate inode's unique * number in glusterfs. */ @@ -1164,6 +1210,47 @@ int gf_d_type_from_ia_type(ia_type_t type); int +gf_syncfs(int fd); + +int gf_nanosleep(uint64_t nsec); +static inline time_t +gf_time(void) +{ + return time(NULL); +} + +/* Return delta value in microseconds. */ + +static inline double +gf_tvdiff(struct timeval *start, struct timeval *end) +{ + struct timeval t; + + if (start->tv_usec > end->tv_usec) + t.tv_sec = end->tv_sec - 1, t.tv_usec = end->tv_usec + 1000000; + else + t.tv_sec = end->tv_sec, t.tv_usec = end->tv_usec; + + return (double)(t.tv_sec - start->tv_sec) * 1e6 + + (double)(t.tv_usec - start->tv_usec); +} + +/* Return delta value in nanoseconds. */ + +static inline double +gf_tsdiff(struct timespec *start, struct timespec *end) +{ + struct timespec t; + + if (start->tv_nsec > end->tv_nsec) + t.tv_sec = end->tv_sec - 1, t.tv_nsec = end->tv_nsec + 1000000000; + else + t.tv_sec = end->tv_sec, t.tv_nsec = end->tv_nsec; + + return (double)(t.tv_sec - start->tv_sec) * 1e9 + + (double)(t.tv_nsec - start->tv_nsec); +} + #endif /* _COMMON_UTILS_H */ diff --git a/libglusterfs/src/glusterfs/dict.h b/libglusterfs/src/glusterfs/dict.h index ce1c2100276..d0467c6dfb6 100644 --- a/libglusterfs/src/glusterfs/dict.h +++ b/libglusterfs/src/glusterfs/dict.h @@ -25,9 +25,6 @@ typedef struct _data_pair data_pair_t; #define dict_add_sizen(this, key, value) dict_addn(this, key, SLEN(key), value) -#define dict_get_with_ref_sizen(this, key, value) \ - dict_get_with_refn(this, key, SLEN(key), value) - #define dict_get_sizen(this, key) dict_getn(this, key, SLEN(key)) #define dict_del_sizen(this, key) dict_deln(this, key, SLEN(key)) @@ -98,7 +95,7 @@ struct _data { char *data; gf_atomic_t refcount; gf_dict_data_type_t data_type; - int32_t len; + uint32_t len; gf_boolean_t is_static; }; @@ -122,6 +119,8 @@ struct _dict { gf_lock_t lock; data_pair_t *members_internal; data_pair_t free_pair; + /* Variable to store total keylen + value->len */ + uint32_t totkvlen; }; typedef gf_boolean_t (*dict_match_t)(dict_t *d, char *k, data_t *v, void *data); @@ -136,6 +135,7 @@ int32_t dict_set(dict_t *this, char *key, data_t *value); int32_t dict_setn(dict_t *this, char *key, const int keylen, data_t *value); + /* function to set a new key/value pair (without checking for duplicate) */ int32_t dict_add(dict_t *this, char *key, data_t *value); @@ -143,8 +143,6 @@ int32_t dict_addn(dict_t *this, char *key, const int keylen, data_t *value); int dict_get_with_ref(dict_t *this, char *key, data_t **data); -int -dict_get_with_refn(dict_t *this, char *key, const int keylen, data_t **data); data_t * dict_get(dict_t *this, char *key); data_t * diff --git a/libglusterfs/src/glusterfs/fd.h b/libglusterfs/src/glusterfs/fd.h index 28906d34e4d..3ffaaa60504 100644 --- a/libglusterfs/src/glusterfs/fd.h +++ b/libglusterfs/src/glusterfs/fd.h @@ -106,6 +106,9 @@ fd_ref(fd_t *fd); void fd_unref(fd_t *fd); +void +fd_close(fd_t *fd); + fd_t * fd_create(struct _inode *inode, pid_t pid); diff --git a/libglusterfs/src/glusterfs/gf-event.h b/libglusterfs/src/glusterfs/gf-event.h index c0f05e7c83b..40f8fbdf10a 100644 --- a/libglusterfs/src/glusterfs/gf-event.h +++ b/libglusterfs/src/glusterfs/gf-event.h @@ -12,6 +12,7 @@ #define _GF_EVENT_H_ #include <pthread.h> +#include "common-utils.h" #include "list.h" struct event_pool; @@ -31,6 +32,9 @@ typedef void (*event_handler_t)(int fd, int idx, int gen, void *data, #define EVENT_EPOLL_SLOTS 1024 #define EVENT_MAX_THREADS 1024 +/* See rpcsvc.h to check why. */ +GF_STATIC_ASSERT(EVENT_MAX_THREADS % __BITS_PER_LONG == 0); + struct event_pool { struct event_ops *ops; diff --git a/libglusterfs/src/glusterfs/globals.h b/libglusterfs/src/glusterfs/globals.h index e50ee3046af..b22eaae6c2f 100644 --- a/libglusterfs/src/glusterfs/globals.h +++ b/libglusterfs/src/glusterfs/globals.h @@ -45,7 +45,7 @@ 1 /* MIN is the fresh start op-version, mostly \ should not change */ #define GD_OP_VERSION_MAX \ - GD_OP_VERSION_8_0 /* MAX VERSION is the maximum \ + GD_OP_VERSION_9_0 /* MAX VERSION is the maximum \ count in VME table, should \ keep changing with \ introduction of newer \ @@ -122,6 +122,8 @@ #define GD_OP_VERSION_8_0 80000 /* Op-version for GlusterFS 8.0 */ +#define GD_OP_VERSION_9_0 90000 /* Op-version for GlusterFS 9.0 */ + #define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0 #include "glusterfs/xlator.h" diff --git a/libglusterfs/src/glusterfs/glusterfs-acl.h b/libglusterfs/src/glusterfs/glusterfs-acl.h index cae55e8062f..987bf5fab0b 100644 --- a/libglusterfs/src/glusterfs/glusterfs-acl.h +++ b/libglusterfs/src/glusterfs/glusterfs-acl.h @@ -143,7 +143,7 @@ gf_posix_acl_get_key(const acl_type_t type) return acl_key; } -static inline const acl_type_t +static inline acl_type_t gf_posix_acl_get_type(const char *key) { acl_type_t type = 0; diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index bbb66d264eb..e6425618b7f 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -43,6 +43,9 @@ #define GF_YES 1 #define GF_NO 0 +#define IS_ERROR(ret) ((ret) < 0) +#define IS_SUCCESS(ret) ((ret) >= 0) + #ifndef O_LARGEFILE /* savannah bug #20053, patch for compiling on darwin */ #define O_LARGEFILE 0100000 /* from bits/fcntl.h */ @@ -422,7 +425,7 @@ static const char *const FOP_PRI_STRINGS[] = {"HIGH", "NORMAL", "LOW", "LEAST"}; static inline const char * fop_pri_to_string(gf_fop_pri_t pri) { - if (pri < 0) + if (IS_ERROR(pri)) return "UNSPEC"; if (pri >= GF_FOP_PRI_MAX) @@ -463,6 +466,8 @@ typedef struct _server_cmdline server_cmdline_t; #define GF_OPTION_DISABLE _gf_false #define GF_OPTION_DEFERRED 2 +typedef enum { _gf_none, _gf_memcheck, _gf_drd } gf_valgrind_tool; + struct _cmd_args { /* basic options */ char *volfile_server; @@ -555,7 +560,8 @@ struct _cmd_args { /* Run this process with valgrind? Might want to prevent calling * functions that prevent valgrind from working correctly, like * dlclose(). */ - int valgrind; + gf_valgrind_tool vgtool; + int localtime_logging; /* For the subdir mount */ @@ -727,12 +733,13 @@ struct _glusterfs_ctx { } stats; struct list_head volfile_list; - /* Add members to manage janitor threads for cleanup fd */ struct list_head janitor_fds; - pthread_cond_t janitor_cond; - pthread_mutex_t janitor_lock; + pthread_cond_t fd_cond; + pthread_mutex_t fd_lock; pthread_t janitor; + /* The variable is use to save total posix xlator count */ + uint32_t pxl_count; char volume_id[GF_UUID_BUF_SIZE]; /* Used only in protocol/client */ }; diff --git a/libglusterfs/src/glusterfs/latency.h b/libglusterfs/src/glusterfs/latency.h index ed47b1f0cbc..4d601bbcbd6 100644 --- a/libglusterfs/src/glusterfs/latency.h +++ b/libglusterfs/src/glusterfs/latency.h @@ -11,13 +11,23 @@ #ifndef __LATENCY_H__ #define __LATENCY_H__ -#include "glusterfs/glusterfs.h" +#include <inttypes.h> +#include <time.h> -typedef struct fop_latency { - double min; /* min time for the call (microseconds) */ - double max; /* max time for the call (microseconds) */ - double total; /* total time (microseconds) */ +typedef struct _gf_latency { + uint64_t min; /* min time for the call (nanoseconds) */ + uint64_t max; /* max time for the call (nanoseconds) */ + uint64_t total; /* total time (nanoseconds) */ uint64_t count; -} fop_latency_t; +} gf_latency_t; +gf_latency_t * +gf_latency_new(size_t n); + +void +gf_latency_reset(gf_latency_t *lat); + +void +gf_latency_update(gf_latency_t *lat, struct timespec *begin, + struct timespec *end); #endif /* __LATENCY_H__ */ diff --git a/libglusterfs/src/glusterfs/mem-pool.h b/libglusterfs/src/glusterfs/mem-pool.h index 90fb8820c74..e5b3276d047 100644 --- a/libglusterfs/src/glusterfs/mem-pool.h +++ b/libglusterfs/src/glusterfs/mem-pool.h @@ -202,6 +202,24 @@ out: return dup_mem; } +#ifdef GF_DISABLE_MEMPOOL + +/* No-op memory pool enough to fit current API without massive redesign. */ + +struct mem_pool { + unsigned long sizeof_type; +}; + +#define mem_pools_init() \ + do { \ + } while (0) +#define mem_pools_fini() \ + do { \ + } while (0) +#define mem_pool_thread_destructor(pool_list) (void)pool_list + +#else /* !GF_DISABLE_MEMPOOL */ + /* kind of 'header' for the actual mem_pool_shared structure, this might make * it possible to dump some more details in a statedump */ struct mem_pool { @@ -209,10 +227,11 @@ struct mem_pool { unsigned long sizeof_type; unsigned long count; /* requested pool size (unused) */ char *name; - gf_atomic_t active; /* current allocations */ + char *xl_name; + gf_atomic_t active; /* current allocations */ #ifdef DEBUG - gf_atomic_t hit; /* number of allocations served from pt_pool */ - gf_atomic_t miss; /* number of std allocs due to miss */ + gf_atomic_t hit; /* number of allocations served from pt_pool */ + gf_atomic_t miss; /* number of std allocs due to miss */ #endif struct list_head owner; /* glusterfs_ctx_t->mempool_list */ glusterfs_ctx_t *ctx; /* take ctx->lock when updating owner */ @@ -286,6 +305,10 @@ void mem_pools_init(void); /* start the pool_sweeper thread */ void mem_pools_fini(void); /* cleanup memory pools */ +void +mem_pool_thread_destructor(per_thread_pool_list_t *pool_list); + +#endif /* GF_DISABLE_MEMPOOL */ struct mem_pool * mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type, @@ -308,9 +331,6 @@ void mem_pool_destroy(struct mem_pool *pool); void -mem_pool_thread_destructor(per_thread_pool_list_t *pool_list); - -void gf_mem_acct_enable_set(void *ctx); #endif /* _MEM_POOL_H */ diff --git a/libglusterfs/src/glusterfs/mem-types.h b/libglusterfs/src/glusterfs/mem-types.h index 36cf7820ad5..d45d5b68c91 100644 --- a/libglusterfs/src/glusterfs/mem-types.h +++ b/libglusterfs/src/glusterfs/mem-types.h @@ -133,6 +133,7 @@ enum gf_common_mem_types_ { gf_common_volfile_t, gf_common_mt_mgmt_v3_lock_timer_t, /* used only in one location */ gf_common_mt_server_cmdline_t, /* used only in one location */ + gf_common_mt_latency_t, gf_common_mt_end }; #endif diff --git a/libglusterfs/src/glusterfs/stack.h b/libglusterfs/src/glusterfs/stack.h index 17585508a22..536a330d38b 100644 --- a/libglusterfs/src/glusterfs/stack.h +++ b/libglusterfs/src/glusterfs/stack.h @@ -45,6 +45,9 @@ typedef int32_t (*ret_fn_t)(call_frame_t *frame, call_frame_t *prev_frame, xlator_t *this, int32_t op_ret, int32_t op_errno, ...); +void +gf_frame_latency_update(call_frame_t *frame); + struct call_pool { union { struct list_head all_frames; @@ -149,8 +152,6 @@ struct _call_stack { } while (0); struct xlator_fops; -void -gf_update_latency(call_frame_t *frame); static inline void FRAME_DESTROY(call_frame_t *frame) @@ -158,7 +159,7 @@ FRAME_DESTROY(call_frame_t *frame) void *local = NULL; if (frame->root->ctx->measure_latency) - gf_update_latency(frame); + gf_frame_latency_update(frame); list_del_init(&frame->frames); if (frame->local) { @@ -429,6 +430,7 @@ call_stack_alloc_groups(call_stack_t *stack, int ngrps) if (ngrps <= SMALL_GROUP_COUNT) { stack->groups = stack->groups_small; } else { + GF_FREE(stack->groups_large); stack->groups_large = GF_CALLOC(ngrps, sizeof(gid_t), gf_common_mt_groups_t); if (!stack->groups_large) @@ -442,6 +444,12 @@ call_stack_alloc_groups(call_stack_t *stack, int ngrps) } static inline int +call_stack_groups_capacity(call_stack_t *stack) +{ + return max(stack->ngrps, SMALL_GROUP_COUNT); +} + +static inline int call_frames_count(call_stack_t *call_stack) { call_frame_t *pos; diff --git a/libglusterfs/src/glusterfs/statedump.h b/libglusterfs/src/glusterfs/statedump.h index 89d04f94587..ce082706bdf 100644 --- a/libglusterfs/src/glusterfs/statedump.h +++ b/libglusterfs/src/glusterfs/statedump.h @@ -127,4 +127,6 @@ gf_proc_dump_xlator_meminfo(xlator_t *this, strfd_t *strfd); void gf_proc_dump_xlator_profile(xlator_t *this, strfd_t *strfd); +void +gf_latency_statedump_and_reset(char *key, gf_latency_t *lat); #endif /* STATEDUMP_H */ diff --git a/libglusterfs/src/glusterfs/store.h b/libglusterfs/src/glusterfs/store.h index c8be544e164..a1f70c7b840 100644 --- a/libglusterfs/src/glusterfs/store.h +++ b/libglusterfs/src/glusterfs/store.h @@ -95,7 +95,7 @@ int32_t gf_store_iter_get_matching(gf_store_iter_t *iter, char *key, char **value); int32_t -gf_store_iter_destroy(gf_store_iter_t *iter); +gf_store_iter_destroy(gf_store_iter_t **iter); char * gf_store_strerror(gf_store_op_errno_t op_errno); diff --git a/libglusterfs/src/glusterfs/syncop.h b/libglusterfs/src/glusterfs/syncop.h index 3614d969264..4e9241a32fc 100644 --- a/libglusterfs/src/glusterfs/syncop.h +++ b/libglusterfs/src/glusterfs/syncop.h @@ -16,6 +16,8 @@ #include <ucontext.h> #include "glusterfs/dict.h" // for dict_t #include "glusterfs/stack.h" // for call_frame_t, STACK_DESTROY, STACK_... +#include "glusterfs/timer.h" + #define SYNCENV_PROC_MAX 16 #define SYNCENV_PROC_MIN 2 #define SYNCPROC_IDLE_TIME 600 @@ -29,9 +31,15 @@ #define SYNCOPCTX_PID 0x00000008 #define SYNCOPCTX_LKOWNER 0x00000010 +#ifdef HAVE_TSAN_API +/* Currently hardcoded within thread context maintained by the sanitizer. */ +#define TSAN_THREAD_NAMELEN 64 +#endif + struct synctask; struct syncproc; struct syncenv; +struct synccond; typedef int (*synctask_cbk_t)(int ret, call_frame_t *frame, void *opaque); @@ -55,9 +63,12 @@ struct synctask { call_frame_t *opframe; synctask_cbk_t synccbk; synctask_fn_t syncfn; - synctask_state_t state; + struct timespec *delta; + gf_timer_t *timer; + struct synccond *synccond; void *opaque; void *stack; + synctask_state_t state; int woken; int slept; int ret; @@ -65,6 +76,13 @@ struct synctask { uid_t uid; gid_t gid; +#ifdef HAVE_TSAN_API + struct { + void *fiber; + char name[TSAN_THREAD_NAMELEN]; + } tsan; +#endif + ucontext_t ctx; struct syncproc *proc; @@ -77,6 +95,14 @@ struct synctask { struct syncproc { pthread_t processor; + +#ifdef HAVE_TSAN_API + struct { + void *fiber; + char name[TSAN_THREAD_NAMELEN]; + } tsan; +#endif + ucontext_t sched; struct syncenv *env; struct synctask *current; @@ -85,19 +111,21 @@ struct syncproc { /* hosts the scheduler thread and framework for executing synctasks */ struct syncenv { struct syncproc proc[SYNCENV_PROC_MAX]; - int procs; + + pthread_mutex_t mutex; + pthread_cond_t cond; struct list_head runq; - int runcount; struct list_head waitq; - int waitcount; + + int procs; + int procs_idle; + + int runcount; int procmin; int procmax; - pthread_mutex_t mutex; - pthread_cond_t cond; - size_t stacksize; int destroy; /* FLAG to mark syncenv is in destroy mode @@ -123,6 +151,13 @@ struct synclock { }; typedef struct synclock synclock_t; +struct synccond { + pthread_mutex_t pmutex; + pthread_cond_t pcond; + struct list_head waitq; +}; +typedef struct synccond synccond_t; + struct syncbarrier { gf_boolean_t initialized; /*Set on successful initialization*/ pthread_mutex_t guard; /* guard the remaining members, pair @cond */ @@ -219,7 +254,7 @@ struct syncopctx { #define __yield(args) \ do { \ if (args->task) { \ - synctask_yield(args->task); \ + synctask_yield(args->task, NULL); \ } else { \ pthread_mutex_lock(&args->mutex); \ { \ @@ -240,7 +275,7 @@ struct syncopctx { task = synctask_get(); \ stb->task = task; \ if (task) \ - frame = task->opframe; \ + frame = copy_frame(task->opframe); \ else \ frame = syncop_create_frame(THIS); \ \ @@ -261,10 +296,7 @@ struct syncopctx { STACK_WIND_COOKIE(frame, cbk, (void *)stb, subvol, fn_op, params); \ \ __yield(stb); \ - if (task) \ - STACK_RESET(frame->root); \ - else \ - STACK_DESTROY(frame->root); \ + STACK_DESTROY(frame->root); \ } while (0) /* @@ -313,7 +345,9 @@ synctask_join(struct synctask *task); void synctask_wake(struct synctask *task); void -synctask_yield(struct synctask *task); +synctask_yield(struct synctask *task, struct timespec *delta); +void +synctask_sleep(int32_t secs); void synctask_waitfor(struct synctask *task, int count); @@ -411,6 +445,24 @@ synclock_trylock(synclock_t *lock); int synclock_unlock(synclock_t *lock); +int32_t +synccond_init(synccond_t *cond); + +void +synccond_destroy(synccond_t *cond); + +int +synccond_wait(synccond_t *cond, synclock_t *lock); + +int +synccond_timedwait(synccond_t *cond, synclock_t *lock, struct timespec *delta); + +void +synccond_signal(synccond_t *cond); + +void +synccond_broadcast(synccond_t *cond); + int syncbarrier_init(syncbarrier_t *barrier); int diff --git a/libglusterfs/src/glusterfs/syscall.h b/libglusterfs/src/glusterfs/syscall.h index 6b33c141a5e..b6d3ab4f2ad 100644 --- a/libglusterfs/src/glusterfs/syscall.h +++ b/libglusterfs/src/glusterfs/syscall.h @@ -96,18 +96,27 @@ int sys_unlink(const char *pathname); int +sys_unlinkat(int dfd, const char *pathname); + +int sys_rmdir(const char *pathname); int sys_symlink(const char *oldpath, const char *newpath); int +sys_symlinkat(const char *oldpath, int dirfd, const char *newpath); + +int sys_rename(const char *oldpath, const char *newpath); int sys_link(const char *oldpath, const char *newpath); int +sys_linkat(int oldfd, const char *oldpath, int newfd, const char *newpath); + +int sys_chmod(const char *path, mode_t mode); int @@ -257,4 +266,13 @@ ssize_t sys_copy_file_range(int fd_in, off64_t *off_in, int fd_out, off64_t *off_out, size_t len, unsigned int flags); +int +sys_kill(pid_t pid, int sig); + +#ifdef __FreeBSD__ +int +sys_sysctl(const int *name, u_int namelen, void *oldp, size_t *oldlenp, + const void *newp, size_t newlen); +#endif + #endif /* __SYSCALL_H__ */ diff --git a/libglusterfs/src/glusterfs/xlator.h b/libglusterfs/src/glusterfs/xlator.h index c94bd75cf1c..4fd3abdaeff 100644 --- a/libglusterfs/src/glusterfs/xlator.h +++ b/libglusterfs/src/glusterfs/xlator.h @@ -702,6 +702,8 @@ typedef size_t (*cbk_inodectx_size_t)(xlator_t *this, inode_t *inode); typedef size_t (*cbk_fdctx_size_t)(xlator_t *this, fd_t *fd); +typedef void (*cbk_fdclose_t)(xlator_t *this, fd_t *fd); + struct xlator_cbks { cbk_forget_t forget; cbk_release_t release; @@ -712,6 +714,8 @@ struct xlator_cbks { cbk_ictxmerge_t ictxmerge; cbk_inodectx_size_t ictxsize; cbk_fdctx_size_t fdctxsize; + cbk_fdclose_t fdclose; + cbk_fdclose_t fdclosedir; }; typedef int32_t (*dumpop_priv_t)(xlator_t *this); @@ -801,7 +805,7 @@ struct _xlator { struct { /* for latency measurement */ - fop_latency_t latencies[GF_FOP_MAXVALUE]; + gf_latency_t latencies[GF_FOP_MAXVALUE]; /* for latency measurement */ fop_metrics_t metrics[GF_FOP_MAXVALUE]; diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c index 64e77e095a2..13f298eb3bd 100644 --- a/libglusterfs/src/graph.c +++ b/libglusterfs/src/graph.c @@ -41,7 +41,7 @@ _gf_dump_details (int argc, char **argv) { extern FILE *gf_log_logfile; int i = 0; - char timestr[64]; + char timestr[GF_TIMESTR_SIZE]; time_t utime = 0; pid_t mypid = 0; struct utsname uname_buf = {{0, }, }; @@ -482,7 +482,7 @@ fill_uuid(char *uuid, int size, struct timeval tv) char hostname[50] = { 0, }; - char now_str[64]; + char now_str[GF_TIMESTR_SIZE]; if (gethostname(hostname, sizeof(hostname) - 1) != 0) { gf_msg("graph", GF_LOG_ERROR, errno, LG_MSG_GETHOSTNAME_FAILED, @@ -490,9 +490,8 @@ fill_uuid(char *uuid, int size, struct timeval tv) hostname[sizeof(hostname) - 1] = '\0'; } - gf_time_fmt(now_str, sizeof now_str, tv.tv_sec, gf_timefmt_dirent); - snprintf(uuid, size, "%s-%d-%s:%" GF_PRI_SUSECONDS, hostname, getpid(), - now_str, tv.tv_usec); + gf_time_fmt_tv(now_str, sizeof now_str, &tv, gf_timefmt_dirent); + snprintf(uuid, size, "%s-%d-%s", hostname, getpid(), now_str); return; } diff --git a/libglusterfs/src/latency.c b/libglusterfs/src/latency.c index 15b397c3799..ce4b0e8255d 100644 --- a/libglusterfs/src/latency.c +++ b/libglusterfs/src/latency.c @@ -16,35 +16,32 @@ #include "glusterfs/glusterfs.h" #include "glusterfs/statedump.h" -void -gf_update_latency(call_frame_t *frame) +gf_latency_t * +gf_latency_new(size_t n) { - double elapsed; - struct timespec *begin, *end; - - fop_latency_t *lat; - - begin = &frame->begin; - end = &frame->end; + int i = 0; + gf_latency_t *lat = NULL; - if (!(begin->tv_sec && end->tv_sec)) - goto out; + lat = GF_MALLOC(n * sizeof(*lat), gf_common_mt_latency_t); + if (!lat) + return NULL; - elapsed = (end->tv_sec - begin->tv_sec) * 1e9 + - (end->tv_nsec - begin->tv_nsec); + for (i = 0; i < n; i++) { + gf_latency_reset(lat + i); + } + return lat; +} - if (frame->op < 0 || frame->op >= GF_FOP_MAXVALUE) { - gf_log("[core]", GF_LOG_WARNING, "Invalid frame op value: %d", - frame->op); +void +gf_latency_update(gf_latency_t *lat, struct timespec *begin, + struct timespec *end) +{ + if (!(begin->tv_sec && end->tv_sec)) { + /*Measure latency might have been enabled/disabled during the op*/ return; } - /* Can happen mostly at initiator xlator, as STACK_WIND/UNWIND macros - set it right anyways for those frames */ - if (!frame->op) - frame->op = frame->root->op; - - lat = &frame->this->stats.interval.latencies[frame->op]; + double elapsed = gf_tsdiff(begin, end); if (lat->max < elapsed) lat->max = elapsed; @@ -54,42 +51,34 @@ gf_update_latency(call_frame_t *frame) lat->total += elapsed; lat->count++; -out: - return; } void -gf_proc_dump_latency_info(xlator_t *xl) +gf_latency_reset(gf_latency_t *lat) { - char key_prefix[GF_DUMP_MAX_BUF_LEN]; - char key[GF_DUMP_MAX_BUF_LEN]; - int i; - - snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.latency", xl->name); - gf_proc_dump_add_section("%s", key_prefix); - - for (i = 0; i < GF_FOP_MAXVALUE; i++) { - gf_proc_dump_build_key(key, key_prefix, "%s", (char *)gf_fop_list[i]); - - fop_latency_t *lat = &xl->stats.interval.latencies[i]; + if (!lat) + return; + memset(lat, 0, sizeof(*lat)); + lat->min = ULLONG_MAX; + /* make sure 'min' is set to high value, so it would be + properly set later */ +} - /* Doesn't make sense to continue if there are no fops - came in the given interval */ - if (!lat->count) - continue; +void +gf_frame_latency_update(call_frame_t *frame) +{ + gf_latency_t *lat; + /* Can happen mostly at initiator xlator, as STACK_WIND/UNWIND macros + set it right anyways for those frames */ + if (!frame->op) + frame->op = frame->root->op; - gf_proc_dump_write( - key, "AVG:%.03f,CNT:%" PRId64 ",TOTAL:%.03f,MIN:%.03f,MAX:%.03f", - (lat->total / lat->count), lat->count, lat->total, lat->min, - lat->max); + if (frame->op < 0 || frame->op >= GF_FOP_MAXVALUE) { + gf_log("[core]", GF_LOG_WARNING, "Invalid frame op value: %d", + frame->op); + return; } - memset(xl->stats.interval.latencies, 0, - sizeof(xl->stats.interval.latencies)); - - /* make sure 'min' is set to high value, so it would be - properly set later */ - for (i = 0; i < GF_FOP_MAXVALUE; i++) { - xl->stats.interval.latencies[i].min = 0xffffffff; - } + lat = &frame->this->stats.interval.latencies[frame->op]; + gf_latency_update(lat, &frame->begin, &frame->end); } diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym index dc7382ba749..5f18cd56cbe 100644 --- a/libglusterfs/src/libglusterfs.sym +++ b/libglusterfs/src/libglusterfs.sym @@ -451,6 +451,7 @@ gf_event_unregister_close fd_anonymous fd_anonymous_with_flags fd_bind +fd_close fd_create fd_create_uint64 __fd_ctx_del @@ -584,6 +585,7 @@ gf_dirent_free gf_dirent_orig_offset gf_dm_hashfn gf_dnscache_init +gf_dnscache_deinit gf_errno_to_error gf_error_to_errno _gf_event @@ -937,6 +939,12 @@ syncbarrier_destroy syncbarrier_init syncbarrier_wait syncbarrier_wake +synccond_init +synccond_destroy +synccond_wait +synccond_timedwait +synccond_signal +synccond_broadcast syncenv_destroy syncenv_new synclock_destroy @@ -1014,6 +1022,7 @@ synctask_new synctask_new1 synctask_set synctask_setid +synctask_sleep synctask_wake synctask_yield sys_access @@ -1039,6 +1048,7 @@ sys_futimes sys_lchown sys_lgetxattr sys_link +sys_linkat sys_llistxattr sys_lremovexattr sys_lseek @@ -1062,13 +1072,17 @@ sys_rmdir sys_stat sys_statvfs sys_symlink +sys_symlinkat sys_truncate sys_unlink +sys_unlinkat sys_utimensat sys_write sys_writev sys_socket sys_accept +sys_kill +sys_sysctl tbf_init tbf_throttle timespec_now @@ -1169,4 +1183,11 @@ glusterfs_process_svc_detach mgmt_is_multiplexed_daemon xlator_is_cleanup_starting gf_nanosleep +gf_syncfs graph_total_client_xlator +get_xattrs_to_heal +gf_latency_statedump_and_reset +gf_latency_new +gf_latency_reset +gf_latency_update +gf_frame_latency_update diff --git a/libglusterfs/src/logging.c b/libglusterfs/src/logging.c index 3d7d31772df..a930d3e3b63 100644 --- a/libglusterfs/src/logging.c +++ b/libglusterfs/src/logging.c @@ -35,7 +35,6 @@ #define GF_LOG_CONTROL_FILE "/etc/glusterfs/logger.conf" #define GF_LOG_BACKTRACE_DEPTH 5 #define GF_LOG_BACKTRACE_SIZE 4096 -#define GF_LOG_TIMESTR_SIZE 256 #define GF_MAX_SLOG_PAIR_COUNT 100 #include "glusterfs/logging.h" @@ -760,7 +759,7 @@ _gf_log_callingfn(const char *domain, const char *file, const char *function, xlator_t *this = THIS; char *logline = NULL; char *msg = NULL; - char timestr[GF_LOG_TIMESTR_SIZE] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char *callstr = NULL; @@ -820,12 +819,12 @@ _gf_log_callingfn(const char *domain, const char *file, const char *function, if (-1 == ret) goto out; - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); - ret = gf_asprintf( - &logline, "[%s.%" GF_PRI_SUSECONDS "] %c [%s:%d:%s] %s %d-%s: %s\n", - timestr, tv.tv_usec, gf_level_strings[level], basename, line, function, - callstr, ((this->graph) ? this->graph->id : 0), domain, msg); + ret = gf_asprintf(&logline, "[%s] %c [%s:%d:%s] %s %d-%s: %s\n", timestr, + gf_level_strings[level], basename, line, function, + callstr, ((this->graph) ? this->graph->id : 0), domain, + msg); if (-1 == ret) { goto out; } @@ -1088,7 +1087,7 @@ _gf_msg_nomem(const char *domain, const char *file, const char *function, char msg[2048] = { 0, }; - char timestr[GF_LOG_TIMESTR_SIZE] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; glusterfs_ctx_t *ctx = NULL; @@ -1116,20 +1115,20 @@ _gf_msg_nomem(const char *domain, const char *file, const char *function, ret = gettimeofday(&tv, NULL); if (-1 == ret) goto out; - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); /* TODO: Currently we print in the enhanced format, with a message ID * of 0. Need to enhance this to support format as configured */ wlen = snprintf( msg, sizeof msg, - "[%s.%" GF_PRI_SUSECONDS "] %c [MSGID: %" PRIu64 + "[%s] %c [MSGID: %" PRIu64 "]" " [%s:%d:%s] %s: no memory " "available for size (%" GF_PRI_SIZET ") current memory usage in kilobytes %ld" " [call stack follows]\n", - timestr, tv.tv_usec, gf_level_strings[level], (uint64_t)0, basename, - line, function, domain, size, + timestr, gf_level_strings[level], (uint64_t)0, basename, line, function, + domain, size, (!getrusage(RUSAGE_SELF, &r_usage) ? r_usage.ru_maxrss : 0)); if (-1 == wlen) { ret = -1; @@ -1275,7 +1274,7 @@ gf_log_glusterlog(glusterfs_ctx_t *ctx, const char *domain, const char *file, int errnum, uint64_t msgid, char **appmsgstr, char *callstr, struct timeval tv, int graph_id, gf_log_format_t fmt) { - char timestr[GF_LOG_TIMESTR_SIZE] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char *header = NULL; @@ -1286,7 +1285,7 @@ gf_log_glusterlog(glusterfs_ctx_t *ctx, const char *domain, const char *file, gf_log_rotate(ctx); /* format the time stamp */ - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); /* generate footer */ if (errnum) { @@ -1302,40 +1301,35 @@ gf_log_glusterlog(glusterfs_ctx_t *ctx, const char *domain, const char *file, if (fmt == gf_logformat_traditional) { if (!callstr) { ret = gf_asprintf(&header, - "[%s.%" GF_PRI_SUSECONDS - "] %c [%s:%d:%s]" + "[%s] %c [%s:%d:%s]" " %d-%s: %s", - timestr, tv.tv_usec, gf_level_strings[level], - file, line, function, graph_id, domain, - *appmsgstr); + timestr, gf_level_strings[level], file, line, + function, graph_id, domain, *appmsgstr); } else { ret = gf_asprintf(&header, - "[%s.%" GF_PRI_SUSECONDS - "] %c [%s:%d:%s] %s" + "[%s] %c [%s:%d:%s] %s" " %d-%s: %s", - timestr, tv.tv_usec, gf_level_strings[level], - file, line, function, callstr, graph_id, domain, - *appmsgstr); + timestr, gf_level_strings[level], file, line, + function, callstr, graph_id, domain, *appmsgstr); } } else { /* gf_logformat_withmsgid */ /* CEE log format unsupported in logger_glusterlog, so just * print enhanced log format */ if (!callstr) { ret = gf_asprintf(&header, - "[%s.%" GF_PRI_SUSECONDS "] %c [MSGID: %" PRIu64 + "[%s] %c [MSGID: %" PRIu64 "]" " [%s:%d:%s] %d-%s: %s", - timestr, tv.tv_usec, gf_level_strings[level], - msgid, file, line, function, graph_id, domain, - *appmsgstr); + timestr, gf_level_strings[level], msgid, file, + line, function, graph_id, domain, *appmsgstr); } else { ret = gf_asprintf(&header, - "[%s.%" GF_PRI_SUSECONDS "] %c [MSGID: %" PRIu64 + "[%s] %c [MSGID: %" PRIu64 "]" " [%s:%d:%s] %s %d-%s: %s", - timestr, tv.tv_usec, gf_level_strings[level], - msgid, file, line, function, callstr, graph_id, - domain, *appmsgstr); + timestr, gf_level_strings[level], msgid, file, + line, function, callstr, graph_id, domain, + *appmsgstr); } } if (-1 == ret) { @@ -1386,39 +1380,36 @@ gf_syslog_log_repetitions(const char *domain, const char *file, int graph_id) { int priority; - char timestr_latest[GF_LOG_TIMESTR_SIZE] = { + char timestr_latest[GF_TIMESTR_SIZE] = { 0, }; - char timestr_oldest[GF_LOG_TIMESTR_SIZE] = { + char timestr_oldest[GF_TIMESTR_SIZE] = { 0, }; SET_LOG_PRIO(level, priority); - gf_time_fmt(timestr_latest, sizeof timestr_latest, latest.tv_sec, - gf_timefmt_FT); - gf_time_fmt(timestr_oldest, sizeof timestr_oldest, oldest.tv_sec, - gf_timefmt_FT); + gf_time_fmt_tv(timestr_latest, sizeof timestr_latest, &latest, + gf_timefmt_FT); + gf_time_fmt_tv(timestr_oldest, sizeof timestr_oldest, &oldest, + gf_timefmt_FT); if (errnum) { - syslog( - priority, - "The message \"[MSGID: %" PRIu64 - "] [%s:%d:%s] " - "%d-%s: %s [%s] \" repeated %d times between %s.%" GF_PRI_SUSECONDS - " and %s.%" GF_PRI_SUSECONDS, - msgid, file, line, function, graph_id, domain, *appmsgstr, - strerror(errnum), refcount, timestr_oldest, oldest.tv_usec, - timestr_latest, latest.tv_usec); + syslog(priority, + "The message \"[MSGID: %" PRIu64 + "] [%s:%d:%s] " + "%d-%s: %s [%s] \" repeated %d times between %s" + " and %s", + msgid, file, line, function, graph_id, domain, *appmsgstr, + strerror(errnum), refcount, timestr_oldest, timestr_latest); } else { syslog(priority, "The message \"[MSGID: %" PRIu64 "] [%s:%d:%s] " - "%d-%s: %s \" repeated %d times between %s.%" GF_PRI_SUSECONDS - " and %s.%" GF_PRI_SUSECONDS, + "%d-%s: %s \" repeated %d times between %s" + " and %s", msgid, file, line, function, graph_id, domain, *appmsgstr, - refcount, timestr_oldest, oldest.tv_usec, timestr_latest, - latest.tv_usec); + refcount, timestr_oldest, timestr_latest); } return 0; } @@ -1432,10 +1423,10 @@ gf_glusterlog_log_repetitions(glusterfs_ctx_t *ctx, const char *domain, struct timeval latest, int graph_id) { int ret = 0; - char timestr_latest[GF_LOG_TIMESTR_SIZE] = { + char timestr_latest[GF_TIMESTR_SIZE] = { 0, }; - char timestr_oldest[GF_LOG_TIMESTR_SIZE] = { + char timestr_oldest[GF_TIMESTR_SIZE] = { 0, }; char errstr[256] = { @@ -1459,21 +1450,17 @@ gf_glusterlog_log_repetitions(glusterfs_ctx_t *ctx, const char *domain, goto err; } - gf_time_fmt(timestr_latest, sizeof timestr_latest, latest.tv_sec, - gf_timefmt_FT); + gf_time_fmt_tv(timestr_latest, sizeof timestr_latest, &latest, + gf_timefmt_FT); - gf_time_fmt(timestr_oldest, sizeof timestr_oldest, oldest.tv_sec, - gf_timefmt_FT); + gf_time_fmt_tv(timestr_oldest, sizeof timestr_oldest, &oldest, + gf_timefmt_FT); if (errnum) snprintf(errstr, sizeof(errstr) - 1, " [%s]", strerror(errnum)); - ret = gf_asprintf(&footer, - "%s\" repeated %d times between" - " [%s.%" GF_PRI_SUSECONDS "] and [%s.%" GF_PRI_SUSECONDS - "]", - errstr, refcount, timestr_oldest, oldest.tv_usec, - timestr_latest, latest.tv_usec); + ret = gf_asprintf(&footer, "%s\" repeated %d times between [%s] and [%s]", + errstr, refcount, timestr_oldest, timestr_latest); if (-1 == ret) { ret = -1; goto err; @@ -2013,7 +2000,7 @@ _gf_log(const char *domain, const char *file, const char *function, int line, const char *basename = NULL; FILE *new_logfile = NULL; va_list ap; - char timestr[GF_LOG_TIMESTR_SIZE] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; struct timeval tv = { @@ -2100,12 +2087,11 @@ log: if (-1 == ret) goto out; - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); - ret = gf_asprintf( - &logline, "[%s.%" GF_PRI_SUSECONDS "] %c [%s:%d:%s] %d-%s: %s\n", - timestr, tv.tv_usec, gf_level_strings[level], basename, line, function, - ((this->graph) ? this->graph->id : 0), domain, msg); + ret = gf_asprintf(&logline, "[%s] %c [%s:%d:%s] %d-%s: %s\n", timestr, + gf_level_strings[level], basename, line, function, + ((this->graph) ? this->graph->id : 0), domain, msg); if (-1 == ret) { goto err; } @@ -2225,7 +2211,7 @@ int gf_cmd_log(const char *domain, const char *fmt, ...) { va_list ap; - char timestr[64]; + char timestr[GF_TIMESTR_SIZE]; struct timeval tv = { 0, }; @@ -2258,10 +2244,9 @@ gf_cmd_log(const char *domain, const char *fmt, ...) goto out; } - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); - ret = gf_asprintf(&logline, "[%s.%" GF_PRI_SUSECONDS "] %s : %s\n", timestr, - tv.tv_usec, domain, msg); + ret = gf_asprintf(&logline, "[%s] %s : %s\n", timestr, domain, msg); if (ret == -1) { goto out; } diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c index d77be55db5a..2d5a12b0a00 100644 --- a/libglusterfs/src/mem-pool.c +++ b/libglusterfs/src/mem-pool.c @@ -362,6 +362,30 @@ free: FREE(ptr); } +#if defined(GF_DISABLE_MEMPOOL) + +struct mem_pool * +mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type, + unsigned long count, char *name) +{ + struct mem_pool *new; + + new = GF_MALLOC(sizeof(struct mem_pool), gf_common_mt_mem_pool); + if (!new) + return NULL; + + new->sizeof_type = sizeof_type; + return new; +} + +void +mem_pool_destroy(struct mem_pool *pool) +{ + GF_FREE(pool); +} + +#else /* !GF_DISABLE_MEMPOOL */ + static pthread_mutex_t pool_lock = PTHREAD_MUTEX_INITIALIZER; static struct list_head pool_threads; static pthread_mutex_t pool_free_lock = PTHREAD_MUTEX_INITIALIZER; @@ -371,7 +395,6 @@ static size_t pool_list_size; static __thread per_thread_pool_list_t *thread_pool_list = NULL; -#if !defined(GF_DISABLE_MEMPOOL) #define N_COLD_LISTS 1024 #define POOL_SWEEP_SECS 30 @@ -550,25 +573,17 @@ mem_pools_preinit(void) static __attribute__((destructor)) void mem_pools_postfini(void) { - per_thread_pool_list_t *pool_list, *next; - - /* This is part of a process shutdown (or dlclose()) which means that - * most probably all threads should be stopped. However this is not the - * case for gluster and there are even legitimate situations in which we - * could have some threads alive. What is sure is that none of those - * threads should be using anything from this library, so destroying - * everything here should be fine and safe. */ - - list_for_each_entry_safe(pool_list, next, &pool_threads, thr_list) - { - mem_pool_thread_destructor(pool_list); - } - - list_for_each_entry_safe(pool_list, next, &pool_free_threads, thr_list) - { - list_del(&pool_list->thr_list); - FREE(pool_list); - } + /* TODO: This function should destroy all per thread memory pools that + * are still alive, but this is not possible right now because glibc + * starts calling destructors as soon as exit() is called, and + * gluster doesn't ensure that all threads have been stopped before + * calling exit(). Existing threads would crash when they try to use + * memory or they terminate if we destroy things here. + * + * When we propertly terminate all threads, we can add the needed + * code here. Till then we need to leave the memory allocated. Most + * probably this function will be executed on process termination, + * so the memory will be released anyway by the system. */ } /* Call mem_pools_init() once threading has been configured completely. This @@ -625,21 +640,29 @@ mem_pools_fini(void) pthread_mutex_unlock(&init_mutex); } -#else void -mem_pools_init(void) -{ -} -void -mem_pools_fini(void) -{ -} -void -mem_pool_thread_destructor(per_thread_pool_list_t *pool_list) +mem_pool_destroy(struct mem_pool *pool) { -} + if (!pool) + return; -#endif + /* remove this pool from the owner (glusterfs_ctx_t) */ + LOCK(&pool->ctx->lock); + { + list_del(&pool->owner); + } + UNLOCK(&pool->ctx->lock); + + /* free this pool, but keep the mem_pool_shared */ + GF_FREE(pool); + + /* + * Pools are now permanent, so the mem_pool->pool is kept around. All + * of the objects *in* the pool will eventually be freed via the + * pool-sweeper thread, and this way we don't have to add a lot of + * reference-counting complexity. + */ +} struct mem_pool * mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type, @@ -690,6 +713,7 @@ mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type, new->sizeof_type = sizeof_type; new->count = count; new->name = name; + new->xl_name = THIS->name; new->pool = pool; GF_ATOMIC_INIT(new->active, 0); #ifdef DEBUG @@ -707,21 +731,6 @@ mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type, return new; } -void * -mem_get0(struct mem_pool *mem_pool) -{ - void *ptr = mem_get(mem_pool); - if (ptr) { -#if defined(GF_DISABLE_MEMPOOL) - memset(ptr, 0, mem_pool->sizeof_type); -#else - memset(ptr, 0, AVAILABLE_SIZE(mem_pool->pool->power_of_two)); -#endif - } - - return ptr; -} - per_thread_pool_list_t * mem_get_pool_list(void) { @@ -830,6 +839,23 @@ mem_get_from_pool(struct mem_pool *mem_pool) return retval; } +#endif /* GF_DISABLE_MEMPOOL */ + +void * +mem_get0(struct mem_pool *mem_pool) +{ + void *ptr = mem_get(mem_pool); + if (ptr) { +#if defined(GF_DISABLE_MEMPOOL) + memset(ptr, 0, mem_pool->sizeof_type); +#else + memset(ptr, 0, AVAILABLE_SIZE(mem_pool->pool->power_of_two)); +#endif + } + + return ptr; +} + void * mem_get(struct mem_pool *mem_pool) { @@ -874,6 +900,14 @@ mem_put(void *ptr) /* Not one of ours; don't touch it. */ return; } + + if (!hdr->pool_list) { + gf_msg_callingfn("mem-pool", GF_LOG_CRITICAL, EINVAL, + LG_MSG_INVALID_ARG, + "invalid argument hdr->pool_list NULL"); + return; + } + pool_list = hdr->pool_list; pt_pool = &pool_list->pools[hdr->power_of_two - POOL_SMALLEST]; @@ -896,27 +930,3 @@ mem_put(void *ptr) } #endif /* GF_DISABLE_MEMPOOL */ } - -void -mem_pool_destroy(struct mem_pool *pool) -{ - if (!pool) - return; - - /* remove this pool from the owner (glusterfs_ctx_t) */ - LOCK(&pool->ctx->lock); - { - list_del(&pool->owner); - } - UNLOCK(&pool->ctx->lock); - - /* free this pool, but keep the mem_pool_shared */ - GF_FREE(pool); - - /* - * Pools are now permanent, so the mem_pool->pool is kept around. All - * of the objects *in* the pool will eventually be freed via the - * pool-sweeper thread, and this way we don't have to add a lot of - * reference-counting complexity. - */ -} diff --git a/libglusterfs/src/monitoring.c b/libglusterfs/src/monitoring.c index 45e3d776903..fbb68dc8622 100644 --- a/libglusterfs/src/monitoring.c +++ b/libglusterfs/src/monitoring.c @@ -113,15 +113,15 @@ dump_latency_and_count(xlator_t *xl, int fd) dprintf(fd, "%s.interval.%s.fail_count %" PRIu64 "\n", xl->name, gf_fop_list[index], cbk); } - if (xl->stats.interval.latencies[index].count != 0.0) { + if (xl->stats.interval.latencies[index].count != 0) { dprintf(fd, "%s.interval.%s.latency %lf\n", xl->name, gf_fop_list[index], - (xl->stats.interval.latencies[index].total / + (((double)xl->stats.interval.latencies[index].total) / xl->stats.interval.latencies[index].count)); - dprintf(fd, "%s.interval.%s.max %lf\n", xl->name, + dprintf(fd, "%s.interval.%s.max %" PRIu64 "\n", xl->name, gf_fop_list[index], xl->stats.interval.latencies[index].max); - dprintf(fd, "%s.interval.%s.min %lf\n", xl->name, + dprintf(fd, "%s.interval.%s.min %" PRIu64 "\n", xl->name, gf_fop_list[index], xl->stats.interval.latencies[index].min); } diff --git a/libglusterfs/src/stack.c b/libglusterfs/src/stack.c index 371f60c4436..1531f0da43f 100644 --- a/libglusterfs/src/stack.c +++ b/libglusterfs/src/stack.c @@ -92,7 +92,7 @@ gf_proc_dump_call_frame(call_frame_t *call_frame, const char *key_buf, ...) }; int ret = -1; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; int len; @@ -162,7 +162,7 @@ gf_proc_dump_call_stack(call_stack_t *call_stack, const char *key_buf, ...) va_list ap; call_frame_t *trav; int32_t i = 1, cnt = 0; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; int len; diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c index 5e0f04f3217..65f0eb5c7f3 100644 --- a/libglusterfs/src/statedump.c +++ b/libglusterfs/src/statedump.c @@ -199,6 +199,40 @@ gf_proc_dump_write(char *key, char *value, ...) return ret; } +void +gf_latency_statedump_and_reset(char *key, gf_latency_t *lat) +{ + /* Doesn't make sense to continue if there are no fops + came in the given interval */ + if (!lat || !lat->count) + return; + gf_proc_dump_write(key, + "AVG:%lf CNT:%" PRIu64 " TOTAL:%" PRIu64 " MIN:%" PRIu64 + " MAX:%" PRIu64, + (((double)lat->total) / lat->count), lat->count, + lat->total, lat->min, lat->max); + gf_latency_reset(lat); +} + +void +gf_proc_dump_xl_latency_info(xlator_t *xl) +{ + char key_prefix[GF_DUMP_MAX_BUF_LEN]; + char key[GF_DUMP_MAX_BUF_LEN]; + int i; + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.latency", xl->name); + gf_proc_dump_add_section("%s", key_prefix); + + for (i = 0; i < GF_FOP_MAXVALUE; i++) { + gf_proc_dump_build_key(key, key_prefix, "%s", (char *)gf_fop_list[i]); + + gf_latency_t *lat = &xl->stats.interval.latencies[i]; + + gf_latency_statedump_and_reset(key, lat); + } +} + static void gf_proc_dump_xlator_mem_info(xlator_t *xl) { @@ -270,7 +304,7 @@ gf_proc_dump_xlator_mem_info_only_in_use(xlator_t *xl) void gf_proc_dump_mem_info() { -#ifdef HAVE_MALLOC_STATS +#ifdef HAVE_MALLINFO struct mallinfo info; memset(&info, 0, sizeof(struct mallinfo)); @@ -296,7 +330,7 @@ gf_proc_dump_mem_info_to_dict(dict_t *dict) { if (!dict) return; -#ifdef HAVE_MALLOC_STATS +#ifdef HAVE_MALLINFO struct mallinfo info; int ret = -1; @@ -349,26 +383,13 @@ gf_proc_dump_mem_info_to_dict(dict_t *dict) void gf_proc_dump_mempool_info(glusterfs_ctx_t *ctx) { +#ifdef GF_DISABLE_MEMPOOL + gf_proc_dump_write("built with --disable-mempool", " so no memory pools"); +#else struct mem_pool *pool = NULL; gf_proc_dump_add_section("mempool"); -#if defined(OLD_MEM_POOLS) - list_for_each_entry(pool, &ctx->mempool_list, global_list) - { - gf_proc_dump_write("-----", "-----"); - gf_proc_dump_write("pool-name", "%s", pool->name); - gf_proc_dump_write("hot-count", "%d", pool->hot_count); - gf_proc_dump_write("cold-count", "%d", pool->cold_count); - gf_proc_dump_write("padded_sizeof", "%lu", pool->padded_sizeof_type); - gf_proc_dump_write("alloc-count", "%" PRIu64, pool->alloc_count); - gf_proc_dump_write("max-alloc", "%d", pool->max_alloc); - - gf_proc_dump_write("pool-misses", "%" PRIu64, pool->pool_misses); - gf_proc_dump_write("cur-stdalloc", "%d", pool->curr_stdalloc); - gf_proc_dump_write("max-stdalloc", "%d", pool->max_stdalloc); - } -#else LOCK(&ctx->lock); { list_for_each_entry(pool, &ctx->mempool_list, owner) @@ -377,6 +398,7 @@ gf_proc_dump_mempool_info(glusterfs_ctx_t *ctx) gf_proc_dump_write("-----", "-----"); gf_proc_dump_write("pool-name", "%s", pool->name); + gf_proc_dump_write("xlator-name", "%s", pool->xl_name); gf_proc_dump_write("active-count", "%" GF_PRI_ATOMIC, active); gf_proc_dump_write("sizeof-type", "%lu", pool->sizeof_type); gf_proc_dump_write("padded-sizeof", "%d", @@ -387,15 +409,13 @@ gf_proc_dump_mempool_info(glusterfs_ctx_t *ctx) } } UNLOCK(&ctx->lock); - - /* TODO: details of (struct mem_pool_shared) pool->pool */ -#endif +#endif /* GF_DISABLE_MEMPOOL */ } void gf_proc_dump_mempool_info_to_dict(glusterfs_ctx_t *ctx, dict_t *dict) { -#if defined(OLD_MEM_POOLS) +#ifndef GF_DISABLE_MEMPOOL struct mem_pool *pool = NULL; char key[GF_DUMP_MAX_BUF_LEN] = { 0, @@ -406,51 +426,47 @@ gf_proc_dump_mempool_info_to_dict(glusterfs_ctx_t *ctx, dict_t *dict) if (!ctx || !dict) return; - list_for_each_entry(pool, &ctx->mempool_list, global_list) + LOCK(&ctx->lock); { - snprintf(key, sizeof(key), "pool%d.name", count); - ret = dict_set_str(dict, key, pool->name); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.hotcount", count); - ret = dict_set_int32(dict, key, pool->hot_count); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.coldcount", count); - ret = dict_set_int32(dict, key, pool->cold_count); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.paddedsizeof", count); - ret = dict_set_uint64(dict, key, pool->padded_sizeof_type); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.alloccount", count); - ret = dict_set_uint64(dict, key, pool->alloc_count); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.max_alloc", count); - ret = dict_set_int32(dict, key, pool->max_alloc); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.max-stdalloc", count); - ret = dict_set_int32(dict, key, pool->max_stdalloc); - if (ret) - return; - - snprintf(key, sizeof(key), "pool%d.pool-misses", count); - ret = dict_set_uint64(dict, key, pool->pool_misses); - if (ret) - return; - count++; + list_for_each_entry(pool, &ctx->mempool_list, owner) + { + int64_t active = GF_ATOMIC_GET(pool->active); + + snprintf(key, sizeof(key), "pool%d.name", count); + ret = dict_set_str(dict, key, pool->name); + if (ret) + goto out; + + snprintf(key, sizeof(key), "pool%d.active-count", count); + ret = dict_set_uint64(dict, key, active); + if (ret) + goto out; + + snprintf(key, sizeof(key), "pool%d.sizeof-type", count); + ret = dict_set_uint64(dict, key, pool->sizeof_type); + if (ret) + goto out; + + snprintf(key, sizeof(key), "pool%d.padded-sizeof", count); + ret = dict_set_uint64(dict, key, 1 << pool->pool->power_of_two); + if (ret) + goto out; + + snprintf(key, sizeof(key), "pool%d.size", count); + ret = dict_set_uint64(dict, key, + (1 << pool->pool->power_of_two) * active); + if (ret) + goto out; + + snprintf(key, sizeof(key), "pool%d.shared-pool", count); + ret = dict_set_static_ptr(dict, key, pool->pool); + if (ret) + goto out; + } } - ret = dict_set_int32(dict, "mempool-count", count); -#endif +out: + UNLOCK(&ctx->lock); +#endif /* !GF_DISABLE_MEMPOOL */ } void @@ -485,7 +501,7 @@ gf_proc_dump_single_xlator_info(xlator_t *trav) return; if (ctx->measure_latency) - gf_proc_dump_latency_info(trav); + gf_proc_dump_xl_latency_info(trav); gf_proc_dump_xlator_mem_info(trav); @@ -782,7 +798,7 @@ gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx) char brick_name[PATH_MAX] = { 0, }; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char sign_string[512] = { @@ -842,7 +858,7 @@ gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx) ? dump_options.dump_path : ((ctx->statedump_path != NULL) ? ctx->statedump_path : DEFAULT_VAR_RUN_DIRECTORY)), - brick_name, getpid(), (uint64_t)time(NULL)); + brick_name, getpid(), (uint64_t)gf_time()); if ((ret < 0) || (ret >= sizeof(path))) { goto out; } @@ -861,10 +877,7 @@ gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx) // continue even though gettimeofday() has failed ret = gettimeofday(&tv, NULL); if (0 == ret) { - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); - len = strlen(timestr); - snprintf(timestr + len, sizeof timestr - len, ".%" GF_PRI_SUSECONDS, - tv.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); } len = snprintf(sign_string, sizeof(sign_string), "DUMP-START-TIME: %s\n", @@ -913,10 +926,7 @@ gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx) ret = gettimeofday(&tv, NULL); if (0 == ret) { - gf_time_fmt(timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT); - len = strlen(timestr); - snprintf(timestr + len, sizeof timestr - len, ".%" GF_PRI_SUSECONDS, - tv.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT); } len = snprintf(sign_string, sizeof(sign_string), "\nDUMP-END-TIME: %s", @@ -1035,7 +1045,7 @@ gf_proc_dump_xlator_profile(xlator_t *this, strfd_t *strfd) { gf_dump_strfd = strfd; - gf_proc_dump_latency_info(this); + gf_proc_dump_xl_latency_info(this); gf_dump_strfd = NULL; } diff --git a/libglusterfs/src/store.c b/libglusterfs/src/store.c index f444741ef0c..5c316b9291a 100644 --- a/libglusterfs/src/store.c +++ b/libglusterfs/src/store.c @@ -649,23 +649,25 @@ out: } int32_t -gf_store_iter_destroy(gf_store_iter_t *iter) +gf_store_iter_destroy(gf_store_iter_t **iter) { int32_t ret = -1; - if (!iter) + if (!(*iter)) return 0; /* gf_store_iter_new will not return a valid iter object with iter->file * being NULL*/ - ret = fclose(iter->file); + ret = fclose((*iter)->file); if (ret) gf_msg("", GF_LOG_ERROR, errno, LG_MSG_FILE_OP_FAILED, "Unable" " to close file: %s, ret: %d", - iter->filepath, ret); + (*iter)->filepath, ret); + + GF_FREE(*iter); + *iter = NULL; - GF_FREE(iter); return ret; } diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c index 0de53c6f4cf..df20cec559f 100644 --- a/libglusterfs/src/syncop.c +++ b/libglusterfs/src/syncop.c @@ -11,6 +11,10 @@ #include "glusterfs/syncop.h" #include "glusterfs/libglusterfs-messages.h" +#ifdef HAVE_TSAN_API +#include <sanitizer/tsan_interface.h> +#endif + int syncopctx_setfsuid(void *uid) { @@ -154,10 +158,14 @@ out: return ret; } +void * +syncenv_processor(void *thdata); + static void __run(struct synctask *task) { struct syncenv *env = NULL; + int32_t total, ret, i; env = task->env; @@ -173,7 +181,6 @@ __run(struct synctask *task) env->runcount--; break; case SYNCTASK_WAIT: - env->waitcount--; break; case SYNCTASK_DONE: gf_msg(task->xl->name, GF_LOG_WARNING, 0, LG_MSG_COMPLETED_TASK, @@ -187,8 +194,27 @@ __run(struct synctask *task) } list_add_tail(&task->all_tasks, &env->runq); - env->runcount++; task->state = SYNCTASK_RUN; + + env->runcount++; + + total = env->procs + env->runcount - env->procs_idle; + if (total > env->procmax) { + total = env->procmax; + } + if (total > env->procs) { + for (i = 0; i < env->procmax; i++) { + if (env->proc[i].env == NULL) { + env->proc[i].env = env; + ret = gf_thread_create(&env->proc[i].processor, NULL, + syncenv_processor, &env->proc[i], + "sproc%d", i); + if ((ret < 0) || (++env->procs >= total)) { + break; + } + } + } + } } static void @@ -210,7 +236,6 @@ __wait(struct synctask *task) gf_msg(task->xl->name, GF_LOG_WARNING, 0, LG_MSG_REWAITING_TASK, "re-waiting already waiting " "task"); - env->waitcount--; break; case SYNCTASK_DONE: gf_msg(task->xl->name, GF_LOG_WARNING, 0, LG_MSG_COMPLETED_TASK, @@ -223,12 +248,11 @@ __wait(struct synctask *task) } list_add_tail(&task->all_tasks, &env->waitq); - env->waitcount++; task->state = SYNCTASK_WAIT; } void -synctask_yield(struct synctask *task) +synctask_yield(struct synctask *task, struct timespec *delta) { xlator_t *oldTHIS = THIS; @@ -237,9 +261,16 @@ synctask_yield(struct synctask *task) task->proc->sched.uc_flags &= ~_UC_TLSBASE; #endif + task->delta = delta; + if (task->state != SYNCTASK_DONE) { task->state = SYNCTASK_SUSPEND; } + +#ifdef HAVE_TSAN_API + __tsan_switch_to_fiber(task->proc->tsan.fiber, 0); +#endif + if (swapcontext(&task->ctx, &task->proc->sched) < 0) { gf_msg("syncop", GF_LOG_ERROR, errno, LG_MSG_SWAPCONTEXT_FAILED, "swapcontext failed"); @@ -249,6 +280,35 @@ synctask_yield(struct synctask *task) } void +synctask_sleep(int32_t secs) +{ + struct timespec delta; + struct synctask *task; + + task = synctask_get(); + + if (task == NULL) { + sleep(secs); + } else { + delta.tv_sec = secs; + delta.tv_nsec = 0; + + synctask_yield(task, &delta); + } +} + +static void +__synctask_wake(struct synctask *task) +{ + task->woken = 1; + + if (task->slept) + __run(task); + + pthread_cond_broadcast(&task->env->cond); +} + +void synctask_wake(struct synctask *task) { struct syncenv *env = NULL; @@ -257,13 +317,18 @@ synctask_wake(struct synctask *task) pthread_mutex_lock(&env->mutex); { - task->woken = 1; + if (task->timer != NULL) { + if (gf_timer_call_cancel(task->xl->ctx, task->timer) != 0) { + goto unlock; + } - if (task->slept) - __run(task); + task->timer = NULL; + task->synccond = NULL; + } - pthread_cond_broadcast(&env->cond); + __synctask_wake(task); } +unlock: pthread_mutex_unlock(&env->mutex); } @@ -282,7 +347,7 @@ synctask_wrap(void) task->state = SYNCTASK_DONE; - synctask_yield(task); + synctask_yield(task, NULL); } void @@ -301,6 +366,10 @@ synctask_destroy(struct synctask *task) pthread_cond_destroy(&task->cond); } +#ifdef HAVE_TSAN_API + __tsan_destroy_fiber(task->tsan.fiber); +#endif + GF_FREE(task); } @@ -411,6 +480,13 @@ synctask_create(struct syncenv *env, size_t stacksize, synctask_fn_t fn, makecontext(&newtask->ctx, (void (*)(void))synctask_wrap, 0); +#ifdef HAVE_TSAN_API + newtask->tsan.fiber = __tsan_create_fiber(0); + snprintf(newtask->tsan.name, TSAN_THREAD_NAMELEN, "<synctask of %s>", + this->name); + __tsan_set_fiber_name(newtask->tsan.fiber, newtask->tsan.name); +#endif + newtask->state = SYNCTASK_INIT; newtask->slept = 1; @@ -422,11 +498,6 @@ synctask_create(struct syncenv *env, size_t stacksize, synctask_fn_t fn, } synctask_wake(newtask); - /* - * Make sure someone's there to execute anything we just put on the - * run queue. - */ - syncenv_scale(env); return newtask; err: @@ -520,8 +591,12 @@ syncenv_task(struct syncproc *proc) goto unlock; } - sleep_till.tv_sec = time(NULL) + SYNCPROC_IDLE_TIME; + env->procs_idle++; + + sleep_till.tv_sec = gf_time() + SYNCPROC_IDLE_TIME; ret = pthread_cond_timedwait(&env->cond, &env->mutex, &sleep_till); + + env->procs_idle--; } task = list_entry(env->runq.next, struct synctask, all_tasks); @@ -540,6 +615,34 @@ unlock: return task; } +static void +synctask_timer(void *data) +{ + struct synctask *task = data; + struct synccond *cond; + + cond = task->synccond; + if (cond != NULL) { + pthread_mutex_lock(&cond->pmutex); + + list_del_init(&task->waitq); + task->synccond = NULL; + + pthread_mutex_unlock(&cond->pmutex); + + task->ret = -ETIMEDOUT; + } + + pthread_mutex_lock(&task->env->mutex); + + gf_timer_call_cancel(task->xl->ctx, task->timer); + task->timer = NULL; + + __synctask_wake(task); + + pthread_mutex_unlock(&task->env->mutex); +} + void synctask_switchto(struct synctask *task) { @@ -555,6 +658,10 @@ synctask_switchto(struct synctask *task) task->ctx.uc_flags &= ~_UC_TLSBASE; #endif +#ifdef HAVE_TSAN_API + __tsan_switch_to_fiber(task->tsan.fiber, 0); +#endif + if (swapcontext(&task->proc->sched, &task->ctx) < 0) { gf_msg("syncop", GF_LOG_ERROR, errno, LG_MSG_SWAPCONTEXT_FAILED, "swapcontext failed"); @@ -572,7 +679,14 @@ synctask_switchto(struct synctask *task) } else { task->slept = 1; __wait(task); + + if (task->delta != NULL) { + task->timer = gf_timer_call_after(task->xl->ctx, *task->delta, + synctask_timer, task); + } } + + task->delta = NULL; } pthread_mutex_unlock(&env->mutex); } @@ -580,63 +694,27 @@ synctask_switchto(struct synctask *task) void * syncenv_processor(void *thdata) { - struct syncenv *env = NULL; struct syncproc *proc = NULL; struct synctask *task = NULL; proc = thdata; - env = proc->env; - for (;;) { - task = syncenv_task(proc); - if (!task) - break; +#ifdef HAVE_TSAN_API + proc->tsan.fiber = __tsan_create_fiber(0); + snprintf(proc->tsan.name, TSAN_THREAD_NAMELEN, "<sched of syncenv@%p>", + proc); + __tsan_set_fiber_name(proc->tsan.fiber, proc->tsan.name); +#endif + while ((task = syncenv_task(proc)) != NULL) { synctask_switchto(task); - - syncenv_scale(env); } - return NULL; -} - -void -syncenv_scale(struct syncenv *env) -{ - int diff = 0; - int scale = 0; - int i = 0; - int ret = 0; - - pthread_mutex_lock(&env->mutex); - { - if (env->procs > env->runcount) - goto unlock; - - scale = env->runcount; - if (scale > env->procmax) - scale = env->procmax; - if (scale > env->procs) - diff = scale - env->procs; - while (diff) { - diff--; - for (; (i < env->procmax); i++) { - if (env->proc[i].processor == 0) - break; - } +#ifdef HAVE_TSAN_API + __tsan_destroy_fiber(proc->tsan.fiber); +#endif - env->proc[i].env = env; - ret = gf_thread_create(&env->proc[i].processor, NULL, - syncenv_processor, &env->proc[i], - "sproc%03hx", env->procs & 0x3ff); - if (ret) - break; - env->procs++; - i++; - } - } -unlock: - pthread_mutex_unlock(&env->mutex); + return NULL; } /* The syncenv threads are cleaned up in this routine. @@ -715,12 +793,13 @@ syncenv_new(size_t stacksize, int procmin, int procmax) newenv->stacksize = stacksize; newenv->procmin = procmin; newenv->procmax = procmax; + newenv->procs_idle = 0; for (i = 0; i < newenv->procmin; i++) { newenv->proc[i].env = newenv; ret = gf_thread_create(&newenv->proc[i].processor, NULL, syncenv_processor, &newenv->proc[i], "sproc%d", - newenv->procs); + i); if (ret) break; newenv->procs++; @@ -810,7 +889,7 @@ __synclock_lock(struct synclock *lock) task->woken = 0; list_add_tail(&task->waitq, &lock->waitq); pthread_mutex_unlock(&lock->guard); - synctask_yield(task); + synctask_yield(task, NULL); /* task is removed from waitq in unlock, * under lock->guard.*/ pthread_mutex_lock(&lock->guard); @@ -963,6 +1042,136 @@ synclock_unlock(synclock_t *lock) return ret; } +/* Condition variables */ + +int32_t +synccond_init(synccond_t *cond) +{ + int32_t ret; + + INIT_LIST_HEAD(&cond->waitq); + + ret = pthread_mutex_init(&cond->pmutex, NULL); + if (ret != 0) { + return -ret; + } + + ret = pthread_cond_init(&cond->pcond, NULL); + if (ret != 0) { + pthread_mutex_destroy(&cond->pmutex); + } + + return -ret; +} + +void +synccond_destroy(synccond_t *cond) +{ + pthread_cond_destroy(&cond->pcond); + pthread_mutex_destroy(&cond->pmutex); +} + +int +synccond_timedwait(synccond_t *cond, synclock_t *lock, struct timespec *delta) +{ + struct timespec now; + struct synctask *task = NULL; + int ret; + + task = synctask_get(); + + if (task == NULL) { + if (delta != NULL) { + timespec_now_realtime(&now); + timespec_adjust_delta(&now, *delta); + } + + pthread_mutex_lock(&cond->pmutex); + + if (delta == NULL) { + ret = -pthread_cond_wait(&cond->pcond, &cond->pmutex); + } else { + ret = -pthread_cond_timedwait(&cond->pcond, &cond->pmutex, &now); + } + } else { + pthread_mutex_lock(&cond->pmutex); + + list_add_tail(&task->waitq, &cond->waitq); + task->synccond = cond; + + ret = synclock_unlock(lock); + if (ret == 0) { + pthread_mutex_unlock(&cond->pmutex); + + synctask_yield(task, delta); + + ret = synclock_lock(lock); + if (ret == 0) { + ret = task->ret; + } + task->ret = 0; + + return ret; + } + + list_del_init(&task->waitq); + } + + pthread_mutex_unlock(&cond->pmutex); + + return ret; +} + +int +synccond_wait(synccond_t *cond, synclock_t *lock) +{ + return synccond_timedwait(cond, lock, NULL); +} + +void +synccond_signal(synccond_t *cond) +{ + struct synctask *task; + + pthread_mutex_lock(&cond->pmutex); + + if (!list_empty(&cond->waitq)) { + task = list_first_entry(&cond->waitq, struct synctask, waitq); + list_del_init(&task->waitq); + + pthread_mutex_unlock(&cond->pmutex); + + synctask_wake(task); + } else { + pthread_cond_signal(&cond->pcond); + + pthread_mutex_unlock(&cond->pmutex); + } +} + +void +synccond_broadcast(synccond_t *cond) +{ + struct list_head list; + struct synctask *task; + + INIT_LIST_HEAD(&list); + + pthread_mutex_lock(&cond->pmutex); + + list_splice_init(&cond->waitq, &list); + pthread_cond_broadcast(&cond->pcond); + + pthread_mutex_unlock(&cond->pmutex); + + while (!list_empty(&list)) { + task = list_first_entry(&list, struct synctask, waitq); + list_del_init(&task->waitq); + + synctask_wake(task); + } +} + /* Barriers */ int @@ -1032,7 +1241,7 @@ __syncbarrier_wait(struct syncbarrier *barrier, int waitfor) /* called within a synctask */ list_add_tail(&task->waitq, &barrier->waitq); pthread_mutex_unlock(&barrier->guard); - synctask_yield(task); + synctask_yield(task, NULL); pthread_mutex_lock(&barrier->guard); } else { /* called by a non-synctask */ @@ -2881,12 +3090,13 @@ syncop_seek(xlator_t *subvol, fd_t *fd, off_t offset, gf_seek_what_t what, SYNCOP(subvol, (&args), syncop_seek_cbk, subvol->fops->seek, fd, offset, what, xdata_in); - if (*off) - *off = args.offset; - - if (args.op_ret == -1) + if (args.op_ret < 0) { return -args.op_errno; - return args.op_ret; + } else { + if (off) + *off = args.offset; + return args.op_ret; + } } int diff --git a/libglusterfs/src/syscall.c b/libglusterfs/src/syscall.c index 1d88c8adac1..04400f98b6c 100644 --- a/libglusterfs/src/syscall.c +++ b/libglusterfs/src/syscall.c @@ -13,6 +13,10 @@ #include "glusterfs/mem-pool.h" #include "glusterfs/libglusterfs-messages.h" +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#include <signal.h> +#endif #include <sys/types.h> #include <utime.h> #include <sys/time.h> @@ -214,6 +218,15 @@ sys_unlink(const char *pathname) } int +sys_unlinkat(int dfd, const char *pathname) +{ +#ifdef GF_SOLARIS_HOST_OS + return FS_RET_CHECK0(solaris_unlinkat(dfd, pathname, 0), errno); +#endif + return FS_RET_CHECK0(unlinkat(dfd, pathname, 0), errno); +} + +int sys_rmdir(const char *pathname) { return FS_RET_CHECK0(rmdir(pathname), errno); @@ -226,6 +239,12 @@ sys_symlink(const char *oldpath, const char *newpath) } int +sys_symlinkat(const char *oldpath, int dirfd, const char *newpath) +{ + return FS_RET_CHECK0(symlinkat(oldpath, dirfd, newpath), errno); +} + +int sys_rename(const char *oldpath, const char *newpath) { #ifdef GF_SOLARIS_HOST_OS @@ -253,6 +272,12 @@ sys_link(const char *oldpath, const char *newpath) } int +sys_linkat(int oldfd, const char *oldpath, int newfd, const char *newpath) +{ + return FS_RET_CHECK0(linkat(oldfd, oldpath, newfd, newpath, 0), errno); +} + +int sys_chmod(const char *path, mode_t mode) { return FS_RET_CHECK0(chmod(path, mode), errno); @@ -485,7 +510,7 @@ sys_lsetxattr(const char *path, const char *name, const void *value, #endif #ifdef GF_BSD_HOST_OS - return FS_RET_CHECK0( + return FS_RET_CHECK( extattr_set_link(path, EXTATTR_NAMESPACE_USER, name, value, size), errno); #endif @@ -603,7 +628,7 @@ sys_fsetxattr(int filedes, const char *name, const void *value, size_t size, #endif #ifdef GF_BSD_HOST_OS - return FS_RET_CHECK0( + return FS_RET_CHECK( extattr_set_fd(filedes, EXTATTR_NAMESPACE_USER, name, value, size), errno); #endif @@ -828,7 +853,24 @@ sys_copy_file_range(int fd_in, off64_t *off_in, int fd_out, off64_t *off_out, return syscall(SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags); #else - return -ENOSYS; + errno = ENOSYS; + return -1; #endif /* HAVE_COPY_FILE_RANGE_SYS */ #endif /* HAVE_COPY_FILE_RANGE */ } + +#ifdef __FreeBSD__ +int +sys_kill(pid_t pid, int sig) +{ + return FS_RET_CHECK0(kill(pid, sig), errno); +} + +int +sys_sysctl(const int *name, u_int namelen, void *oldp, size_t *oldlenp, + const void *newp, size_t newlen) +{ + return FS_RET_CHECK0(sysctl(name, namelen, oldp, oldlenp, newp, newlen), + errno); +} +#endif diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c index 90a993afdd2..9a2582d45d5 100644 --- a/libglusterfs/src/xlator.c +++ b/libglusterfs/src/xlator.c @@ -184,9 +184,11 @@ xlator_volopt_dynload(char *xlator_type, void **dl_handle, volume_opt_list_t *opt_list) { int ret = -1; + int flag = 0; char *name = NULL; void *handle = NULL; xlator_api_t *xlapi = NULL; + volume_option_t *opt = NULL; GF_VALIDATE_OR_GOTO("xlator", xlator_type, out); @@ -194,8 +196,10 @@ xlator_volopt_dynload(char *xlator_type, void **dl_handle, * need this check */ if (!strstr(xlator_type, "rpc-transport")) ret = gf_asprintf(&name, "%s/%s.so", XLATORDIR, xlator_type); - else + else { + flag = 1; ret = gf_asprintf(&name, "%s/%s.so", XLATORPARENTDIR, xlator_type); + } if (-1 == ret) { goto out; } @@ -211,18 +215,29 @@ xlator_volopt_dynload(char *xlator_type, void **dl_handle, goto out; } - /* check new struct first, and then check this */ - xlapi = dlsym(handle, "xlator_api"); - if (!xlapi) { - gf_smsg("xlator", GF_LOG_ERROR, 0, LG_MSG_DLSYM_ERROR, "error=%s", - dlerror(), NULL); - goto out; - } + if (flag == 0) { + /* check new struct first, and then check this */ + xlapi = dlsym(handle, "xlator_api"); + if (!xlapi) { + gf_smsg("xlator", GF_LOG_ERROR, 0, LG_MSG_DLSYM_ERROR, "error=%s", + dlerror(), NULL); + goto out; + } - opt_list->given_opt = xlapi->options; - if (!opt_list->given_opt) { - gf_smsg("xlator", GF_LOG_ERROR, 0, LG_MSG_LOAD_FAILED, NULL); - goto out; + opt_list->given_opt = xlapi->options; + if (!opt_list->given_opt) { + gf_smsg("xlator", GF_LOG_ERROR, 0, LG_MSG_LOAD_FAILED, NULL); + goto out; + } + } else { + opt = dlsym(handle, "options"); + if (!opt) { + gf_smsg("xlator", GF_LOG_ERROR, 0, LG_MSG_DLSYM_ERROR, "error=%s", + dlerror(), NULL); + goto out; + } + + opt_list->given_opt = opt; } *dl_handle = handle; @@ -245,6 +260,7 @@ xlator_dynload_apis(xlator_t *xl) void *handle = NULL; volume_opt_list_t *vol_opt = NULL; xlator_api_t *xlapi = NULL; + int i = 0; handle = xl->dlhandle; @@ -342,6 +358,10 @@ xlator_dynload_apis(xlator_t *xl) memcpy(xl->op_version, xlapi->op_version, sizeof(uint32_t) * GF_MAX_RELEASES); + for (i = 0; i < GF_FOP_MAXVALUE; i++) { + gf_latency_reset(&xl->stats.interval.latencies[i]); + } + ret = 0; out: return ret; @@ -806,7 +826,7 @@ xlator_members_free(xlator_t *xl) GF_FREE(xl->name); GF_FREE(xl->type); - if (!(xl->ctx && xl->ctx->cmd_args.valgrind) && xl->dlhandle) + if (!(xl->ctx && xl->ctx->cmd_args.vgtool != _gf_none) && xl->dlhandle) dlclose(xl->dlhandle); if (xl->options) dict_unref(xl->options); @@ -937,6 +957,8 @@ xlator_mem_cleanup(xlator_t *this) xlator_list_t **trav_p = NULL; xlator_t *top = NULL; xlator_t *victim = NULL; + glusterfs_graph_t *graph = NULL; + gf_boolean_t graph_cleanup = _gf_false; if (this->call_cleanup || !this->ctx) return; @@ -944,6 +966,12 @@ xlator_mem_cleanup(xlator_t *this) this->call_cleanup = 1; ctx = this->ctx; + inode_table = this->itable; + if (inode_table) { + inode_table_destroy(inode_table); + this->itable = NULL; + } + xlator_call_fini(trav); while (prev) { @@ -952,12 +980,6 @@ xlator_mem_cleanup(xlator_t *this) prev = trav; } - inode_table = this->itable; - if (inode_table) { - inode_table_destroy(inode_table); - this->itable = NULL; - } - if (this->fini) { this->fini(this); } @@ -967,17 +989,28 @@ xlator_mem_cleanup(xlator_t *this) if (ctx->active) { top = ctx->active->first; LOCK(&ctx->volfile_lock); - /* TODO here we have leak for xlator node in a graph */ - /* Need to move only top xlator from a graph */ for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) { victim = (*trav_p)->xlator; if (victim->call_cleanup && !strcmp(victim->name, this->name)) { + graph_cleanup = _gf_true; (*trav_p) = (*trav_p)->next; break; } } UNLOCK(&ctx->volfile_lock); } + + if (graph_cleanup) { + prev = this; + graph = ctx->active; + pthread_mutex_lock(&graph->mutex); + while (prev) { + trav = prev->next; + GF_FREE(prev); + prev = trav; + } + pthread_mutex_unlock(&graph->mutex); + } } void @@ -1544,6 +1577,11 @@ graph_total_client_xlator(glusterfs_graph_t *graph) } xl = graph->first; + if (!strcmp(xl->type, "protocol/server")) { + gf_msg_debug(xl->name, 0, "Return because it is a server graph"); + return 0; + } + while (xl) { if (strcmp(xl->type, "protocol/client") == 0) { count++; @@ -4,7 +4,29 @@ # i.e. where we are interested in the result of a command, # we have to run the command in an if-statement. -ORIGIN=${GLUSTER_ORIGIN:-origin} +UPSTREAM=${GLUSTER_UPSTREAM} +if [ "x$UPSTREAM" -eq "x" ]; then + for rmt in $(git remote); do + rmt_repo=$(git remote show $rmt -n | grep Fetch | awk '{ print $3 }'); + if [ $rmt_repo -eq "git@github:gluster/glusterfs" ]; then + UPSTREAM=$rmt + echo "Picked $rmt as upstream remote" + break + fi + done +fi + +USER_REPO=${GLUSTER_USER_REPO:-origin} +if [ "x${USER_REPO}" -eq "x${UPSTREAM}" ] ; then + echo "When you submit patches, it should get submitted to your fork, not to upstream directly" + echo "If you are not sure, check `for rmt in $(git remote); do git remote show $rmt -n; done`" + echo "And pick the correct remote you would like to push to and do `export GLUSTER_USER_REPO=$rmt`" + echo "" + echo "Exiting..." + exit 1 +fi + + while getopts "v" opt; do case $opt in @@ -18,7 +40,7 @@ done shift $((OPTIND-1)) -branch="master"; +branch="devel"; set_hooks_commit_msg() { @@ -50,21 +72,21 @@ is_num() backport_id_message() { echo "" - echo "This commit is to a non-master branch, and hence is treated as a backport." + echo "This commit is to a non-devel branch, and hence is treated as a backport." echo "" echo "For backports we would like to retain the same gerrit Change-Id across" echo "branches. On auto inspection it is found that a gerrit Change-Id is" - echo "missing, or the Change-Id is not found on your local master" + echo "missing, or the Change-Id is not found on your local devel branch" echo "" echo "This could mean a few things:" echo " 1. This is not a backport, hence choose Y on the prompt to proceed" - echo " 2. Your $ORIGIN master is not up to date, hence the script is unable" - echo " to find the corresponding Change-Id on master. Either choose N," + echo " 2. Your $USER_REPO/devel is not up to date, hence the script is unable" + echo " to find the corresponding Change-Id on devel. Either choose N," echo " 'git fetch', and try again, OR if you are sure you used the" echo " same Change-Id, choose Y at the prompt to proceed" echo " 3. You commented or removed the Change-Id in your commit message after" echo " cherry picking the commit. Choose N, fix the commit message to" - echo " use the same Change-Id as master (git commit --amend), resubmit" + echo " use the same Change-Id as 'devel' (git commit --amend), resubmit" echo "" } @@ -72,8 +94,8 @@ check_backport() { moveon='N' - # Backports are never made to master - if [ $branch = "master" ]; then + # Backports are never made to 'devel' + if [ $branch = "devel" ]; then return; fi @@ -86,22 +108,22 @@ check_backport() echo -n "Did not find a Change-Id for a possible backport. Continue (y/N): " read moveon else - # Search master for the same change ID (rebase_changes has run, so we + # Search 'devel' for the same change ID (rebase_changes has run, so we # should never not find a Change-Id) - mchangeid=$(git log $ORIGIN/master --format='%b' --grep="^Change-Id: ${changeid}" | grep ${changeid} | awk '{print $2}') + mchangeid=$(git log $UPSTREAM/devel --format='%b' --grep="^Change-Id: ${changeid}" | grep ${changeid} | awk '{print $2}') - # Check if we found the change ID on master, else throw a message to + # Check if we found the change ID on 'devel', else throw a message to # decide if we should continue. - # NOTE: If master was not rebased, we will not find the Change-ID and + # NOTE: If 'devel' was not rebased, we will not find the Change-ID and # could hit a false positive case here (or if someone checks out some - # other branch as master). + # other branch as 'devel'). if [ "${mchangeid}" = "${changeid}" ]; then moveon="Y" else backport_id_message; echo "Change-Id of commit: $changeid" - echo "Change-Id on master: $mchangeid" - echo -n "Did not find mentioned Change-Id on master for a possible backport. Continue (y/N): " + echo "Change-Id on devel: $mchangeid" + echo -n "Did not find mentioned Change-Id on 'devel' for a possible backport. Continue (y/N): " read moveon fi fi @@ -116,7 +138,7 @@ check_backport() rebase_changes() { - GIT_EDITOR=$0 git rebase -i $ORIGIN/$branch; + GIT_EDITOR=$0 git rebase -i $UPSTREAM/$branch; } @@ -154,15 +176,12 @@ rebase_changes() # [[:space:]]+ # Followed by 1 or more spaces # -# (gluster\/glusterfs|bz)? -# Followed by nothing, 'gluster/glusterfs' or 'bz' -# # # # Followed by # # # [[:digit:]]+ # Followed by 1 or more digits -REFRE="^[[:space:]]*(Fixes|Updates)(:)?[[:space:]]+(gluster\/glusterfs|bz)?#[[:digit:]]+" +REFRE="^[[:space:]]*(Fixes|Updates)(:)?[[:space:]]+#[[:digit:]]+" editor_mode() { @@ -181,22 +200,17 @@ editor_mode() while true; do echo Commit: "\"$(head -n 1 $1)\"" - echo -n "Reference (Bugzilla ID or Github Issue ID): " - read bug - if [ -z "$bug" ]; then + echo -n "Github Issue ID: " + read issue + if [ -z "$issue" ]; then return; fi - if ! is_num "$bug"; then - echo "Invalid reference ID ($bug)!!!"; + if ! is_num "$issue"; then + echo "Invalid Github Issue ID!!!"; continue; fi - bz_string="bz" - if [ $bug -lt 742000 ]; then - bz_string="" - fi - - echo "Select yes '(y)' if this patch fixes the bug/feature completely," + echo "Select yes '(y)' if this patch fixes the issue/feature completely," echo -n "or is the last of the patchset which brings feature (Y/n): " read fixes fixes_string="Fixes" @@ -204,7 +218,7 @@ editor_mode() fixes_string="Updates" fi - sed "/^Change-Id:/{p; s/^.*$/${fixes_string}: ${bz_string}#${bug}/;}" $1 > $1.new && \ + sed "/^Change-Id:/{p; s/^.*$/${fixes_string}: #${issue}/;}" $1 > $1.new && \ mv $1.new $1; return; done @@ -220,7 +234,7 @@ EOF assert_diverge() { - git diff $ORIGIN/$branch..HEAD | grep -q .; + git diff $UPSTREAM/$branch..HEAD | grep -q .; } @@ -229,34 +243,24 @@ warn_reference_missing() echo "" echo "=== Missing a reference in commit! ===" echo "" - echo "Gluster commits are made with a reference to a bug or a github issue" - echo "" - echo "Submissions that are enhancements (IOW, not functional" - echo "bug fixes, but improvements of any nature to the code) are tracked" - echo "using github issues [1]." + echo "Gluster commits are made with a reference to a github issue" echo "" - echo "Submissions that are bug fixes are tracked using Bugzilla [2]." + echo "A check on the commit message, reveals that there is no " + echo "github issue referenced in the commit message." echo "" - echo "A check on the commit message, reveals that there is no bug or" - echo "github issue referenced in the commit message" + echo "https://github.com/gluster/glusterfs/issues/new" echo "" - echo "[1] https://github.com/gluster/glusterfs/issues/new" - echo "[2] https://bugzilla.redhat.com/enter_bug.cgi?product=GlusterFS" + echo "Please open an issue and reference the same in the commit message " + echo "using the following tags:" echo "" - echo "Please file an issue or a bug report and reference the same in the" - echo "commit message using the following tags:" - echo "GitHub Issues:" - echo "\"Fixes: gluster/glusterfs#n\" OR \"Updates: gluster/glusterfs#n\"," - echo "\"Fixes: #n\" OR \"Updates: #n\"," - echo "Bugzilla ID:" - echo "\"Fixes: bz#n\" OR \"Updates: bz#n\"," - echo "where n is the issue or bug number" + echo "\"Fixes: #NNNN\" OR \"Updates: #NNNN\"," + echo "where NNNN is the issue id" echo "" echo "You may abort the submission choosing 'N' below and use" echo "'git commit --amend' to add the issue reference before posting" echo "to gerrit." echo "" - echo -n "Missing reference to a bug or a github issue. Continue (y/N): " + echo -n "Missing reference to a github issue. Continue (y/N): " read moveon if [ "${moveon}" = 'Y' ] || [ "${moveon}" = 'y' ]; then return; @@ -276,7 +280,7 @@ main() return; fi - git fetch $ORIGIN; + git fetch $UPSTREAM; rebase_changes; @@ -287,9 +291,9 @@ main() # see note above variable REFRE for regex elaboration reference=$(git log -n1 --format='%b' | grep -iow -E "${REFRE}" | awk -F '#' '{print $2}'); - # If this is a commit against master and does not have a bug ID or a github + # If this is a commit against 'devel' and does not have a github # issue reference. Warn the contributor that one of the 2 is required - if [ -z "${reference}" ] && [ $branch = "master" ]; then + if [ -z "${reference}" ] && [ $branch = "devel" ]; then warn_reference_missing; fi @@ -317,9 +321,9 @@ main() fi if [ -z "${reference}" ]; then - $drier git push $ORIGIN HEAD:refs/for/$branch/rfc; + $drier git push $USER_REPO HEAD:temp_${branch}/$(date +%Y-%m-%d_%s); else - $drier git push $ORIGIN HEAD:refs/for/$branch/ref-${reference}; + $drier git push $USER_REPO HEAD:issue${reference}_${branch}; fi } diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am index 81a96476883..35c9db07e7f 100644 --- a/rpc/rpc-lib/src/Makefile.am +++ b/rpc/rpc-lib/src/Makefile.am @@ -2,7 +2,7 @@ lib_LTLIBRARIES = libgfrpc.la libgfrpc_la_SOURCES = auth-unix.c rpcsvc-auth.c rpcsvc.c auth-null.c \ rpc-transport.c xdr-rpc.c xdr-rpcclnt.c rpc-clnt.c auth-glusterfs.c \ - rpc-drc.c $(CONTRIBDIR)/sunrpc/xdr_sizeof.c rpc-clnt-ping.c \ + rpc-drc.c rpc-clnt-ping.c \ autoscale-threads.c mgmt-pmap.c EXTRA_DIST = libgfrpc.sym diff --git a/rpc/rpc-lib/src/libgfrpc.sym b/rpc/rpc-lib/src/libgfrpc.sym index 54d1be1112f..e026d80259b 100644 --- a/rpc/rpc-lib/src/libgfrpc.sym +++ b/rpc/rpc-lib/src/libgfrpc.sym @@ -65,3 +65,4 @@ rpc_transport_unix_options_build rpc_transport_unref rpc_clnt_mgmt_pmap_signout rpcsvc_autoscale_threads +rpcsvc_statedump diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h index 7275d7568b6..0cb5862e9a9 100644 --- a/rpc/rpc-lib/src/protocol-common.h +++ b/rpc/rpc-lib/src/protocol-common.h @@ -309,6 +309,7 @@ enum glusterd_mgmt_v3_procnum { GLUSTERD_MGMT_V3_PRE_VALIDATE, GLUSTERD_MGMT_V3_BRICK_OP, GLUSTERD_MGMT_V3_COMMIT, + GLUSTERD_MGMT_V3_POST_COMMIT, GLUSTERD_MGMT_V3_POST_VALIDATE, GLUSTERD_MGMT_V3_UNLOCK, GLUSTERD_MGMT_V3_MAXVALUE, diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c index 2298ef6394f..31f17841bea 100644 --- a/rpc/rpc-lib/src/rpc-clnt-ping.c +++ b/rpc/rpc-lib/src/rpc-clnt-ping.c @@ -122,7 +122,7 @@ rpc_clnt_ping_timer_expired(void *rpc_ptr) goto out; } - clock_gettime(CLOCK_REALTIME, ¤t); + timespec_now_realtime(¤t); pthread_mutex_lock(&conn->lock); { unref = rpc_clnt_remove_ping_timer_locked(rpc); diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index dac707664df..517037c4a5d 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -97,7 +97,7 @@ call_bail(void *data) struct saved_frame *saved_frame = NULL; struct saved_frame *trav = NULL; struct saved_frame *tmp = NULL; - char frame_sent[256] = { + char frame_sent[GF_TIMESTR_SIZE] = { 0, }; struct timespec timeout = { @@ -105,7 +105,6 @@ call_bail(void *data) }; char peerid[UNIX_PATH_MAX] = {0}; gf_boolean_t need_unref = _gf_false; - int len; GF_VALIDATE_OR_GOTO("client", data, out); @@ -165,11 +164,8 @@ call_bail(void *data) list_for_each_entry_safe(trav, tmp, &list, list) { - gf_time_fmt(frame_sent, sizeof frame_sent, trav->saved_at.tv_sec, - gf_timefmt_FT); - len = strlen(frame_sent); - snprintf(frame_sent + len, sizeof(frame_sent) - len, - ".%" GF_PRI_SUSECONDS, trav->saved_at.tv_usec); + gf_time_fmt_tv(frame_sent, sizeof frame_sent, &trav->saved_at, + gf_timefmt_FT); gf_log(conn->name, GF_LOG_ERROR, "bailing out frame type(%s), op(%s(%d)), xid = 0x%x, " @@ -317,20 +313,15 @@ saved_frames_unwind(struct saved_frames *saved_frames) { struct saved_frame *trav = NULL; struct saved_frame *tmp = NULL; - char timestr[1024] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; - int len; list_splice_init(&saved_frames->lk_sf.list, &saved_frames->sf.list); list_for_each_entry_safe(trav, tmp, &saved_frames->sf.list, list) { - gf_time_fmt(timestr, sizeof timestr, trav->saved_at.tv_sec, - gf_timefmt_FT); - len = strlen(timestr); - snprintf(timestr + len, sizeof(timestr) - len, ".%" GF_PRI_SUSECONDS, - trav->saved_at.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, &trav->saved_at, gf_timefmt_FT); if (!trav->rpcreq || !trav->rpcreq->prog) continue; @@ -928,7 +919,7 @@ rpc_clnt_notify(rpc_transport_t *trans, void *mydata, } case RPC_TRANSPORT_MSG_RECEIVED: { - clock_gettime(CLOCK_REALTIME, &conn->last_received); + timespec_now_realtime(&conn->last_received); pollin = data; if (pollin->is_reply) @@ -942,8 +933,7 @@ rpc_clnt_notify(rpc_transport_t *trans, void *mydata, } case RPC_TRANSPORT_MSG_SENT: { - clock_gettime(CLOCK_REALTIME, &conn->last_sent); - + timespec_now_realtime(&conn->last_sent); ret = 0; break; } @@ -960,6 +950,7 @@ rpc_clnt_notify(rpc_transport_t *trans, void *mydata, conn->config.remote_port = 0; conn->connected = 1; conn->disconnected = 0; + pthread_cond_broadcast(&conn->cond); } pthread_mutex_unlock(&conn->lock); @@ -1005,6 +996,7 @@ rpc_clnt_connection_init(struct rpc_clnt *clnt, glusterfs_ctx_t *ctx, conn = &clnt->conn; pthread_mutex_init(&clnt->conn.lock, NULL); + pthread_cond_init(&clnt->conn.cond, NULL); conn->name = gf_strdup(name); if (!conn->name) { @@ -1830,6 +1822,7 @@ rpc_clnt_destroy(struct rpc_clnt *rpc) saved_frames_destroy(saved_frames); pthread_mutex_destroy(&rpc->lock); pthread_mutex_destroy(&rpc->conn.lock); + pthread_cond_destroy(&rpc->conn.cond); /* mem-pool should be destroyed, otherwise, it will cause huge memory leaks */ diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index 0d0b115578c..2945265200b 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -127,6 +127,7 @@ struct rpc_clnt_config { struct rpc_clnt_connection { pthread_mutex_t lock; + pthread_cond_t cond; rpc_transport_t *trans; struct rpc_clnt_config config; gf_timer_t *reconnect; diff --git a/rpc/rpc-lib/src/rpc-drc.c b/rpc/rpc-lib/src/rpc-drc.c index d083db24fc5..de8dc630626 100644 --- a/rpc/rpc-lib/src/rpc-drc.c +++ b/rpc/rpc-lib/src/rpc-drc.c @@ -190,7 +190,7 @@ rpcsvc_get_drc_client(rpcsvc_drc_globals_t *drc, if (!client) goto out; - client->ref = 0; + GF_ATOMIC_INIT(client->ref, 0); client->sock_union = (union gf_sock_union) * sockaddr; client->op_count = 0; INIT_LIST_HEAD(&client->client_list); @@ -246,7 +246,7 @@ static drc_client_t * rpcsvc_drc_client_ref(drc_client_t *client) { GF_ASSERT(client); - client->ref++; + GF_ATOMIC_INC(client->ref); return client; } @@ -261,11 +261,12 @@ rpcsvc_drc_client_ref(drc_client_t *client) static drc_client_t * rpcsvc_drc_client_unref(rpcsvc_drc_globals_t *drc, drc_client_t *client) { + uint32_t refcount; + GF_ASSERT(drc); - GF_ASSERT(client->ref); - client->ref--; - if (!client->ref) { + refcount = GF_ATOMIC_DEC(client->ref); + if (!refcount) { drc->client_count--; rpcsvc_remove_drc_client(client); client = NULL; @@ -589,7 +590,7 @@ rpcsvc_drc_priv(rpcsvc_drc_globals_t *drc) } gf_proc_dump_build_key(key, "client", "%d.ref_count", i); - gf_proc_dump_write(key, "%d", client->ref); + gf_proc_dump_write(key, "%" PRIu32, GF_ATOMIC_GET(client->ref)); gf_proc_dump_build_key(key, "client", "%d.op_count", i); gf_proc_dump_write(key, "%d", client->op_count); i++; diff --git a/rpc/rpc-lib/src/rpc-drc.h b/rpc/rpc-lib/src/rpc-drc.h index 6aaede0828a..ce66430809b 100644 --- a/rpc/rpc-lib/src/rpc-drc.h +++ b/rpc/rpc-lib/src/rpc-drc.h @@ -24,7 +24,7 @@ struct drc_client { struct rb_table *rbtree; /* no. of ops currently cached */ uint32_t op_count; - uint32_t ref; + gf_atomic_uint32_t ref; struct list_head client_list; }; diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index 81d40c8ec0e..39910d481bf 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -13,6 +13,7 @@ #include <glusterfs/dict.h> #include <glusterfs/byte-order.h> #include <glusterfs/compat-errno.h> +#include <glusterfs/statedump.h> #include "xdr-rpc.h" #include <glusterfs/iobuf.h> #include "xdr-common.h" @@ -41,6 +42,10 @@ #include "xdr-rpcclnt.h" #include <glusterfs/glusterfs-acl.h> +#ifndef PTHREAD_MUTEX_ADAPTIVE_NP +#define PTHREAD_MUTEX_ADAPTIVE_NP PTHREAD_MUTEX_DEFAULT +#endif + static struct rpcsvc_program gluster_dump_prog; #define rpcsvc_alloc_request(svc, request) \ @@ -66,59 +71,33 @@ rpcsvc_request_handler(void *arg); static int rpcsvc_match_subnet_v4(const char *addrtok, const char *ipaddr); -void +static void rpcsvc_toggle_queue_status(rpcsvc_program_t *prog, - rpcsvc_request_queue_t *queue, char status[]) + rpcsvc_request_queue_t *queue, + unsigned long status[]) { - int queue_index = 0, status_index = 0, set_bit = 0; - - if (queue != &prog->request_queue[0]) { - queue_index = (queue - &prog->request_queue[0]); - } - - status_index = queue_index / 8; - set_bit = queue_index % 8; + unsigned queue_index = queue - prog->request_queue; - status[status_index] ^= (1 << set_bit); - - return; + status[queue_index / __BITS_PER_LONG] ^= (1UL << (queue_index % + __BITS_PER_LONG)); } int rpcsvc_get_free_queue_index(rpcsvc_program_t *prog) { - int queue_index = 0, max_index = 0, i = 0; - unsigned int right_most_unset_bit = 0; + unsigned i, j = 0; - right_most_unset_bit = 8; - - max_index = gf_roof(EVENT_MAX_THREADS, 8) / 8; - for (i = 0; i < max_index; i++) { - if (prog->request_queue_status[i] == 0) { - right_most_unset_bit = 0; + for (i = 0; i < EVENT_MAX_THREADS / __BITS_PER_LONG; i++) + if (prog->request_queue_status[i] != ULONG_MAX) { + j = __builtin_ctzl(~prog->request_queue_status[i]); break; - } else { - /* get_rightmost_set_bit (sic)*/ - right_most_unset_bit = __builtin_ctz( - ~prog->request_queue_status[i]); - if (right_most_unset_bit < 8) { - break; - } } - } - if (right_most_unset_bit > 7) { - queue_index = -1; - } else { - queue_index = i * 8; - queue_index += right_most_unset_bit; - } - - if (queue_index != -1) { - prog->request_queue_status[i] |= (0x1 << right_most_unset_bit); - } + if (i == EVENT_MAX_THREADS / __BITS_PER_LONG) + return -1; - return queue_index; + prog->request_queue_status[i] |= (1UL << j); + return i * __BITS_PER_LONG + j; } rpcsvc_notify_wrapper_t * @@ -363,6 +342,10 @@ rpcsvc_program_actor(rpcsvc_request_t *req) goto err; } + if (svc->xl->ctx->measure_latency) { + timespec_now(&req->begin); + } + req->ownthread = program->ownthread; req->synctask = program->synctask; @@ -1512,10 +1495,18 @@ rpcsvc_submit_generic(rpcsvc_request_t *req, struct iovec *proghdr, size_t hdrlen = 0; char new_iobref = 0; rpcsvc_drc_globals_t *drc = NULL; + gf_latency_t *lat = NULL; if ((!req) || (!req->trans)) return -1; + if (req->prog && req->begin.tv_sec) { + if ((req->procnum >= 0) && (req->procnum < req->prog->numactors)) { + timespec_now(&req->end); + lat = &req->prog->latencies[req->procnum]; + gf_latency_update(lat, &req->begin, &req->end); + } + } trans = req->trans; for (i = 0; i < hdrcount; i++) { @@ -1846,6 +1837,15 @@ rpcsvc_submit_message(rpcsvc_request_t *req, struct iovec *proghdr, iobref); } +void +rpcsvc_program_destroy(rpcsvc_program_t *program) +{ + if (program) { + GF_FREE(program->latencies); + GF_FREE(program); + } +} + int rpcsvc_program_unregister(rpcsvc_t *svc, rpcsvc_program_t *program) { @@ -1903,8 +1903,7 @@ rpcsvc_program_unregister(rpcsvc_t *svc, rpcsvc_program_t *program) ret = 0; out: - if (prog) - GF_FREE(prog); + rpcsvc_program_destroy(prog); if (ret == -1) { if (program) { @@ -2289,6 +2288,11 @@ rpcsvc_program_register(rpcsvc_t *svc, rpcsvc_program_t *program, } memcpy(newprog, program, sizeof(*program)); + newprog->latencies = gf_latency_new(program->numactors); + if (!newprog->latencies) { + rpcsvc_program_destroy(newprog); + goto out; + } INIT_LIST_HEAD(&newprog->program); pthread_mutexattr_init(&thr_attr); @@ -3234,6 +3238,48 @@ out: return ret; } +void +rpcsvc_program_dump(rpcsvc_program_t *prog) +{ + char key_prefix[GF_DUMP_MAX_BUF_LEN]; + char key[GF_DUMP_MAX_BUF_LEN]; + int i; + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s", prog->progname); + gf_proc_dump_add_section("%s", key_prefix); + + gf_proc_dump_build_key(key, key_prefix, "program-number"); + gf_proc_dump_write(key, "%d", prog->prognum); + + gf_proc_dump_build_key(key, key_prefix, "program-version"); + gf_proc_dump_write(key, "%d", prog->progver); + + strncat(key_prefix, ".latency", + sizeof(key_prefix) - strlen(key_prefix) - 1); + + for (i = 0; i < prog->numactors; i++) { + gf_proc_dump_build_key(key, key_prefix, "%s", prog->actors[i].procname); + gf_latency_statedump_and_reset(key, &prog->latencies[i]); + } +} + +void +rpcsvc_statedump(rpcsvc_t *svc) +{ + rpcsvc_program_t *prog = NULL; + int ret = 0; + ret = pthread_rwlock_tryrdlock(&svc->rpclock); + if (ret) + return; + { + list_for_each_entry(prog, &svc->programs, program) + { + rpcsvc_program_dump(prog); + } + } + pthread_rwlock_unlock(&svc->rpclock); +} + static rpcsvc_actor_t gluster_dump_actors[GF_DUMP_MAXVALUE] = { [GF_DUMP_NULL] = {"NULL", NULL, NULL, GF_DUMP_NULL, DRC_NA, 0}, [GF_DUMP_DUMP] = {"DUMP", rpcsvc_dump, NULL, GF_DUMP_DUMP, DRC_NA, 0}, diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h index a88fc841b60..7b3030926c8 100644 --- a/rpc/rpc-lib/src/rpcsvc.h +++ b/rpc/rpc-lib/src/rpcsvc.h @@ -264,6 +264,8 @@ struct rpcsvc_request { gf_boolean_t ownthread; gf_boolean_t synctask; + struct timespec begin; /*req handling start time*/ + struct timespec end; /*req handling end time*/ }; #define rpcsvc_request_program(req) ((rpcsvc_program_t *)((req)->prog)) @@ -419,6 +421,7 @@ struct rpcsvc_program { supported by the system. */ /* Program specific state handed to actors */ void *private; + gf_latency_t *latencies; /*Tracks latency statistics for the rpc call*/ /* This upcall is provided by the program during registration. * It is used to notify the program about events like connection being @@ -457,7 +460,7 @@ struct rpcsvc_program { gf_boolean_t alive; gf_boolean_t synctask; - char request_queue_status[EVENT_MAX_THREADS / 8 + 1]; + unsigned long request_queue_status[EVENT_MAX_THREADS / __BITS_PER_LONG]; }; typedef struct rpcsvc_cbk_program { @@ -686,4 +689,6 @@ rpcsvc_autoscale_threads(glusterfs_ctx_t *ctx, rpcsvc_t *rpc, int incr); extern int rpcsvc_destroy(rpcsvc_t *svc); +void +rpcsvc_statedump(rpcsvc_t *svc); #endif diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c index a295e6a9bab..ed8b473be23 100644 --- a/rpc/rpc-transport/socket/src/socket.c +++ b/rpc/rpc-transport/socket/src/socket.c @@ -1154,7 +1154,7 @@ __socket_reset(rpc_transport_t *this) SSL_free(priv->ssl_ssl); priv->ssl_ssl = NULL; } - if (priv->use_ssl && priv->ssl_ctx) { + if (priv->ssl_ctx) { SSL_CTX_free(priv->ssl_ctx); priv->ssl_ctx = NULL; } @@ -2950,6 +2950,13 @@ socket_event_handler(int fd, int idx, int gen, void *data, int poll_in, socket_dump_info(sa, priv->is_server, priv->use_ssl, priv->sock, this->name, "disconnecting from"); + /* Dump the SSL error stack to clear any errors that may otherwise + * resurface in the future. + */ + if (priv->use_ssl && priv->ssl_ssl) { + ssl_dump_error_stack(this->name); + } + /* Logging has happened already in earlier cases */ gf_log("transport", ((ret >= 0) ? GF_LOG_INFO : GF_LOG_DEBUG), "EPOLLERR - disconnecting (sock:%d) (%s)", priv->sock, @@ -4161,6 +4168,34 @@ static void __attribute__((destructor)) fini_openssl_mt(void) ERR_free_strings(); } +/* The function returns 0 if AES bit is enabled on the CPU */ +static int +ssl_check_aes_bit(void) +{ + FILE *fp = fopen("/proc/cpuinfo", "r"); + int ret = 1; + size_t len = 0; + char *line = NULL; + char *match = NULL; + + GF_ASSERT(fp != NULL); + + while (getline(&line, &len, fp) > 0) { + if (!strncmp(line, "flags", 5)) { + match = strstr(line, " aes"); + if ((match != NULL) && ((match[4] == ' ') || (match[4] == 0))) { + ret = 0; + break; + } + } + } + + free(line); + fclose(fp); + + return ret; +} + static int ssl_setup_connection_params(rpc_transport_t *this) { @@ -4171,6 +4206,7 @@ ssl_setup_connection_params(rpc_transport_t *this) char *cipher_list = DEFAULT_CIPHER_LIST; char *dh_param = DEFAULT_DH_PARAM; char *ec_curve = DEFAULT_EC_CURVE; + gf_boolean_t dh_flag = _gf_false; priv = this->private; @@ -4179,6 +4215,14 @@ ssl_setup_connection_params(rpc_transport_t *this) return 0; } + if (!priv->ssl_enabled && !priv->mgmt_ssl) { + return 0; + } + + if (!ssl_check_aes_bit()) { + cipher_list = "AES128:" DEFAULT_CIPHER_LIST; + } + priv->ssl_own_cert = DEFAULT_CERT_PATH; if (dict_get_str_sizen(this->options, SSL_OWN_CERT_OPT, &optstr) == 0) { if (!priv->ssl_enabled) { @@ -4225,28 +4269,25 @@ ssl_setup_connection_params(rpc_transport_t *this) priv->crl_path = gf_strdup(optstr); } - gf_log(this->name, priv->ssl_enabled ? GF_LOG_INFO : GF_LOG_DEBUG, - "SSL support on the I/O path is %s", - priv->ssl_enabled ? "ENABLED" : "NOT enabled"); - gf_log(this->name, priv->mgmt_ssl ? GF_LOG_INFO : GF_LOG_DEBUG, - "SSL support for glusterd is %s", - priv->mgmt_ssl ? "ENABLED" : "NOT enabled"); - if (!priv->mgmt_ssl) { if (!dict_get_int32_sizen(this->options, SSL_CERT_DEPTH_OPT, &cert_depth)) { - gf_log(this->name, GF_LOG_INFO, "using certificate depth %d", - cert_depth); } } else { cert_depth = this->ctx->ssl_cert_depth; - gf_log(this->name, GF_LOG_INFO, "using certificate depth %d", - cert_depth); } + gf_log(this->name, priv->ssl_enabled ? GF_LOG_INFO : GF_LOG_DEBUG, + "SSL support for MGMT is %s IO path is %s certificate depth is %d " + "for peer %s", + (priv->mgmt_ssl ? "ENABLED" : "NOT enabled"), + (priv->ssl_enabled ? "ENABLED" : "NOT enabled"), cert_depth, + this->peerinfo.identifier); + if (!dict_get_str_sizen(this->options, SSL_CIPHER_LIST_OPT, &cipher_list)) { gf_log(this->name, GF_LOG_INFO, "using cipher list %s", cipher_list); } if (!dict_get_str_sizen(this->options, SSL_DH_PARAM_OPT, &dh_param)) { + dh_flag = _gf_true; gf_log(this->name, GF_LOG_INFO, "using DH parameters %s", dh_param); } if (!dict_get_str_sizen(this->options, SSL_EC_CURVE_OPT, &ec_curve)) { @@ -4281,12 +4322,15 @@ ssl_setup_connection_params(rpc_transport_t *this) #ifdef SSL_OP_NO_COMPRESSION SSL_CTX_set_options(priv->ssl_ctx, SSL_OP_NO_COMPRESSION); #endif - - if ((bio = BIO_new_file(dh_param, "r")) == NULL) { - gf_log(this->name, GF_LOG_INFO, - "failed to open %s, " - "DH ciphers are disabled", - dh_param); + /* Upload file to bio wrapper only if dh param is configured + */ + if (dh_flag) { + if ((bio = BIO_new_file(dh_param, "r")) == NULL) { + gf_log(this->name, GF_LOG_ERROR, + "failed to open %s, " + "DH ciphers are disabled", + dh_param); + } } if (bio != NULL) { @@ -4603,7 +4647,7 @@ fini(rpc_transport_t *this) SSL_free(priv->ssl_ssl); priv->ssl_ssl = NULL; } - if (priv->use_ssl && priv->ssl_ctx) { + if (priv->ssl_ctx) { SSL_CTX_free(priv->ssl_ctx); priv->ssl_ctx = NULL; } diff --git a/rpc/xdr/src/glusterd1-xdr.x b/rpc/xdr/src/glusterd1-xdr.x index 02ebec26c01..b631dea3502 100644 --- a/rpc/xdr/src/glusterd1-xdr.x +++ b/rpc/xdr/src/glusterd1-xdr.x @@ -202,6 +202,21 @@ struct gd1_mgmt_v3_commit_rsp { string op_errstr<>; } ; +struct gd1_mgmt_v3_post_commit_req { + unsigned char uuid[16]; + int op; + opaque dict<>; +} ; + +struct gd1_mgmt_v3_post_commit_rsp { + unsigned char uuid[16]; + int op; + int op_ret; + int op_errno; + opaque dict<>; + string op_errstr<>; +} ; + struct gd1_mgmt_v3_post_val_req { unsigned char uuid[16]; int op; diff --git a/rpc/xdr/src/libgfxdr.sym b/rpc/xdr/src/libgfxdr.sym index dd4ac8562bc..8fa0e0ddd8a 100644 --- a/rpc/xdr/src/libgfxdr.sym +++ b/rpc/xdr/src/libgfxdr.sym @@ -28,6 +28,8 @@ xdr_gd1_mgmt_v3_brick_op_req xdr_gd1_mgmt_v3_brick_op_rsp xdr_gd1_mgmt_v3_commit_req xdr_gd1_mgmt_v3_commit_rsp +xdr_gd1_mgmt_v3_post_commit_req +xdr_gd1_mgmt_v3_post_commit_rsp xdr_gd1_mgmt_v3_lock_req xdr_gd1_mgmt_v3_lock_rsp xdr_gd1_mgmt_v3_post_val_req diff --git a/run-tests.sh b/run-tests.sh index d3ff78d6915..e2a1655d8e0 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -328,6 +328,7 @@ function get_bug_list_for_disabled_test () function run_tests() { RES=0 + FLAKY='' FAILED='' TESTS_NEEDED_RETRY='' GENERATED_CORE='' @@ -349,16 +350,16 @@ function run_tests() timeout_cmd_exists="no" fi - for t in $(find ${regression_testsdir}/tests -name '*.t' \ - | LC_COLLATE=C sort) ; do + all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort)) + all_tests_cnt=${#all_tests[@]} + for t in "${all_tests[@]}" ; do old_cores=$(ls /*-*.core 2> /dev/null | wc -l) total_tests=$((total_tests+1)) if match $t "$@" ; then selected_tests=$((selected_tests+1)) echo - echo $section_separator$section_separator - if [[ $(get_test_status $t) == "BAD_TEST" ]] || \ - [[ $(get_test_status $t) == "BRICK_MUX_BAD_TEST" ]] && \ + echo $section_separator "(${total_tests} / ${all_tests_cnt})" $section_separator + if [[ $(get_test_status $t) =~ "BAD_TEST" ]] && \ [[ $skip_bad_tests == "yes" ]] then skipped_bad_tests=$((skipped_bad_tests+1)) @@ -433,9 +434,17 @@ function run_tests() TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} " fi + + if [ ${TMP_RES} -ne 0 ] ; then - RES=${TMP_RES} - FAILED="${FAILED}${t} " + if [[ "$t" == *"tests/000-flaky/"* ]]; then + FLAKY="${FLAKY}${t} " + echo "FAILURE -> SUCCESS: Flaky test" + TMP_RES=0 + else + RES=${TMP_RES} + FAILED="${FAILED}${t} " + fi fi new_cores=$(ls /*-*.core 2> /dev/null | wc -l) @@ -470,8 +479,10 @@ function run_tests() echo "$key - ${ELAPSEDTIMEMAP["$key"]} second" done | sort -rn -k3 - # Output the errors into a file + # initialize the output file echo > "${result_output}" + + # Output the errors into a file if [ ${RES} -ne 0 ] ; then FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u ) FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' ) @@ -484,7 +495,13 @@ function run_tests() TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u ) RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' ) if [ ${RETRY_COUNT} -ne 0 ] ; then - echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" + echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}" + fi + + FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u ) + RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' ) + if [ ${RETRY_COUNT} -ne 0 ] ; then + echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}" fi echo @@ -513,8 +530,38 @@ function run_head_tests() run_tests "$htests" } -function parse_args () { - args=`getopt frcbkphHno:t: "$@"` +function show_usage () +{ + cat <<EOF +Usage: $0 <opts> [<glob>|<bzid>]... + +Options: + +-f force +-h skip tests altering from HEAD +-H run only tests altering from HEAD +-r retry failed tests +-R do not retry failed tests +-c dont't exit on failure +-b don't skip bad tests +-k don't skip known bugs +-p don't keep logs from preceding runs +-o OUTPUT +-t TIMEOUT +-n skip NFS tests +--help +EOF +} + +usage="no" + +function parse_args () +{ + args=`getopt -u -l help frRcbkphHno:t: "$@"` + if ! [ $? -eq 0 ]; then + show_usage + exit 1 + fi set -- $args while [ $# -gt 0 ]; do case "$1" in @@ -522,6 +569,7 @@ function parse_args () { -h) head="no" ;; -H) head="only" ;; -r) retry="yes" ;; + -R) retry="no" ;; -c) exit_on_failure="no" ;; -b) skip_bad_tests="no" ;; -k) skip_known_bugs="no" ;; @@ -529,6 +577,7 @@ function parse_args () { -o) result_output="$2"; shift;; -t) run_timeout="$2"; shift;; -n) nfs_tests="no";; + --help) usage="yes" ;; --) shift; break;; esac shift @@ -543,6 +592,10 @@ echo # Get user options parse_args "$@" +if [ x"$usage" == x"yes" ]; then + show_usage + exit 0 +fi # Make sure we're running as the root user check_user diff --git a/tests/00-geo-rep/00-georep-verify-non-root-setup.t b/tests/00-geo-rep/00-georep-verify-non-root-setup.t index 27ed82732c9..a55fd3e5e6a 100644 --- a/tests/00-geo-rep/00-georep-verify-non-root-setup.t +++ b/tests/00-geo-rep/00-georep-verify-non-root-setup.t @@ -5,7 +5,7 @@ . $(dirname $0)/../geo-rep.rc . $(dirname $0)/../env.rc -SCRIPT_TIMEOUT=500 +SCRIPT_TIMEOUT=600 ### Basic Non-root geo-rep setup test with Distribute Replicate volumes @@ -144,12 +144,15 @@ TEST pidof glusterd; TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3}; TEST $CLI volume start $META_VOL TEST mkdir -p $META_MNT +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "3" brick_count ${META_VOL} TEST glusterfs -s $H0 --volfile-id $META_VOL $META_MNT ##Mount master +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "4" brick_count $GMV0 TEST glusterfs -s $H0 --volfile-id $GMV0 $M0 ##Mount slave +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "4" brick_count $GSV0 TEST glusterfs -s $H0 --volfile-id $GSV0 $M1 ## Check status of mount-broker @@ -189,6 +192,8 @@ TEST gluster-georep-sshkey generate TEST $GEOREP_CLI $master $slave_url create push-pem +#check for session creation +EXPECT_WITHIN $GEO_REP_TIMEOUT 4 check_status_non_root "Created" #Config gluster-command-dir TEST $GEOREP_CLI $master $slave_url config gluster-command-dir ${GLUSTER_CMD_DIR} @@ -240,21 +245,25 @@ TEST ! $GEOREP_CLI $master $slave_url config ssh-port 22a TEST $GEOREP_CLI $master $slave config ssh-port 22 #Hybrid directory rename test BZ#1763439 + TEST $GEOREP_CLI $master $slave_url config change_detector xsync -mkdir ${master_mnt}/dir1 -mkdir ${master_mnt}/dir1/dir2 -mkdir ${master_mnt}/dir1/dir3 -mkdir ${master_mnt}/hybrid_d1 +#verify master and slave mount -EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/hybrid_d1 -EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/dir1 -EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/dir1/dir2 -EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/dir1/dir3 +EXPECT_WITHIN $CHECK_MOUNT_TIMEOUT "^1$" check_mounted ${master_mnt} +EXPECT_WITHIN $CHECK_MOUNT_TIMEOUT "^1$" check_mounted ${slave_mnt} + +#Create test data for hybrid crawl +TEST mkdir ${master_mnt}/dir1 +TEST mkdir ${master_mnt}/dir1/dir2 +TEST mkdir ${master_mnt}/dir1/dir3 +TEST mkdir ${master_mnt}/hybrid_d1 mv ${master_mnt}/hybrid_d1 ${master_mnt}/hybrid_rn_d1 mv ${master_mnt}/dir1/dir2 ${master_mnt}/rn_dir2 mv ${master_mnt}/dir1/dir3 ${master_mnt}/dir1/rn_dir3 +#Verify hybrid crawl data on slave +EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/dir1 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/hybrid_rn_d1 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/rn_dir2 EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/dir1/rn_dir3 diff --git a/tests/00-geo-rep/00-georep-verify-setup.t b/tests/00-geo-rep/00-georep-verify-setup.t index cdfccd35db5..0d46c04102d 100644 --- a/tests/00-geo-rep/00-georep-verify-setup.t +++ b/tests/00-geo-rep/00-georep-verify-setup.t @@ -5,7 +5,8 @@ . $(dirname $0)/../geo-rep.rc . $(dirname $0)/../env.rc -SCRIPT_TIMEOUT=300 +SCRIPT_TIMEOUT=400 +GEO_REP_TIMEOUT=200 ##Cleanup and start glusterd cleanup; diff --git a/tests/00-geo-rep/georep-upgrade.t b/tests/00-geo-rep/georep-upgrade.t new file mode 100644 index 00000000000..7523068ed50 --- /dev/null +++ b/tests/00-geo-rep/georep-upgrade.t @@ -0,0 +1,79 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc + +SCRIPT_TIMEOUT=500 + +############################################################################################### +#Before upgrade +############################################################################################### +brick=/bricks/brick1 +epoch1=$(date '+%s') +sleep 1 +epoch2=$(date '+%s') +mkdir -p /bricks/brick1/.glusterfs/changelogs/htime +mkdir -p /bricks/brick1/.glusterfs/changelogs + +#multiple htime files(changelog enable/disable scenario) +TEST touch /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch1 +TEST touch /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch2 + +#changelog files +TEST touch /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch1 +TEST touch /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch2 + +htime_file1=/bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch1 +htime_file2=/bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch2 + +#data inside htime files before upgrade +data1=/bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch1 +data2=/bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch2 + +#data inside htime files after upgrade +updated_data1=/bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m/%d')`/CHANGELOG.$epoch1 +updated_data2=/bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m/%d')`/CHANGELOG.$epoch2 + +echo -n $data1>$htime_file1 +echo -n $data2>$htime_file2 + +echo "Before upgrade:" +EXPECT '1' echo $(grep $data1 $htime_file1 | wc -l) +EXPECT '1' echo $(grep $data2 $htime_file2 | wc -l) + +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch1 | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch1 | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch2 | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch2 | wc -l) +############################################################################################### +#Upgrade +############################################################################################### +### This needed to be fixed as this very vague finding a file with name in '/' +### multiple file with same name can exist +### for temp fix picking only 1st result +TEST upgrade_script=$(find / -type f -name glusterfs-georep-upgrade.py -print | head -n 1) +TEST python3 $upgrade_script $brick + +############################################################################################### +#After upgrade +############################################################################################### +echo "After upgrade:" +EXPECT '1' echo $(grep $updated_data1 $htime_file1 | wc -l) +EXPECT '1' echo $(grep $updated_data2 $htime_file2 | wc -l) + +#Check directory structure inside changelogs +TEST ! ls /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch1 +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch1 | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch1.bak | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y')` | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m')` | wc -l) +EXPECT '2' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m/%d')` | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m/%d')`/CHANGELOG.$epoch1 | wc -l) + +TEST ! ls /bricks/brick1/.glusterfs/changelogs/CHANGELOG.$epoch2 +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch2 | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/htime/HTIME.$epoch2.bak | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y')` | wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m')`| wc -l) +EXPECT '1' echo $(ls /bricks/brick1/.glusterfs/changelogs/`echo $(date '+%Y/%m/%d')`/CHANGELOG.$epoch2 | wc -l) + +TEST rm -rf /bricks diff --git a/tests/basic/afr/split-brain-favorite-child-policy.t b/tests/000-flaky/basic_afr_split-brain-favorite-child-policy.t index c268c125610..77d82a4996f 100644 --- a/tests/basic/afr/split-brain-favorite-child-policy.t +++ b/tests/000-flaky/basic_afr_split-brain-favorite-child-policy.t @@ -1,8 +1,8 @@ #!/bin/bash #Test the split-brain resolution CLI commands. -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc cleanup; diff --git a/tests/basic/changelog/changelog-snapshot.t b/tests/000-flaky/basic_changelog_changelog-snapshot.t index 7742db48cdd..f6cd0b04d47 100644 --- a/tests/basic/changelog/changelog-snapshot.t +++ b/tests/000-flaky/basic_changelog_changelog-snapshot.t @@ -1,7 +1,7 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../snapshot.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../snapshot.rc cleanup; ROLLOVER_TIME=3 diff --git a/tests/basic/distribute/rebal-all-nodes-migrate.t b/tests/000-flaky/basic_distribute_rebal-all-nodes-migrate.t index acc4ffefecc..eb5d3305ac1 100644 --- a/tests/basic/distribute/rebal-all-nodes-migrate.t +++ b/tests/000-flaky/basic_distribute_rebal-all-nodes-migrate.t @@ -1,8 +1,8 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../cluster.rc -. $(dirname $0)/../../dht.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../cluster.rc +. $(dirname $0)/../dht.rc # Check if every single rebalance process migrated some files @@ -10,11 +10,9 @@ function cluster_rebal_all_nodes_migrated_files { val=0 a=$($CLI_1 volume rebalance $V0 status | grep "completed" | awk '{print $2}'); -# echo $a b=($a) for i in "${b[@]}" do -# echo "$i"; if [ "$i" -eq "0" ]; then echo "false"; val=1; diff --git a/tests/basic/ec/ec-quorum-count-partial-failure.t b/tests/000-flaky/basic_ec_ec-quorum-count-partial-failure.t index 79f5825ae10..42808ce0c0e 100755..100644 --- a/tests/basic/ec/ec-quorum-count-partial-failure.t +++ b/tests/000-flaky/basic_ec_ec-quorum-count-partial-failure.t @@ -1,7 +1,7 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc #This test checks that partial failure of fop results in main fop failure only cleanup; diff --git a/tests/basic/mount-nfs-auth.t b/tests/000-flaky/basic_mount-nfs-auth.t index 3d4a9cff00b..3d4a9cff00b 100755..100644 --- a/tests/basic/mount-nfs-auth.t +++ b/tests/000-flaky/basic_mount-nfs-auth.t diff --git a/tests/bugs/core/multiplex-limit-issue-151.t b/tests/000-flaky/bugs_core_multiplex-limit-issue-151.t index dc9013061b0..5a88f97d726 100644 --- a/tests/bugs/core/multiplex-limit-issue-151.t +++ b/tests/000-flaky/bugs_core_multiplex-limit-issue-151.t @@ -1,8 +1,8 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../traps.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../traps.rc +. $(dirname $0)/../volume.rc function count_up_bricks { $CLI --xml volume status all | grep '<status>1' | wc -l diff --git a/tests/bugs/distribute/bug-1117851.t b/tests/000-flaky/bugs_distribute_bug-1117851.t index 62cb6b66ab4..5980bf2fd4b 100755..100644 --- a/tests/bugs/distribute/bug-1117851.t +++ b/tests/000-flaky/bugs_distribute_bug-1117851.t @@ -2,8 +2,8 @@ SCRIPT_TIMEOUT=250 -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc create_files () { for i in {1..1000}; do diff --git a/tests/bugs/distribute/bug-1122443.t b/tests/000-flaky/bugs_distribute_bug-1122443.t index 906be7072bd..abd37082b33 100644 --- a/tests/bugs/distribute/bug-1122443.t +++ b/tests/000-flaky/bugs_distribute_bug-1122443.t @@ -1,8 +1,8 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc -. $(dirname $0)/../../dht.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../dht.rc make_files() { mkdir $1 && \ @@ -42,8 +42,8 @@ TEST glusterfs -s $H0 --volfile-id $V0 $M0 TEST make_files $M0/subdir # Get mtime before migration -BEFORE="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')" - +BEFORE="$(stat -c %n:%Y $M0/subdir/* | sort | tr '\n' ',')" +echo $BEFORE # Migrate brick TEST $CLI volume add-brick $V0 $H0:$B0/${V0}1 TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 start @@ -51,9 +51,10 @@ EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 commit # Get mtime after migration -EXPECT_WITHIN 5 RECONNECTED bug_1113050_workaround $M0/subdir/* -AFTER="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')" - +EXPECT_WITHIN 30 RECONNECTED bug_1113050_workaround $M0/subdir/symlink +sleep 3 +AFTER="$(stat -c %n:%Y $M0/subdir/* | sort | tr '\n' ',')" +echo $AFTER # Check if mtime is unchanged TEST [ "$AFTER" == "$BEFORE" ] diff --git a/tests/bugs/glusterd/bug-857330/common.rc b/tests/000-flaky/bugs_glusterd_bug-857330/common.rc index d0aa4b1a640..bd122eff18c 100644 --- a/tests/bugs/glusterd/bug-857330/common.rc +++ b/tests/000-flaky/bugs_glusterd_bug-857330/common.rc @@ -1,4 +1,4 @@ -. $(dirname $0)/../../../include.rc +. $(dirname $0)/../../include.rc UUID_REGEX='[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' diff --git a/tests/bugs/glusterd/bug-857330/normal.t b/tests/000-flaky/bugs_glusterd_bug-857330/normal.t index ad0c8844fae..6c1cf54ec3c 100755 --- a/tests/bugs/glusterd/bug-857330/normal.t +++ b/tests/000-flaky/bugs_glusterd_bug-857330/normal.t @@ -1,7 +1,7 @@ #!/bin/bash . $(dirname $0)/common.rc -. $(dirname $0)/../../../volume.rc +. $(dirname $0)/../../volume.rc cleanup; TEST glusterd @@ -14,7 +14,7 @@ TEST $CLI volume start $V0; TEST glusterfs -s $H0 --volfile-id=$V0 $M0; -TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \ +TEST $PYTHON $(dirname $0)/../../utils/create-files.py \ --multi -b 10 -d 10 -n 10 $M0; EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 diff --git a/tests/bugs/glusterd/bug-857330/xml.t b/tests/000-flaky/bugs_glusterd_bug-857330/xml.t index 8383d2a0711..11785adacdb 100755 --- a/tests/bugs/glusterd/bug-857330/xml.t +++ b/tests/000-flaky/bugs_glusterd_bug-857330/xml.t @@ -1,7 +1,7 @@ #!/bin/bash . $(dirname $0)/common.rc -. $(dirname $0)/../../../volume.rc +. $(dirname $0)/../../volume.rc cleanup; @@ -15,7 +15,7 @@ TEST $CLI volume start $V0; TEST glusterfs -s $H0 --volfile-id=$V0 $M0; -TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \ +TEST $PYTHON $(dirname $0)/../../utils/create-files.py \ --multi -b 10 -d 10 -n 10 $M0; EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 diff --git a/tests/bugs/glusterd/quorum-value-check.t b/tests/000-flaky/bugs_glusterd_quorum-value-check.t index c701f08bbd5..a431b8c4fd4 100755..100644 --- a/tests/bugs/glusterd/quorum-value-check.t +++ b/tests/000-flaky/bugs_glusterd_quorum-value-check.t @@ -1,7 +1,7 @@ #!/bin/bash -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc #G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST diff --git a/tests/bugs/nfs/bug-1116503.t b/tests/000-flaky/bugs_nfs_bug-1116503.t index dd3998df150..fc50021acc7 100644 --- a/tests/bugs/nfs/bug-1116503.t +++ b/tests/000-flaky/bugs_nfs_bug-1116503.t @@ -3,9 +3,9 @@ # Verify that mounting NFS over UDP (MOUNT service only) works. # -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc -. $(dirname $0)/../../nfs.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../nfs.rc #G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST diff --git a/tests/features/lock-migration/lkmigration-set-option.t b/tests/000-flaky/features_lock-migration_lkmigration-set-option.t index 4340438591f..1327ef3579f 100644 --- a/tests/features/lock-migration/lkmigration-set-option.t +++ b/tests/000-flaky/features_lock-migration_lkmigration-set-option.t @@ -1,7 +1,7 @@ #!/bin/bash # Test to check -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc #Check lock-migration set option sanity cleanup; diff --git a/tests/afr.rc b/tests/afr.rc index 35f352df78f..241789903ba 100644 --- a/tests/afr.rc +++ b/tests/afr.rc @@ -105,3 +105,19 @@ function get_quorum_type() local repl_id="$3" cat $m/.meta/graphs/active/$v-replicate-$repl_id/private|grep quorum-type|awk '{print $3}' } + +function afr_private_key_value() +{ + local v=$1 + local m=$2 + local replica_id=$3 + local key=$4 +#xargs at the end will strip leading spaces + grep -E "^${key} = " $m/.meta/graphs/active/${v}-replicate-${replica_id}/private | cut -f2 -d'=' | xargs +} + +function afr_anon_entry_count() +{ + local b=$1 + ls $b/.glusterfs-anonymous-inode* | wc -l +} diff --git a/tests/basic/afr/afr-anon-inode-no-quorum.t b/tests/basic/afr/afr-anon-inode-no-quorum.t new file mode 100644 index 00000000000..896ba0c9b2c --- /dev/null +++ b/tests/basic/afr/afr-anon-inode-no-quorum.t @@ -0,0 +1,63 @@ +#!/bin/bash + +#Test that anon-inode entry is not cleaned up as long as there exists at least +#one valid entry +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../afr.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume heal $V0 disable +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.readdir-ahead off +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 cluster.entry-self-heal off +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 + +TEST touch $M0/a $M0/b + +gfid_a=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/a)) +gfid_b=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/b)) +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST mv $M0/a $M0/a-new +TEST mv $M0/b $M0/b-new + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0 +TEST ! ls $M0/a +TEST ! ls $M0/b +anon_inode_name=$(ls -a $B0/${V0}0 | grep glusterfs-anonymous-inode) +TEST stat $B0/${V0}0/$anon_inode_name/$gfid_a +TEST stat $B0/${V0}0/$anon_inode_name/$gfid_b +#Make sure index heal doesn't happen after enabling heal +TEST setfattr -x trusted.afr.$V0-client-0 $B0/${V0}1 +TEST rm -f $B0/${V0}1/.glusterfs/indices/xattrop/* +TEST $CLI volume heal $V0 enable +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 +TEST $CLI volume heal $V0 +#Allow time for a scan +sleep 5 +TEST stat $B0/${V0}0/$anon_inode_name/$gfid_a +TEST stat $B0/${V0}0/$anon_inode_name/$gfid_b +inum_b=$(STAT_INO $B0/${V0}0/$anon_inode_name/$gfid_b) +TEST rm -f $M0/a-new +TEST stat $M0/b-new + +TEST $CLI volume heal $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/${V0}0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/${V0}1 +EXPECT "$inum_b" STAT_INO $B0/${V0}0/b-new + +cleanup diff --git a/tests/basic/afr/afr-anon-inode.t b/tests/basic/afr/afr-anon-inode.t new file mode 100644 index 00000000000..f4cf37a2fa0 --- /dev/null +++ b/tests/basic/afr/afr-anon-inode.t @@ -0,0 +1,114 @@ +#!/bin/bash +#Tests that afr-anon-inode test cases work fine as expected +#These are cases where in entry-heal/name-heal we dont know entry for an inode +#so these inodes are kept in a special directory + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../afr.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0..2} +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0; +EXPECT "^1$" afr_private_key_value $V0 $M0 0 "use-anonymous-inode" +TEST $CLI volume set $V0 cluster.use-anonymous-inode no +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^0$" afr_private_key_value $V0 $M0 0 "use-anonymous-inode" +TEST $CLI volume set $V0 cluster.use-anonymous-inode yes +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "use-anonymous-inode" +TEST mkdir -p $M0/d1/b $M0/d2/a +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST mv $M0/d2/a $M0/d1 +TEST mv $M0/d1/b $M0/d2 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +anon_inode_name=$(ls -a $B0/${V0}0 | grep glusterfs-anonymous-inode) +TEST [[ -d $B0/${V0}1/$anon_inode_name ]] +TEST [[ -d $B0/${V0}2/$anon_inode_name ]] +anon_gfid=$(gf_get_gfid_xattr $B0/${V0}0/$anon_inode_name) +EXPECT "$anon_gfid" gf_get_gfid_xattr $B0/${V0}1/$anon_inode_name +EXPECT "$anon_gfid" gf_get_gfid_xattr $B0/${V0}2/$anon_inode_name + +TEST ! ls $M0/$anon_inode_name +EXPECT "^4$" echo $(ls -a $M0 | wc -l) + +#Test purging code path by shd +TEST $CLI volume heal $V0 disable +TEST mkdir $M0/l0 $M0/l1 $M0/l2 +TEST touch $M0/del-file $M0/del-file-nolink $M0/l0/file +TEST ln $M0/del-file $M0/del-file-link +TEST ln $M0/l0/file $M0/l1/file-link1 +TEST ln $M0/l0/file $M0/l2/file-link2 +TEST mkdir -p $M0/del-recursive-dir/d1 + +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST rm -f $M0/del-file $M0/del-file-nolink +TEST rm -rf $M0/del-recursive-dir +TEST mv $M0/d1/a $M0/d2 +TEST mv $M0/l0/file $M0/l0/renamed-file +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status $V0 0 + +nolink_gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/del-file-nolink)) +link_gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/del-file)) +dir_gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/del-recursive-dir)) +rename_dir_gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/d1/a)) +rename_file_gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/l0/file)) +TEST ! stat $M0/del-file +TEST stat $B0/${V0}0/$anon_inode_name/$link_gfid +TEST ! stat $M0/del-file-nolink +TEST ! stat $B0/${V0}0/$anon_inode_name/$nolink_gfid +TEST ! stat $M0/del-recursive-dir +TEST stat $B0/${V0}0/$anon_inode_name/$dir_gfid +TEST ! stat $M0/d1/a +TEST stat $B0/${V0}0/$anon_inode_name/$rename_dir_gfid +TEST ! stat $M0/l0/file +TEST stat $B0/${V0}0/$anon_inode_name/$rename_file_gfid + +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST mv $M0/l1/file-link1 $M0/l1/renamed-file-link1 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status $V0 1 +TEST ! stat $M0/l1/file-link1 +TEST stat $B0/${V0}1/$anon_inode_name/$rename_file_gfid + +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST mv $M0/l2/file-link2 $M0/l2/renamed-file-link2 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status $V0 2 +TEST ! stat $M0/l2/file-link2 +TEST stat $B0/${V0}2/$anon_inode_name/$rename_file_gfid + +#Simulate only anon-inodes present in all bricks +TEST rm -f $M0/l0/renamed-file $M0/l1/renamed-file-link1 $M0/l2/renamed-file-link2 + +#Test that shd doesn't cleanup anon-inodes when some bricks are down +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST $CLI volume heal $V0 enable +$CLI volume heal $V0 +sleep 5 #Allow time for completion of one scan +TEST stat $B0/${V0}0/$anon_inode_name/$link_gfid +TEST stat $B0/${V0}0/$anon_inode_name/$rename_dir_gfid +TEST stat $B0/${V0}0/$anon_inode_name/$dir_gfid +rename_dir_inum=$(STAT_INO $B0/${V0}0/$anon_inode_name/$rename_dir_gfid) + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "^1$" afr_child_up_status $V0 1 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/${V0}0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/${V0}1 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/${V0}2 + +#Test that rename indeed happened instead of rmdir/mkdir +renamed_dir_inum=$(STAT_INO $B0/${V0}0/d2/a) +EXPECT "$rename_dir_inum" echo $renamed_dir_inum +cleanup; diff --git a/tests/basic/afr/durability-off.t b/tests/basic/afr/durability-off.t index 155ffa09ef0..6e0f18b88f8 100644 --- a/tests/basic/afr/durability-off.t +++ b/tests/basic/afr/durability-off.t @@ -26,6 +26,8 @@ TEST $CLI volume heal $V0 EXPECT_WITHIN $HEAL_TIMEOUT "0" get_pending_heal_count $V0 EXPECT "^0$" echo $($CLI volume profile $V0 info | grep -w FSYNC | wc -l) +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1 #Test that fsyncs happen when durability is on TEST $CLI volume set $V0 cluster.ensure-durability on TEST $CLI volume set $V0 performance.strict-write-ordering on diff --git a/tests/basic/afr/entry-self-heal-anon-dir-off.t b/tests/basic/afr/entry-self-heal-anon-dir-off.t new file mode 100644 index 00000000000..7bb6ee14193 --- /dev/null +++ b/tests/basic/afr/entry-self-heal-anon-dir-off.t @@ -0,0 +1,459 @@ +#!/bin/bash + +#This file checks if missing entry self-heal and entry self-heal are working +#as expected. +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../afr.rc + +cleanup; + +function get_file_type { + stat -c "%a:%F:%g:%t:%T:%u" $1 +} + +function diff_dirs { + diff <(ls $1 | sort) <(ls $2 | sort) +} + +function heal_status { + local f1_path="${1}/${3}" + local f2_path="${2}/${3}" + local insync="" + diff_dirs $f1_path $f2_path + if [ $? -eq 0 ]; + then + insync="Y" + else + insync="N" + fi + local xattr11=$(get_hex_xattr trusted.afr.$V0-client-0 $f1_path) + local xattr12=$(get_hex_xattr trusted.afr.$V0-client-1 $f1_path) + local xattr21=$(get_hex_xattr trusted.afr.$V0-client-0 $f2_path) + local xattr22=$(get_hex_xattr trusted.afr.$V0-client-1 $f2_path) + local dirty1=$(get_hex_xattr trusted.afr.dirty $f1_path) + local dirty2=$(get_hex_xattr trusted.afr.dirty $f2_path) + if [ -z $xattr11 ]; then xattr11="000000000000000000000000"; fi + if [ -z $xattr12 ]; then xattr12="000000000000000000000000"; fi + if [ -z $xattr21 ]; then xattr21="000000000000000000000000"; fi + if [ -z $xattr22 ]; then xattr22="000000000000000000000000"; fi + if [ -z $dirty1 ]; then dirty1="000000000000000000000000"; fi + if [ -z $dirty2 ]; then dirty2="000000000000000000000000"; fi + echo ${insync}${xattr11}${xattr12}${xattr21}${xattr22}${dirty1}${dirty2} +} + +function is_heal_done { + local zero_xattr="000000000000000000000000" + if [ "$(heal_status $@)" == "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" ]; + then + echo "Y" + else + echo "N" + fi +} + +function print_pending_heals { + local result=":" + for i in "$@"; + do + if [ "N" == $(is_heal_done $B0/${V0}0 $B0/${V0}1 $i) ]; + then + result="$result:$i" + fi + done +#To prevent any match for EXPECT_WITHIN, print a char non-existent in file-names + if [ $result == ":" ]; then result="~"; fi + echo $result +} + +zero_xattr="000000000000000000000000" +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume heal $V0 disable +TEST $CLI volume set $V0 cluster.use-anonymous-inode off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.readdir-ahead off +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 cluster.data-self-heal on +TEST $CLI volume set $V0 cluster.metadata-self-heal on +TEST $CLI volume set $V0 cluster.entry-self-heal on +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 --use-readdirp=no $M0 +cd $M0 +#_me_ is dir on which missing entry self-heal happens, _heal is where dir self-heal happens +#spb is split-brain, fool is all fool + +#source_self_accusing means there exists source and a sink which self-accuses. +#This simulates failures where fops failed on the bricks without it going down. +#Something like EACCESS/EDQUOT etc + +TEST mkdir spb_heal spb spb_me_heal spb_me fool_heal fool_me v1_fool_heal v1_fool_me source_creations_heal source_deletions_heal source_creations_me source_deletions_me v1_dirty_me v1_dirty_heal source_self_accusing +TEST mkfifo source_deletions_heal/fifo +TEST mknod source_deletions_heal/block b 4 5 +TEST mknod source_deletions_heal/char c 1 5 +TEST touch source_deletions_heal/file +TEST ln -s source_deletions_heal/file source_deletions_heal/slink +TEST mkdir source_deletions_heal/dir1 +TEST mkdir source_deletions_heal/dir1/dir2 + +TEST mkfifo source_deletions_me/fifo +TEST mknod source_deletions_me/block b 4 5 +TEST mknod source_deletions_me/char c 1 5 +TEST touch source_deletions_me/file +TEST ln -s source_deletions_me/file source_deletions_me/slink +TEST mkdir source_deletions_me/dir1 +TEST mkdir source_deletions_me/dir1/dir2 + +TEST mkfifo source_self_accusing/fifo +TEST mknod source_self_accusing/block b 4 5 +TEST mknod source_self_accusing/char c 1 5 +TEST touch source_self_accusing/file +TEST ln -s source_self_accusing/file source_self_accusing/slink +TEST mkdir source_self_accusing/dir1 +TEST mkdir source_self_accusing/dir1/dir2 + +TEST kill_brick $V0 $H0 $B0/${V0}0 + +TEST touch spb_heal/0 spb/0 spb_me_heal/0 spb_me/0 fool_heal/0 fool_me/0 v1_fool_heal/0 v1_fool_me/0 v1_dirty_heal/0 v1_dirty_me/0 +TEST rm -rf source_deletions_heal/fifo source_deletions_heal/block source_deletions_heal/char source_deletions_heal/file source_deletions_heal/slink source_deletions_heal/dir1 +TEST rm -rf source_deletions_me/fifo source_deletions_me/block source_deletions_me/char source_deletions_me/file source_deletions_me/slink source_deletions_me/dir1 +TEST rm -rf source_self_accusing/fifo source_self_accusing/block source_self_accusing/char source_self_accusing/file source_self_accusing/slink source_self_accusing/dir1 + +#Test that the files are deleted +TEST ! stat $B0/${V0}1/source_deletions_heal/fifo +TEST ! stat $B0/${V0}1/source_deletions_heal/block +TEST ! stat $B0/${V0}1/source_deletions_heal/char +TEST ! stat $B0/${V0}1/source_deletions_heal/file +TEST ! stat $B0/${V0}1/source_deletions_heal/slink +TEST ! stat $B0/${V0}1/source_deletions_heal/dir1 +TEST ! stat $B0/${V0}1/source_deletions_me/fifo +TEST ! stat $B0/${V0}1/source_deletions_me/block +TEST ! stat $B0/${V0}1/source_deletions_me/char +TEST ! stat $B0/${V0}1/source_deletions_me/file +TEST ! stat $B0/${V0}1/source_deletions_me/slink +TEST ! stat $B0/${V0}1/source_deletions_me/dir1 +TEST ! stat $B0/${V0}1/source_self_accusing/fifo +TEST ! stat $B0/${V0}1/source_self_accusing/block +TEST ! stat $B0/${V0}1/source_self_accusing/char +TEST ! stat $B0/${V0}1/source_self_accusing/file +TEST ! stat $B0/${V0}1/source_self_accusing/slink +TEST ! stat $B0/${V0}1/source_self_accusing/dir1 + + +TEST mkfifo source_creations_heal/fifo +TEST mknod source_creations_heal/block b 4 5 +TEST mknod source_creations_heal/char c 1 5 +TEST touch source_creations_heal/file +TEST ln -s source_creations_heal/file source_creations_heal/slink +TEST mkdir source_creations_heal/dir1 +TEST mkdir source_creations_heal/dir1/dir2 + +TEST mkfifo source_creations_me/fifo +TEST mknod source_creations_me/block b 4 5 +TEST mknod source_creations_me/char c 1 5 +TEST touch source_creations_me/file +TEST ln -s source_creations_me/file source_creations_me/slink +TEST mkdir source_creations_me/dir1 +TEST mkdir source_creations_me/dir1/dir2 + +$CLI volume stop $V0 + +#simulate fool fool scenario for fool_* dirs +setfattr -x trusted.afr.$V0-client-0 $B0/${V0}1/{fool_heal,fool_me} +setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}1/{fool_heal,fool_me} +setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}1/{v1_fool_heal,v1_fool_me} + +#Simulate v1-dirty(self-accusing but no pending ops on others) scenario for v1-dirty +setfattr -x trusted.afr.$V0-client-0 $B0/${V0}1/v1_dirty_{heal,me} +setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}1/v1_dirty_{heal,me} + +$CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +TEST kill_brick $V0 $H0 $B0/${V0}1 + +TEST touch spb_heal/1 spb/0 spb_me_heal/1 spb_me/0 fool_heal/1 fool_me/1 v1_fool_heal/1 v1_fool_me/1 + +$CLI volume stop $V0 + +#simulate fool fool scenario for fool_* dirs +setfattr -x trusted.afr.$V0-client-1 $B0/${V0}0/{fool_heal,fool_me} +setfattr -n trusted.afr.dirty -v 0x000000000000000000000001 $B0/${V0}1/{fool_heal,fool_me} +setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}1/{v1_fool_heal,v1_fool_me} + +#simulate self-accusing for source_self_accusing +TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000006 $B0/${V0}0/source_self_accusing + +$CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +# Check if conservative merges happened correctly on _me_ dirs +TEST stat spb_me_heal/1 +TEST stat $B0/${V0}0/spb_me_heal/1 +TEST stat $B0/${V0}1/spb_me_heal/1 + +TEST stat spb_me_heal/0 +TEST stat $B0/${V0}0/spb_me_heal/0 +TEST stat $B0/${V0}1/spb_me_heal/0 + +TEST stat fool_me/1 +TEST stat $B0/${V0}0/fool_me/1 +TEST stat $B0/${V0}1/fool_me/1 + +TEST stat fool_me/0 +TEST stat $B0/${V0}0/fool_me/0 +TEST stat $B0/${V0}1/fool_me/0 + +TEST stat v1_fool_me/0 +TEST stat $B0/${V0}0/v1_fool_me/0 +TEST stat $B0/${V0}1/v1_fool_me/0 + +TEST stat v1_fool_me/1 +TEST stat $B0/${V0}0/v1_fool_me/1 +TEST stat $B0/${V0}1/v1_fool_me/1 + +TEST stat v1_dirty_me/0 +TEST stat $B0/${V0}0/v1_dirty_me/0 +TEST stat $B0/${V0}1/v1_dirty_me/0 + +#Check if files that have gfid-mismatches in _me_ are giving EIO +TEST ! stat spb_me/0 + +#Check if stale files are deleted on access +TEST ! stat source_deletions_me/fifo +TEST ! stat $B0/${V0}0/source_deletions_me/fifo +TEST ! stat $B0/${V0}1/source_deletions_me/fifo +TEST ! stat source_deletions_me/block +TEST ! stat $B0/${V0}0/source_deletions_me/block +TEST ! stat $B0/${V0}1/source_deletions_me/block +TEST ! stat source_deletions_me/char +TEST ! stat $B0/${V0}0/source_deletions_me/char +TEST ! stat $B0/${V0}1/source_deletions_me/char +TEST ! stat source_deletions_me/file +TEST ! stat $B0/${V0}0/source_deletions_me/file +TEST ! stat $B0/${V0}1/source_deletions_me/file +TEST ! stat source_deletions_me/file +TEST ! stat $B0/${V0}0/source_deletions_me/file +TEST ! stat $B0/${V0}1/source_deletions_me/file +TEST ! stat source_deletions_me/dir1/dir2 +TEST ! stat $B0/${V0}0/source_deletions_me/dir1/dir2 +TEST ! stat $B0/${V0}1/source_deletions_me/dir1/dir2 +TEST ! stat source_deletions_me/dir1 +TEST ! stat $B0/${V0}0/source_deletions_me/dir1 +TEST ! stat $B0/${V0}1/source_deletions_me/dir1 + +#Test if the files created as part of access are healed correctly +r=$(get_file_type source_creations_me/fifo) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/fifo +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/fifo +TEST [ -p source_creations_me/fifo ] + +r=$(get_file_type source_creations_me/block) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/block +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/block +EXPECT "^4 5$" stat -c "%t %T" $B0/${V0}1/source_creations_me/block +EXPECT "^4 5$" stat -c "%t %T" $B0/${V0}0/source_creations_me/block +TEST [ -b source_creations_me/block ] + +r=$(get_file_type source_creations_me/char) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/char +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/char +EXPECT "^1 5$" stat -c "%t %T" $B0/${V0}1/source_creations_me/char +EXPECT "^1 5$" stat -c "%t %T" $B0/${V0}0/source_creations_me/char +TEST [ -c source_creations_me/char ] + +r=$(get_file_type source_creations_me/file) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/file +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/file +TEST [ -f source_creations_me/file ] + +r=$(get_file_type source_creations_me/slink) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/slink +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/slink +TEST [ -h source_creations_me/slink ] + +r=$(get_file_type source_creations_me/dir1/dir2) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/dir1/dir2 +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/dir1/dir2 +TEST [ -d source_creations_me/dir1/dir2 ] + +r=$(get_file_type source_creations_me/dir1) +EXPECT "$r" get_file_type $B0/${V0}0/source_creations_me/dir1 +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_me/dir1 +TEST [ -d source_creations_me/dir1 ] + +#Trigger heal and check _heal dirs are healed properly +#Trigger change in event generation number. That way inodes would get refreshed during lookup +TEST kill_brick $V0 $H0 $B0/${V0}1 +$CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +TEST stat spb_heal +TEST stat spb_me_heal +TEST stat fool_heal +TEST stat fool_me +TEST stat v1_fool_heal +TEST stat v1_fool_me +TEST stat source_deletions_heal +TEST stat source_deletions_me +TEST stat source_self_accusing +TEST stat source_creations_heal +TEST stat source_creations_me +TEST stat v1_dirty_heal +TEST stat v1_dirty_me +TEST $CLI volume stop $V0 +TEST rm -rf $B0/${V0}{0,1}/.glusterfs/indices/xattrop/* + +$CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +#Create base entry in indices/xattrop +echo "Data" > $M0/FILE +rm -f $M0/FILE +EXPECT "1" count_index_entries $B0/${V0}0 +EXPECT "1" count_index_entries $B0/${V0}1 + +TEST $CLI volume stop $V0; + +#Create entries for fool_heal and fool_me to ensure they are fully healed and dirty xattrs erased, before triggering index heal +create_brick_xattrop_entry $B0/${V0}0 fool_heal fool_me source_creations_heal/dir1 + +$CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +$CLI volume heal $V0 enable +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 + +TEST $CLI volume heal $V0; +EXPECT_WITHIN $HEAL_TIMEOUT "~" print_pending_heals spb_heal spb_me_heal fool_heal fool_me v1_fool_heal v1_fool_me source_deletions_heal source_deletions_me source_creations_heal source_creations_me v1_dirty_heal v1_dirty_me source_self_accusing + +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 spb_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 spb_me_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 fool_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 fool_me +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 v1_fool_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 v1_fool_me +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 source_deletions_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 source_deletions_me +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 source_self_accusing +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 source_creations_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 source_creations_me +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 v1_dirty_heal +EXPECT "Y${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" heal_status $B0/${V0}0 $B0/${V0}1 v1_dirty_me + +#Don't access the files/dirs from mount point as that may cause self-heals +# Check if conservative merges happened correctly on heal dirs +TEST stat $B0/${V0}0/spb_heal/1 +TEST stat $B0/${V0}1/spb_heal/1 + +TEST stat $B0/${V0}0/spb_heal/0 +TEST stat $B0/${V0}1/spb_heal/0 + +TEST stat $B0/${V0}0/fool_heal/1 +TEST stat $B0/${V0}1/fool_heal/1 + +TEST stat $B0/${V0}0/fool_heal/0 +TEST stat $B0/${V0}1/fool_heal/0 + +TEST stat $B0/${V0}0/v1_fool_heal/0 +TEST stat $B0/${V0}1/v1_fool_heal/0 + +TEST stat $B0/${V0}0/v1_fool_heal/1 +TEST stat $B0/${V0}1/v1_fool_heal/1 + +TEST stat $B0/${V0}0/v1_dirty_heal/0 +TEST stat $B0/${V0}1/v1_dirty_heal/0 + +#Check if files that have gfid-mismatches in spb are giving EIO +TEST ! stat spb/0 + +#Check if stale files are deleted on access +TEST ! stat $B0/${V0}0/source_deletions_heal/fifo +TEST ! stat $B0/${V0}1/source_deletions_heal/fifo +TEST ! stat $B0/${V0}0/source_deletions_heal/block +TEST ! stat $B0/${V0}1/source_deletions_heal/block +TEST ! stat $B0/${V0}0/source_deletions_heal/char +TEST ! stat $B0/${V0}1/source_deletions_heal/char +TEST ! stat $B0/${V0}0/source_deletions_heal/file +TEST ! stat $B0/${V0}1/source_deletions_heal/file +TEST ! stat $B0/${V0}0/source_deletions_heal/file +TEST ! stat $B0/${V0}1/source_deletions_heal/file +TEST ! stat $B0/${V0}0/source_deletions_heal/dir1/dir2 +TEST ! stat $B0/${V0}1/source_deletions_heal/dir1/dir2 +TEST ! stat $B0/${V0}0/source_deletions_heal/dir1 +TEST ! stat $B0/${V0}1/source_deletions_heal/dir1 + +#Check if stale files are deleted on access +TEST ! stat $B0/${V0}0/source_self_accusing/fifo +TEST ! stat $B0/${V0}1/source_self_accusing/fifo +TEST ! stat $B0/${V0}0/source_self_accusing/block +TEST ! stat $B0/${V0}1/source_self_accusing/block +TEST ! stat $B0/${V0}0/source_self_accusing/char +TEST ! stat $B0/${V0}1/source_self_accusing/char +TEST ! stat $B0/${V0}0/source_self_accusing/file +TEST ! stat $B0/${V0}1/source_self_accusing/file +TEST ! stat $B0/${V0}0/source_self_accusing/file +TEST ! stat $B0/${V0}1/source_self_accusing/file +TEST ! stat $B0/${V0}0/source_self_accusing/dir1/dir2 +TEST ! stat $B0/${V0}1/source_self_accusing/dir1/dir2 +TEST ! stat $B0/${V0}0/source_self_accusing/dir1 +TEST ! stat $B0/${V0}1/source_self_accusing/dir1 + +#Test if the files created as part of full self-heal correctly +r=$(get_file_type $B0/${V0}0/source_creations_heal/fifo) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/fifo +TEST [ -p $B0/${V0}0/source_creations_heal/fifo ] +EXPECT "^4 5$" stat -c "%t %T" $B0/${V0}1/source_creations_heal/block +EXPECT "^4 5$" stat -c "%t %T" $B0/${V0}0/source_creations_heal/block + +r=$(get_file_type $B0/${V0}0/source_creations_heal/block) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/block + +r=$(get_file_type $B0/${V0}0/source_creations_heal/char) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/char +EXPECT "^1 5$" stat -c "%t %T" $B0/${V0}1/source_creations_heal/char +EXPECT "^1 5$" stat -c "%t %T" $B0/${V0}0/source_creations_heal/char + +r=$(get_file_type $B0/${V0}0/source_creations_heal/file) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/file +TEST [ -f $B0/${V0}0/source_creations_heal/file ] + +r=$(get_file_type source_creations_heal/file $B0/${V0}0/slink) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/file slink +TEST [ -h $B0/${V0}0/source_creations_heal/slink ] + +r=$(get_file_type $B0/${V0}0/source_creations_heal/dir1/dir2) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/dir1/dir2 +TEST [ -d $B0/${V0}0/source_creations_heal/dir1/dir2 ] + +r=$(get_file_type $B0/${V0}0/source_creations_heal/dir1) +EXPECT "$r" get_file_type $B0/${V0}1/source_creations_heal/dir1 +TEST [ -d $B0/${V0}0/source_creations_heal/dir1 ] + +cd - + +#Anonymous directory shouldn't be created +TEST mkdir $M0/rename-dir +before_rename=$(STAT_INO $B0/${V0}1/rename-dir) +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST mv $M0/rename-dir $M0/new-name +TEST $CLI volume start $V0 force +#'spb' is in split-brain so pending-heal-count will be 2 +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 +after_rename=$(STAT_INO $B0/${V0}1/new-name) +EXPECT "0" echo $(ls -a $B0/${V0}0/ | grep anonymous-inode | wc -l) +EXPECT "0" echo $(ls -a $B0/${V0}1/ | grep anonymous-inode | wc -l) +EXPECT_NOT "$before_rename" echo $after_rename +cleanup diff --git a/tests/basic/afr/gfid-mismatch-resolution-with-fav-child-policy.t b/tests/basic/afr/gfid-mismatch-resolution-with-fav-child-policy.t index f4aa351e461..35e295dc170 100644 --- a/tests/basic/afr/gfid-mismatch-resolution-with-fav-child-policy.t +++ b/tests/basic/afr/gfid-mismatch-resolution-with-fav-child-policy.t @@ -168,8 +168,8 @@ TEST [ "$gfid_1" != "$gfid_2" ] #We know that second brick has the bigger size file BIGGER_FILE_MD5=$(md5sum $B0/${V0}1/f3 | cut -d\ -f1) -TEST ls $M0/f3 -TEST cat $M0/f3 +TEST ls $M0 #Trigger entry heal via readdir inode refresh +TEST cat $M0/f3 #Trigger data heal via readv inode refresh EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 #gfid split-brain should be resolved @@ -215,8 +215,8 @@ TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 2 -TEST ls $M0/f4 -TEST cat $M0/f4 +TEST ls $M0 #Trigger entry heal via readdir inode refresh +TEST cat $M0/f4 #Trigger data heal via readv inode refresh EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 #gfid split-brain should be resolved @@ -227,4 +227,3 @@ HEALED_MD5=$(md5sum $B0/${V0}2/f4 | cut -d\ -f1) TEST [ "$MAJORITY_MD5" == "$HEALED_MD5" ] cleanup; -#G_TESTDEF_TEST_STATUS_NETBSD7=1501390 diff --git a/tests/basic/afr/halo.t b/tests/basic/afr/halo.t new file mode 100644 index 00000000000..3f61f5a0402 --- /dev/null +++ b/tests/basic/afr/halo.t @@ -0,0 +1,61 @@ +#!/bin/bash +#Tests that halo basic functionality works as expected + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../afr.rc + +cleanup; + +function get_up_child() +{ + if [ "1" == $(afr_private_key_value $V0 $M0 0 "child_up\[0\]") ]; + then + echo 0 + elif [ "1" == $(afr_private_key_value $V0 $M0 0 "child_up\[1\]") ] + then + echo 1 + fi +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume set $V0 cluster.halo-enabled yes +TEST $CLI volume set $V0 cluster.halo-max-replicas 1 +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0; +EXPECT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[0\]" +EXPECT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[1\]" +EXPECT_NOT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[0\]" +EXPECT_NOT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[1\]" + +up_id=$(get_up_child) +TEST [[ ! -z "$up_id" ]] + +down_id=$((1-up_id)) + +TEST kill_brick $V0 $H0 $B0/${V0}${up_id} +#As max-replicas is configured to be 1, down_child should be up now +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[${down_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "child_up\[${down_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" afr_private_key_value $V0 $M0 0 "halo_child_up\[${up_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" afr_private_key_value $V0 $M0 0 "child_up\[${up_id}\]" +EXPECT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[${up_id}\]" +EXPECT_NOT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[${down_id}\]" + +#Bring the brick back up and the state should be restored +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[${up_id}\]" + +up_id=$(get_up_child) +TEST [[ ! -z "$up_id" ]] +down_id=$((1-up_id)) +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[${down_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" afr_private_key_value $V0 $M0 0 "child_up\[${down_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "halo_child_up\[${up_id}\]" +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" afr_private_key_value $V0 $M0 0 "child_up\[${up_id}\]" +EXPECT_NOT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[0\]" +EXPECT_NOT "^-1$" afr_private_key_value $V0 $M0 0 "child_latency\[1\]" + +cleanup; diff --git a/tests/basic/afr/rename-data-loss.t b/tests/basic/afr/rename-data-loss.t new file mode 100644 index 00000000000..256ee2aafce --- /dev/null +++ b/tests/basic/afr/rename-data-loss.t @@ -0,0 +1,72 @@ +#!/bin/bash +#Self-heal tests +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../afr.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/brick{0,1} +TEST $CLI volume set $V0 write-behind off +TEST $CLI volume set $V0 self-heal-daemon off +TEST $CLI volume set $V0 data-self-heal off +TEST $CLI volume set $V0 metadata-self-heal off +TEST $CLI volume set $V0 entry-self-heal off +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0; + +cd $M0 +TEST `echo "line1" >> file1` +TEST mkdir dir1 +TEST mkdir dir2 +TEST mkdir -p dir1/dira/dirb +TEST `echo "line1">>dir1/dira/dirb/file1` +TEST mkdir delete_me +TEST `echo "line1" >> delete_me/file1` + +#brick0 has witnessed the second write while brick1 is down. +TEST kill_brick $V0 $H0 $B0/brick1 +TEST `echo "line2" >> file1` +TEST `echo "line2" >> dir1/dira/dirb/file1` +TEST `echo "line2" >> delete_me/file1` + +#Toggle the bricks that are up/down. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1 +TEST kill_brick $V0 $H0 $B0/brick0 + +#Rename when the 'source' brick0 for data-selfheals is down. +mv file1 file2 +mv dir1/dira dir2 + +#Delete a dir when brick0 is down. +rm -rf delete_me +cd - + +#Bring everything up and trigger heal +TEST $CLI volume set $V0 self-heal-daemon on +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 +TEST $CLI volume heal $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/brick0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_anon_entry_count $B0/brick1 + +#Remount to avoid reading from caches +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0; +EXPECT "line2" tail -1 $M0/file2 +EXPECT "line2" tail -1 $M0/dir2/dira/dirb/file1 +TEST ! stat $M0/delete_me/file1 +TEST ! stat $M0/delete_me + +anon_inode_name=$(ls -a $B0/brick0 | grep glusterfs-anonymous-inode) +TEST [[ -d $B0/brick0/$anon_inode_name ]] +TEST [[ -d $B0/brick1/$anon_inode_name ]] +cleanup diff --git a/tests/basic/afr/split-brain-favorite-child-policy-client-side-healing.t b/tests/basic/afr/split-brain-favorite-child-policy-client-side-healing.t new file mode 100644 index 00000000000..7c249c4bcbd --- /dev/null +++ b/tests/basic/afr/split-brain-favorite-child-policy-client-side-healing.t @@ -0,0 +1,124 @@ +#!/bin/bash + +#Test the client side split-brain resolution +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +GET_MDATA_PATH=$(dirname $0)/../../utils +build_tester $GET_MDATA_PATH/get-mdata-xattr.c + +TEST glusterd +TEST pidof glusterd + +count_files () { + ls $1 | wc -l +} + +#Create replica 2 volume +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume heal $V0 disable +TEST $CLI volume set $V0 cluster.quorum-type fixed +TEST $CLI volume set $V0 cluster.quorum-count 1 +TEST $CLI volume set $V0 cluster.metadata-self-heal on +TEST $CLI volume set $V0 cluster.data-self-heal on +TEST $CLI volume set $V0 cluster.entry-self-heal on + +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +TEST mkdir $M0/data +TEST touch $M0/data/file + + +############ Client side healing using favorite-child-policy = mtime ################# +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST dd if=/dev/urandom of=$M0/data/file bs=1024 count=1024 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST dd if=/dev/urandom of=$M0/data/file bs=1024 count=1024 + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +mtime1=$(get_mtime $B0/${V0}0/data/file) +mtime2=$(get_mtime $B0/${V0}1/data/file) +if (( $(echo "$mtime1 > $mtime2" | bc -l) )); then + LATEST_MTIME_MD5=$(md5sum $B0/${V0}0/data/file | cut -d\ -f1) +else + LATEST_MTIME_MD5=$(md5sum $B0/${V0}1/data/file | cut -d\ -f1) +fi + +#file will be in split-brain +cat $M0/data/file > /dev/null +EXPECT "1" echo $? + +TEST $CLI volume set $V0 cluster.favorite-child-policy mtime +TEST $CLI volume start $V0 force + +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" afr_get_split_brain_count $V0 +cat $M0/data/file > /dev/null +EXPECT "0" echo $? +M0_MD5=$(md5sum $M0/data/file | cut -d\ -f1) +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_get_split_brain_count $V0 +TEST [ "$LATEST_MTIME_MD5" == "$M0_MD5" ] + +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +B0_MD5=$(md5sum $B0/${V0}0/data/file | cut -d\ -f1) +B1_MD5=$(md5sum $B0/${V0}1/data/file | cut -d\ -f1) +TEST [ "$LATEST_MTIME_MD5" == "$B0_MD5" ] +TEST [ "$LATEST_MTIME_MD5" == "$B1_MD5" ] + +############ Client side directory conservative merge ################# +TEST $CLI volume reset $V0 cluster.favorite-child-policy +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST touch $M0/data/test +files=$(count_files $M0/data) +EXPECT "2" echo $files +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST touch $M0/data/test1 +files=$(count_files $M0/data) +EXPECT "2" echo $files + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +#data dir will be in entry split-brain +ls $M0/data > /dev/null +EXPECT "2" echo $? + +TEST $CLI volume set $V0 cluster.favorite-child-policy mtime + +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" afr_get_split_brain_count $V0 + + +ls $M0/data > /dev/null +EXPECT "0" echo $? + +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" afr_get_split_brain_count $V0 +#Entry Split-brain is gone, but data self-heal is pending on the files +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 + +cat $M0/data/test > /dev/null +cat $M0/data/test1 > /dev/null + +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +files=$(count_files $M0/data) +EXPECT "3" echo $files + +TEST force_umount $M0 +TEST rm $GET_MDATA_PATH/get-mdata-xattr + +cleanup diff --git a/tests/basic/distribute/spare_file_rebalance.t b/tests/basic/distribute/spare_file_rebalance.t new file mode 100644 index 00000000000..061c02f7392 --- /dev/null +++ b/tests/basic/distribute/spare_file_rebalance.t @@ -0,0 +1,51 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../dht.rc + +# Initialize +#------------------------------------------------------------ +cleanup; + +# Start glusterd +TEST glusterd; +TEST pidof glusterd; +TEST $CLI volume info; + +# Create a volume +TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2}; + +# Verify volume creation +EXPECT "$V0" volinfo_field $V0 'Volume Name'; +EXPECT 'Created' volinfo_field $V0 'Status'; + +# Start volume and verify successful start +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; + +#------------------------------------------------------------ + +# Test case - Create sparse files on MP and verify +# file info after rebalance +#------------------------------------------------------------ + +# Create some sparse files and get their size +TEST cd $M0; +dd if=/dev/urandom of=sparse_file bs=10k count=1 seek=2M +cp --sparse=always sparse_file sparse_file_3; + +# Add a 3rd brick +TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3; + +# Trigger rebalance +TEST $CLI volume rebalance $V0 start force; +EXPECT_WITHIN $REBALANCE_TIMEOUT "0" rebalance_completed; + +# Compare original and rebalanced files +TEST cd $B0/${V0}2 +TEST cmp sparse_file $B0/${V0}3/sparse_file_3 +EXPECT_WITHIN 30 ""; + +cleanup; diff --git a/tests/basic/ec/ec-quorum-count.t b/tests/basic/ec/ec-quorum-count.t index 56b5329c411..9310ebbb8f2 100644 --- a/tests/basic/ec/ec-quorum-count.t +++ b/tests/basic/ec/ec-quorum-count.t @@ -11,6 +11,8 @@ TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} TEST $CLI volume create $V1 $H0:$B0/${V1}{0..5} TEST $CLI volume set $V0 disperse.eager-lock-timeout 5 TEST $CLI volume set $V0 performance.flush-behind off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.heal-wait-qlength 0 #Should fail on non-disperse volume TEST ! $CLI volume set $V1 disperse.quorum-count 5 diff --git a/tests/basic/ec/ec-reset-brick.t b/tests/basic/ec/ec-reset-brick.t new file mode 100644 index 00000000000..f1a625df4ff --- /dev/null +++ b/tests/basic/ec/ec-reset-brick.t @@ -0,0 +1,50 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup +function num_entries { + ls -l $1 | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +mkdir $M0/dir +touch $M0/dir/{1..10} + +mkdir $M0/dir/dir1 +touch $M0/dir/dir1/{1..10} + +#kill brick process +TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}5 start +EXPECT_WITHIN $CHILD_UP_TIMEOUT "5" ec_child_up_count $V0 0 + +#reset-brick by removing all the data and create dir again +rm -rf $B0/${V0}5 +mkdir $B0/${V0}5 + +#start brick process and heal by commiting reset-brick +TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}5 $H0:$B0/${V0}5 commit force + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count_shd $V0 0 + +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} + +EXPECT "^12$" num_entries $B0/${V0}5/dir +EXPECT "^11$" num_entries $B0/${V0}5/dir/dir1 + +ec_version=$(get_hex_xattr trusted.ec.version $B0/${V0}0) +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}1 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}2 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}3 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}4 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}5 + +cleanup; diff --git a/tests/basic/ec/gfapi-ec-open-truncate.c b/tests/basic/ec/gfapi-ec-open-truncate.c index 22f17f436b6..fb16807003a 100644 --- a/tests/basic/ec/gfapi-ec-open-truncate.c +++ b/tests/basic/ec/gfapi-ec-open-truncate.c @@ -150,7 +150,8 @@ main(int argc, char *argv[]) for (i = 0; i < 20; i++) { ret = system( "[ $(for i in $(pgrep glusterfsd); do ls -l /proc/$i/fd | grep " - "\"[.]glusterfs\" | grep -v health_check; done | wc -l) == 3 ]"); + "\"[.]glusterfs\" | grep -v \".glusterfs/[0-9a-f][0-9a-f]\" | grep " + "-v health_check; done | wc -l) == 3 ]"); if (WIFEXITED(ret) && WEXITSTATUS(ret)) { printf("Ret value of system: %d\n, ifexited: %d, exitstatus: %d", ret, WIFEXITED(ret), WEXITSTATUS(ret)); diff --git a/tests/basic/fencing/afr-lock-heal-advanced.t b/tests/basic/fencing/afr-lock-heal-advanced.t index 8a7a208db29..8a5b5989b5e 100644 --- a/tests/basic/fencing/afr-lock-heal-advanced.t +++ b/tests/basic/fencing/afr-lock-heal-advanced.t @@ -4,6 +4,7 @@ . $(dirname $0)/../../volume.rc cleanup; +PROCESS_UP_TIMEOUT=90 function is_gfapi_program_alive() { @@ -19,9 +20,14 @@ function is_gfapi_program_alive() function get_active_lock_count { brick=$1 + i1=$2 + i2=$3 + pattern="ACTIVE.*client-${brick: -1}" + sdump=$(generate_brick_statedump $V0 $H0 $brick) - lock_count="$(grep ACTIVE $sdump| wc -l)" - echo "$lock_count" + lock_count1="$(egrep "$i1" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + lock_count2="$(egrep "$i2" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + echo "$((lock_count1+lock_count2))" } TEST glusterd @@ -49,6 +55,11 @@ TEST [ $client_pid ] TEST sleep 5 # By now, the client would have opened an fd on FILE1 and FILE2 and waiting for a SIGUSR1. EXPECT "Y" is_gfapi_program_alive $client_pid +gfid_str1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE1)) +inode1="FILE1|gfid:$gfid_str1" +gfid_str2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE2)) +inode2="FILE2|gfid:$gfid_str2" + # Kill brick-3 and let client-1 take lock on both files. TEST kill_brick $V0 $H0 $B0/${V0}2 TEST kill -SIGUSR1 $client_pid @@ -56,15 +67,15 @@ TEST kill -SIGUSR1 $client_pid EXPECT "Y" is_gfapi_program_alive $client_pid # Check lock is present on brick-1 and brick-2 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 $inode1 $inode2 # Restart brick-3 and check that the lock has healed on it. TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 $inode1 $inode2 #------------------------------------------------------------------------------ # Kill same brick before heal completes the first time and check it completes the second time. @@ -80,7 +91,7 @@ TEST kill_brick $V0 $H0 $B0/${V0}0 TEST $CLI volume reset $V0 delay-gen TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 #------------------------------------------------------------------------------ # Kill 2 bricks and bring it back. The fds must be marked bad. diff --git a/tests/basic/fencing/afr-lock-heal-basic.t b/tests/basic/fencing/afr-lock-heal-basic.t index 5ac05c7aec6..69131af085d 100644 --- a/tests/basic/fencing/afr-lock-heal-basic.t +++ b/tests/basic/fencing/afr-lock-heal-basic.t @@ -45,6 +45,9 @@ TEST sleep 5 # By now, the 2 clients would have opened an fd on FILE and waitin EXPECT "Y" is_gfapi_program_alive $client1_pid EXPECT "Y" is_gfapi_program_alive $client2_pid +gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE)) +inode="FILE|gfid:$gfid_str" + # Kill brick-3 and let client-1 take lock on the file. TEST kill_brick $V0 $H0 $B0/${V0}2 TEST kill -SIGUSR1 $client1_pid @@ -53,9 +56,9 @@ EXPECT "Y" is_gfapi_program_alive $client1_pid # Check lock is present on brick-1 and brick-2 b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) +c1_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) -c1_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c1_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c1_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b2" ] # Restart brick-3 and check that the lock has healed on it. @@ -64,7 +67,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. Also wait for lock heal. b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) -c1_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c1_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b3" ] # Kill brick-1 and let client-2 preempt the lock on bricks 2 and 3. @@ -80,11 +83,11 @@ TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fai # Check that all bricks now have locks from client 2 only. b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) +c2_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) +c2_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) -c2_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c2_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c2_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c2_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b2" ] TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b3" ] TEST [ "$c2_lock_on_b1" != "$c1_lock_on_b1" ] diff --git a/tests/basic/fuse/active-io-graph-switch.t b/tests/basic/fuse/active-io-graph-switch.t new file mode 100644 index 00000000000..6ec3e1fcbfa --- /dev/null +++ b/tests/basic/fuse/active-io-graph-switch.t @@ -0,0 +1,65 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +TESTS_EXPECTED_IN_LOOP=12 + +function perform_io_on_mount { + local m="$1" + local f="$2" + local lockfile="$3" + while [ -f "$m/$lockfile" ]; + do + dd if=/dev/zero of=$m/$f bs=1M count=1 + done +} + +function perform_graph_switch { + for i in {1..3} + do + TEST_IN_LOOP $CLI volume set $V0 performance.stat-prefetch off + sleep 3 + TEST_IN_LOOP $CLI volume set $V0 performance.stat-prefetch on + sleep 3 + done +} + +function count_files { + ls $M0 | wc -l +} + +cleanup; +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} +TEST $CLI volume set $V0 flush-behind off +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +TEST touch $M0/lock +for i in {1..100}; do perform_io_on_mount $M0 $i lock & done +EXPECT_WITHIN 5 "101" count_files + +perform_graph_switch +TEST rm -f $M0/lock +wait +EXPECT "100" count_files +TEST rm -f $M0/{1..100} +EXPECT "0" count_files + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +#Repeat the tests with reader-thread-count +TEST $GFS --reader-thread-count=10 --volfile-id=/$V0 --volfile-server=$H0 $M0 +TEST touch $M0/lock +for i in {1..100}; do perform_io_on_mount $M0 $i lock & done +EXPECT_WITHIN 5 "101" count_files + +perform_graph_switch +TEST rm -f $M0/lock +wait +EXPECT "100" count_files +TEST rm -f $M0/{1..100} +EXPECT "0" count_files + +cleanup diff --git a/tests/basic/gfapi/gfapi-copy-file-range.t b/tests/basic/gfapi/gfapi-copy-file-range.t index c24c1433edf..a56d3a58e07 100644 --- a/tests/basic/gfapi/gfapi-copy-file-range.t +++ b/tests/basic/gfapi/gfapi-copy-file-range.t @@ -5,20 +5,22 @@ cleanup; +mkfs.xfs 2>&1 | grep reflink +if [ $? -ne 0 ]; then + SKIP_TESTS + exit +fi + + TEST glusterd +TEST truncate -s 2G $B0/xfs_image # for now, a xfs filesystem with reflink support is created. # In future, better to make changes in MKFS_LOOP so that, # once can create a xfs filesystem with reflink enabled in # generic and simple way, instead of doing below steps each # time. -TEST truncate -s 2G $B0/xfs_image -mkfs.xfs 2>&1 | grep reflink -if [ $? -eq 0 ]; then - mkfs.xfs -f -i size=512 -m reflink=1 $B0/xfs_image; -else - mkfs.xfs -f -i size=512 $B0/xfs_image; -fi +TEST mkfs.xfs -f -i size=512 -m reflink=1 $B0/xfs_image; TEST mkdir $B0/bricks TEST mount -t xfs -o loop $B0/xfs_image $B0/bricks diff --git a/tests/basic/gfapi/gfapi-graph-switch-open-fd.t b/tests/basic/gfapi/gfapi-graph-switch-open-fd.t new file mode 100644 index 00000000000..2e666be7ec7 --- /dev/null +++ b/tests/basic/gfapi/gfapi-graph-switch-open-fd.t @@ -0,0 +1,44 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +TEST glusterd + +TEST $CLI volume create $V0 replica 3 ${H0}:$B0/brick{0..2}; +EXPECT 'Created' volinfo_field $V0 'Status'; + +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0; +TEST touch $M0/sync +logdir=`gluster --print-logdir` + +TEST build_tester $(dirname $0)/gfapi-keep-writing.c -lgfapi + + +#Launch a program to keep doing writes on an fd +./$(dirname $0)/gfapi-keep-writing ${H0} $V0 $logdir/gfapi-async-calls-test.log sync & +p=$! +sleep 1 #Let some writes go through +#Check if graph switch will lead to any pending markers for ever +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.read-ahead off + + +TEST rm -f $M0/sync #Make sure the glfd is closed +TEST wait #Wait for background process to die +#Goal is to check if there is permanent FOOL changelog +sleep 5 +EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/brick0/glfs_test.txt trusted.afr.dirty +EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/brick1/glfs_test.txt trusted.afr.dirty +EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/brick2/glfs_test.txt trusted.afr.dirty + +cleanup_tester $(dirname $0)/gfapi-async-calls-test + +cleanup; diff --git a/tests/basic/gfapi/gfapi-keep-writing.c b/tests/basic/gfapi/gfapi-keep-writing.c new file mode 100644 index 00000000000..91b59cea02b --- /dev/null +++ b/tests/basic/gfapi/gfapi-keep-writing.c @@ -0,0 +1,129 @@ +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +#define LOG_ERR(msg) \ + do { \ + fprintf(stderr, "%s : Error (%s)\n", msg, strerror(errno)); \ + } while (0) + +glfs_t * +init_glfs(const char *hostname, const char *volname, const char *logfile) +{ + int ret = -1; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + LOG_ERR("glfs_new failed"); + return NULL; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + LOG_ERR("glfs_set_volfile_server failed"); + goto out; + } + + ret = glfs_set_logging(fs, logfile, 7); + if (ret < 0) { + LOG_ERR("glfs_set_logging failed"); + goto out; + } + + ret = glfs_init(fs); + if (ret < 0) { + LOG_ERR("glfs_init failed"); + goto out; + } + + ret = 0; +out: + if (ret) { + glfs_fini(fs); + fs = NULL; + } + + return fs; +} + +int +glfs_test_function(const char *hostname, const char *volname, + const char *logfile, const char *syncfile) +{ + int ret = -1; + int flags = O_CREAT | O_RDWR; + glfs_t *fs = NULL; + glfs_fd_t *glfd = NULL; + const char *buff = "This is from my prog\n"; + const char *filename = "glfs_test.txt"; + struct stat buf = {0}; + + fs = init_glfs(hostname, volname, logfile); + if (fs == NULL) { + LOG_ERR("init_glfs failed"); + return -1; + } + + glfd = glfs_creat(fs, filename, flags, 0644); + if (glfd == NULL) { + LOG_ERR("glfs_creat failed"); + goto out; + } + + while (glfs_stat(fs, syncfile, &buf) == 0) { + ret = glfs_write(glfd, buff, strlen(buff), flags); + if (ret < 0) { + LOG_ERR("glfs_write failed"); + goto out; + } + } + + ret = glfs_close(glfd); + if (ret < 0) { + LOG_ERR("glfs_write failed"); + goto out; + } + +out: + ret = glfs_fini(fs); + if (ret) { + LOG_ERR("glfs_fini failed"); + } + + return ret; +} + +int +main(int argc, char *argv[]) +{ + int ret = 0; + char *hostname = NULL; + char *volname = NULL; + char *logfile = NULL; + char *syncfile = NULL; + + if (argc != 5) { + fprintf(stderr, "Invalid argument\n"); + exit(1); + } + + hostname = argv[1]; + volname = argv[2]; + logfile = argv[3]; + syncfile = argv[4]; + + ret = glfs_test_function(hostname, volname, logfile, syncfile); + if (ret) { + LOG_ERR("glfs_test_function failed"); + } + + return ret; +} diff --git a/tests/basic/gfapi/gfapi-ssl-load-volfile-test.c b/tests/basic/gfapi/gfapi-ssl-load-volfile-test.c new file mode 100644 index 00000000000..7beb8dd1fe4 --- /dev/null +++ b/tests/basic/gfapi/gfapi-ssl-load-volfile-test.c @@ -0,0 +1,127 @@ +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +#define LOG_ERR(msg) \ + do { \ + fprintf(stderr, "%s : Error (%s)\n", msg, strerror(errno)); \ + } while (0) + +glfs_t * +init_glfs(const char *hostname, const char *volname, const char *volfile, + const char *logfile) +{ + int ret = -1; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + LOG_ERR("glfs_new failed"); + return NULL; + } + + ret = glfs_set_volfile(fs, volfile); + if (ret < 0) { + LOG_ERR("glfs_set_volfile failed"); + goto out; + } + + ret = glfs_set_logging(fs, logfile, 7); + if (ret < 0) { + LOG_ERR("glfs_set_logging failed"); + goto out; + } + + ret = glfs_init(fs); + if (ret < 0) { + LOG_ERR("glfs_init failed"); + goto out; + } + + ret = 0; +out: + if (ret) { + glfs_fini(fs); + fs = NULL; + } + + return fs; +} + +int +glfs_test_function(const char *hostname, const char *volname, + const char *volfile, const char *logfile) +{ + int ret = -1; + int flags = O_CREAT | O_RDWR; + glfs_t *fs = NULL; + glfs_fd_t *glfd = NULL; + const char *buff = "This is from my prog\n"; + const char *filename = "glfs_test.txt"; + + fs = init_glfs(hostname, volname, volfile, logfile); + if (fs == NULL) { + LOG_ERR("init_glfs failed"); + return -1; + } + + glfd = glfs_creat(fs, filename, flags, 0644); + if (glfd == NULL) { + LOG_ERR("glfs_creat failed"); + goto out; + } + + ret = glfs_write(glfd, buff, strlen(buff), flags); + if (ret < 0) { + LOG_ERR("glfs_write failed"); + goto out; + } + + ret = glfs_close(glfd); + if (ret < 0) { + LOG_ERR("glfs_write failed"); + goto out; + } + +out: + ret = glfs_fini(fs); + if (ret) { + LOG_ERR("glfs_fini failed"); + } + + return ret; +} + +int +main(int argc, char *argv[]) +{ + int ret = 0; + char *hostname = NULL; + char *volname = NULL; + char *volfile = NULL; + char *logfile = NULL; + + if (argc != 5) { + fprintf(stderr, "Invalid argument\n"); + exit(1); + } + + hostname = argv[1]; + volname = argv[2]; + volfile = argv[3]; + logfile = argv[4]; + + ret = glfs_test_function(hostname, volname, volfile, logfile); + if (ret) { + LOG_ERR("glfs_test_function failed"); + } + + return ret; +} diff --git a/tests/basic/gfapi/gfapi-ssl-load-volfile-test.t b/tests/basic/gfapi/gfapi-ssl-load-volfile-test.t new file mode 100755 index 00000000000..8e94df9d321 --- /dev/null +++ b/tests/basic/gfapi/gfapi-ssl-load-volfile-test.t @@ -0,0 +1,76 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../traps.rc +. $(dirname $0)/../../ssl.rc + +cleanup; + +sed -e "s,@@HOSTNAME@@,${H0},g" -e "s,@@BRICKPATH@@,${B0}/brick1,g" \ + -e "s,@@SSL@@,off,g" \ + $(dirname ${0})/protocol-client-ssl.vol.in \ + > $(dirname ${0})/protocol-client-ssl.vol + +TEST create_self_signed_certs + +TEST glusterd + +TEST $CLI volume create $V0 $H0:$B0/brick1; +EXPECT 'Created' volinfo_field $V0 'Status'; + +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" online_brick_count + +logdir=`gluster --print-logdir` + +TEST build_tester $(dirname $0)/gfapi-ssl-load-volfile-test.c -lgfapi + +# Run test without I/O or management encryption +TEST $(dirname $0)/gfapi-ssl-load-volfile-test $H0 $V0 \ + $(dirname ${0})/protocol-client-ssl.vol \ + $logdir/gfapi-ssl-load-volfile-test.log + +# Enable management encryption +touch $GLUSTERD_WORKDIR/secure-access + +killall_gluster + +TEST glusterd +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" online_brick_count + +# Run test with management encryption (No I/O encryption) +TEST $(dirname $0)/gfapi-ssl-load-volfile-test $H0 $V0 \ + $(dirname ${0})/protocol-client-ssl.vol \ + $logdir/gfapi-ssl-load-volfile-test.log + +# Enable I/O encryption +TEST $CLI volume set $V0 server.ssl on + +killall_gluster + +sed -e "s,@@HOSTNAME@@,${H0},g" -e "s,@@BRICKPATH@@,${B0}/brick1,g" \ + -e "s,@@SSL@@,on,g" \ + $(dirname ${0})/protocol-client-ssl.vol.in \ + > $(dirname ${0})/protocol-client-ssl.vol + +TEST glusterd +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" online_brick_count + +# Run test without I/O or management encryption +TEST $(dirname $0)/gfapi-ssl-load-volfile-test $H0 $V0 \ + $(dirname ${0})/protocol-client-ssl.vol \ + $logdir/gfapi-ssl-load-volfile-test.log + +cleanup_tester $(dirname $0)/gfapi-ssl-load-volfile-test + +TEST $CLI volume stop $V0 +TEST $CLI volume delete $V0 + +cleanup; + +# NetBSD build scripts are not up to date therefore this test +# is failing in NetBSD. Therefore skipping the test in NetBSD +# as of now. +#G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=000000 diff --git a/tests/basic/gfapi/protocol-client-ssl.vol.in b/tests/basic/gfapi/protocol-client-ssl.vol.in new file mode 100644 index 00000000000..cdc0c9d0671 --- /dev/null +++ b/tests/basic/gfapi/protocol-client-ssl.vol.in @@ -0,0 +1,15 @@ +# +# This .vol file expects that there is +# +# 1. GlusterD listening on @@HOSTNAME@@ +# 2. a volume that provides a brick on @@BRICKPATH@@ +# 3. the volume with the brick has been started +# +volume test + type protocol/client + option remote-host @@HOSTNAME@@ + option remote-subvolume @@BRICKPATH@@ + option transport-type socket + option transport.socket.ssl-enabled @@SSL@@ +end-volume + diff --git a/tests/basic/metadisp/fsyncdir.c b/tests/basic/metadisp/fsyncdir.c new file mode 100644 index 00000000000..62b532b9ce4 --- /dev/null +++ b/tests/basic/metadisp/fsyncdir.c @@ -0,0 +1,29 @@ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <fcntl.h> + +int +main(int argc, char **argv) +{ + int pfd; + + pfd = open(argv[1], O_RDONLY | O_DIRECTORY); + if (pfd == (-1)) { + perror("open"); + return EXIT_FAILURE; + } + + if (rename(argv[2], argv[3]) == (-1)) { + perror("rename"); + return EXIT_FAILURE; + } + + if (fsync(pfd) == (-1)) { + perror("fsync"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/tests/basic/metadisp/ftruncate.c b/tests/basic/metadisp/ftruncate.c new file mode 100644 index 00000000000..c9185212c31 --- /dev/null +++ b/tests/basic/metadisp/ftruncate.c @@ -0,0 +1,34 @@ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <fcntl.h> + +int +main(int argc, char **argv) +{ + int pfd; + + pfd = open(argv[1], O_RDWR); + if (pfd == (-1)) { + perror("open"); + return EXIT_FAILURE; + } + + if (ftruncate(pfd, 0) == (-1)) { + perror("ftruncate"); + return EXIT_FAILURE; + } + + if (write(pfd, "hello", 5) == (-1)) { + perror("write"); + return EXIT_FAILURE; + } + + if (fsync(pfd) == (-1)) { + perror("fsync"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/tests/basic/metadisp/fxattr.c b/tests/basic/metadisp/fxattr.c new file mode 100644 index 00000000000..e552057778a --- /dev/null +++ b/tests/basic/metadisp/fxattr.c @@ -0,0 +1,107 @@ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <fcntl.h> +#include <string.h> +#include <sys/types.h> +#include <sys/xattr.h> + +static char MY_XATTR[] = "user.fxtest"; +static char *PROGRAM; +#define CONSUME(v) \ + do { \ + if (!argc) { \ + fprintf(stderr, "missing argument\n"); \ + return EXIT_FAILURE; \ + } \ + v = argv[0]; \ + ++argv; \ + --argc; \ + } while (0) + +static int +do_get(int argc, char **argv, int fd) +{ + char *value; + int ret; + char buf[1024]; + + CONSUME(value); + + ret = fgetxattr(fd, MY_XATTR, buf, sizeof(buf)); + if (ret == (-1)) { + perror("fgetxattr"); + return EXIT_FAILURE; + } + + if (strncmp(buf, value, ret) != 0) { + fprintf(stderr, "data mismatch\n"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static int +do_set(int argc, char **argv, int fd) +{ + char *value; + int ret; + + CONSUME(value); + + ret = fsetxattr(fd, MY_XATTR, value, strlen(value), 0); + if (ret == (-1)) { + perror("fsetxattr"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static int +do_remove(int argc, char **argv, int fd) +{ + int ret; + + ret = fremovexattr(fd, MY_XATTR); + if (ret == (-1)) { + perror("femovexattr"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int +main(int argc, char **argv) +{ + int fd; + char *path; + char *cmd; + + CONSUME(PROGRAM); + CONSUME(path); + CONSUME(cmd); + + fd = open(path, O_RDWR); + if (fd == (-1)) { + perror("open"); + return EXIT_FAILURE; + } + + if (strcmp(cmd, "get") == 0) { + return do_get(argc, argv, fd); + } + + if (strcmp(cmd, "set") == 0) { + return do_set(argc, argv, fd); + } + + if (strcmp(cmd, "remove") == 0) { + return do_remove(argc, argv, fd); + } + + return EXIT_SUCCESS; +} diff --git a/tests/basic/metadisp/gfs-fsetxattr.c b/tests/basic/metadisp/gfs-fsetxattr.c new file mode 100644 index 00000000000..63578bc528f --- /dev/null +++ b/tests/basic/metadisp/gfs-fsetxattr.c @@ -0,0 +1,141 @@ +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int gfapi = 1; + +int +main(int argc, char *argv[]) +{ + glfs_t *fs = NULL; + int ret = 0; + int i = 0; + glfs_fd_t *fd = NULL; + char *topdir = "topdir", *filename = "file1"; + char *buf = NULL; + char *logfile = NULL; + char *hostname = NULL; + char *basename = NULL; + char *dir1 = NULL, *dir2 = NULL, *filename1 = NULL, *filename2 = NULL; + struct stat sb = { + 0, + }; + + if (argc != 5) { + fprintf( + stderr, + "Expect following args %s <hostname> <Vol> <log file> <basename>\n", + argv[0]); + return -1; + } + + hostname = argv[1]; + logfile = argv[3]; + basename = argv[4]; + + fs = glfs_new(argv[2]); + if (!fs) { + fprintf(stderr, "glfs_new: returned NULL (%s)\n", strerror(errno)); + return -1; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + fprintf(stderr, "glfs_set_volfile_server failed ret:%d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = glfs_set_logging(fs, logfile, 7); + if (ret < 0) { + fprintf(stderr, "glfs_set_logging failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = glfs_init(fs); + if (ret < 0) { + fprintf(stderr, "glfs_init failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = asprintf(&dir1, "%s-dir", basename); + if (ret < 0) { + fprintf(stderr, "cannot construct filename (%s)", strerror(errno)); + return ret; + } + + ret = glfs_mkdir(fs, dir1, 0755); + if (ret < 0) { + fprintf(stderr, "mkdir(%s): %s\n", dir1, strerror(errno)); + return -1; + } + + fd = glfs_opendir(fs, dir1); + if (!fd) { + fprintf(stderr, "/: %s\n", strerror(errno)); + return -1; + } + + ret = glfs_fsetxattr(fd, "user.dirfattr", "fsetxattr", 9, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr(%s): %d (%s)\n", dir1, ret, strerror(errno)); + return -1; + } + + ret = glfs_closedir(fd); + if (ret < 0) { + fprintf(stderr, "glfs_closedir failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = asprintf(&filename1, "%s-file", basename); + if (ret < 0) { + fprintf(stderr, "cannot construct filename (%s)", strerror(errno)); + return ret; + } + + ret = asprintf(&filename2, "%s-file-renamed", basename); + if (ret < 0) { + fprintf(stderr, "cannot construct filename (%s)", strerror(errno)); + return ret; + } + + fd = glfs_creat(fs, filename1, O_RDWR, 0644); + if (!fd) { + fprintf(stderr, "%s: (%p) %s\n", filename1, fd, strerror(errno)); + return -1; + } + + ret = glfs_rename(fs, filename1, filename2); + if (ret < 0) { + fprintf(stderr, "glfs_rename failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = glfs_lstat(fs, filename2, &sb); + if (ret < 0) { + fprintf(stderr, "glfs_lstat failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } + + ret = glfs_fsetxattr(fd, "user.filefattr", "fsetxattr", 9, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr(%s): %d (%s)\n", dir1, ret, strerror(errno)); + return -1; + } + + ret = glfs_close(fd); + if (ret < 0) { + fprintf(stderr, "glfs_close failed with ret: %d (%s)\n", ret, + strerror(errno)); + return -1; + } +} diff --git a/tests/basic/metadisp/metadisp.t b/tests/basic/metadisp/metadisp.t new file mode 100644 index 00000000000..894ffe07226 --- /dev/null +++ b/tests/basic/metadisp/metadisp.t @@ -0,0 +1,316 @@ +#!/usr/bin/env bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + + +# Considering `--enable-metadisp` is an option for `./configure`, +# which is disabled by default, this test will never pass regression. +# But to see the value of this test, run below after configuring +# with above option : +# `prove -vmfe '/bin/bash' tests/basic/metadisp/metadisp.t` + +#G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST + +cleanup; + +TEST mkdir -p $B0/b0/{0,1} + +TEST setfattr -n trusted.glusterfs.volume-id -v 0xddab9eece7b64a95b07351a1f748f56f ${B0}/b0/0 +TEST setfattr -n trusted.glusterfs.volume-id -v 0xddab9eece7b64a95b07351a1f748f56f ${B0}/b0/1 + +TEST $GFS --volfile=$(dirname $0)/metadisp.vol --volfile-id=$V0 $M0; + +NUM_FILES=40 +TEST touch $M0/{1..${NUM_FILES}} + +# each drive should get 40 files +TEST [ $(dir -1 $B0/b0/0/ | wc -l) -eq $NUM_FILES ] +TEST [ $(dir -1 $B0/b0/1/ | wc -l) -eq $NUM_FILES ] + +# now write some data to a file +echo "hello" > $M0/3 +filename=$$ +echo "hello" > /tmp/metadisp-write-${filename} +checksum=$(md5sum /tmp/metadisp-write-${filename} | awk '{print $1}') +TEST [ "$(md5sum $M0/3 | awk '{print $1}')" == "$checksum" ] + +# check that the backend file exists on b1 +gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/b0/*/3)) +TEST [ $(dir -1 $B0/b0/1/$gfid | wc -l) -eq 1 ] + +# check that the backend file matches the frontend +TEST [ "$(md5sum $B0/b0/1/$gfid | awk '{print $1}')" == "$checksum" ] + +# delete the file +TEST rm $M0/3 + +# ensure the frontend and backend files are cleaned up +TEST ! -e $M0/3 +TEST ! [ stat $B0/b*/*/$gfid ] + +# Test TRUNCATE + WRITE flow +echo "hello" | tee $M0/4 +echo "goo" | tee $M0/4 +filename=$$ +echo "goo" | tee /tmp/metadisp-truncate-${filename} +checksum=$(md5sum /tmp/metadisp-truncate-${filename} | awk '{print $1}') +TEST [ "$(md5sum $M0/4 | awk '{print $1}')" == "$checksum" ] + +# Test mkdir + rmdir. +TEST mkdir $M0/rmdir_me +nfiles=$(ls -d $B0/b*/*/rmdir_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] +TEST rmdir $M0/rmdir_me +nfiles=$(ls -d $B0/b*/*/rmdir_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] + +# Test rename. +TEST touch $M0/rename_me +nfiles=$(ls $B0/b*/*/rename_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] +nfiles=$(ls $B0/b*/*/such_rename 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] +TEST mv $M0/rename_me $M0/such_rename +nfiles=$(ls $B0/b*/*/rename_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] +nfiles=$(ls $B0/b*/*/such_rename 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] + +# Test rename of a file that doesn't exist. +TEST ! mv $M0/does-not-exist $M0/neither-does-this + + +# cleanup all the other files. +TEST rm -v $M0/1 $M0/2 $M0/{4..${NUM_FILES}} +TEST rm $M0/such_rename +TEST [ $(ls /d/backends/b0/0/ | wc -l) -eq 0 ] +TEST [ $(ls /d/backends/b0/1/ | wc -l) -eq 0 ] + +# Test CREATE flow +NUM_FILES=40 +TEST touch $M0/{1..${NUM_FILES}} +TEST [ $(ls /d/backends/b0/0/ | wc -l) -eq $NUM_FILES ] +TEST [ $(ls /d/backends/b0/1/ | wc -l) -eq $NUM_FILES ] + +# Test UNLINK flow +# No drives should have any files +TEST rm -v $M0/{1..${NUM_FILES}} +TEST [ $(ls /d/backends/b0/0/ | wc -l) -eq 0 ] +TEST [ $(ls /d/backends/b0/1/ | wc -l) -eq 0 ] + +# Test CREATE + WRITE + READ flow +filename=$$ +dd if=/dev/urandom of=/tmp/${filename} bs=1M count=10 +checksum=$(md5sum /tmp/${filename} | awk '{print $1}') +TEST cp -v /tmp/${filename} $M0/1 +TEST cp -v /tmp/${filename} $M0/2 +TEST cp -v /tmp/${filename} $M0/3 +TEST cp -v /tmp/${filename} $M0/4 +TEST [ "$(md5sum $M0/1 | awk '{print $1}')" == "$checksum" ] +TEST [ "$(md5sum $M0/2 | awk '{print $1}')" == "$checksum" ] +TEST [ "$(md5sum $M0/3 | awk '{print $1}')" == "$checksum" ] +TEST [ "$(md5sum $M0/4 | awk '{print $1}')" == "$checksum" ] + +# Test TRUNCATE + WRITE flow +TEST dd if=/dev/zero of=$M0/1 bs=1M count=20 + +# Check that readdir stats the files properly and we get the correct sizes +TEST [ $(find $M0 -size +9M | wc -l) -eq 4 ]; + +# Test mkdir + rmdir. +TEST mkdir $M0/rmdir_me +nfiles=$(ls -d $B0/b*/*/rmdir_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] +TEST rmdir $M0/rmdir_me +nfiles=$(ls -d $B0/b*/*/rmdir_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] + +# Test rename. +# Still flaky, so disabled until it can be debugged. +TEST touch $M0/rename_me +nfiles=$(ls $B0/b*/*/rename_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] +nfiles=$(ls $B0/b*/*/such_rename 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] +TEST mv $M0/rename_me $M0/such_rename +nfiles=$(ls $B0/b*/*/rename_me 2> /dev/null | wc -l) +TEST [ "$nfiles" = "0" ] +nfiles=$(ls $B0/b*/*/such_rename 2> /dev/null | wc -l) +TEST [ "$nfiles" = "1" ] + +# Test rename of a file that doesn't exist. +TEST ! mv $M0/does-not-exist $M0/neither-does-this + +# Test rename over an existing file. +ok=yes +for i in $(seq 0 9); do + echo foo > $M0/src$i + echo bar > $M0/dst$i +done +for i in $(seq 0 9); do + mv $M0/src$i $M0/dst$i +done +for i in $(seq 0 9); do + nfiles=$(cat $B0/b0/*/dst$i | wc -l) + if [ "$nfiles" = "2" ]; then + echo "COLLISION on dst$i" + (ls -l $B0/b0/*/dst$i; cat $B0/b0/*/dst$i) | sed "/^/s// /" + ok=no + fi +done +EXPECT "yes" echo $ok + +# Test rename of a directory. +count_copies () { + ls -d $B0/b?/?/$1 2> /dev/null | wc -l +} +TEST mkdir $M0/foo_dir +EXPECT 1 count_copies foo_dir +EXPECT 0 count_copies bar_dir +TEST mv $M0/foo_dir $M0/bar_dir +EXPECT 0 count_copies foo_dir +EXPECT 1 count_copies bar_dir + +for x in $(seq 0 99); do + touch $M0/target$x + ln -s $M0/target$x $M0/link$x +done +on_0=$(ls $B0/b*/0/link* | wc -l) +on_1=$(ls $B0/b*/1/link* | wc -l) +TEST [ "$on_0" -eq 100 ] +TEST [ "$on_1" -eq 0 ] +TEST [ "$(ls -l $M0/link* | wc -l)" = 100 ] + +# Test (hard) link. +_test_hardlink () { + local b + local has_src + local has_dst + local src_inum + local dst_inum + touch $M0/hardsrc$1 + ln $M0/hardsrc$1 $M0/harddst$1 + for b in $B0/b{0}/{0,1}; do + [ -f $b/hardsrc$1 ]; has_src=$? + [ -f $b/harddst$1 ]; has_dst=$? + if [ "$has_src" != "$has_dst" ]; then + echo "MISSING $b/hardxxx$1 $has_src $has_dst" + return + fi + if [ "$has_src$has_dst" = "00" ]; then + src_inum=$(stat -c '%i' $b/hardsrc$1) + dst_inum=$(stat -c '%i' $b/harddst$1) + if [ "$dst_inum" != "$src_inum" ]; then + echo "MISMATCH $b/hardxx$i $src_inum $dst_inum" + return + fi + fi + done + echo "OK" +} + +test_hardlink () { + local result=$(_test_hardlink $*) + # [ "$result" = "OK" ] || echo $result > /dev/tty + echo $result +} + +# Do this multiple times to make sure colocation isn't a fluke. +EXPECT "OK" test_hardlink 0 +EXPECT "OK" test_hardlink 1 +EXPECT "OK" test_hardlink 2 +EXPECT "OK" test_hardlink 3 +EXPECT "OK" test_hardlink 4 +EXPECT "OK" test_hardlink 5 +EXPECT "OK" test_hardlink 6 +EXPECT "OK" test_hardlink 7 +EXPECT "OK" test_hardlink 8 +EXPECT "OK" test_hardlink 9 + +# Test remove hardlink source. ensure deleting one file +# doesn't delete the data unless link-count is 1 +TEST mkdir $M0/hardlink +TEST touch $M0/hardlink/fileA +echo "data" >> $M0/hardlink/fileA +checksum=$(md5sum $M0/hardlink/fileA | awk '{print $1}') +TEST ln $M0/hardlink/fileA $M0/hardlink/fileB +TEST [ $(dir -1 $M0/hardlink/ | wc -l) -eq 2 ] +TEST rm $M0/hardlink/fileA +TEST [ $(dir -1 $M0/hardlink/ | wc -l) -eq 1 ] +TEST [ "$(md5sum $M0/hardlink/fileB | awk '{print $1}')" == "$checksum" ] + +# +# FIXME: statfs values look ok but the test is bad +# +# Test statfs. If we're doing it right, the numbers for the mountpoint should be +# double those for the brick filesystem times the number of bricks, +# but unless we're on a completely idle +# system (which never happens) the numbers can change even while this function +# runs and that would trip us up. Do a sloppy comparison to deal with that. +#compare_fields () { +# val1=$(df $1 | grep / | awk "{print \$$3}") +# val2=$(df $2 | grep / | awk "{print \$$3}") +# [ "$val2" -gt "$(((val1/(29/10))*19/10))" -a "$val2" -lt "$(((val1/(31/10))*21/10))" ] +#} + +#brick_df=$(df $B0 | grep /) +#mount_df=$(df $M0 | grep /) +#TEST compare_fields $B0 $M0 2 # Total blocks +#TEST compare_fields $B0 $M0 3 # Used +#TEST compare_fields $B0 $M0 4 # Available + +# Test removexattr. +#RXATTR_FILE=$(get_file_not_on_disk0 rxtest) +#TEST setfattr -n user.foo -v bar $M0/$RXATTR_FILE +#TEST getfattr -n user.foo $B0/b0/1/$RXATTR_FILE +#TEST setfattr -x user.foo $M0/$RXATTR_FILE +#TEST ! getfattr -n user.foo $B0/b0/1/$RXATTR_FILE + +# Test fsyncdir. We can't really test whether it's doing the right thing, +# but we can test that it doesn't fail and we can hand-check that it's calling +# down to all of the disks instead of just one. +# +# P.S. There's no fsyncdir test in the rest of Gluster, so who even knows if +# other translators are handling it correctly? + +#FSYNCDIR_EXE=$(dirname $0)/fsyncdir +#build_tester ${FSYNCDIR_EXE}.c +#TEST touch $M0/fsyncdir_src +#TEST $FSYNCDIR_EXE $M0 $M0/fsyncdir_src $M0/fsyncdir_dst +#TEST rm -f $FSYNCDIR_EXE + +# Test fsetxattr, fgetxattr, fremovexattr (in that order). +FXATTR_FILE=$M0/fxfile1 +TEST touch $FXATTR_FILE +FXATTR_EXE=$(dirname $0)/fxattr +build_tester ${FXATTR_EXE}.c +TEST ! getfattr -n user.fxtest $FXATTR_FILE +TEST $FXATTR_EXE $FXATTR_FILE set value1 +TEST getfattr -n user.fxtest $FXATTR_FILE +TEST setfattr -n user.fxtest -v value2 $FXATTR_FILE +TEST $FXATTR_EXE $FXATTR_FILE get value2 +TEST $FXATTR_EXE $FXATTR_FILE remove +TEST ! getfattr -n user.fxtest $FXATTR_FILE +TEST rm -f $FXATTR_EXE + +# Test ftruncate +FTRUNCATE_EXE=$(dirname $0)/ftruncate +build_tester ${FTRUNCATE_EXE}.c +FTRUNCATE_FILE=$M0/ftfile1 +TEST dd if=/dev/urandom of=$FTRUNCATE_FILE count=1 bs=1MB +TEST $FTRUNCATE_EXE $FTRUNCATE_FILE +#gfid=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/b0/*/ftfile1)) + +# Test fallocate, discard, zerofill. Actually we don't so much check that these +# *work* as that they don't throw any errors (especially ENOENT because the +# file's not on disk zero). +FALLOC_FILE=fatest1 +TEST touch $M0/$FALLOC_FILE +TEST fallocate -l $((4096*5)) $M0/$FALLOC_FILE +TEST fallocate -p -o 4096 -l 4096 $M0/$FALLOC_FILE +# This actually fails with "operation not supported" on most filesystems, so +# don't leave it enabled except to test changes. +#TEST fallocate -z -o $((4096*3)) -l 4096 $M0/$FALLOC_FILE + +#cleanup; diff --git a/tests/basic/metadisp/metadisp.vol b/tests/basic/metadisp/metadisp.vol new file mode 100644 index 00000000000..58ae2f6f2a8 --- /dev/null +++ b/tests/basic/metadisp/metadisp.vol @@ -0,0 +1,14 @@ +volume posix-0 + type storage/posix + option directory /d/backends/b0/0 +end-volume + +volume posix-1 + type storage/posix + option directory /d/backends/b0/1 +end-volume + +volume metadisp-0 + type features/metadisp + subvolumes posix-0 posix-1 +end-volume diff --git a/tests/basic/open-behind/open-behind.t b/tests/basic/open-behind/open-behind.t new file mode 100644 index 00000000000..5e865d602e2 --- /dev/null +++ b/tests/basic/open-behind/open-behind.t @@ -0,0 +1,183 @@ +#!/bin/bash + +WD="$(dirname "${0}")" + +. ${WD}/../../include.rc +. ${WD}/../../volume.rc + +function assign() { + local _assign_var="${1}" + local _assign_value="${2}" + + printf -v "${_assign_var}" "%s" "${_assign_value}" +} + +function pipe_create() { + local _pipe_create_var="${1}" + local _pipe_create_name + local _pipe_create_fd + + _pipe_create_name="$(mktemp -u)" + mkfifo "${_pipe_create_name}" + exec {_pipe_create_fd}<>"${_pipe_create_name}" + rm "${_pipe_create_name}" + + assign "${_pipe_create_var}" "${_pipe_create_fd}" +} + +function pipe_close() { + local _pipe_close_fd="${!1}" + + exec {_pipe_close_fd}>&- +} + +function tester_start() { + declare -ag tester + local tester_in + local tester_out + + pipe_create tester_in + pipe_create tester_out + + ${WD}/tester <&${tester_in} >&${tester_out} & + + tester=("$!" "${tester_in}" "${tester_out}") +} + +function tester_send() { + declare -ag tester + local tester_res + local tester_extra + + echo "${*}" >&${tester[1]} + + read -t 3 -u ${tester[2]} tester_res tester_extra + echo "${tester_res} ${tester_extra}" + if [[ "${tester_res}" == "OK" ]]; then + return 0 + fi + + return 1 +} + +function tester_stop() { + declare -ag tester + local tester_res + + tester_send "quit" + + tester_res=0 + if ! wait ${tester[0]}; then + tester_res=$? + fi + + unset tester + + return ${tester_res} +} + +function count_open() { + local file="$(realpath "${B0}/${V0}/${1}")" + local count="0" + local inode + local ref + + inode="$(stat -c %i "${file}")" + + for fd in /proc/${BRICK_PID}/fd/*; do + ref="$(readlink "${fd}")" + if [[ "${ref}" == "${B0}/${V0}/"* ]]; then + if [[ "$(stat -c %i "${ref}")" == "${inode}" ]]; then + count="$((${count} + 1))" + fi + fi + done + + echo "${count}" +} + +cleanup + +TEST build_tester ${WD}/tester.c ${WD}/tester-fd.c + +TEST glusterd +TEST pidof glusterd +TEST ${CLI} volume create ${V0} ${H0}:${B0}/${V0} +TEST ${CLI} volume set ${V0} flush-behind off +TEST ${CLI} volume set ${V0} write-behind off +TEST ${CLI} volume set ${V0} quick-read off +TEST ${CLI} volume set ${V0} stat-prefetch on +TEST ${CLI} volume set ${V0} io-cache off +TEST ${CLI} volume set ${V0} open-behind on +TEST ${CLI} volume set ${V0} lazy-open off +TEST ${CLI} volume set ${V0} read-after-open off +TEST ${CLI} volume start ${V0} + +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +BRICK_PID="$(get_brick_pid ${V0} ${H0} ${B0}/${V0})" + +TEST touch "${M0}/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_start + +TEST tester_send fd open 0 "${M0}/test" +EXPECT_WITHIN 5 "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${CLI} volume set ${V0} lazy-open on +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +sleep 2 +EXPECT "0" count_open "/test" +TEST tester_send fd write 0 "test" +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +EXPECT "test" tester_send fd read 0 64 +# Even though read-after-open is disabled, use-anonymous-fd is also disabled, +# so reads need to open the file first. +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +TEST tester_send fd open 1 "${M0}/test" +EXPECT "2" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "1" count_open "/test" +TEST tester_send fd close 1 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST ${CLI} volume set ${V0} read-after-open on +TEST ${GFS} --volfile-id=/${V0} --volfile-server=${H0} ${M0}; + +TEST tester_send fd open 0 "${M0}/test" +EXPECT "0" count_open "/test" +EXPECT "test" tester_send fd read 0 64 +EXPECT "1" count_open "/test" +TEST tester_send fd close 0 +EXPECT_WITHIN 5 "0" count_open "/test" + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +TEST tester_stop + +cleanup diff --git a/tests/basic/open-behind/tester-fd.c b/tests/basic/open-behind/tester-fd.c new file mode 100644 index 00000000000..00f02bc5b0a --- /dev/null +++ b/tests/basic/open-behind/tester-fd.c @@ -0,0 +1,99 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "tester.h" + +#include <stdlib.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> + +static int32_t +fd_open(context_t *ctx, command_t *cmd) +{ + obj_t *obj; + int32_t fd; + + obj = cmd->args[0].obj.ref; + + fd = open(cmd->args[1].str.data, O_RDWR); + if (fd < 0) { + return error(errno, "open() failed"); + } + + obj->type = OBJ_TYPE_FD; + obj->fd = fd; + + out_ok("%d", fd); + + return 0; +} + +static int32_t +fd_close(context_t *ctx, command_t *cmd) +{ + obj_t *obj; + + obj = cmd->args[0].obj.ref; + obj->type = OBJ_TYPE_NONE; + + if (close(obj->fd) != 0) { + return error(errno, "close() failed"); + } + + out_ok(); + + return 0; +} + +static int32_t +fd_write(context_t *ctx, command_t *cmd) +{ + ssize_t len, ret; + + len = strlen(cmd->args[1].str.data); + ret = write(cmd->args[0].obj.ref->fd, cmd->args[1].str.data, len); + if (ret < 0) { + return error(errno, "write() failed"); + } + + out_ok("%zd", ret); + + return 0; +} + +static int32_t +fd_read(context_t *ctx, command_t *cmd) +{ + char data[cmd->args[1].num.value + 1]; + ssize_t ret; + + ret = read(cmd->args[0].obj.ref->fd, data, cmd->args[1].num.value); + if (ret < 0) { + return error(errno, "read() failed"); + } + + data[ret] = 0; + + out_ok("%zd %s", ret, data); + + return 0; +} + +command_t fd_commands[] = { + {"open", fd_open, CMD_ARGS(ARG_VAL(OBJ_TYPE_NONE), ARG_STR(1024))}, + {"close", fd_close, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD))}, + {"write", fd_write, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD), ARG_STR(1024))}, + {"read", fd_read, CMD_ARGS(ARG_VAL(OBJ_TYPE_FD), ARG_NUM(0, 1024))}, + CMD_END}; diff --git a/tests/basic/open-behind/tester.c b/tests/basic/open-behind/tester.c new file mode 100644 index 00000000000..b2da71c8385 --- /dev/null +++ b/tests/basic/open-behind/tester.c @@ -0,0 +1,444 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "tester.h" + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> + +static void * +mem_alloc(size_t size) +{ + void *ptr; + + ptr = malloc(size); + if (ptr == NULL) { + error(ENOMEM, "Failed to allocate memory (%zu bytes)", size); + } + + return ptr; +} + +static void +mem_free(void *ptr) +{ + free(ptr); +} + +static bool +buffer_create(context_t *ctx, size_t size) +{ + ctx->buffer.base = mem_alloc(size); + if (ctx->buffer.base == NULL) { + return false; + } + + ctx->buffer.size = size; + ctx->buffer.len = 0; + ctx->buffer.pos = 0; + + return true; +} + +static void +buffer_destroy(context_t *ctx) +{ + mem_free(ctx->buffer.base); + ctx->buffer.size = 0; + ctx->buffer.len = 0; +} + +static int32_t +buffer_get(context_t *ctx) +{ + ssize_t len; + + if (ctx->buffer.pos >= ctx->buffer.len) { + len = read(0, ctx->buffer.base, ctx->buffer.size); + if (len < 0) { + return error(errno, "read() failed"); + } + if (len == 0) { + return 0; + } + + ctx->buffer.len = len; + ctx->buffer.pos = 0; + } + + return ctx->buffer.base[ctx->buffer.pos++]; +} + +static int32_t +str_skip_spaces(context_t *ctx, int32_t current) +{ + while ((current > 0) && (current != '\n') && isspace(current)) { + current = buffer_get(ctx); + } + + return current; +} + +static int32_t +str_token(context_t *ctx, char *buffer, uint32_t size, int32_t current) +{ + uint32_t len; + + current = str_skip_spaces(ctx, current); + + len = 0; + while ((size > 0) && (current > 0) && (current != '\n') && + !isspace(current)) { + len++; + *buffer++ = current; + size--; + current = buffer_get(ctx); + } + + if (len == 0) { + return error(ENODATA, "Expecting a token"); + } + + if (size == 0) { + return error(ENOBUFS, "Token too long"); + } + + *buffer = 0; + + return current; +} + +static int32_t +str_number(context_t *ctx, uint64_t min, uint64_t max, uint64_t *value, + int32_t current) +{ + char text[32], *ptr; + uint64_t num; + + current = str_token(ctx, text, sizeof(text), current); + if (current > 0) { + num = strtoul(text, &ptr, 0); + if ((*ptr != 0) || (num < min) || (num > max)) { + return error(ERANGE, "Invalid number"); + } + *value = num; + } + + return current; +} + +static int32_t +str_eol(context_t *ctx, int32_t current) +{ + current = str_skip_spaces(ctx, current); + if (current != '\n') { + return error(EINVAL, "Expecting end of command"); + } + + return current; +} + +static void +str_skip(context_t *ctx, int32_t current) +{ + while ((current > 0) && (current != '\n')) { + current = buffer_get(ctx); + } +} + +static int32_t +cmd_parse_obj(context_t *ctx, arg_t *arg, int32_t current) +{ + obj_t *obj; + uint64_t id; + + current = str_number(ctx, 0, ctx->obj_count, &id, current); + if (current <= 0) { + return current; + } + + obj = &ctx->objs[id]; + if (obj->type != arg->obj.type) { + if (obj->type != OBJ_TYPE_NONE) { + return error(EBUSY, "Object is in use"); + } + return error(ENOENT, "Object is not defined"); + } + + arg->obj.ref = obj; + + return current; +} + +static int32_t +cmd_parse_num(context_t *ctx, arg_t *arg, int32_t current) +{ + return str_number(ctx, arg->num.min, arg->num.max, &arg->num.value, + current); +} + +static int32_t +cmd_parse_str(context_t *ctx, arg_t *arg, int32_t current) +{ + return str_token(ctx, arg->str.data, arg->str.size, current); +} + +static int32_t +cmd_parse_args(context_t *ctx, command_t *cmd, int32_t current) +{ + arg_t *arg; + + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_OBJ: + current = cmd_parse_obj(ctx, arg, current); + break; + case ARG_TYPE_NUM: + current = cmd_parse_num(ctx, arg, current); + break; + case ARG_TYPE_STR: + current = cmd_parse_str(ctx, arg, current); + break; + default: + return error(EINVAL, "Unknown argument type"); + } + } + + if (current < 0) { + return current; + } + + current = str_eol(ctx, current); + if (current <= 0) { + return error(EINVAL, "Syntax error"); + } + + return cmd->handler(ctx, cmd); +} + +static int32_t +cmd_parse(context_t *ctx, command_t *cmds) +{ + char text[32]; + command_t *cmd; + int32_t current; + + cmd = cmds; + do { + current = str_token(ctx, text, sizeof(text), buffer_get(ctx)); + if (current <= 0) { + return current; + } + + while (cmd->name != NULL) { + if (strcmp(cmd->name, text) == 0) { + if (cmd->handler != NULL) { + return cmd_parse_args(ctx, cmd, current); + } + cmd = cmd->cmds; + break; + } + cmd++; + } + } while (cmd->name != NULL); + + str_skip(ctx, current); + + return error(ENOTSUP, "Unknown command"); +} + +static void +cmd_fini(context_t *ctx, command_t *cmds) +{ + command_t *cmd; + arg_t *arg; + + for (cmd = cmds; cmd->name != NULL; cmd++) { + if (cmd->handler == NULL) { + cmd_fini(ctx, cmd->cmds); + } else { + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_STR: + mem_free(arg->str.data); + arg->str.data = NULL; + break; + default: + break; + } + } + } + } +} + +static bool +cmd_init(context_t *ctx, command_t *cmds) +{ + command_t *cmd; + arg_t *arg; + + for (cmd = cmds; cmd->name != NULL; cmd++) { + if (cmd->handler == NULL) { + if (!cmd_init(ctx, cmd->cmds)) { + return false; + } + } else { + for (arg = cmd->args; arg->type != ARG_TYPE_NONE; arg++) { + switch (arg->type) { + case ARG_TYPE_STR: + arg->str.data = mem_alloc(arg->str.size); + if (arg->str.data == NULL) { + return false; + } + break; + default: + break; + } + } + } + } + + return true; +} + +static bool +objs_create(context_t *ctx, uint32_t count) +{ + uint32_t i; + + ctx->objs = mem_alloc(sizeof(obj_t) * count); + if (ctx->objs == NULL) { + return false; + } + ctx->obj_count = count; + + for (i = 0; i < count; i++) { + ctx->objs[i].type = OBJ_TYPE_NONE; + } + + return true; +} + +static int32_t +objs_destroy(context_t *ctx) +{ + uint32_t i; + int32_t err; + + err = 0; + for (i = 0; i < ctx->obj_count; i++) { + if (ctx->objs[i].type != OBJ_TYPE_NONE) { + err = error(ENOTEMPTY, "Objects not destroyed"); + break; + } + } + + mem_free(ctx->objs); + ctx->objs = NULL; + ctx->obj_count = 0; + + return err; +} + +static context_t * +init(size_t size, uint32_t objs, command_t *cmds) +{ + context_t *ctx; + + ctx = mem_alloc(sizeof(context_t)); + if (ctx == NULL) { + goto failed; + } + + if (!buffer_create(ctx, size)) { + goto failed_ctx; + } + + if (!objs_create(ctx, objs)) { + goto failed_buffer; + } + + if (!cmd_init(ctx, cmds)) { + goto failed_objs; + } + + ctx->active = true; + + return ctx; + +failed_objs: + cmd_fini(ctx, cmds); + objs_destroy(ctx); +failed_buffer: + buffer_destroy(ctx); +failed_ctx: + mem_free(ctx); +failed: + return NULL; +} + +static int32_t +fini(context_t *ctx, command_t *cmds) +{ + int32_t ret; + + cmd_fini(ctx, cmds); + buffer_destroy(ctx); + + ret = objs_destroy(ctx); + + ctx->active = false; + + return ret; +} + +static int32_t +exec_quit(context_t *ctx, command_t *cmd) +{ + ctx->active = false; + + return 0; +} + +static command_t commands[] = {{"fd", NULL, CMD_SUB(fd_commands)}, + {"quit", exec_quit, CMD_ARGS()}, + CMD_END}; + +int32_t +main(int32_t argc, char *argv[]) +{ + context_t *ctx; + int32_t res; + + ctx = init(1024, 16, commands); + if (ctx == NULL) { + return 1; + } + + do { + res = cmd_parse(ctx, commands); + if (res < 0) { + out_err(-res); + } + } while (ctx->active); + + res = fini(ctx, commands); + if (res >= 0) { + out_ok(); + return 0; + } + + out_err(-res); + + return 1; +} diff --git a/tests/basic/open-behind/tester.h b/tests/basic/open-behind/tester.h new file mode 100644 index 00000000000..64e940c78fc --- /dev/null +++ b/tests/basic/open-behind/tester.h @@ -0,0 +1,145 @@ +/* + Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __TESTER_H__ +#define __TESTER_H__ + +#include <stdio.h> +#include <inttypes.h> +#include <stdbool.h> + +enum _obj_type; +typedef enum _obj_type obj_type_t; + +enum _arg_type; +typedef enum _arg_type arg_type_t; + +struct _buffer; +typedef struct _buffer buffer_t; + +struct _obj; +typedef struct _obj obj_t; + +struct _context; +typedef struct _context context_t; + +struct _arg; +typedef struct _arg arg_t; + +struct _command; +typedef struct _command command_t; + +enum _obj_type { OBJ_TYPE_NONE, OBJ_TYPE_FD }; + +enum _arg_type { ARG_TYPE_NONE, ARG_TYPE_OBJ, ARG_TYPE_NUM, ARG_TYPE_STR }; + +struct _buffer { + char *base; + uint32_t size; + uint32_t len; + uint32_t pos; +}; + +struct _obj { + obj_type_t type; + union { + int32_t fd; + }; +}; + +struct _context { + obj_t *objs; + buffer_t buffer; + uint32_t obj_count; + bool active; +}; + +struct _arg { + arg_type_t type; + union { + struct { + obj_type_t type; + obj_t *ref; + } obj; + struct { + uint64_t value; + uint64_t min; + uint64_t max; + } num; + struct { + uint32_t size; + char *data; + } str; + }; +}; + +struct _command { + const char *name; + int32_t (*handler)(context_t *ctx, command_t *cmd); + union { + arg_t *args; + command_t *cmds; + }; +}; + +#define msg(_stream, _fmt, _args...) \ + do { \ + fprintf(_stream, _fmt "\n", ##_args); \ + fflush(_stream); \ + } while (0) + +#define msg_out(_fmt, _args...) msg(stdout, _fmt, ##_args) +#define msg_err(_err, _fmt, _args...) \ + ({ \ + int32_t __msg_err = (_err); \ + msg(stderr, "[%4u:%-15s] " _fmt, __LINE__, __FUNCTION__, __msg_err, \ + ##_args); \ + -__msg_err; \ + }) + +#define error(_err, _fmt, _args...) msg_err(_err, "E(%4d) " _fmt, ##_args) +#define warn(_err, _fmt, _args...) msg_err(_err, "W(%4d) " _fmt, ##_args) +#define info(_err, _fmt, _args...) msg_err(_err, "I(%4d) " _fmt, ##_args) + +#define out_ok(_args...) msg_out("OK " _args) +#define out_err(_err) msg_out("ERR %d", _err) + +#define ARG_END \ + { \ + ARG_TYPE_NONE \ + } + +#define CMD_ARGS1(_x, _args...) \ + .args = (arg_t[]) { _args } +#define CMD_ARGS(_args...) CMD_ARGS1(, ##_args, ARG_END) + +#define CMD_SUB(_cmds) .cmds = _cmds + +#define CMD_END \ + { \ + NULL, NULL, CMD_SUB(NULL) \ + } + +#define ARG_VAL(_type) \ + { \ + ARG_TYPE_OBJ, .obj = {.type = _type } \ + } +#define ARG_NUM(_min, _max) \ + { \ + ARG_TYPE_NUM, .num = {.min = _min, .max = _max } \ + } +#define ARG_STR(_size) \ + { \ + ARG_TYPE_STR, .str = {.size = _size } \ + } + +extern command_t fd_commands[]; + +#endif /* __TESTER_H__ */
\ No newline at end of file diff --git a/tests/basic/posix/zero-fill-enospace.c b/tests/basic/posix/zero-fill-enospace.c index 1371ff59a5f..b1f142c6be9 100644 --- a/tests/basic/posix/zero-fill-enospace.c +++ b/tests/basic/posix/zero-fill-enospace.c @@ -1,4 +1,5 @@ #include <stdio.h> +#include <stdlib.h> #include <glusterfs/api/glfs.h> #include <glusterfs/api/glfs-handles.h> @@ -8,7 +9,7 @@ main(int argc, char *argv[]) glfs_t *fs = NULL; glfs_fd_t *fd = NULL; int ret = 1; - int size = 0; + off_t size = 0; if (argc != 6) { fprintf(stderr, @@ -45,12 +46,12 @@ main(int argc, char *argv[]) goto out; } - size = atoi(argv[5]); + size = strtol(argv[5], NULL, 10); if (size < 0) { fprintf(stderr, "Wrong size %s", argv[5]); goto out; } - ret = glfs_zerofill(fd, 0, atoi(argv[5])); + ret = glfs_zerofill(fd, 0, size); if (ret <= 0) { fprintf(stderr, "glfs_zerofill: returned %d\n", ret); goto out; diff --git a/tests/basic/quick-read-with-upcall.t b/tests/basic/quick-read-with-upcall.t index 2012053d125..dfb751dfcdb 100644 --- a/tests/basic/quick-read-with-upcall.t +++ b/tests/basic/quick-read-with-upcall.t @@ -15,8 +15,8 @@ TEST $CLI volume create $V0 $H0:$B0/${V0}{1..2}; TEST $CLI volume start $V0 # Mount FUSE without selinux: -TEST glusterfs -s $H0 --volfile-id $V0 $M0; -TEST glusterfs -s $H0 --volfile-id $V0 $M1; +TEST glusterfs -s $H0 --volfile-id $V0 --direct-io-mode=enable $M0; +TEST glusterfs -s $H0 --volfile-id $V0 --direct-io-mode=enable $M1; D0="test-message0"; D1="test-message1"; @@ -37,16 +37,13 @@ TEST write_to "$M0/test.txt" "$D1" EXPECT "$D1" cat $M0/test.txt EXPECT "$D0" cat $M1/test.txt -# This is 3.7 for no good reason. We could have kept this to -# any number above 2 seconds. Noticed that when it is 2 seconds, or -# less, there is a possibility of not getting a lookup on the same inode. -sleep 3.7 +sleep 1 # TODO: This line normally fails EXPECT "$D1" cat $M1/test.txt TEST $CLI volume set $V0 features.cache-invalidation on -TEST $CLI volume set $V0 performance.qr-cache-timeout 15 +TEST $CLI volume set $V0 performance.quick-read-cache-timeout 15 TEST $CLI volume set $V0 performance.md-cache-timeout 15 TEST write_to "$M0/test1.txt" "$D0" @@ -73,7 +70,3 @@ EXPECT "$D0" cat $M1/test2.txt TEST write_to "$M0/test2.txt" "$D1" EXPECT "$D1" cat $M0/test2.txt EXPECT "$D1" cat $M1/test2.txt - - -#G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=1718191 -#G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=1718191 diff --git a/tests/basic/volume-snap-scheduler.t b/tests/basic/volume-snap-scheduler.t new file mode 100644 index 00000000000..a638c5cc46a --- /dev/null +++ b/tests/basic/volume-snap-scheduler.t @@ -0,0 +1,49 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +cleanup; + +TEST glusterd; +TEST pidof glusterd; + +TEST $CLI volume create $V0 replica 2 $H0:$B0/${GMV0}{1,2,3,4}; +TEST $CLI volume start $V0 + +## Create, start and mount meta_volume as +## snap_scheduler expects shared storage to be enabled. +## This test is very basic in nature not creating any snapshot +## and purpose is to validate snap scheduling commands. + +TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3}; +TEST $CLI volume start $META_VOL +TEST mkdir -p $META_MNT +TEST glusterfs -s $H0 --volfile-id $META_VOL $META_MNT + +##function to check status +function check_status_scheduler() +{ + local key=$1 + snap_scheduler.py status | grep -F "$key" | wc -l +} + +##Basic snap_scheduler command test init/enable/disable/list + +TEST snap_scheduler.py init + +TEST snap_scheduler.py enable + +EXPECT 1 check_status_scheduler "Enabled" + +TEST snap_scheduler.py disable + +EXPECT 1 check_status_scheduler "Disabled" + +TEST snap_scheduler.py list + +TEST $CLI volume stop $V0; + +TEST $CLI volume delete $V0; + +cleanup; diff --git a/tests/basic/volume.t b/tests/basic/volume.t index 7648ae1d6e1..27fe093d07d 100755..100644 --- a/tests/basic/volume.t +++ b/tests/basic/volume.t @@ -11,7 +11,6 @@ TEST $CLI volume info; TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1,2,3,4,5,6}; - EXPECT "$V0" volinfo_field $V0 'Volume Name'; EXPECT 'Created' volinfo_field $V0 'Status'; EXPECT '6' brick_count $V0 @@ -28,10 +27,34 @@ EXPECT '6' brick_count $V0 TEST $CLI volume top $V0 read-perf bs 4096 count 1000 TEST $CLI volume top $V0 write-perf bs 1048576 count 2 -TEST $CLI volume stop $V0; -EXPECT 'Stopped' volinfo_field $V0 'Status'; +TEST touch $M0/foo + +# statedump path should be a directory, setting it to a file path should fail + +TEST ! $CLI v set $V0 server.statedump-path $M0/foo; +EXPECT '/var/run/gluster' $CLI v get $V0 server.statedump-path + +#set the statedump path to an existing ditectory which should succeed +TEST mkdir $D0/level; +TEST $CLI v set $V0 server.statedump-path $D0/level +EXPECT '/level' volinfo_field $V0 'server.statedump-path' + +ret=$(ls $D0/level | wc -l); +TEST [ $ret == 0 ] +TEST $CLI v statedump $V0; +ret=$(ls $D0/level | wc -l); +TEST ! [ $ret == 0 ] + +#set the statedump path to a non - existing directory which should fail +TEST ! $CLI v set $V0 server.statedump-path /root/test +EXPECT '/level' volinfo_field $V0 'server.statedump-path' + +TEST rm -rf $D0/level + +TEST $CLI volume stop $V0 +EXPECT 'Stopped' volinfo_field $V0 'Status' -TEST $CLI volume delete $V0; -TEST ! $CLI volume info $V0; +TEST $CLI volume delete $V0 +TEST ! $CLI volume info $V0 cleanup; diff --git a/tests/bugs/bug-1064147.t b/tests/bugs/bug-1064147.t new file mode 100755 index 00000000000..27ffde4eb44 --- /dev/null +++ b/tests/bugs/bug-1064147.t @@ -0,0 +1,72 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +# Initialize +#------------------------------------------------------------ +cleanup; + +# Start glusterd +TEST glusterd; +TEST pidof glusterd; +TEST $CLI volume info; + +# Create a volume +TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2} + +# Verify volume creation +EXPECT "$V0" volinfo_field $V0 'Volume Name'; +EXPECT 'Created' volinfo_field $V0 'Status'; + +# Start volume and verify successful start +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; +#------------------------------------------------------------ + +# Test case 1 - Subvolume down + Healing +#------------------------------------------------------------ +# Kill 2nd brick process +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "1" online_brick_count + +# Change root permissions +TEST chmod 444 $M0 + +# Store permission for comparision +TEST permission_new=`stat -c "%A" $M0` + +# Bring up the killed brick process +TEST $CLI volume start $V0 force + +# Perform lookup +sleep 5 +TEST ls $M0 + +# Check brick permissions +TEST brick_perm=`stat -c "%A" $B0/${V0}2` +TEST [ ${brick_perm} = ${permission_new} ] +#------------------------------------------------------------ + +# Test case 2 - Add-brick + Healing +#------------------------------------------------------------ +# Change root permissions +TEST chmod 777 $M0 + +# Store permission for comparision +TEST permission_new_2=`stat -c "%A" $M0` + +# Add a 3rd brick +TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3 + +# Perform lookup +sleep 5 +TEST ls $M0 + +# Check permissions on the new brick +TEST brick_perm2=`stat -c "%A" $B0/${V0}3` + +TEST [ ${brick_perm2} = ${permission_new_2} ] + +cleanup; diff --git a/tests/bugs/bug-1371806.t b/tests/bugs/bug-1371806.t index 7dc1613a4f2..08180525650 100644 --- a/tests/bugs/bug-1371806.t +++ b/tests/bugs/bug-1371806.t @@ -28,6 +28,7 @@ TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3,4,5} TEST $CLI volume start $V0 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "6" online_brick_count TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; diff --git a/tests/bugs/bug-1371806_acl.t b/tests/bugs/bug-1371806_acl.t index a2e5af49bb0..c39165628cc 100644 --- a/tests/bugs/bug-1371806_acl.t +++ b/tests/bugs/bug-1371806_acl.t @@ -39,6 +39,7 @@ TEST pidof glusterd TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3,4,5} TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG TEST $CLI volume start $V0 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "6" online_brick_count TEST glusterfs --volfile-id=$V0 --acl --volfile-server=$H0 --entry-timeout=0 $M0; diff --git a/tests/bugs/ctime/issue-832.t b/tests/bugs/ctime/issue-832.t new file mode 100755 index 00000000000..740f731ab73 --- /dev/null +++ b/tests/bugs/ctime/issue-832.t @@ -0,0 +1,32 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../traps.rc + +#Trigger trusted.glusterfs.mdata setting codepath and see things work as expected +cleanup + +TEST_USER=test-ctime-user +TEST_UID=27341 + +TEST useradd -o -M -u ${TEST_UID} ${TEST_USER} +push_trapfunc "userdel --force ${TEST_USER}" + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 $H0:$B0/$V0 +TEST $CLI volume start $V0 + +$GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +echo abc > $M0/test +TEST chmod 755 $M0/ +TEST chmod 744 $M0/test +TEST setfattr -x trusted.glusterfs.mdata $B0/$V0/test +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +$GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +su ${TEST_USER} -c "cat $M0/test" +TEST getfattr -n trusted.glusterfs.mdata $B0/$V0/test + +cleanup diff --git a/tests/bugs/distribute/bug-1600379.t b/tests/bugs/distribute/bug-1600379.t new file mode 100644 index 00000000000..8d2f6154100 --- /dev/null +++ b/tests/bugs/distribute/bug-1600379.t @@ -0,0 +1,54 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# Initialize +#------------------------------------------------------------ +cleanup; + +# Start glusterd +TEST glusterd; +TEST pidof glusterd; +TEST $CLI volume info; + +# Create a volume +TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2} + +# Verify volume creation +EXPECT "$V0" volinfo_field $V0 'Volume Name'; +EXPECT 'Created' volinfo_field $V0 'Status'; + +# Start volume and verify successful start +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; +#------------------------------------------------------------ + +# Test case - Remove xattr from killed brick on lookup +#------------------------------------------------------------ +# Create a dir and set custom xattr +TEST mkdir $M0/testdir +TEST setfattr -n user.attr -v val $M0/testdir +xattr_val=`getfattr -d $B0/${V0}2/testdir | awk '{print $1}'`; +TEST ${xattr_val}='user.attr="val"'; + +# Kill 2nd brick process +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "1" online_brick_count + +# Remove custom xattr +TEST setfattr -x user.attr $M0/testdir + +# Bring up the killed brick process +TEST $CLI volume start $V0 force + +# Perform lookup +sleep 5 +TEST ls $M0/testdir + +# Check brick xattrs +xattr_val_2=`getfattr -d $B0/${V0}2/testdir`; +TEST [ ${xattr_val_2} = ''] ; + +cleanup; diff --git a/tests/bugs/distribute/issue-1327.t b/tests/bugs/distribute/issue-1327.t new file mode 100755 index 00000000000..acd8c8c6614 --- /dev/null +++ b/tests/bugs/distribute/issue-1327.t @@ -0,0 +1,33 @@ +#!/bin/bash + +SCRIPT_TIMEOUT=250 + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../dht.rc + +cleanup + +TEST glusterd +TEST pidof glusterd + +BRICK1=$B0/${V0}-0 +BRICK2=$B0/${V0}-1 + +TEST $CLI volume create $V0 $H0:$BRICK1 $H0:$BRICK2 +TEST $CLI volume start $V0 + +TEST glusterfs -s $H0 --volfile-id $V0 $M0 +TEST mkdir $M0/dir + +#remove dir from one of the brick +TEST rmdir $BRICK2/dir + +#safe cache timeout for lookup to be triggered +sleep 2 + +TEST ls $M0/dir + +TEST stat $BRICK2/dir + +cleanup diff --git a/tests/bugs/fuse/many-groups-for-acl.t b/tests/bugs/fuse/many-groups-for-acl.t index d959f750ee0..a51b1bc7267 100755 --- a/tests/bugs/fuse/many-groups-for-acl.t +++ b/tests/bugs/fuse/many-groups-for-acl.t @@ -38,6 +38,13 @@ do done TEST useradd -o -M -u ${NEW_UID} -g ${NEW_GID} -G ${NEW_USER}-${NEW_GIDS} ${NEW_USER} +# Linux < 3.8 exports only first 32 gids of pid to userspace +kernel_exports_few_gids=0 +if [ "$OSTYPE" = Linux ] && \ + su -m ${NEW_USER} -c "grep ^Groups: /proc/self/status | wc -w | xargs -I@ expr @ - 1 '<' $LAST_GID - $NEW_GID + 1" > /dev/null; then + kernel_exports_few_gids=1 +fi + # preparation done, start the tests TEST glusterd @@ -48,6 +55,8 @@ TEST $CLI volume set $V0 nfs.disable off TEST $CLI volume set ${V0} server.manage-gids off TEST $CLI volume start ${V0} +# This is just a synchronization hack to make sure the bricks are +# up before going on. EXPECT_WITHIN ${NFS_EXPORT_TIMEOUT} "1" is_nfs_export_available # mount the volume with POSIX ACL support, without --resolve-gids @@ -69,8 +78,8 @@ TEST [ $? -eq 0 ] su -m ${NEW_USER} -c "touch ${M0}/first-32-gids-2/success > /dev/null" TEST [ $? -eq 0 ] -su -m ${NEW_USER} -c "touch ${M0}/gid-64/failure > /dev/null" -TEST [ $? -ne 0 ] +su -m ${NEW_USER} -c "touch ${M0}/gid-64/success--if-all-gids-exported > /dev/null" +TEST [ $? -eq $kernel_exports_few_gids ] su -m ${NEW_USER} -c "touch ${M0}/gid-120/failure > /dev/null" TEST [ $? -ne 0 ] diff --git a/tests/bugs/glusterd/brick-mux-validation-in-cluster.t b/tests/bugs/glusterd/brick-mux-validation-in-cluster.t index 4e570381701..b6af487a791 100644 --- a/tests/bugs/glusterd/brick-mux-validation-in-cluster.t +++ b/tests/bugs/glusterd/brick-mux-validation-in-cluster.t @@ -7,6 +7,20 @@ function count_brick_processes { pgrep glusterfsd | wc -l } +function count_brick_pids { + $CLI_1 --xml volume status all | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \ + | grep -v "N/A" | sort | uniq | wc -l +} + +function count_N/A_brick_pids { + $CLI_1 --xml volume status all | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \ + | grep -- '\-1' | sort | uniq | wc -l +} + +function check_peers { + $CLI_2 peer status | grep 'Peer in Cluster (Connected)' | wc -l +} + cleanup; TEST launch_cluster 3 @@ -48,4 +62,47 @@ TEST $CLI_1 volume stop $V1 EXPECT 3 count_brick_processes -cleanup +TEST $CLI_1 volume stop $META_VOL + +TEST $CLI_1 volume delete $META_VOL +TEST $CLI_1 volume delete $V0 +TEST $CLI_1 volume delete $V1 + +#bug-1773856 - Brick process fails to come up with brickmux on + +TEST $CLI_1 volume create $V0 $H1:$B1/${V0}1 $H2:$B2/${V0}1 $H3:$B3/${V0}1 force +TEST $CLI_1 volume start $V0 + + +EXPECT 3 count_brick_processes + +#create and start a new volume +TEST $CLI_1 volume create $V1 $H1:$B1/${V1}2 $H2:$B2/${V1}2 $H3:$B3/${V1}2 force +TEST $CLI_1 volume start $V1 + +EXPECT 3 count_brick_processes + +V2=patchy2 +TEST $CLI_1 volume create $V2 $H1:$B1/${V2}3 $H2:$B2/${V2}3 $H3:$B3/${V2}3 force +TEST $CLI_1 volume start $V2 + +EXPECT 3 count_brick_processes + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids + +TEST kill_node 1 + +sleep 10 + +EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers; + +$CLI_2 volume set $V0 performance.readdir-ahead on +$CLI_2 volume set $V1 performance.readdir-ahead on + +TEST $glusterd_1; +EXPECT_WITHIN $PROBE_TIMEOUT 2 peer_count + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_brick_pids +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 count_N/A_brick_pids + +cleanup; diff --git a/tests/bugs/glusterd/brick-mux-validation.t b/tests/bugs/glusterd/brick-mux-validation.t index 03a476823ca..61b0455f9a8 100644 --- a/tests/bugs/glusterd/brick-mux-validation.t +++ b/tests/bugs/glusterd/brick-mux-validation.t @@ -24,7 +24,7 @@ TEST $CLI volume create $V0 $H0:$B0/${V0}{1..3} TEST $CLI volume start $V0 EXPECT 1 count_brick_processes -EXPECT 1 count_brick_pids +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 1 count_brick_pids EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 online_brick_count pkill gluster @@ -101,4 +101,4 @@ TEST $CLI_IGNORE_PARTITION volume reset-brick $V1 $H0:$B0/${V1}1 $H0:$B0/${V1}1 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 online_brick_count EXPECT 1 count_brick_processes -cleanup;
\ No newline at end of file +cleanup; diff --git a/tests/bugs/glusterd/brick-mux.t b/tests/bugs/glusterd/brick-mux.t index eeaa3ebfea8..927940534c1 100644 --- a/tests/bugs/glusterd/brick-mux.t +++ b/tests/bugs/glusterd/brick-mux.t @@ -39,7 +39,7 @@ TEST glusterd EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 online_brick_count EXPECT 1 count_brick_processes -TEST $CLI volume set $V1 performance.cache-size 32MB +TEST $CLI volume set $V1 performance.io-cache-size 32MB TEST $CLI volume stop $V1 TEST $CLI volume start $V1 diff --git a/tests/bugs/glusterd/brick-order-check-add-brick.t b/tests/bugs/glusterd/brick-order-check-add-brick.t new file mode 100644 index 00000000000..0be31dac768 --- /dev/null +++ b/tests/bugs/glusterd/brick-order-check-add-brick.t @@ -0,0 +1,61 @@ +#!/bin/bash +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../cluster.rc +. $(dirname $0)/../../snapshot.rc + +cleanup; + +TEST verify_lvm_version; +#Create cluster with 3 nodes +TEST launch_cluster 3 -NO_DEBUG -NO_FORCE +TEST setup_lvm 3 + +TEST $CLI_1 peer probe $H2 +TEST $CLI_1 peer probe $H3 +EXPECT_WITHIN $PROBE_TIMEOUT 2 peer_count + +TEST $CLI_1 volume create $V0 replica 3 $H1:$L1/$V0 $H2:$L2/$V0 $H3:$L3/$V0 +EXPECT '1 x 3 = 3' volinfo_field $V0 'Number of Bricks' +EXPECT 'Created' volinfo_field $V0 'Status' + +TEST $CLI_1 volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +#add-brick with or without mentioning the replica count should not fail +TEST $CLI_1 volume add-brick $V0 replica 3 $H1:$L1/${V0}_1 $H2:$L2/${V0}_1 $H3:$L3/${V0}_1 +EXPECT '2 x 3 = 6' volinfo_field $V0 'Number of Bricks' + +TEST $CLI_1 volume add-brick $V0 $H1:$L1/${V0}_2 $H2:$L2/${V0}_2 $H3:$L3/${V0}_2 +EXPECT '3 x 3 = 9' volinfo_field $V0 'Number of Bricks' + +#adding bricks from same host should fail the brick order check +TEST ! $CLI_1 volume add-brick $V0 $H1:$L1/${V0}_3 $H1:$L1/${V0}_4 $H1:$L1/${V0}_5 +EXPECT '3 x 3 = 9' volinfo_field $V0 'Number of Bricks' + +#adding bricks from same host with force should succeed +TEST $CLI_1 volume add-brick $V0 $H1:$L1/${V0}_3 $H1:$L1/${V0}_4 $H1:$L1/${V0}_5 force +EXPECT '4 x 3 = 12' volinfo_field $V0 'Number of Bricks' + +TEST $CLI_1 volume stop $V0 +TEST $CLI_1 volume delete $V0 + +TEST $CLI_1 volume create $V0 replica 2 $H1:$L1/${V0}1 $H2:$L2/${V0}1 +EXPECT '1 x 2 = 2' volinfo_field $V0 'Number of Bricks' +EXPECT 'Created' volinfo_field $V0 'Status' + +TEST $CLI_1 volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +#Add-brick with Increasing replica count +TEST $CLI_1 volume add-brick $V0 replica 3 $H3:$L3/${V0}1 +EXPECT '1 x 3 = 3' volinfo_field $V0 'Number of Bricks' + +#Add-brick with Increasing replica count from same host should fail +TEST ! $CLI_1 volume add-brick $V0 replica 5 $H1:$L1/${V0}2 $H1:$L1/${V0}3 + +#adding multiple bricks from same host should fail the brick order check +TEST ! $CLI_1 volume add-brick $V0 replica 3 $H1:$L1/${V0}{4..6} $H2:$L2/${V0}{7..9} +EXPECT '1 x 3 = 3' volinfo_field $V0 'Number of Bricks' + +cleanup diff --git a/tests/bugs/glusterd/check_elastic_server.t b/tests/bugs/glusterd/check_elastic_server.t index 8e9e4e5b0eb..41d2140aa2b 100644 --- a/tests/bugs/glusterd/check_elastic_server.t +++ b/tests/bugs/glusterd/check_elastic_server.t @@ -23,6 +23,9 @@ EXPECT 'Created' cluster_volinfo_field 1 $V0 'Status'; $CLI_1 volume start $V0 EXPECT 'Started' cluster_volinfo_field 1 $V0 'Status'; +#Mount invalid volume +TEST ! glusterfs -s $H1 --volfile-id=$V0_NA $M0; + #Mount FUSE TEST glusterfs -s $H1 --volfile-id=$V0 $M0; diff --git a/tests/bugs/glusterd/mgmt-handshake-and-volume-sync-post-glusterd-restart.t b/tests/bugs/glusterd/mgmt-handshake-and-volume-sync-post-glusterd-restart.t index fdc0a73f60c..8001359e6b3 100644 --- a/tests/bugs/glusterd/mgmt-handshake-and-volume-sync-post-glusterd-restart.t +++ b/tests/bugs/glusterd/mgmt-handshake-and-volume-sync-post-glusterd-restart.t @@ -4,7 +4,7 @@ . $(dirname $0)/../../cluster.rc function check_peers { -$CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l +eval \$CLI_$1 peer status | grep 'Peer in Cluster (Connected)' | wc -l } cleanup @@ -36,23 +36,35 @@ TEST [[ $OP_VERS_ORIG == $OP_VERS_NEW ]] #bug-948686 - volume sync after bringing up the killed node TEST $CLI_1 peer probe $H3 -EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers; +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 1 +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 2 +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 3 TEST $CLI_1 volume create $V0 replica 2 $H1:$B1/$V0 $H1:$B1/${V0}_1 $H2:$B2/$V0 $H3:$B3/$V0 TEST $CLI_1 volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field_1 $V0 'Status' TEST glusterfs --volfile-server=$H1 --volfile-id=$V0 $M0 #kill a node TEST kill_node 3 +EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers 1 +EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers 2 #modify volume config to see change in volume-sync TEST $CLI_1 volume set $V0 write-behind off #add some files to the volume to see effect of volume-heal cmd TEST touch $M0/{1..100}; TEST $CLI_1 volume stop $V0; +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 'Stopped' volinfo_field_1 $V0 'Status' + TEST $glusterd_3; -EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 1 +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 2 +EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers 3 + +sleep 5 TEST $CLI_3 volume start $V0; +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field_1 $V0 'Status' TEST $CLI_2 volume stop $V0; TEST $CLI_2 volume delete $V0; diff --git a/tests/bugs/glusterd/enable-shared-storage-and-remove-brick-validation.t b/tests/bugs/glusterd/remove-brick-validation.t index 11ed0d94d79..a0ff4ff6a24 100644 --- a/tests/bugs/glusterd/enable-shared-storage-and-remove-brick-validation.t +++ b/tests/bugs/glusterd/remove-brick-validation.t @@ -18,20 +18,6 @@ TEST $CLI_1 peer probe $H2; EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count 1 -#test case for bug 1266818 - disabling enable-shared-storage option -##should not delete user created volume with name glusterd_shared_storage - -## creating a volume with name glusterd_shared_storage -TEST $CLI_1 volume create glusterd_shared_storage $H1:$B1/${V0}0 $H2:$B2/${V0}1 -TEST $CLI_1 volume start glusterd_shared_storage - -## disabling enable-shared-storage should not succeed and should not delete the -## user created volume with name "glusterd_shared_storage" -TEST ! $CLI_1 volume all enable-shared-storage disable - -## volume with name should exist -TEST $CLI_1 volume info glusterd_shared_storage - #testcase: bug-1245045-remove-brick-validation TEST $CLI_1 peer probe $H3; diff --git a/tests/bugs/glusterd/removing-multiple-bricks-in-single-remove-brick-command.t b/tests/bugs/glusterd/removing-multiple-bricks-in-single-remove-brick-command.t index 20c84d26b9c..00beab59137 100644 --- a/tests/bugs/glusterd/removing-multiple-bricks-in-single-remove-brick-command.t +++ b/tests/bugs/glusterd/removing-multiple-bricks-in-single-remove-brick-command.t @@ -49,6 +49,7 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 #Create a 3x3 dist-rep volume TEST $CLI volume create $V1 replica 3 $H0:$B0/${V1}{0,1,2,3,4,5,6,7,8}; TEST $CLI volume start $V1 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "9" brick_count ${V1} # Mount FUSE and create file/directory TEST glusterfs -s $H0 --volfile-id $V1 $M0 diff --git a/tests/bugs/glusterd/serialize-shd-manager-glusterd-restart.t b/tests/bugs/glusterd/serialize-shd-manager-glusterd-restart.t index 3a27c2a9d1b..a871e112d87 100644 --- a/tests/bugs/glusterd/serialize-shd-manager-glusterd-restart.t +++ b/tests/bugs/glusterd/serialize-shd-manager-glusterd-restart.t @@ -4,7 +4,7 @@ . $(dirname $0)/../../cluster.rc function check_peers { -count=`$CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l` +count=`$CLI_3 peer status | grep 'Peer in Cluster (Connected)' | wc -l` echo $count } diff --git a/tests/bugs/glusterd/validating-options-for-replicated-volume.t b/tests/bugs/glusterd/validating-options-for-replicated-volume.t index ee231338ff1..ddc80b17870 100644 --- a/tests/bugs/glusterd/validating-options-for-replicated-volume.t +++ b/tests/bugs/glusterd/validating-options-for-replicated-volume.t @@ -61,10 +61,15 @@ TEST ! $CLI volume set $V0 background-self-heal-count " " TEST $CLI volume set $V0 background-self-heal-count 10 EXPECT "10" volume_option $V0 cluster.background-self-heal-count -TEST ! $CLI volume set $V0 cache-size "" -TEST ! $CLI volume set $V0 cache-size " " -TEST $CLI volume set $V0 cache-size 512MB -EXPECT "512MB" volume_option $V0 performance.cache-size +TEST ! $CLI volume set $V0 io-cache-size "" +TEST ! $CLI volume set $V0 io-cache-size " " +TEST $CLI volume set $V0 io-cache-size 64MB +EXPECT "64MB" volume_option $V0 performance.io-cache-size + +TEST ! $CLI volume set $V0 quick-read-cache-size "" +TEST ! $CLI volume set $V0 quick-read-cache-size " " +TEST $CLI volume set $V0 quick-read-cache-size 512MB +EXPECT "512MB" volume_option $V0 performance.quick-read-cache-size TEST ! $CLI volume set $V0 self-heal-daemon "" TEST ! $CLI volume set $V0 self-heal-daemon " " diff --git a/tests/bugs/glusterfs-server/bug-873549.t b/tests/bugs/glusterfs-server/bug-873549.t index a3b2f9c9bf7..8b5534728fd 100644 --- a/tests/bugs/glusterfs-server/bug-873549.t +++ b/tests/bugs/glusterfs-server/bug-873549.t @@ -10,7 +10,7 @@ TEST $CLI volume info; TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2}; -TEST $CLI volume set $V0 performance.cache-size 512MB +TEST $CLI volume set $V0 performance.quick-read-cache-size 512MB TEST $CLI volume start $V0 TEST $CLI volume statedump $V0 all diff --git a/tests/bugs/glusterfs/bug-873962-spb.t b/tests/bugs/glusterfs/bug-873962-spb.t index db84a223089..db71cc0f6fe 100644 --- a/tests/bugs/glusterfs/bug-873962-spb.t +++ b/tests/bugs/glusterfs/bug-873962-spb.t @@ -14,6 +14,7 @@ TEST $CLI volume set $V0 performance.io-cache off TEST $CLI volume set $V0 performance.write-behind off TEST $CLI volume set $V0 performance.stat-prefetch off TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.open-behind off TEST $CLI volume set $V0 cluster.background-self-heal-count 0 TEST $CLI volume start $V0 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable diff --git a/tests/bugs/logging/bug-823081.t b/tests/bugs/logging/bug-823081.t index 0ed8f4c26c1..bd1965d2d49 100755 --- a/tests/bugs/logging/bug-823081.t +++ b/tests/bugs/logging/bug-823081.t @@ -22,20 +22,20 @@ function set_tail () set_tail $V0; TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2}; -tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-` +tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 6-` TEST [[ \"$tail\" == \"$tail_success\" ]] TEST ! $CLI volume create $V0 $H0:$B0/${V0}{1,2}; -tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-` +tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 6-` TEST [[ \"$tail\" == \"$tail_failure\" ]] set_tail $V1; TEST gluster volume create $V1 $H0:$B0/${V1}{1,2} force; -tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-` +tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 6-` TEST [[ \"$tail\" == \"$tail_success_force\" ]] TEST ! gluster volume create $V1 $H0:$B0/${V1}{1,2} force; -tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-` +tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 6-` TEST [[ \"$tail\" == \"$tail_failure_force\" ]] cleanup; diff --git a/tests/bugs/replicate/bug-1101647.t b/tests/bugs/replicate/bug-1101647.t index 8f420eec012..708bc1a1e29 100644 --- a/tests/bugs/replicate/bug-1101647.t +++ b/tests/bugs/replicate/bug-1101647.t @@ -12,6 +12,8 @@ TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2}; TEST $CLI volume start $V0; TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 #Create base entry in indices/xattrop echo "Data">$M0/file diff --git a/tests/bugs/replicate/bug-1586020-mark-dirty-for-entry-txn-on-quorum-failure.t b/tests/bugs/replicate/bug-1586020-mark-dirty-for-entry-txn-on-quorum-failure.t index 26f90497d6f..49c4dea4e9c 100644 --- a/tests/bugs/replicate/bug-1586020-mark-dirty-for-entry-txn-on-quorum-failure.t +++ b/tests/bugs/replicate/bug-1586020-mark-dirty-for-entry-txn-on-quorum-failure.t @@ -53,8 +53,6 @@ TEST ! ls $B0/${V0}1/file$i TEST ls $B0/${V0}2/file$i dirty=$(get_hex_xattr trusted.afr.dirty $B0/${V0}2) TEST [ "$dirty" != "000000000000000000000000" ] -EXPECT "000000010000000100000000" get_hex_xattr trusted.afr.$V0-client-0 $B0/${V0}2/file$i -EXPECT "000000010000000100000000" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}2/file$i TEST $CLI volume set $V0 self-heal-daemon on EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status diff --git a/tests/bugs/replicate/bug-1744548-heal-timeout.t b/tests/bugs/replicate/bug-1744548-heal-timeout.t index c208112c8b0..011535066f9 100644 --- a/tests/bugs/replicate/bug-1744548-heal-timeout.t +++ b/tests/bugs/replicate/bug-1744548-heal-timeout.t @@ -25,14 +25,14 @@ TEST ! $CLI volume heal $V0 TEST $CLI volume profile $V0 start TEST $CLI volume profile $V0 info clear TEST $CLI volume heal $V0 enable -# Each brick does 3 opendirs, corresponding to dirty, xattrop and entry-changes -EXPECT_WITHIN $HEAL_TIMEOUT "^333$" get_cumulative_opendir_count +# Each brick does 4 opendirs, corresponding to dirty, xattrop and entry-changes, anonymous-inode +EXPECT_WITHIN 4 "^444$" get_cumulative_opendir_count # Check that a change in heal-timeout is honoured immediately. TEST $CLI volume set $V0 cluster.heal-timeout 5 sleep 10 # Two crawls must have happened. -EXPECT_WITHIN $HEAL_TIMEOUT "^999$" get_cumulative_opendir_count +EXPECT_WITHIN $CHILD_UP_TIMEOUT "^121212$" get_cumulative_opendir_count # shd must not heal if it is disabled and heal-timeout is changed. TEST $CLI volume heal $V0 disable diff --git a/tests/bugs/replicate/issue-1254-prioritize-enospc.t b/tests/bugs/replicate/issue-1254-prioritize-enospc.t new file mode 100644 index 00000000000..fab94b71b27 --- /dev/null +++ b/tests/bugs/replicate/issue-1254-prioritize-enospc.t @@ -0,0 +1,80 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup + +function create_bricks { + TEST truncate -s 100M $B0/brick0 + TEST truncate -s 100M $B0/brick1 + TEST truncate -s 20M $B0/brick2 + LO1=`SETUP_LOOP $B0/brick0` + TEST [ $? -eq 0 ] + TEST MKFS_LOOP $LO1 + LO2=`SETUP_LOOP $B0/brick1` + TEST [ $? -eq 0 ] + TEST MKFS_LOOP $LO2 + LO3=`SETUP_LOOP $B0/brick2` + TEST [ $? -eq 0 ] + TEST MKFS_LOOP $LO3 + TEST mkdir -p $B0/${V0}0 $B0/${V0}1 $B0/${V0}2 + TEST MOUNT_LOOP $LO1 $B0/${V0}0 + TEST MOUNT_LOOP $LO2 $B0/${V0}1 + TEST MOUNT_LOOP $LO3 $B0/${V0}2 +} + +function create_files { + local i=1 + while (true) + do + touch $M0/file$i + if [ -e $B0/${V0}2/file$i ]; + then + ((i++)) + else + break + fi + done +} + +TESTS_EXPECTED_IN_LOOP=13 + +#Arbiter volume: Check for ENOSPC when arbiter brick becomes full# +TEST glusterd +create_bricks +TEST $CLI volume create $V0 replica 3 arbiter 1 $H0:$B0/${V0}{0,1,2} +TEST $CLI volume start $V0 +TEST $CLI volume set $V0 performance.write-behind off +TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0 + +create_files +TEST kill_brick $V0 $H0 $B0/${V0}1 +error1=$(touch $M0/file-1 2>&1) +EXPECT "No space left on device" echo $error1 +error2=$(mkdir $M0/dir-1 2>&1) +EXPECT "No space left on device" echo $error2 +error3=$((echo "Test" > $M0/file-3) 2>&1) +EXPECT "No space left on device" echo $error3 + +cleanup + +#Replica-3 volume: Check for ENOSPC when one of the brick becomes full# +#Keeping the third brick of lower size to simulate disk full scenario# +TEST glusterd +create_bricks +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} +TEST $CLI volume start $V0 +TEST $CLI volume set $V0 performance.write-behind off +TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0 + +create_files +TEST kill_brick $V0 $H0 $B0/${V0}1 +error1=$(touch $M0/file-1 2>&1) +EXPECT "No space left on device" echo $error1 +error2=$(mkdir $M0/dir-1 2>&1) +EXPECT "No space left on device" echo $error2 +error3=$((cat /dev/zero > $M0/file1) 2>&1) +EXPECT "No space left on device" echo $error3 + +cleanup diff --git a/tests/bugs/replicate/mdata-heal-no-xattrs.t b/tests/bugs/replicate/mdata-heal-no-xattrs.t new file mode 100644 index 00000000000..d3b0c504c80 --- /dev/null +++ b/tests/bugs/replicate/mdata-heal-no-xattrs.t @@ -0,0 +1,59 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}; +TEST $CLI volume set $V0 cluster.self-heal-daemon off +TEST $CLI volume start $V0 + +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2 +echo "Data">$M0/FILE +ret=$? +TEST [ $ret -eq 0 ] + +# Change permission on brick-0: simulates the case where there is metadata +# mismatch but no pending xattrs. This brick will become the source for heal. +TEST chmod +x $B0/$V0"0"/FILE + +# Add gfid to xattrop +xattrop_b0=$(afr_get_index_path $B0/$V0"0") +base_entry_b0=`ls $xattrop_b0` +gfid_str_FILE=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/$V0"0"/FILE)) +TEST ln $xattrop_b0/$base_entry_b0 $xattrop_b0/$gfid_str_FILE +EXPECT_WITHIN $HEAL_TIMEOUT "^1$" get_pending_heal_count $V0 + +TEST $CLI volume set $V0 cluster.self-heal-daemon on +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2 +TEST $CLI volume heal $V0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +# Brick-0 should contain xattrs blaming other 2 bricks. +# The values will be zero because heal is over. +EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.$V0-client-1 $B0/${V0}0/FILE +EXPECT "000000000000000000000000" get_hex_xattr trusted.afr.$V0-client-2 $B0/${V0}0/FILE +TEST ! getfattr -n trusted.afr.$V0-client-0 $B0/${V0}0/FILE + +# Brick-1 and Brick-2 must not contain any afr xattrs. +TEST ! getfattr -n trusted.afr.$V0-client-0 $B0/${V0}1/FILE +TEST ! getfattr -n trusted.afr.$V0-client-1 $B0/${V0}1/FILE +TEST ! getfattr -n trusted.afr.$V0-client-2 $B0/${V0}1/FILE +TEST ! getfattr -n trusted.afr.$V0-client-0 $B0/${V0}2/FILE +TEST ! getfattr -n trusted.afr.$V0-client-1 $B0/${V0}2/FILE +TEST ! getfattr -n trusted.afr.$V0-client-2 $B0/${V0}2/FILE + +# check permission bits. +EXPECT '755' stat -c %a $B0/${V0}0/FILE +EXPECT '755' stat -c %a $B0/${V0}1/FILE +EXPECT '755' stat -c %a $B0/${V0}2/FILE + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +cleanup; diff --git a/tests/bugs/shard/bug-1696136.c b/tests/bugs/shard/bug-1696136.c index b9e8d1375e5..cb650535b09 100644 --- a/tests/bugs/shard/bug-1696136.c +++ b/tests/bugs/shard/bug-1696136.c @@ -87,8 +87,9 @@ main(int argc, char *argv[]) goto out; } - offset = atoi(argv[4]); - len = atoi(argv[5]); + /* Note that off_t is signed but size_t isn't. */ + offset = strtol(argv[4], NULL, 10); + len = strtoul(argv[5], NULL, 10); fd = glfs_open(fs, argv[6], O_RDWR); if (fd == NULL) { diff --git a/tests/bugs/shard/bug-shard-discard.c b/tests/bugs/shard/bug-shard-discard.c index 15dca6c2181..6fa93fb89d1 100644 --- a/tests/bugs/shard/bug-shard-discard.c +++ b/tests/bugs/shard/bug-shard-discard.c @@ -50,8 +50,9 @@ main(int argc, char *argv[]) goto out; } - off = atoi(argv[4]); - len = atoi(argv[5]); + /* Note that off_t is signed but size_t isn't. */ + off = strtol(argv[4], NULL, 10); + len = strtoul(argv[5], NULL, 10); ret = glfs_discard(fd, off, len); if (ret <= 0) { diff --git a/tests/bugs/shard/issue-1243.t b/tests/bugs/shard/issue-1243.t new file mode 100644 index 00000000000..ba22d2b74fe --- /dev/null +++ b/tests/bugs/shard/issue-1243.t @@ -0,0 +1,43 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1} +TEST $CLI volume set $V0 features.shard on +TEST $CLI volume set $V0 features.shard-block-size 4MB +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.strict-o-direct on +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0 + +TEST $CLI volume set $V0 md-cache-timeout 10 + +# Write data into a file such that its size crosses shard-block-size +TEST dd if=/dev/zero of=$M0/foo bs=1048576 count=8 oflag=direct + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0 + +# Execute a setxattr on the file. +TEST setfattr -n trusted.libvirt -v some-value $M0/foo + +# Size of the file should be the aggregated size, not the shard-block-size +EXPECT '8388608' stat -c %s $M0/foo + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0 + +# Execute a removexattr on the file. +TEST setfattr -x trusted.libvirt $M0/foo + +# Size of the file should be the aggregated size, not the shard-block-size +EXPECT '8388608' stat -c %s $M0/foo +cleanup diff --git a/tests/bugs/shard/issue-1281.t b/tests/bugs/shard/issue-1281.t new file mode 100644 index 00000000000..9704caa8944 --- /dev/null +++ b/tests/bugs/shard/issue-1281.t @@ -0,0 +1,34 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1} +TEST $CLI volume set $V0 features.shard on +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0 + +#Open a file and store descriptor in fd = 5 +exec 5>$M0/foo + +#Unlink the same file which is opened in prev step +TEST unlink $M0/foo + +#Write something on the file using the open fd = 5 +echo "issue-1281" >&5 + +#Write on the descriptor should be succesful +EXPECT 0 echo $? + +#Close the fd = 5 +exec 5>&- + +cleanup diff --git a/tests/bugs/shard/issue-1425.t b/tests/bugs/shard/issue-1425.t new file mode 100644 index 00000000000..bbe82c0e5b2 --- /dev/null +++ b/tests/bugs/shard/issue-1425.t @@ -0,0 +1,45 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +FILE_COUNT_TIME=5 + +function get_file_count { + ls $1* | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 $H0:$B0/${V0}0 +TEST $CLI volume set $V0 features.shard on +TEST $CLI volume set $V0 features.shard-block-size 4MB +TEST $CLI volume start $V0 +TEST $CLI volume profile $V0 start + +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0 + +TEST fallocate -l 20M $M0/foo +gfid_new=$(get_gfid_string $M0/foo) + +# Check for the base shard +TEST stat $M0/foo +TEST stat $B0/${V0}0/foo + +# There should be 4 associated shards +EXPECT_WITHIN $FILE_COUNT_TIME 4 get_file_count $B0/${V0}0/.shard/$gfid_new + +# There should be 1+4 shards and we expect 4 lookups less than on the build without this patch +EXPECT "21" echo `$CLI volume profile $V0 info incremental | grep -w LOOKUP | awk '{print $8}'` + +# Delete the base shard and check shards get cleaned up +TEST unlink $M0/foo + +TEST ! stat $M0/foo +TEST ! stat $B0/${V0}0/foo + +# There should be no shards now +EXPECT_WITHIN $FILE_COUNT_TIME 0 get_file_count $B0/${V0}0/.shard/$gfid_new +cleanup diff --git a/tests/bugs/shard/shard-fallocate.c b/tests/bugs/shard/shard-fallocate.c index 45b9ce00509..cb0714e8564 100644 --- a/tests/bugs/shard/shard-fallocate.c +++ b/tests/bugs/shard/shard-fallocate.c @@ -87,8 +87,9 @@ main(int argc, char *argv[]) goto out; } - offset = atoi(argv[4]); - len = atoi(argv[5]); + /* Note that off_t is signed but size_t isn't. */ + offset = strtol(argv[4], NULL, 10); + len = strtoul(argv[5], NULL, 10); fd = glfs_open(fs, argv[6], O_RDWR); if (fd == NULL) { diff --git a/tests/bugs/snapshot/bug-1111041.t b/tests/bugs/snapshot/bug-1111041.t index f771d64f2a3..efda9688d8b 100755 --- a/tests/bugs/snapshot/bug-1111041.t +++ b/tests/bugs/snapshot/bug-1111041.t @@ -11,6 +11,10 @@ function is_snapd_running { $CLI volume status $1 | grep "Snapshot Daemon" | wc -l; } +function snapd_pid { + $CLI volume status $V0 | grep "Snapshot Daemon" | awk {'print $8'} +} + TEST glusterd; TEST pidof glusterd; @@ -25,14 +29,12 @@ TEST $CLI volume set $V0 features.uss enable; EXPECT "1" is_snapd_running $V0 -SNAPD_PID=$($CLI volume status $V0 | grep "Snapshot Daemon" | awk {'print $8'}); +SNAPD_PID=$(snapd_pid); TEST [ $SNAPD_PID -gt 0 ] kill -9 $SNAPD_PID -SNAPD_PID=$($CLI volume status $V0 | grep "Snapshot Daemon" | awk {'print $8'}); - -TEST [ $SNAPD_PID = 'N/A' ] +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^N/A$" snapd_pid cleanup ; diff --git a/tests/bugs/snapshot/bug-1597662.t b/tests/bugs/snapshot/bug-1597662.t index dc87d17a0ef..f582930476a 100644 --- a/tests/bugs/snapshot/bug-1597662.t +++ b/tests/bugs/snapshot/bug-1597662.t @@ -34,12 +34,13 @@ function is_snap_path EXPECT "1" is_snap_path $CLI snapshot deactivate snap1; - +EXPECT_WITHIN ${PROCESS_DOWN_TIMEOUT} 'Stopped' snapshot_status snap1 # snap is deactivated so snap_path should not exist EXPECT "0" is_snap_path # activate snap again $CLI snapshot activate snap1; +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} 'Started' snapshot_status snap1 # snap is active so snap_path should exist EXPECT "1" is_snap_path diff --git a/tests/bugs/write-behind/issue-884.c b/tests/bugs/write-behind/issue-884.c new file mode 100644 index 00000000000..e9c33b351ad --- /dev/null +++ b/tests/bugs/write-behind/issue-884.c @@ -0,0 +1,267 @@ + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +#include <assert.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <pthread.h> + +#include <glusterfs/api/glfs.h> + +/* Based on a reproducer by Stefan Ring. It seems to be quite sensible to any + * timing modification, so the code has been maintained as is, only with minor + * changes. */ + +struct glfs *glfs; + +pthread_mutex_t the_mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t the_cond = PTHREAD_COND_INITIALIZER; + +typedef struct _my_aiocb { + int64_t size; + volatile int64_t seq; + int which; +} my_aiocb; + +typedef struct _worker_data { + my_aiocb cb; + struct iovec iov; + int64_t offset; +} worker_data; + +typedef struct { + worker_data wdata[2]; + + volatile unsigned busy; +} all_data_t; + +all_data_t all_data; + +static void +completion_fnc(struct glfs_fd *fd, ssize_t ret, struct glfs_stat *pre, + struct glfs_stat *post, void *arg) +{ + void *the_thread; + my_aiocb *cb = (my_aiocb *)arg; + long seq = cb->seq; + + assert(ret == cb->size); + + pthread_mutex_lock(&the_mutex); + pthread_cond_broadcast(&the_cond); + + all_data.busy &= ~(1 << cb->which); + cb->seq = -1; + + the_thread = (void *)pthread_self(); + printf("worker %d is done from thread %p, seq %ld!\n", cb->which, + the_thread, seq); + + pthread_mutex_unlock(&the_mutex); +} + +static void +init_wdata(worker_data *data, int which) +{ + data->cb.which = which; + data->cb.seq = -1; + + data->iov.iov_base = malloc(1024 * 1024); + memset(data->iov.iov_base, 6, + 1024 * 1024); /* tail part never overwritten */ +} + +static void +init() +{ + all_data.busy = 0; + + init_wdata(&all_data.wdata[0], 0); + init_wdata(&all_data.wdata[1], 1); +} + +static void +do_write(struct glfs_fd *fd, int content, int size, int64_t seq, + worker_data *wdata, const char *name) +{ + int ret; + + wdata->cb.size = size; + wdata->cb.seq = seq; + + if (content >= 0) + memset(wdata->iov.iov_base, content, size); + wdata->iov.iov_len = size; + + pthread_mutex_lock(&the_mutex); + printf("(%d) dispatching write \"%s\", offset %lx, len %x, seq %ld\n", + wdata->cb.which, name, (long)wdata->offset, size, (long)seq); + pthread_mutex_unlock(&the_mutex); + ret = glfs_pwritev_async(fd, &wdata->iov, 1, wdata->offset, 0, + completion_fnc, &wdata->cb); + assert(ret >= 0); +} + +#define IDLE 0 // both workers must be idle +#define ANY 1 // use any worker, other one may be busy + +int +get_worker(int waitfor, int64_t excl_seq) +{ + int which; + + pthread_mutex_lock(&the_mutex); + + while (waitfor == IDLE && (all_data.busy & 3) != 0 || + waitfor == ANY && + ((all_data.busy & 3) == 3 || + excl_seq >= 0 && (all_data.wdata[0].cb.seq == excl_seq || + all_data.wdata[1].cb.seq == excl_seq))) + pthread_cond_wait(&the_cond, &the_mutex); + + if (!(all_data.busy & 1)) + which = 0; + else + which = 1; + + all_data.busy |= (1 << which); + + pthread_mutex_unlock(&the_mutex); + + return which; +} + +static int +doit(struct glfs_fd *fd) +{ + int ret; + int64_t seq = 0; + int64_t offset = 0; // position in file, in blocks + int64_t base = 0x1000; // where to place the data, in blocks + + int async_mode = ANY; + + init(); + + for (;;) { + int which; + worker_data *wdata; + + // for growing to the first offset + for (;;) { + int gap = base + 0x42 - offset; + if (!gap) + break; + if (gap > 80) + gap = 80; + + which = get_worker(IDLE, -1); + wdata = &all_data.wdata[which]; + + wdata->offset = offset << 9; + do_write(fd, 0, gap << 9, seq++, wdata, "gap-filling"); + + offset += gap; + } + + // 8700 + which = get_worker(IDLE, -1); + wdata = &all_data.wdata[which]; + + wdata->offset = (base + 0x42) << 9; + do_write(fd, 1, 62 << 9, seq++, wdata, "!8700"); + + // 8701 + which = get_worker(IDLE, -1); + wdata = &all_data.wdata[which]; + + wdata->offset = (base + 0x42) << 9; + do_write(fd, 2, 55 << 9, seq++, wdata, "!8701"); + + // 8702 + which = get_worker(async_mode, -1); + wdata = &all_data.wdata[which]; + + wdata->offset = (base + 0x79) << 9; + do_write(fd, 3, 54 << 9, seq++, wdata, "!8702"); + + // 8703 + which = get_worker(async_mode, -1); + wdata = &all_data.wdata[which]; + + wdata->offset = (base + 0xaf) << 9; + do_write(fd, 4, 81 << 9, seq++, wdata, "!8703"); + + // 8704 + // this writes both 5s and 6s + // the range of 5s is the one that overwrites 8703 + + which = get_worker(async_mode, seq - 1); + wdata = &all_data.wdata[which]; + + memset(wdata->iov.iov_base, 5, 81 << 9); + wdata->offset = (base + 0xaf) << 9; + do_write(fd, -1, 1623 << 9, seq++, wdata, "!8704"); + + offset = base + 0x706; + base += 0x1000; + if (base >= 0x100000) + break; + } + + printf("done!\n"); + fflush(stdout); + + pthread_mutex_lock(&the_mutex); + + while ((all_data.busy & 3) != 0) + pthread_cond_wait(&the_cond, &the_mutex); + + pthread_mutex_unlock(&the_mutex); + + ret = glfs_close(fd); + assert(ret >= 0); + /* + ret = glfs_fini(glfs); + assert(ret >= 0); + */ + return 0; +} + +int +main(int argc, char *argv[]) +{ + int ret; + int open_flags = O_RDWR | O_DIRECT | O_TRUNC; + struct glfs_fd *fd; + + glfs = glfs_new(argv[1]); + if (!glfs) { + printf("glfs_new!\n"); + goto out; + } + ret = glfs_set_volfile_server(glfs, "tcp", "localhost", 24007); + if (ret < 0) { + printf("set_volfile!\n"); + goto out; + } + ret = glfs_init(glfs); + if (ret) { + printf("init!\n"); + goto out; + } + fd = glfs_open(glfs, argv[2], open_flags); + if (!fd) { + printf("open!\n"); + goto out; + } + srand(time(NULL)); + return doit(fd); +out: + return 1; +} diff --git a/tests/bugs/write-behind/issue-884.t b/tests/bugs/write-behind/issue-884.t new file mode 100755 index 00000000000..2bcf7d15265 --- /dev/null +++ b/tests/bugs/write-behind/issue-884.t @@ -0,0 +1,40 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test tries to detect a race condition in write-behind. It's based on a +# reproducer written by Stefan Ring that is able to hit it sometimes. On my +# system, it happened around 10% of the runs. This means that if this bug +# appears again, this test will fail once every 10 runs. Most probably this +# failure will be hidden by the automatic test retry of the testing framework. +# +# Please, if this test fails, it needs to be analyzed in detail. + +function run() { + "${@}" >/dev/null +} + +cleanup + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 $H0:$B0/$V0 +# This makes it easier to hit the issue +TEST $CLI volume set $V0 client-log-level TRACE +TEST $CLI volume start $V0 + +TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0 + +build_tester $(dirname $0)/issue-884.c -lgfapi + +TEST touch $M0/testfile + +# This program generates a file of 535694336 bytes with a fixed pattern +TEST run $(dirname $0)/issue-884 $V0 testfile + +# This is the md5sum of the expected pattern without corruption +EXPECT "ad105f9349345a70fc697632cbb5eec8" echo "$(md5sum $B0/$V0/testfile | awk '{ print $1; }')" + +cleanup diff --git a/tests/cluster.rc b/tests/cluster.rc index 2d826d07e39..34f5b02398f 100644 --- a/tests/cluster.rc +++ b/tests/cluster.rc @@ -11,7 +11,7 @@ function launch_cluster() { define_backends $count; define_hosts $count; define_glusterds $count $2; - define_clis $count; + define_clis $count $3; start_glusterds; } @@ -90,6 +90,20 @@ function kill_glusterd() { kill `cat $pidfile`; } +function restart_glusterd() { + local index=$1 + local b + local pidfile + local g + + b="B$index" + pidfile="${!b}/glusterd.pid" + + kill `cat $pidfile` + + g="glusterd_${index}" + ${!g} +} function kill_node() { local index=$1; @@ -134,8 +148,13 @@ function define_clis() { lopt1="--log-file=$logdir/$logfile1" - eval "CLI_$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock $lopt'"; - eval "CLI$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock $lopt1'"; + if [ "$2" == "-NO_FORCE" ]; then + eval "CLI_$i='$CLI_NO_FORCE --glusterd-sock=${!b}/glusterd/gd.sock $lopt'"; + eval "CLI$i='$CLI_NO_FORCE --glusterd-sock=${!b}/glusterd/gd.sock $lopt1'"; + else + eval "CLI_$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock $lopt'"; + eval "CLI$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock $lopt1'"; + fi done } diff --git a/tests/env.rc.in b/tests/env.rc.in index c7472a7988d..0478d66aec6 100644 --- a/tests/env.rc.in +++ b/tests/env.rc.in @@ -2,7 +2,7 @@ prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ -PATH=@sbindir@:$PATH +PATH=@bindir@:@sbindir@:$PATH export PATH GLUSTERD_PIDFILEDIR=@localstatedir@/run/gluster diff --git a/tests/features/flock_interrupt.t b/tests/features/flock_interrupt.t index 964a4bc20ef..b8717e30dfb 100644 --- a/tests/features/flock_interrupt.t +++ b/tests/features/flock_interrupt.t @@ -28,6 +28,5 @@ flock $M0/testfile sleep 6 & { sleep 0.3; flock -w 2 $M0/testfile true; echo ok EXPECT_WITHIN 4 ok cat got_lock; ## Finish up -sleep 7; rm -f got_lock; cleanup; diff --git a/tests/features/fuse-lru-limit.t b/tests/features/fuse-lru-limit.t index 9f1211660ce..dd6be2d5397 100644 --- a/tests/features/fuse-lru-limit.t +++ b/tests/features/fuse-lru-limit.t @@ -10,6 +10,7 @@ TEST pidof glusterd TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1} TEST $CLI volume start $V0 TEST glusterfs -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "2" online_brick_count EXPECT "1" get_mount_active_size_value $V0 $M0 EXPECT "0" get_mount_lru_size_value $V0 $M0 diff --git a/tests/features/interrupt.t b/tests/features/interrupt.t index bd70ff87545..067eb1b7486 100644 --- a/tests/features/interrupt.t +++ b/tests/features/interrupt.t @@ -43,18 +43,22 @@ function test_interrupt { # If the test helper fails (which is considered a setup error, not failure of the test # case itself), kill will be invoked without argument, and that will be the actual # error which is caught. - TEST "./$(dirname $0)/open_and_sleep $M0/testfile | { sleep 0.1; xargs -n1 kill -INT; }" + TEST "./$(dirname $0)/open_and_sleep $M0/testfile-$handlebool | { sleep 0.1; xargs -n1 kill -INT; }" TEST "grep -E '$logpattern' $log_file" # Basic sanity check, making sure filesystem has not crashed. - TEST test -f $M0/testfile + TEST test -f $M0/testfile-$handlebool } # Theoretically FLUSH might finish before INTERRUPT is handled, -# in which case we'd get the "no handler found" message (but it's unlikely). -test_interrupt yes 'FLUSH.*interrupt handler triggered|INTERRUPT.*no handler found' +# in which case we'd get the "no handler found" message instead of +# "interrupt handler triggered" (but it's unlikely). +# If that's observed, the pattern can be changed to +# 'FLUSH.*interrupt handler triggered|[I]NTERRUPT.*no handler found' +# to fix the test. +test_interrupt yes '[F]LUSH.*interrupt handler triggered' EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 -test_interrupt no 'INTERRUPT.*no handler found' +test_interrupt no '[I]NTERRUPT.*no handler found' ## Finish up TEST $CLI volume stop $V0; diff --git a/tests/features/ssl-authz.t b/tests/features/ssl-authz.t index 132b598ff69..497083e5a3a 100755 --- a/tests/features/ssl-authz.t +++ b/tests/features/ssl-authz.t @@ -67,13 +67,14 @@ echo "Memory consumption for glusterfsd process" for i in $(seq 1 100); do gluster v heal $V0 info >/dev/null done - +#Wait to cleanup memory +sleep 10 end=`pmap -x $glusterfsd_pid | grep total | awk -F " " '{print $4}'` diff=$((end-start)) -# If memory consumption is more than 5M some leak in SSL code path +# If memory consumption is more than 15M some leak in SSL code path -TEST [ $diff -lt 5000 ] +TEST [ $diff -lt 15000 ] # Set ssl-allow to a wildcard that includes our identity. diff --git a/tests/features/ssl-ciphers.t b/tests/features/ssl-ciphers.t index e4bcdf51095..b70fe360e02 100644 --- a/tests/features/ssl-ciphers.t +++ b/tests/features/ssl-ciphers.t @@ -110,8 +110,15 @@ EXPECT "N" openssl_connect -ssl2 -connect $H0:$BRICK_PORT # Test SSLv3 protocol fails EXPECT "N" openssl_connect -ssl3 -connect $H0:$BRICK_PORT -# Test TLSv1 protocol fails -EXPECT "N" openssl_connect -tls1 -connect $H0:$BRICK_PORT +# Test TLSv1 protocol based on openssl version +cmd="openssl version" +ver=$(eval $cmd | awk -F " " '{print $2}' | grep "^1.1") +if [ "x${ver}" = "x" ]; then + supp="N" +else + supp="Y" +fi +EXPECT "${supp}" openssl_connect -tls1 -connect $H0:$BRICK_PORT # Test a HIGH CBC cipher cph=`check_cipher -cipher AES256-SHA -connect $H0:$BRICK_PORT` diff --git a/tests/features/trash.t b/tests/features/trash.t index 472e909e567..da5b50bc85a 100755 --- a/tests/features/trash.t +++ b/tests/features/trash.t @@ -94,105 +94,105 @@ wildcard_not_exists() { if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi } -# testing glusterd [1-3] +# testing glusterd TEST glusterd TEST pidof glusterd TEST $CLI volume info -# creating distributed volume [4] +# creating distributed volume TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2} -# checking volume status [5-7] +# checking volume status EXPECT "$V0" volinfo_field $V0 'Volume Name' EXPECT 'Created' volinfo_field $V0 'Status' EXPECT '2' brick_count $V0 -# test without enabling trash translator [8] +# test without enabling trash translator TEST start_vol $V0 $M0 -# test on enabling trash translator [9-10] +# test on enabling trash translator TEST $CLI volume set $V0 features.trash on EXPECT 'on' volinfo_field $V0 'features.trash' -# files directly under mount point [11] +# files directly under mount point create_files $M0/file1 $M0/file2 TEST file_exists $V0 file1 file2 -# perform unlink [12] +# perform unlink TEST unlink_op file1 -# perform truncate [13] +# perform truncate TEST truncate_op file2 4 -# create files directory hierarchy and check [14] +# create files directory hierarchy and check mkdir -p $M0/1/2/3 create_files $M0/1/2/3/foo1 $M0/1/2/3/foo2 TEST file_exists $V0 1/2/3/foo1 1/2/3/foo2 -# perform unlink [15] +# perform unlink TEST unlink_op 1/2/3/foo1 -# perform truncate [16] +# perform truncate TEST truncate_op 1/2/3/foo2 4 # create a directory for eliminate pattern mkdir $M0/a -# set the eliminate pattern [17-18] +# set the eliminate pattern TEST $CLI volume set $V0 features.trash-eliminate-path /a EXPECT '/a' volinfo_field $V0 'features.trash-eliminate-path' -# create two files and check [19] +# create two files and check create_files $M0/a/test1 $M0/a/test2 TEST file_exists $V0 a/test1 a/test2 -# remove from eliminate pattern [20] +# remove from eliminate pattern rm -f $M0/a/test1 EXPECT "Y" wildcard_not_exists $M0/.trashcan/a/test1* -# truncate from eliminate path [21-23] +# truncate from eliminate path truncate -s 2 $M0/a/test2 TEST [ -e $M0/a/test2 ] TEST [ `ls -l $M0/a/test2 | awk '{print $5}'` -eq 2 ] EXPECT "Y" wildcard_not_exists $M0/.trashcan/a/test2* -# set internal op on [24-25] +# set internal op on TEST $CLI volume set $V0 features.trash-internal-op on EXPECT 'on' volinfo_field $V0 'features.trash-internal-op' -# again create two files and check [26] +# again create two files and check create_files $M0/inop1 $M0/inop2 TEST file_exists $V0 inop1 inop2 -# perform unlink [27] +# perform unlink TEST unlink_op inop1 -# perform truncate [28] +# perform truncate TEST truncate_op inop2 4 -# remove one brick and restart the volume [28-31] +# remove one brick and restart the volume TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 force EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 TEST $CLI volume stop $V0 TEST start_vol $V0 $M0 $M0/.trashcan -# again create two files and check [33] +# again create two files and check create_files $M0/rebal1 $M0/rebal2 TEST file_exists $V0 rebal1 rebal2 -# add one brick [34-35] +# add one brick TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3 TEST [ -d $B0/${V0}3 ] -# perform rebalance [36] +# perform rebalance TEST $CLI volume rebalance $V0 start force EXPECT_WITHIN $REBALANCE_TIMEOUT "0" rebalance_completed #Find out which file was migrated to the new brick file_name=$(ls $B0/${V0}3/rebal*| xargs basename) -# check whether rebalance was succesful [37-40] +# check whether rebalance was succesful EXPECT "Y" wildcard_exists $B0/${V0}3/$file_name* EXPECT "Y" wildcard_exists $B0/${V0}1/.trashcan/internal_op/$file_name* @@ -201,52 +201,42 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 # force required in case rebalance is not over TEST $CLI volume stop $V0 force -# create a replicated volume [41] +# create a replicated volume TEST $CLI volume create $V1 replica 2 $H0:$B0/${V1}{1,2} -# checking volume status [42-45] +# checking volume status EXPECT "$V1" volinfo_field $V1 'Volume Name' EXPECT 'Replicate' volinfo_field $V1 'Type' EXPECT 'Created' volinfo_field $V1 'Status' EXPECT '2' brick_count $V1 -# enable trash with options and start the replicate volume by disabling automatic self-heal [46-50] +# enable trash with options and start the replicate volume by disabling automatic self-heal TEST $CLI volume set $V1 features.trash on TEST $CLI volume set $V1 features.trash-internal-op on EXPECT 'on' volinfo_field $V1 'features.trash' EXPECT 'on' volinfo_field $V1 'features.trash-internal-op' TEST start_vol $V1 $M1 $M1/.trashcan -# mount and check for trash directory [51] +# mount and check for trash directory TEST [ -d $M1/.trashcan/internal_op ] -# create a file and check [52] +# create a file and check touch $M1/self TEST [ -e $B0/${V1}1/self -a -e $B0/${V1}2/self ] -# kill one brick and delete the file from mount point [53-54] +# kill one brick and delete the file from mount point kill_brick $V1 $H0 $B0/${V1}1 EXPECT_WITHIN ${PROCESS_UP_TIMEOUT} "1" online_brick_count rm -f $M1/self EXPECT "Y" wildcard_exists $B0/${V1}2/.trashcan/self* -# force start the volume and trigger the self-heal manually [55-57] -TEST $CLI volume start $V1 force -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" online_brick_count -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status -# Since we created the file under root of the volume, it will be -# healed automatically - -# check for the removed file in trashcan [58] -EXPECT_WITHIN $HEAL_TIMEOUT "Y" wildcard_exists $B0/${V1}1/.trashcan/internal_op/self* - -# check renaming of trash directory through cli [59-62] +# check renaming of trash directory through cli TEST $CLI volume set $V0 trash-dir abc TEST start_vol $V0 $M0 $M0/abc TEST [ -e $M0/abc -a ! -e $M0/.trashcan ] EXPECT "Y" wildcard_exists $B0/${V0}1/abc/internal_op/rebal* -# ensure that rename and delete operation on trash directory fails [63-65] +# ensure that rename and delete operation on trash directory fails rm -rf $M0/abc/internal_op TEST [ -e $M0/abc/internal_op ] rm -rf $M0/abc/ diff --git a/tests/features/worm.t b/tests/features/worm.t index 8b2574c8a47..40b08cdee02 100755 --- a/tests/features/worm.t +++ b/tests/features/worm.t @@ -72,11 +72,19 @@ sleep 10 TEST `echo "worm 1" >> $M0/file4` TEST ! rm -f $M0/file4 +## Test for state transition if auto-commit-period is 0 +TEST $CLI volume set $V0 features.auto-commit-period 0 +TEST `echo "worm 1" > $M0/file5` +EXPECT '3/10/0' echo $(getfattr -e text --absolute-names --only-value -n "trusted.reten_state" $B0/${V0}1/file5) +EXPECT 'worm 1' cat $M0/file5 +TEST ! rm -f $M0/file5 +TEST $CLI volume set $V0 features.auto-commit-period 5 + ## Test for checking if retention-period is updated on increasing the access time of a WORM-RETAINED file. TEST $CLI volume set $V0 features.worm-files-deletable 1 TEST `echo "worm 1" >> $M0/file1` initial_timestamp=$(date +%s) -current_time_seconds=$(date +%S); +current_time_seconds=$(date +%S | sed 's/^0*//' ); TEST chmod 0444 $M0/file1 EXPECT '3/10/5' echo $(getfattr -e text --absolute-names --only-value -n "trusted.reten_state" $B0/${V0}1/file1) changed_timestamp=$(date +%Y%m%d%H%M --date '60 seconds'); @@ -91,7 +99,7 @@ EXPECT "$initial_timestamp" echo $(stat --printf %X $M0/file1) ## Test for checking if retention-period is updated on decreasing the access time of a WORM-RETAINED file TEST $CLI volume set $V0 features.default-retention-period 120 initial_timestamp=$(date +%s) -current_time_seconds=$(date +%S); +current_time_seconds=$(date +%S | sed 's/^0*//' ); TEST chmod 0444 $M0/file1 EXPECT '3/120/5' echo $(getfattr -e text --absolute-names --only-value -n "trusted.reten_state" $B0/${V0}1/file1) changed_timestamp=$(date +%Y%m%d%H%M --date '60 seconds'); diff --git a/tests/geo-rep.rc b/tests/geo-rep.rc index 84c2c2251c0..9ba4262730e 100644 --- a/tests/geo-rep.rc +++ b/tests/geo-rep.rc @@ -1,4 +1,9 @@ GEO_REP_TIMEOUT=120 +CHECK_MOUNT_TIMEOUT=50 +#check for mount point +function check_mounted () { + df | grep $1 | wc -l +} function check_status_num_rows() { diff --git a/tests/glusterfind/glusterfind-basic.t b/tests/glusterfind/glusterfind-basic.t new file mode 100644 index 00000000000..ccb33fb1fc8 --- /dev/null +++ b/tests/glusterfind/glusterfind-basic.t @@ -0,0 +1,84 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc +. $(dirname $0)/../env.rc + +SCRIPT_TIMEOUT=300 + +##Cleanup and start glusterd +cleanup; +TEST glusterd; +TEST pidof glusterd + +##create .keys +mkdir -p /var/lib/glusterd/glusterfind/.keys + +#create_and_start test_volume +TEST $CLI volume create test-vol $H0:$B0/b1 $H0:$B0/b2 $H0:$B0/b3 +TEST gluster volume start test-vol + +##Mount test-vol +TEST glusterfs -s $H0 --volfile-id test-vol $M0 + +TEST timestamp1=$(date +'%s') + +##Create files and dirs inside the mount point +TEST mkdir -p $M0/dir1 +TEST touch $M0/file1 + +##Glusterfind Create +TEST glusterfind create sess_vol1 test-vol --force + +################################################################################## +#Incremental crawl +################################################################################## +##Glusterfind Pre +TEST glusterfind pre sess_vol1 test-vol output_file.txt + +#Glusterfind Post +TEST glusterfind post sess_vol1 test-vol + +##Glusterfind List +EXPECT '1' echo $(glusterfind list | grep sess_vol1 | wc -l) + +TEST timestamp2=$(date +'%s') + +##Glusterfind Query +TEST glusterfind query test-vol --since-time $timestamp1 --end-time $timestamp2 output_file.txt + +################################################################################# +#Full Crawl +################################################################################# +##Glusterfind Pre +TEST glusterfind pre sess_vol1 test-vol output_file.txt --full --regenerate-outfile +EXPECT '1' echo $(grep 'NEW dir1' output_file.txt | wc -l) +EXPECT '1' echo $(grep 'NEW file1' output_file.txt | wc -l) + +##Glusterfind Query commands +TEST glusterfind query test-vol --full output_file.txt +EXPECT '1' echo $(grep 'NEW dir1' output_file.txt | wc -l) +EXPECT '1' echo $(grep 'NEW file1' output_file.txt | wc -l) + +##using tag, full crawl +TEST glusterfind query test-vol --full --tag-for-full-find NEW output_file.txt +EXPECT '1' echo $(grep 'NEW dir1' output_file.txt | wc -l) +EXPECT '1' echo $(grep 'NEW file1' output_file.txt | wc -l) + +##using -field-separator option, full crawl +glusterfind query test-vol --full output_file.txt --field-separator "==" +EXPECT '1' echo $(grep 'NEW==dir1' output_file.txt | wc -l) +EXPECT '1' echo $(grep 'NEW==file1' output_file.txt | wc -l) + +##Adding or Replacing a Brick from an Existing Glusterfind Session +TEST gluster volume add-brick test-vol $H0:$B0/b4 force + +##To make existing session work after brick add +TEST glusterfind create sess_vol test-vol --force +EXPECT '1' echo $(glusterfind list | grep sess_vol1 | wc -l) + +##glusterfind delete +TEST glusterfind delete sess_vol test-vol + +rm -rf output_file.txt +cleanup; diff --git a/tests/line-coverage/cli-peer-and-volume-operations.t b/tests/line-coverage/cli-peer-and-volume-operations.t index a1545048773..0cf8dbe81f9 100644 --- a/tests/line-coverage/cli-peer-and-volume-operations.t +++ b/tests/line-coverage/cli-peer-and-volume-operations.t @@ -77,6 +77,46 @@ TEST ! $CLI_1 volume status $V0 shd #test explicitly provided options TEST $CLI_1 --timeout=120 --log-level=INFO volume status +#changing timezone to a different one, to check localtime logging feature +TEST export TZ='Asia/Kolkata' +TEST restart_glusterd 1 + +#localtime logging enable +TEST $CLI_1 volume set all cluster.localtime-logging enable +EXPECT '1' logging_time_check $LOGDIR + +#localtime logging disable +TEST $CLI_1 volume set all cluster.localtime-logging disable +EXPECT '0' logging_time_check $LOGDIR + +#changing timezone back to original timezone +TEST export TZ='UTC' + +#negative tests for volume options +#'set' option to enable quota/inode-quota is now depreciated +TEST ! $CLI_1 volume set $V0 quota enable +TEST ! $CLI_1 volume set $V0 inode-quota enable + +#invalid transport type 'rcp' +TEST ! $CLI_1 volume set $V0 config.transport rcp + +#'op-version' option is not valid for a single volume +TEST ! $CLI_1 volume set $V0 cluster.op-version 72000 + +#'op-version' option can't be used with any other option +TEST ! $CLI_1 volume set all cluster.localtime-logging disable cluster.op-version 72000 + +#invalid format of 'op-version' +TEST ! $CLI_1 volume set all cluster.op-version 72-000 + +#provided 'op-version' value is greater than max allowed op-version +op_version=$($CLI_1 volume get all cluster.max-op-version | awk 'NR==3 {print$2}') +op_version=$((op_version+1000)) #this can be any number greater than 0 +TEST ! $CLI_1 volume set all cluster.op-version $op_version + +#provided 'op-verison' value cannot be less than the current cluster op-version value +TEST ! $CLI_1 volume set all cluster.op-version 00000 + # system commnds TEST $CLI_1 system help TEST $CLI_1 system uuid get diff --git a/tests/line-coverage/log-and-brick-ops-negative-case.t b/tests/line-coverage/log-and-brick-ops-negative-case.t new file mode 100644 index 00000000000..d86cb452282 --- /dev/null +++ b/tests/line-coverage/log-and-brick-ops-negative-case.t @@ -0,0 +1,82 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +cleanup +TEST glusterd +TEST pidof glusterd + +#create volumes +TEST $CLI volume create ${V0}_1 $H0:$B0/v{1..2} + +TEST $CLI volume create ${V0}_2 replica 3 arbiter 1 $H0:$B0/v{3..5} + +TEST $CLI volume create ${V0}_3 disperse 3 redundancy 1 $H0:$B0/v{6..8} +TEST $CLI volume start ${V0}_3 +EXPECT 'Started' volinfo_field ${V0}_3 'Status' + +TEST $CLI volume create ${V0}_4 replica 3 $H0:$B0/v{9..14} +TEST $CLI volume start ${V0}_4 +EXPECT 'Started' volinfo_field ${V0}_4 'Status' + +#log rotate option +#provided volume does not exist +TEST ! $CLI volume log ${V0}_5 rotate + +#volume must be started before using log rotate option +TEST ! $CLI volume log ${V0}_1 rotate +TEST $CLI volume start ${V0}_1 +EXPECT 'Started' volinfo_field ${V0}_1 'Status' + +#incorrect brick provided for the volume +TEST ! $CLI volume log ${V0}_1 rotate $H0:$B0/v15 + +#add-brick operations +#volume must be in started to state to increase replica count +TEST ! $CLI volume add-brick ${V0}_2 replica 4 $H0:$B0/v15 +TEST $CLI volume start ${V0}_2 +EXPECT 'Started' volinfo_field ${V0}_2 'Status' + +#incorrect number of bricks for a replica 4 volume +TEST ! $CLI volume add-brick ${V0}_1 replica 4 $H0:$B0/v15 + +#replica count provided is less than the current replica count +TEST ! $CLI volume add-brick ${V0}_2 replica 2 $H0:$B0/v15 + +#dispersed to replicated dispersed not possible +TEST ! $CLI volume add-brick ${V0}_3 replica 2 $H0:$B0/v15 + +#remove-brick operations +#replica count option provided for dispersed vol +TEST ! $CLI volume remove-brick ${V0}_3 replica 2 $H0:$B0/v8 start + +#given replica count is greater than the current replica count +TEST ! $CLI volume remove-brick ${V0}_2 replica 4 $H0:$B0/v5 start + +#number of bricks to be removed, must be a multiple of replica count +TEST ! $CLI volume remove-brick ${V0}_2 replica 3 $H0:$B0/v{3..4} start + +#less number of bricks given to reduce the replica count +TEST ! $CLI volume remove-brick ${V0}_2 replica 1 $H0:$B0/v3 start + +#bricks should be from different subvol +TEST ! $CLI volume remove-brick ${V0}_4 replica 2 $H0:$B0/v{13..14} start + +#arbiter must be removed to reduce replica count +TEST ! $CLI volume remove-brick ${V0}_2 replica 1 $H0:$B0/v{3..4} start + +#removal of bricks is not allowed without reducing the replica count explicitly +TEST ! $CLI volume remove-brick ${V0}_2 replica 3 $H0:$B0/v{3..5} start + +#incorrect brick for given vol +TEST ! $CLI volume remove-brick ${V0}_1 $H0:$B0/v15 start + +#removing all the bricks are not allowed +TEST ! $CLI volume remove-brick ${V0}_1 $H0:$B0/v{1..2} start + +#volume must not be stopped state while removing bricks +TEST $CLI volume stop ${V0}_1 +TEST ! $CLI volume remove-brick ${V0}_1 $H0:$B0/v1 start + +cleanup
\ No newline at end of file diff --git a/tests/volume.rc b/tests/volume.rc index 288d491de39..b38848c0e52 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -193,7 +193,7 @@ function afr_child_up_status_meta { local mnt=$1 local repl=$2 local child=$3 - grep "child_up\[$child\]" $mnt/.meta/graphs/active/$repl/private | awk '{print $3}' + grep -E "^child_up\[$child\]" $mnt/.meta/graphs/active/$repl/private | awk '{print $3}' } function client_connected_status_meta { @@ -992,3 +992,11 @@ function get-xml() { $CLI $1 --xml | xmllint --format - | grep $2 | sed 's/\(<"$2">\|<\/"$2">\)//g' } + +function logging_time_check() +{ + local logdir=$1 + local logfile=`echo ${0##*/}`_glusterd1.log + + cat $logdir/1/$logfile | tail -n 2 | head -n 1 | grep $(date +%H:%M) | wc -l +} diff --git a/tools/gfind_missing_files/gfind_missing_files.sh b/tools/gfind_missing_files/gfind_missing_files.sh index f42fe7b05af..e7aaa0b5dd4 100644 --- a/tools/gfind_missing_files/gfind_missing_files.sh +++ b/tools/gfind_missing_files/gfind_missing_files.sh @@ -61,7 +61,7 @@ mount_slave() parse_cli() { - if [[ $# -ne 4 ]]; then + if [ "$#" -ne 4 ]; then echo "Usage: gfind_missing_files <brick-path> <slave-host> <slave-vol> <OUTFILE>" exit 1 else diff --git a/tools/glusterfind/src/utils.py b/tools/glusterfind/src/utils.py index cc099031ef8..906ebd8f252 100644 --- a/tools/glusterfind/src/utils.py +++ b/tools/glusterfind/src/utils.py @@ -230,7 +230,11 @@ def get_changelog_rollover_time(volumename): try: tree = etree.fromstring(out) - return int(tree.find('volGetopts/Opt/Value').text) + val = tree.find('volGetopts/Opt/Value').text + if val is not None: + # Filter the value by split, as it may be 'X (DEFAULT)' + # and we only need 'X' + return int(val.split(' ', 1)[0]) except ParseError: return DEFAULT_CHANGELOG_INTERVAL diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c index 5462f0c7f53..032ab5c8001 100644 --- a/xlators/cluster/afr/src/afr-common.c +++ b/xlators/cluster/afr/src/afr-common.c @@ -45,7 +45,42 @@ afr_quorum_errno(afr_private_t *priv) return ENOTCONN; } -static void +gf_boolean_t +afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name, + pid_t pid) +{ + if (!__is_root_gfid(pargfid)) { + return _gf_false; + } + + if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) { + /*For backward compatibility /.landfill is private*/ + return _gf_true; + } + + if (pid == GF_CLIENT_PID_GSYNCD) { + /*geo-rep needs to create/sync private directory on slave because + * it appears in changelog*/ + return _gf_false; + } + + if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) { + if (strcmp(name, priv->anon_inode_name) == 0) { + /* anonymous-inode dir is private*/ + return _gf_true; + } + } else { + if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) == + 0) { + /* anonymous-inode dir prefix is private for geo-rep to work*/ + return _gf_true; + } + } + + return _gf_false; +} + +void afr_fill_success_replies(afr_local_t *local, afr_private_t *priv, unsigned char *replies) { @@ -885,7 +920,7 @@ __afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local, metadatamap |= (1 << index); } if (metadatamap_old != metadatamap) { - event = 0; + __afr_inode_need_refresh_set(inode, this); } break; @@ -898,7 +933,7 @@ __afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local, datamap |= (1 << index); } if (datamap_old != datamap) - event = 0; + __afr_inode_need_refresh_set(inode, this); break; default: @@ -1062,34 +1097,6 @@ out: } int -__afr_inode_event_gen_reset_small(inode_t *inode, xlator_t *this) -{ - int ret = -1; - uint16_t datamap = 0; - uint16_t metadatamap = 0; - uint32_t event = 0; - uint64_t val = 0; - afr_inode_ctx_t *ctx = NULL; - - ret = __afr_inode_ctx_get(this, inode, &ctx); - if (ret) - return ret; - - val = ctx->read_subvol; - - metadatamap = (val & 0x000000000000ffff) >> 0; - datamap = (val & 0x00000000ffff0000) >> 16; - event = 0; - - val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) | - (((uint64_t)event) << 32); - - ctx->read_subvol = val; - - return ret; -} - -int __afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data, unsigned char *metadata, int *event_p) { @@ -1160,22 +1167,6 @@ out: } int -__afr_inode_event_gen_reset(inode_t *inode, xlator_t *this) -{ - afr_private_t *priv = NULL; - int ret = -1; - - priv = this->private; - - if (priv->child_count <= 16) - ret = __afr_inode_event_gen_reset_small(inode, this); - else - ret = -1; - - return ret; -} - -int afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data, unsigned char *metadata, int *event_p) { @@ -1241,12 +1232,11 @@ afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this, return 0; } -int +static int afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this, int *spb_choice) { int ret = -1; - GF_VALIDATE_OR_GOTO(this->name, inode, out); LOCK(&inode->lock); @@ -1258,6 +1248,40 @@ out: return ret; } +/* + * frame is used to get the favourite policy. Since + * afr_inode_split_brain_choice_get was called with afr_open, it is possible to + * have a frame with out local->replies. So in that case, frame is passed as + * null, hence this function will handle the frame NULL case. + */ +int +afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this, + call_frame_t *frame, int *spb_subvol) +{ + int ret = -1; + afr_local_t *local = NULL; + afr_private_t *priv = NULL; + + GF_VALIDATE_OR_GOTO("afr", this, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + GF_VALIDATE_OR_GOTO(this->name, inode, out); + GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out); + + priv = this->private; + + ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol); + if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) { + local = frame->local; + *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode, + NULL); + if (*spb_subvol >= 0) { + ret = 0; + } + } + +out: + return ret; +} int afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data, unsigned char *metadata, int event) @@ -1324,30 +1348,22 @@ out: return need_refresh; } -static int -afr_inode_need_refresh_set(inode_t *inode, xlator_t *this) +int +__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this) { int ret = -1; afr_inode_ctx_t *ctx = NULL; - GF_VALIDATE_OR_GOTO(this->name, inode, out); - - LOCK(&inode->lock); - { - ret = __afr_inode_ctx_get(this, inode, &ctx); - if (ret) - goto unlock; - + ret = __afr_inode_ctx_get(this, inode, &ctx); + if (ret == 0) { ctx->need_refresh = _gf_true; } -unlock: - UNLOCK(&inode->lock); -out: + return ret; } int -afr_inode_event_gen_reset(inode_t *inode, xlator_t *this) +afr_inode_need_refresh_set(inode_t *inode, xlator_t *this) { int ret = -1; @@ -1355,7 +1371,7 @@ afr_inode_event_gen_reset(inode_t *inode, xlator_t *this) LOCK(&inode->lock); { - ret = __afr_inode_event_gen_reset(inode, this); + ret = __afr_inode_need_refresh_set(inode, this); } UNLOCK(&inode->lock); out: @@ -1790,7 +1806,7 @@ afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err) ret = afr_inode_get_readable(frame, inode, this, local->readable, &event_generation, local->transaction.type); - if (ret == -EIO || (local->is_read_txn && !event_generation)) { + if (ret == -EIO) { /* No readable subvolume even after refresh ==> splitbrain.*/ if (!priv->fav_child_policy) { err = EIO; @@ -2290,8 +2306,9 @@ afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv, * need is a low probability that multiple clients * won't converge on the same subvolume. */ + gf_uuid_copy(gfid_copy, args->gfid); pid = getpid(); - memcpy(gfid_copy, &pid, sizeof(pid)); + *(pid_t *)gfid_copy ^= pid; } child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) % priv->child_count; @@ -2875,7 +2892,7 @@ afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this, { afr_private_t *priv = NULL; afr_local_t *local = NULL; - int spb_choice = -1; + int spb_subvol = -1; int child_count = -1; if (*read_subvol != -1) @@ -2885,10 +2902,10 @@ afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this, local = frame->local; child_count = priv->child_count; - afr_inode_split_brain_choice_get(local->inode, this, &spb_choice); - if ((spb_choice >= 0) && + afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol); + if ((spb_subvol >= 0) && (AFR_COUNT(success_replies, child_count) == child_count)) { - *read_subvol = spb_choice; + *read_subvol = spb_subvol; } else if (!priv->quorum_count || frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) { *read_subvol = afr_first_up_child(frame, this); @@ -2929,6 +2946,7 @@ afr_lookup_done(call_frame_t *frame, xlator_t *this) 0, }; gf_boolean_t locked_entry = _gf_false; + gf_boolean_t in_flight_create = _gf_false; gf_boolean_t can_interpret = _gf_true; inode_t *parent = NULL; ia_type_t ia_type = IA_INVAL; @@ -2972,17 +2990,12 @@ afr_lookup_done(call_frame_t *frame, xlator_t *this) if (!replies[i].valid) continue; - if (locked_entry && replies[i].op_ret == -1 && - replies[i].op_errno == ENOENT) { - /* Second, check entry is still - "underway" in creation */ - local->op_ret = -1; - local->op_errno = ENOENT; - goto error; - } - - if (replies[i].op_ret == -1) + if (replies[i].op_ret == -1) { + if (locked_entry && replies[i].op_errno == ENOENT) { + in_flight_create = _gf_true; + } continue; + } if (read_subvol == -1 || !readable[read_subvol]) { read_subvol = i; @@ -2992,6 +3005,12 @@ afr_lookup_done(call_frame_t *frame, xlator_t *this) } } + if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) { + local->op_ret = -1; + local->op_errno = ENOENT; + goto error; + } + if (read_subvol == -1) goto error; /* We now have a read_subvol, which is readable[] (if there @@ -3050,7 +3069,7 @@ afr_lookup_done(call_frame_t *frame, xlator_t *this) if (read_subvol == -1) goto cant_interpret; if (ret) { - afr_inode_event_gen_reset(local->inode, this); + afr_inode_need_refresh_set(local->inode, this); dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY); } } else { @@ -3103,7 +3122,7 @@ error: * others in that they must be given higher priority while * returning to the user. * - * The hierarchy is ENODATA > ENOENT > ESTALE > others + * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others */ int @@ -3115,6 +3134,8 @@ afr_higher_errno(int32_t old_errno, int32_t new_errno) return ENOENT; if (old_errno == ESTALE || new_errno == ESTALE) return ESTALE; + if (old_errno == ENOSPC || new_errno == ENOSPC) + return ENOSPC; return new_errno; } @@ -3606,6 +3627,7 @@ afr_discover_unwind(call_frame_t *frame, xlator_t *this) afr_private_t *priv = NULL; afr_local_t *local = NULL; int read_subvol = -1; + int ret = 0; unsigned char *data_readable = NULL; unsigned char *success_replies = NULL; @@ -3627,7 +3649,10 @@ afr_discover_unwind(call_frame_t *frame, xlator_t *this) if (!afr_has_quorum(success_replies, this, frame)) goto unwind; - afr_replies_interpret(frame, this, local->inode, NULL); + ret = afr_replies_interpret(frame, this, local->inode, NULL); + if (ret) { + afr_inode_need_refresh_set(local->inode, this); + } read_subvol = afr_read_subvol_decide(local->inode, this, NULL, data_readable); @@ -3679,7 +3704,7 @@ afr_ta_id_file_check(void *opaque) this = opaque; priv = this->private; - ret = afr_fill_ta_loc(this, &loc); + ret = afr_fill_ta_loc(this, &loc, _gf_false); if (ret) { gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, "Failed to populate thin-arbiter loc for: %s.", loc.name); @@ -3888,11 +3913,7 @@ afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req) afr_read_subvol_get(loc->inode, this, NULL, NULL, &event, AFR_DATA_TRANSACTION, NULL); - if (afr_is_inode_refresh_reqd(loc->inode, this, event, - local->event_generation)) - afr_inode_refresh(frame, this, loc->inode, NULL, afr_discover_do); - else - afr_discover_do(frame, this, 0); + afr_discover_do(frame, this, 0); return 0; out: @@ -3993,11 +4014,10 @@ afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req) return 0; } - if (__is_root_gfid(loc->parent->gfid)) { - if (!strcmp(loc->name, GF_REPLICATE_TRASH_DIR)) { - op_errno = EPERM; - goto out; - } + if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name, + frame->root->pid)) { + op_errno = EPERM; + goto out; } local = AFR_FRAME_INIT(frame, op_errno); @@ -4033,11 +4053,7 @@ afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req) afr_read_subvol_get(loc->parent, this, NULL, NULL, &event, AFR_DATA_TRANSACTION, NULL); - if (afr_is_inode_refresh_reqd(loc->inode, this, event, - local->event_generation)) - afr_inode_refresh(frame, this, loc->parent, NULL, afr_lookup_do); - else - afr_lookup_do(frame, this, 0); + afr_lookup_do(frame, this, 0); return 0; out: @@ -5665,6 +5681,8 @@ afr_priv_dump(xlator_t *this) GF_ATOMIC_GET(priv->pending_reads[i])); sprintf(key, "child_latency[%d]", i); gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]); + sprintf(key, "halo_child_up[%d]", i); + gf_proc_dump_write(key, "%d", priv->halo_child_up[i]); } gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal); gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal); @@ -5677,6 +5695,7 @@ afr_priv_dump(xlator_t *this) priv->background_self_heal_count); gf_proc_dump_write("healers", "%d", priv->healers); gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode); + gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode); if (priv->quorum_count == AFR_QUORUM_AUTO) { gf_proc_dump_write("quorum-type", "auto"); } else if (priv->quorum_count == 0) { @@ -5841,7 +5860,7 @@ find_best_down_child(xlator_t *this) priv = this->private; for (i = 0; i < priv->child_count; i++) { - if (priv->child_up[i] && priv->child_latency[i] >= 0 && + if (!priv->child_up[i] && priv->child_latency[i] >= 0 && priv->child_latency[i] < best_latency) { best_child = i; best_latency = priv->child_latency[i]; @@ -5913,7 +5932,9 @@ __afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx, "), " "marking child down.", child_latency_msec, halo_max_latency_msec); - *event = GF_EVENT_CHILD_DOWN; + if (priv->halo_child_up[idx]) { + *event = GF_EVENT_CHILD_DOWN; + } } } else if (child_latency_msec < halo_max_latency_msec && priv->child_up[idx] == 0) { @@ -5925,7 +5946,9 @@ __afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx, "), " "marking child up.", child_latency_msec, halo_max_latency_msec); - *event = GF_EVENT_CHILD_UP; + if (priv->halo_child_up[idx]) { + *event = GF_EVENT_CHILD_UP; + } } else { gf_log(child_xlator->name, GF_LOG_INFO, "Not marking child %d up, " @@ -5992,7 +6015,10 @@ __afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator, if (child_latency_msec < 0) { /*set to INT64_MAX-1 so that it is found for best_down_child*/ - priv->child_latency[idx] = AFR_HALO_MAX_LATENCY; + priv->halo_child_up[idx] = 1; + if (priv->child_latency[idx] < 0) { + priv->child_latency[idx] = AFR_HALO_MAX_LATENCY; + } } /* @@ -6081,6 +6107,7 @@ __afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx, */ if (child_latency_msec < 0) { priv->child_latency[idx] = child_latency_msec; + priv->halo_child_up[idx] = 0; } priv->child_up[idx] = 0; @@ -6645,6 +6672,8 @@ afr_priv_destroy(afr_private_t *priv) if (!priv) goto out; + + GF_FREE(priv->sh_domain); GF_FREE(priv->last_event); child_count = priv->child_count; @@ -6660,7 +6689,9 @@ afr_priv_destroy(afr_private_t *priv) GF_FREE(priv->local); GF_FREE(priv->pending_key); GF_FREE(priv->children); + GF_FREE(priv->anon_inode); GF_FREE(priv->child_up); + GF_FREE(priv->halo_child_up); GF_FREE(priv->child_latency); LOCK_DESTROY(&priv->lock); diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c index 74f71fdc76a..f8bf8340dab 100644 --- a/xlators/cluster/afr/src/afr-dir-read.c +++ b/xlators/cluster/afr/src/afr-dir-read.c @@ -67,7 +67,8 @@ afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, } int -afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd) +afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, + dict_t *xdata) { afr_private_t *priv = NULL; afr_local_t *local = NULL; @@ -163,8 +164,8 @@ afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol) } static void -afr_readdir_transform_entries(gf_dirent_t *subvol_entries, int subvol, - gf_dirent_t *entries, fd_t *fd) +afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries, + int subvol, gf_dirent_t *entries, fd_t *fd) { int ret = -1; gf_dirent_t *entry = NULL; @@ -182,8 +183,8 @@ afr_readdir_transform_entries(gf_dirent_t *subvol_entries, int subvol, list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list) { - if (__is_root_gfid(fd->inode->gfid) && - !strcmp(entry->d_name, GF_REPLICATE_TRASH_DIR)) { + if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name, + frame->root->pid)) { continue; } @@ -227,8 +228,8 @@ afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, } if (op_ret >= 0) - afr_readdir_transform_entries(subvol_entries, (long)cookie, &entries, - local->fd); + afr_readdir_transform_entries(frame, subvol_entries, (long)cookie, + &entries, local->fd); AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata); diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c index e96b7d0798e..b7cceb79158 100644 --- a/xlators/cluster/afr/src/afr-dir-write.c +++ b/xlators/cluster/afr/src/afr-dir-write.c @@ -119,11 +119,11 @@ __afr_dir_write_finalize(call_frame_t *frame, xlator_t *this) continue; if (local->replies[i].op_ret < 0) { if (local->inode) - afr_inode_event_gen_reset(local->inode, this); + afr_inode_need_refresh_set(local->inode, this); if (local->parent) - afr_inode_event_gen_reset(local->parent, this); + afr_inode_need_refresh_set(local->parent, this); if (local->parent2) - afr_inode_event_gen_reset(local->parent2, this); + afr_inode_need_refresh_set(local->parent2, this); continue; } @@ -345,6 +345,7 @@ afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this) afr_private_t *priv = NULL; int pre_op_count = 0; int failed_count = 0; + unsigned char *success_replies = NULL; local = frame->local; priv = this->private; @@ -360,9 +361,16 @@ afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this) failed_count = AFR_COUNT(local->transaction.failed_subvols, priv->child_count); + /* FOP succeeded on all bricks. */ if (pre_op_count == priv->child_count && !failed_count) return; + /* FOP did not suceed on quorum no. of bricks. */ + success_replies = alloca0(priv->child_count); + afr_fill_success_replies(local, priv, success_replies); + if (!afr_has_quorum(success_replies, this, NULL)) + return; + if (priv->thin_arbiter_count) { /*Mark new entry using ta file*/ local->is_new_entry = _gf_true; diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c index c01b4131d58..1d6e4f3570a 100644 --- a/xlators/cluster/afr/src/afr-inode-write.c +++ b/xlators/cluster/afr/src/afr-inode-write.c @@ -2506,6 +2506,7 @@ afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, call_frame_t *transaction_frame = NULL; int ret = -1; int32_t op_errno = ENOMEM; + int8_t last_fsync = 0; AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out); transaction_frame = copy_frame(frame); @@ -2516,10 +2517,16 @@ afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, if (!local) goto out; - if (xdata) + if (xdata) { local->xdata_req = dict_copy_with_ref(xdata, NULL); - else + if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) { + if (last_fsync) { + local->transaction.disable_delayed_post_op = _gf_true; + } + } + } else { local->xdata_req = dict_new(); + } if (!local->xdata_req) goto out; diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c index a5b004f4258..64856042b65 100644 --- a/xlators/cluster/afr/src/afr-open.c +++ b/xlators/cluster/afr/src/afr-open.c @@ -137,7 +137,7 @@ afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, { afr_private_t *priv = NULL; afr_local_t *local = NULL; - int spb_choice = 0; + int spb_subvol = 0; int event_generation = 0; int ret = 0; int32_t op_errno = 0; @@ -179,9 +179,9 @@ afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, ret = afr_inode_get_readable(frame, local->inode, this, NULL, &event_generation, AFR_DATA_TRANSACTION); if ((ret < 0) && - (afr_inode_split_brain_choice_get(local->inode, this, &spb_choice) == - 0) && - spb_choice < 0) { + (afr_split_brain_read_subvol_get(local->inode, this, NULL, + &spb_subvol) == 0) && + spb_subvol < 0) { afr_inode_refresh(frame, this, local->inode, local->inode->gfid, afr_open_continue); } else { diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c index 772b59f9a2f..6fc2c75145c 100644 --- a/xlators/cluster/afr/src/afr-read-txn.c +++ b/xlators/cluster/afr/src/afr-read-txn.c @@ -164,7 +164,7 @@ afr_ta_read_txn(void *opaque) xdata_rsp = NULL; /* It doesn't. So query thin-arbiter to see if it blames any data brick. */ - ret = afr_fill_ta_loc(this, &loc); + ret = afr_fill_ta_loc(this, &loc, _gf_true); if (ret) { gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, "Failed to populate thin-arbiter loc for: %s.", loc.name); @@ -272,7 +272,7 @@ afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err) int read_subvol = -1; inode_t *inode = NULL; int ret = -1; - int spb_choice = -1; + int spb_subvol = -1; local = frame->local; inode = local->inode; @@ -303,9 +303,9 @@ afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err) local->read_attempted[read_subvol] = 1; readfn: if (read_subvol == -1) { - ret = afr_inode_split_brain_choice_get(inode, this, &spb_choice); - if ((ret == 0) && spb_choice >= 0) - read_subvol = spb_choice; + ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol); + if ((ret == 0) && spb_subvol >= 0) + read_subvol = spb_subvol; } if (read_subvol == -1) { diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c index b2f0ef48011..a580a1584cc 100644 --- a/xlators/cluster/afr/src/afr-self-heal-common.c +++ b/xlators/cluster/afr/src/afr-self-heal-common.c @@ -140,7 +140,7 @@ heal: } } out: - if (gfid_idx && (*gfid_idx == -1) && (ret == 0)) { + if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) { ret = -afr_final_errno(local, priv); } loc_wipe(&loc); @@ -1905,17 +1905,16 @@ int afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid, struct afr_reply *replies) { - afr_private_t *priv = NULL; afr_local_t *local = NULL; dict_t *dict = NULL; - priv = frame->this->private; local = frame->local; - if (local && local->xattr_req) + + if (local->xattr_req) dict = local->xattr_req; return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies, - priv->child_up, dict); + local->child_up, dict); } unsigned int @@ -2751,3 +2750,185 @@ afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources, out: return source; } + +static int +afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + afr_local_t *local = frame->local; + int i = (long)cookie; + + local->replies[i].valid = 1; + local->replies[i].op_ret = op_ret; + local->replies[i].op_errno = op_errno; + if (op_ret == 0) { + local->op_ret = 0; + local->replies[i].poststat = *buf; + local->replies[i].preparent = *preparent; + local->replies[i].postparent = *postparent; + } + if (xdata) { + local->replies[i].xdata = dict_ref(xdata); + } + + syncbarrier_wake(&local->barrier); + return 0; +} + +int +afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode) +{ + call_frame_t *frame = NULL; + afr_local_t *local = NULL; + afr_private_t *priv = this->private; + unsigned char *mkdir_on = alloca0(priv->child_count); + unsigned char *lookup_on = alloca0(priv->child_count); + loc_t loc = {0}; + int32_t op_errno = 0; + int32_t child_op_errno = 0; + struct iatt iatt = {0}; + dict_t *xdata = NULL; + uuid_t anon_inode_gfid = {0}; + int mkdir_count = 0; + int i = 0; + + /*Try to mkdir everywhere and return success if the dir exists on 'child' + */ + + if (!priv->use_anon_inode) { + op_errno = EINVAL; + goto out; + } + + frame = afr_frame_create(this, &op_errno); + if (op_errno) { + goto out; + } + local = frame->local; + if (!local->child_up[child]) { + /*Other bricks may need mkdir so don't error out yet*/ + child_op_errno = ENOTCONN; + } + gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid); + for (i = 0; i < priv->child_count; i++) { + if (!local->child_up[i]) + continue; + + if (priv->anon_inode[i]) { + mkdir_on[i] = 0; + } else { + mkdir_on[i] = 1; + mkdir_count++; + } + } + + if (mkdir_count == 0) { + *linked_inode = inode_find(this->itable, anon_inode_gfid); + if (*linked_inode) { + op_errno = 0; + goto out; + } + } + + loc.parent = inode_ref(this->itable->root); + loc.name = priv->anon_inode_name; + loc.inode = inode_new(this->itable); + if (!loc.inode) { + op_errno = ENOMEM; + goto out; + } + + xdata = dict_new(); + if (!xdata) { + op_errno = ENOMEM; + goto out; + } + + op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true); + if (op_errno) { + goto out; + } + + if (mkdir_count == 0) { + memcpy(lookup_on, local->child_up, priv->child_count); + goto lookup; + } + + AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0, + xdata); + + for (i = 0; i < priv->child_count; i++) { + if (!mkdir_on[i]) { + continue; + } + + if (local->replies[i].op_ret == 0) { + priv->anon_inode[i] = 1; + iatt = local->replies[i].poststat; + } else if (local->replies[i].op_ret < 0 && + local->replies[i].op_errno == EEXIST) { + lookup_on[i] = 1; + } else if (i == child) { + child_op_errno = local->replies[i].op_errno; + } + } + + if (AFR_COUNT(lookup_on, priv->child_count) == 0) { + goto link; + } + +lookup: + AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc, + xdata); + for (i = 0; i < priv->child_count; i++) { + if (!lookup_on[i]) { + continue; + } + + if (local->replies[i].op_ret == 0) { + if (gf_uuid_compare(anon_inode_gfid, + local->replies[i].poststat.ia_gfid) == 0) { + priv->anon_inode[i] = 1; + iatt = local->replies[i].poststat; + } else { + if (i == child) + child_op_errno = EINVAL; + gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA, + "%s has gfid: %s", priv->anon_inode_name, + uuid_utoa(local->replies[i].poststat.ia_gfid)); + } + } else if (i == child) { + child_op_errno = local->replies[i].op_errno; + } + } +link: + if (!gf_uuid_is_null(iatt.ia_gfid)) { + *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt); + if (*linked_inode) { + op_errno = 0; + inode_lookup(*linked_inode); + } else { + op_errno = ENOMEM; + } + goto out; + } + +out: + if (xdata) + dict_unref(xdata); + loc_wipe(&loc); + /*child_op_errno takes precedence*/ + if (child_op_errno == 0) { + child_op_errno = op_errno; + } + + if (child_op_errno && *linked_inode) { + inode_unref(*linked_inode); + *linked_inode = NULL; + } + if (frame) + AFR_STACK_DESTROY(frame); + return -child_op_errno; +} diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c index ac31751997f..64893f441e3 100644 --- a/xlators/cluster/afr/src/afr-self-heal-entry.c +++ b/xlators/cluster/afr/src/afr-self-heal-entry.c @@ -16,54 +16,170 @@ #include <glusterfs/syncop-utils.h> #include <glusterfs/events.h> -static int -afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name, - inode_t *inode, int child, struct afr_reply *replies) +int +afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name, + inode_t *inode, int child, + struct afr_reply *replies, + gf_boolean_t *anon_inode) { afr_private_t *priv = NULL; + afr_local_t *local = NULL; xlator_t *subvol = NULL; int ret = 0; + int i = 0; + char g[64] = {0}; + unsigned char *lookup_success = NULL; + call_frame_t *frame = NULL; + loc_t loc2 = { + 0, + }; loc_t loc = { 0, }; - char g[64]; priv = this->private; - subvol = priv->children[child]; + lookup_success = alloca0(priv->child_count); + uuid_utoa_r(replies[child].poststat.ia_gfid, g); + loc.inode = inode_new(inode->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + + if (replies[child].poststat.ia_type == IA_IFDIR) { + /* This directory may have sub-directory hierarchy which may need to + * be preserved for subsequent heals. So unconditionally move the + * directory to anonymous-inode directory*/ + *anon_inode = _gf_true; + goto anon_inode; + } + + frame = afr_frame_create(this, &ret); + if (!frame) { + ret = -ret; + goto out; + } + local = frame->local; + gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid); + AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc, + NULL); + for (i = 0; i < priv->child_count; i++) { + if (local->replies[i].op_ret == 0) { + lookup_success[i] = 1; + } else if (local->replies[i].op_errno != ENOENT && + local->replies[i].op_errno != ESTALE) { + ret = -local->replies[i].op_errno; + } + } + + if (priv->quorum_count) { + if (afr_has_quorum(lookup_success, this, NULL)) { + *anon_inode = _gf_true; + } + } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) { + *anon_inode = _gf_true; + } else if (ret) { + goto out; + } + +anon_inode: + if (!*anon_inode) { + ret = 0; + goto out; + } loc.parent = inode_ref(dir); gf_uuid_copy(loc.pargfid, dir->gfid); loc.name = name; - loc.inode = inode_ref(inode); - if (replies[child].valid && replies[child].op_ret == 0) { - switch (replies[child].poststat.ia_type) { - case IA_IFDIR: - gf_msg(this->name, GF_LOG_WARNING, 0, - AFR_MSG_EXPUNGING_FILE_OR_DIR, - "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), - name, uuid_utoa_r(replies[child].poststat.ia_gfid, g), - subvol->name); - ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL); - break; - default: - gf_msg(this->name, GF_LOG_WARNING, 0, - AFR_MSG_EXPUNGING_FILE_OR_DIR, - "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid), - name, uuid_utoa_r(replies[child].poststat.ia_gfid, g), - subvol->name); - ret = syncop_unlink(subvol, &loc, NULL, NULL); - break; - } + ret = afr_anon_inode_create(this, child, &loc2.parent); + if (ret < 0) + goto out; + + loc2.name = g; + ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR, + "Rename to %s dir %s/%s (%s) on %s failed", + priv->anon_inode_name, uuid_utoa(dir->gfid), name, g, + subvol->name); + } else { + gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR, + "Rename to %s dir %s/%s (%s) on %s successful", + priv->anon_inode_name, uuid_utoa(dir->gfid), name, g, + subvol->name); } +out: loc_wipe(&loc); + loc_wipe(&loc2); + if (frame) { + AFR_STACK_DESTROY(frame); + } return ret; } int +afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name, + inode_t *inode, int child, struct afr_reply *replies) +{ + char g[64] = {0}; + afr_private_t *priv = NULL; + xlator_t *subvol = NULL; + int ret = 0; + loc_t loc = { + 0, + }; + gf_boolean_t anon_inode = _gf_false; + + priv = this->private; + subvol = priv->children[child]; + + if ((!replies[child].valid) || (replies[child].op_ret < 0)) { + /*Nothing to do*/ + ret = 0; + goto out; + } + + if (priv->use_anon_inode) { + ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child, + replies, &anon_inode); + if (ret < 0 || anon_inode) + goto out; + } + + loc.parent = inode_ref(dir); + loc.inode = inode_new(inode->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + loc.name = name; + switch (replies[child].poststat.ia_type) { + case IA_IFDIR: + gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR, + "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name, + uuid_utoa_r(replies[child].poststat.ia_gfid, g), + subvol->name); + ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL); + break; + default: + gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR, + "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid), + name, uuid_utoa_r(replies[child].poststat.ia_gfid, g), + subvol->name); + ret = syncop_unlink(subvol, &loc, NULL, NULL); + break; + } + +out: + loc_wipe(&loc); + return ret; +} + +int afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source, unsigned char *sources, inode_t *dir, const char *name, inode_t *inode, @@ -76,6 +192,9 @@ afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source, loc_t srcloc = { 0, }; + loc_t anonloc = { + 0, + }; xlator_t *this = frame->this; afr_private_t *priv = NULL; dict_t *xdata = NULL; @@ -86,15 +205,17 @@ afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source, 0, }; unsigned char *newentry = NULL; - char dir_uuid_str[64] = {0}, iatt_uuid_str[64] = {0}; + char iatt_uuid_str[64] = {0}; + char dir_uuid_str[64] = {0}; priv = this->private; iatt = &replies[source].poststat; + uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str); if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) { gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED, "Invalid ia_type (%d) or gfid(%s). source brick=%d, " "pargfid=%s, name=%s", - iatt->ia_type, uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str), source, + iatt->ia_type, iatt_uuid_str, source, uuid_utoa_r(dir->gfid, dir_uuid_str), name); ret = -EINVAL; goto out; @@ -120,14 +241,24 @@ afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source, srcloc.inode = inode_ref(inode); gf_uuid_copy(srcloc.gfid, iatt->ia_gfid); - if (iatt->ia_type != IA_IFDIR) - ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0); - if (iatt->ia_type == IA_IFDIR || ret == -ENOENT || ret == -ESTALE) { + ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0); + if (ret == -ENOENT || ret == -ESTALE) { newentry[dst] = 1; ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies, sources, newentry); if (ret) goto out; + } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) { + // Try rename from hidden directory + ret = afr_anon_inode_create(this, dst, &anonloc.parent); + if (ret < 0) + goto out; + anonloc.inode = inode_ref(inode); + anonloc.name = iatt_uuid_str; + ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL); + if (ret == -ENOENT || ret == -ESTALE) + ret = -1; /*This sets 'mismatch' to true*/ + goto out; } mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type); @@ -166,6 +297,7 @@ out: GF_FREE(linkname); loc_wipe(&loc); loc_wipe(&srcloc); + loc_wipe(&anonloc); return ret; } @@ -578,6 +710,11 @@ afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd, priv = this->private; + if (afr_is_private_directory(priv, fd->inode->gfid, name, + GF_CLIENT_PID_SELF_HEALD)) { + return 0; + } + xattr = dict_new(); if (!xattr) return -ENOMEM; @@ -626,7 +763,7 @@ afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd, replies); if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) { - ret = afr_shd_index_purge(subvol, parent_idx_inode, name, + ret = afr_shd_entry_purge(subvol, parent_idx_inode, name, inode->ia_type); /* Why is ret force-set to 0? We do not care about * index purge failing for full heal as it is quite @@ -756,10 +893,6 @@ afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd, if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; - if (__is_root_gfid(fd->inode->gfid) && - !strcmp(entry->d_name, GF_REPLICATE_TRASH_DIR)) - continue; - ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name, loc.inode, subvol, local->need_full_crawl); @@ -822,7 +955,7 @@ afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry, /* The name indices under the pgfid index dir are guaranteed * to be regular files. Hence the hardcoding. */ - afr_shd_index_purge(subvol, parent->inode, entry->d_name, IA_IFREG); + afr_shd_entry_purge(subvol, parent->inode, entry->d_name, IA_IFREG); ret = 0; goto out; } diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c index f4e31b65bf9..03f43bad16e 100644 --- a/xlators/cluster/afr/src/afr-self-heal-metadata.c +++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c @@ -190,6 +190,59 @@ out: return ret; } +static int +__afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this, + inode_t *inode, + struct afr_reply *replies, + unsigned char *sources) +{ + int ret = 0; + int i = 0; + int m_idx = 0; + afr_private_t *priv = NULL; + int raw[AFR_NUM_CHANGE_LOGS] = {0}; + dict_t *xattr = NULL; + + priv = this->private; + m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION); + raw[m_idx] = 1; + + xattr = dict_new(); + if (!xattr) + return -ENOMEM; + + for (i = 0; i < priv->child_count; i++) { + if (sources[i]) + continue; + ret = dict_set_static_bin(xattr, priv->pending_key[i], raw, + sizeof(int) * AFR_NUM_CHANGE_LOGS); + if (ret) { + ret = -1; + goto out; + } + } + + for (i = 0; i < priv->child_count; i++) { + if (!sources[i]) + continue; + ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO, + "Failed to set pending metadata xattr on child %d for %s", i, + uuid_utoa(inode->gfid)); + goto out; + } + } + + afr_replies_wipe(replies, priv->child_count); + ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies); + +out: + if (xattr) + dict_unref(xattr); + return ret; +} + /* * Look for mismatching uid/gid or mode or user xattrs even if * AFR xattrs don't say so, and pick one arbitrarily as winner. */ @@ -210,6 +263,7 @@ __afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this, }; int source = -1; int sources_count = 0; + int ret = 0; priv = this->private; @@ -300,7 +354,13 @@ __afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this, healed_sinks[i] = 1; } } - + if ((sources_count == priv->child_count) && (source > -1) && + (AFR_COUNT(healed_sinks, priv->child_count) != 0)) { + ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode, + replies, sources); + if (ret < 0) + return ret; + } out: afr_mark_active_sinks(this, sources, locked_on, healed_sinks); return source; diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c index 7d4f2080ec3..834aac86d48 100644 --- a/xlators/cluster/afr/src/afr-self-heal-name.c +++ b/xlators/cluster/afr/src/afr-self-heal-name.c @@ -98,21 +98,12 @@ __afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid, const char *bname, inode_t *inode, struct afr_reply *replies) { - loc_t loc = { - 0, - }; int i = 0; afr_private_t *priv = NULL; - char g[64]; int ret = 0; priv = this->private; - loc.parent = inode_ref(parent); - gf_uuid_copy(loc.pargfid, pargfid); - loc.name = bname; - loc.inode = inode_ref(inode); - for (i = 0; i < priv->child_count; i++) { if (!replies[i].valid) continue; @@ -120,30 +111,10 @@ __afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid, if (replies[i].op_ret) continue; - switch (replies[i].poststat.ia_type) { - case IA_IFDIR: - gf_msg(this->name, GF_LOG_WARNING, 0, - AFR_MSG_EXPUNGING_FILE_OR_DIR, - "expunging dir %s/%s (%s) on %s", uuid_utoa(pargfid), - bname, uuid_utoa_r(replies[i].poststat.ia_gfid, g), - priv->children[i]->name); - - ret |= syncop_rmdir(priv->children[i], &loc, 1, NULL, NULL); - break; - default: - gf_msg(this->name, GF_LOG_WARNING, 0, - AFR_MSG_EXPUNGING_FILE_OR_DIR, - "expunging file %s/%s (%s) on %s", uuid_utoa(pargfid), - bname, uuid_utoa_r(replies[i].poststat.ia_gfid, g), - priv->children[i]->name); - - ret |= syncop_unlink(priv->children[i], &loc, NULL, NULL); - break; - } + ret |= afr_selfheal_entry_delete(this, parent, bname, inode, i, + replies); } - loc_wipe(&loc); - return ret; } @@ -381,7 +352,7 @@ __afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent, ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode, replies, gfid, locked_on, source, sources, is_gfid_absent, &gfid_idx); - if (ret) + if (ret || (gfid_idx < 0)) return ret; ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname, @@ -560,13 +531,15 @@ afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this, struct afr_reply *replies = NULL; inode_t *inode = NULL; int first_idx = -1; + afr_local_t *local = NULL; priv = this->private; + local = frame->local; replies = alloca0(sizeof(*replies) * priv->child_count); inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies, - priv->child_up, NULL); + local->child_up, NULL); if (!inode) return -ENOMEM; diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h index 9746860e36f..48e6dbcfb18 100644 --- a/xlators/cluster/afr/src/afr-self-heal.h +++ b/xlators/cluster/afr/src/afr-self-heal.h @@ -45,13 +45,16 @@ afr_local_t *__local = frame->local; \ afr_private_t *__priv = frame->this->private; \ int __i = 0; \ - int __count = AFR_COUNT(list, __priv->child_count); \ + int __count = 0; \ + unsigned char *__list = alloca(__priv->child_count); \ \ + memcpy(__list, list, sizeof(*__list) * __priv->child_count); \ + __count = AFR_COUNT(__list, __priv->child_count); \ __local->barrier.waitfor = __count; \ afr_local_replies_wipe(__local, __priv); \ \ for (__i = 0; __i < __priv->child_count; __i++) { \ - if (!list[__i]) \ + if (!__list[__i]) \ continue; \ STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i, \ __priv->children[__i], \ @@ -366,4 +369,9 @@ gf_boolean_t afr_is_file_empty_on_all_children(afr_private_t *priv, struct afr_reply *replies); +int +afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name, + inode_t *inode, int child, struct afr_reply *replies); +int +afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode); #endif /* !_AFR_SELFHEAL_H */ diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c index aa774bb8d51..109fd4b7421 100644 --- a/xlators/cluster/afr/src/afr-self-heald.c +++ b/xlators/cluster/afr/src/afr-self-heald.c @@ -94,7 +94,7 @@ __afr_shd_healer_wait(struct subvol_healer *healer) priv = healer->this->private; disabled_loop: - wait_till.tv_sec = time(NULL) + priv->shd.timeout; + wait_till.tv_sec = gf_time() + priv->shd.timeout; while (!healer->rerun) { ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till); @@ -222,7 +222,7 @@ out: } int -afr_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name, +afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name, ia_type_t type) { int ret = 0; @@ -371,7 +371,7 @@ afr_shd_sweep_prepare(struct subvol_healer *healer) event->split_brain_count = 0; event->heal_failed_count = 0; - time(&event->start_time); + event->start_time = gf_time(); event->end_time = 0; _mask_cancellation(); } @@ -386,7 +386,7 @@ afr_shd_sweep_done(struct subvol_healer *healer) event = &healer->crawl_event; shd = &(((afr_private_t *)healer->this->private)->shd); - time(&event->end_time); + event->end_time = gf_time(); history = gf_memdup(event, sizeof(*event)); event->start_time = 0; @@ -424,7 +424,7 @@ afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, ret = afr_shd_selfheal(healer, healer->subvol, gfid); if (ret == -ENOENT || ret == -ESTALE) - afr_shd_index_purge(subvol, parent->inode, entry->d_name, val); + afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val); if (ret == 2) /* If bricks crashed in pre-op after creating indices/xattrop @@ -843,6 +843,176 @@ out: return need_heal; } +static int +afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct subvol_healer *healer = data; + afr_private_t *priv = healer->this->private; + call_frame_t *frame = NULL; + afr_local_t *local = NULL; + int ret = 0; + loc_t loc = {0}; + int count = 0; + int i = 0; + int op_errno = 0; + struct iatt *iatt = NULL; + gf_boolean_t multiple_links = _gf_false; + unsigned char *gfid_present = alloca0(priv->child_count); + unsigned char *entry_present = alloca0(priv->child_count); + char *type = "file"; + + frame = afr_frame_create(healer->this, &ret); + if (!frame) { + ret = -ret; + goto out; + } + local = frame->local; + if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) { + gf_msg_debug(healer->this->name, 0, + "Not all bricks are up. Skipping " + "cleanup of %s on %s", + entry->d_name, subvol->name); + ret = 0; + goto out; + } + + loc.inode = inode_new(parent->inode->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + ret = gf_uuid_parse(entry->d_name, loc.gfid); + if (ret) { + ret = 0; + goto out; + } + AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc, + NULL); + for (i = 0; i < priv->child_count; i++) { + if (local->replies[i].op_ret == 0) { + count++; + gfid_present[i] = 1; + iatt = &local->replies[i].poststat; + if (iatt->ia_type == IA_IFDIR) { + type = "dir"; + } + + if (i == healer->subvol) { + if (local->replies[i].poststat.ia_nlink > 1) { + multiple_links = _gf_true; + } + } + } else if (local->replies[i].op_errno != ENOENT && + local->replies[i].op_errno != ESTALE) { + /*We don't have complete view. Skip the entry*/ + gf_msg_debug(healer->this->name, local->replies[i].op_errno, + "Skipping cleanup of %s on %s", entry->d_name, + subvol->name); + ret = 0; + goto out; + } + } + + /*Inode is deleted from subvol*/ + if (count == 1 || (iatt->ia_type != IA_IFDIR && multiple_links)) { + gf_msg(healer->this->name, GF_LOG_WARNING, 0, + AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type, + priv->anon_inode_name, entry->d_name, subvol->name); + ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name, + iatt->ia_type); + if (ret == -ENOENT || ret == -ESTALE) + ret = 0; + } else if (count > 1) { + loc_wipe(&loc); + loc.parent = inode_ref(parent->inode); + loc.name = entry->d_name; + loc.inode = inode_new(parent->inode->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, + &loc, NULL); + count = 0; + for (i = 0; i < priv->child_count; i++) { + if (local->replies[i].op_ret == 0) { + count++; + entry_present[i] = 1; + iatt = &local->replies[i].poststat; + } else if (local->replies[i].op_errno != ENOENT && + local->replies[i].op_errno != ESTALE) { + /*We don't have complete view. Skip the entry*/ + gf_msg_debug(healer->this->name, local->replies[i].op_errno, + "Skipping cleanup of %s on %s", entry->d_name, + subvol->name); + ret = 0; + goto out; + } + } + for (i = 0; i < priv->child_count; i++) { + if (gfid_present[i] && !entry_present[i]) { + /*Entry is not anonymous on at least one subvol*/ + gf_msg_debug(healer->this->name, 0, + "Valid entry present on %s " + "Skipping cleanup of %s on %s", + priv->children[i]->name, entry->d_name, + subvol->name); + ret = 0; + goto out; + } + } + + gf_msg(healer->this->name, GF_LOG_WARNING, 0, + AFR_MSG_EXPUNGING_FILE_OR_DIR, + "expunging %s %s/%s on all subvols", type, priv->anon_inode_name, + entry->d_name); + ret = 0; + for (i = 0; i < priv->child_count; i++) { + op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent, + entry->d_name, iatt->ia_type); + if (op_errno != ENOENT && op_errno != ESTALE) { + ret |= -op_errno; + } + } + } + +out: + if (frame) + AFR_STACK_DESTROY(frame); + loc_wipe(&loc); + return ret; +} + +static void +afr_cleanup_anon_inode_dir(struct subvol_healer *healer) +{ + int ret = 0; + call_frame_t *frame = NULL; + afr_private_t *priv = healer->this->private; + loc_t loc = {0}; + + ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode); + if (ret) + goto out; + + frame = afr_frame_create(healer->this, &ret); + if (!frame) { + ret = -ret; + goto out; + } + + ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc, + GF_CLIENT_PID_SELF_HEALD, healer, + afr_shd_anon_inode_cleaner, NULL, + priv->shd.max_threads, priv->shd.wait_qlength); +out: + if (frame) + AFR_STACK_DESTROY(frame); + loc_wipe(&loc); + return; +} + void * afr_shd_index_healer(void *data) { @@ -900,6 +1070,10 @@ afr_shd_index_healer(void *data) sleep(1); } while (ret > 0); + if (ret == 0) { + afr_cleanup_anon_inode_dir(healer); + } + if (ret == 0 && pre_crawl_xdata && !healer->crawl_event.heal_failed_count) { afr_shd_ta_check_and_unset_xattrs(this, &loc, healer, @@ -1481,15 +1655,6 @@ afr_xl_op(xlator_t *this, dict_t *input, dict_t *output) case GF_SHD_OP_INDEX_SUMMARY: /* this case has been handled in glfs-heal.c */ break; - case GF_SHD_OP_HEALED_FILES: - case GF_SHD_OP_HEAL_FAILED_FILES: - for (i = 0; i < priv->child_count; i++) { - keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i); - AFR_SET_DICT_AND_LOG(this->name, output, key, keylen, - SOP_NOT_SUPPORTED, - SLEN(SOP_NOT_SUPPORTED)); - } - break; case GF_SHD_OP_SPLIT_BRAIN_FILES: eh_dump(shd->split_brain, output, afr_add_shd_event); break; diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h index 687c28e6472..18db728ea7b 100644 --- a/xlators/cluster/afr/src/afr-self-heald.h +++ b/xlators/cluster/afr/src/afr-self-heald.h @@ -70,6 +70,6 @@ afr_shd_gfid_to_path(xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p); int -afr_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name, +afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name, ia_type_t type); #endif /* !_AFR_SELF_HEALD_H */ diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c index 78438f91331..a51f79b1f43 100644 --- a/xlators/cluster/afr/src/afr-transaction.c +++ b/xlators/cluster/afr/src/afr-transaction.c @@ -124,9 +124,9 @@ afr_release_notify_lock_for_ta(void *opaque) this = (xlator_t *)opaque; priv = this->private; - ret = afr_fill_ta_loc(this, &loc); + ret = afr_fill_ta_loc(this, &loc, _gf_true); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB, + gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, "Failed to populate loc for thin-arbiter."); goto out; } @@ -521,42 +521,6 @@ afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this) local->transaction.pre_op_sources[j] = 0; } -gf_boolean_t -afr_has_arbiter_fop_cbk_quorum(call_frame_t *frame) -{ - afr_local_t *local = NULL; - afr_private_t *priv = NULL; - xlator_t *this = NULL; - gf_boolean_t fop_failed = _gf_false; - unsigned char *pre_op_sources = NULL; - int i = 0; - - local = frame->local; - this = frame->this; - priv = this->private; - pre_op_sources = local->transaction.pre_op_sources; - - /* If the fop failed on the brick, it is not a source. */ - for (i = 0; i < priv->child_count; i++) - if (local->transaction.failed_subvols[i]) - pre_op_sources[i] = 0; - - switch (AFR_COUNT(pre_op_sources, priv->child_count)) { - case 1: - if (pre_op_sources[ARBITER_BRICK_INDEX]) - fop_failed = _gf_true; - break; - case 0: - fop_failed = _gf_true; - break; - } - - if (fop_failed) - return _gf_false; - - return _gf_true; -} - void afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this) { @@ -971,12 +935,8 @@ afr_need_dirty_marking(call_frame_t *frame, xlator_t *this) priv->child_count) return _gf_false; - if (priv->arbiter_count) { - if (!afr_has_arbiter_fop_cbk_quorum(frame)) - need_dirty = _gf_true; - } else if (!afr_has_fop_cbk_quorum(frame)) { + if (!afr_has_fop_cbk_quorum(frame)) need_dirty = _gf_true; - } return need_dirty; } @@ -1026,12 +986,8 @@ afr_handle_quorum(call_frame_t *frame, xlator_t *this) * no split-brain with the fix. The problem is eliminated completely. */ - if (priv->arbiter_count) { - if (afr_has_arbiter_fop_cbk_quorum(frame)) - return; - } else if (afr_has_fop_cbk_quorum(frame)) { + if (afr_has_fop_cbk_quorum(frame)) return; - } if (afr_need_dirty_marking(frame, this)) goto set_response; @@ -1073,7 +1029,7 @@ set_response: } int -afr_fill_ta_loc(xlator_t *this, loc_t *loc) +afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop) { afr_private_t *priv = NULL; @@ -1081,6 +1037,11 @@ afr_fill_ta_loc(xlator_t *this, loc_t *loc) loc->parent = inode_ref(priv->root_inode); gf_uuid_copy(loc->pargfid, loc->parent->gfid); loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX]; + if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) { + /* Except afr_ta_id_file_check() which is path based, all other gluster + * FOPS need gfid.*/ + return -EINVAL; + } gf_uuid_copy(loc->gfid, priv->ta_gfid); loc->inode = inode_new(loc->parent->table); if (!loc->inode) { @@ -1090,86 +1051,6 @@ afr_fill_ta_loc(xlator_t *this, loc_t *loc) return 0; } -int -afr_changelog_thin_arbiter_post_op(xlator_t *this, afr_local_t *local) -{ - int ret = 0; - afr_private_t *priv = NULL; - dict_t *xattr = NULL; - int failed_count = 0; - struct gf_flock flock = { - 0, - }; - loc_t loc = { - 0, - }; - int i = 0; - - priv = this->private; - if (!priv->thin_arbiter_count) - return 0; - - failed_count = AFR_COUNT(local->transaction.failed_subvols, - priv->child_count); - if (!failed_count) - return 0; - - GF_ASSERT(failed_count == 1); - ret = afr_fill_ta_loc(this, &loc); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, - "Failed to populate thin-arbiter loc for: %s.", loc.name); - goto out; - } - - xattr = dict_new(); - if (!xattr) { - ret = -ENOMEM; - goto out; - } - for (i = 0; i < priv->child_count; i++) { - ret = dict_set_static_bin(xattr, priv->pending_key[i], - local->pending[i], - AFR_NUM_CHANGE_LOGS * sizeof(int)); - if (ret) - goto out; - } - - flock.l_type = F_WRLCK; - flock.l_start = 0; - flock.l_len = 0; - - /*TODO: Convert to two domain locking. */ - ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX], - AFR_TA_DOM_NOTIFY, &loc, F_SETLKW, &flock, NULL, NULL); - if (ret) - goto out; - - ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, - GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL); - - if (ret == -EINVAL) { - gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_THIN_ARB, - "Thin-arbiter has denied post-op on %s for gfid %s.", - priv->pending_key[THIN_ARBITER_BRICK_INDEX], - uuid_utoa(local->inode->gfid)); - - } else if (ret) { - gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, - "Post-op on thin-arbiter id file %s failed for gfid %s.", - priv->pending_key[THIN_ARBITER_BRICK_INDEX], - uuid_utoa(local->inode->gfid)); - } - flock.l_type = F_UNLCK; - syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX], AFR_TA_DOM_NOTIFY, - &loc, F_SETLK, &flock, NULL, NULL); -out: - if (xattr) - dict_unref(xattr); - - return ret; -} - static int afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque) { @@ -1264,9 +1145,9 @@ afr_ta_post_op_do(void *opaque) this = local->transaction.frame->this; priv = this->private; - ret = afr_fill_ta_loc(this, &loc); + ret = afr_fill_ta_loc(this, &loc, _gf_true); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB, + gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB, "Failed to populate loc for thin-arbiter."); goto out; } @@ -2466,8 +2347,13 @@ afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this, goto out; } - if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP)) { - /*Only allow writes but shard does [f]xattrops on writes, so + if (local->transaction.disable_delayed_post_op) { + goto out; + } + + if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) && + (local->op != GF_FOP_FSYNC)) { + /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so * they are fine too*/ goto out; } diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c index ec7aa226821..df7366f0a65 100644 --- a/xlators/cluster/afr/src/afr.c +++ b/xlators/cluster/afr/src/afr.c @@ -135,6 +135,27 @@ set_data_self_heal_algorithm(afr_private_t *priv, char *algo) } } +void +afr_handle_anon_inode_options(afr_private_t *priv, dict_t *options) +{ + char *volfile_id_str = NULL; + uuid_t anon_inode_gfid = {0}; + + /*If volume id is not present don't enable anything*/ + if (dict_get_str(options, "volume-id", &volfile_id_str)) + return; + GF_ASSERT(strlen(AFR_ANON_DIR_PREFIX) + strlen(volfile_id_str) <= NAME_MAX); + /*anon_inode_name is not supposed to change once assigned*/ + if (!priv->anon_inode_name[0]) { + snprintf(priv->anon_inode_name, sizeof(priv->anon_inode_name), "%s-%s", + AFR_ANON_DIR_PREFIX, volfile_id_str); + gf_uuid_parse(volfile_id_str, anon_inode_gfid); + /*Flip a bit to make sure volfile-id and anon-gfid are not same*/ + anon_inode_gfid[0] ^= 1; + uuid_utoa_r(anon_inode_gfid, priv->anon_gfid_str); + } +} + int reconfigure(xlator_t *this, dict_t *options) { @@ -168,7 +189,8 @@ reconfigure(xlator_t *this, dict_t *options) bool, out); GF_OPTION_RECONF("data-self-heal", data_self_heal, options, str, out); - gf_string2boolean(data_self_heal, &priv->data_self_heal); + if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1) + goto out; GF_OPTION_RECONF("entry-self-heal", priv->entry_self_heal, options, bool, out); @@ -289,6 +311,10 @@ reconfigure(xlator_t *this, dict_t *options) consistent_io = _gf_false; priv->consistent_io = consistent_io; + afr_handle_anon_inode_options(priv, options); + + GF_OPTION_RECONF("use-anonymous-inode", priv->use_anon_inode, options, bool, + out); if (priv->shd.enabled) { if ((priv->shd.enabled != enabled_old) || (timeout_old != priv->shd.timeout)) @@ -485,7 +511,8 @@ init(xlator_t *this) GF_OPTION_INIT("heal-wait-queue-length", priv->heal_wait_qlen, uint32, out); GF_OPTION_INIT("data-self-heal", data_self_heal, str, out); - gf_string2boolean(data_self_heal, &priv->data_self_heal); + if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1) + goto out; GF_OPTION_INIT("data-self-heal-algorithm", data_self_heal_algorithm, str, out); @@ -539,7 +566,9 @@ init(xlator_t *this) GF_OPTION_INIT("consistent-metadata", priv->consistent_metadata, bool, out); GF_OPTION_INIT("consistent-io", priv->consistent_io, bool, out); + afr_handle_anon_inode_options(priv, this->options); + GF_OPTION_INIT("use-anonymous-inode", priv->use_anon_inode, bool, out); if (priv->quorum_count != 0) priv->consistent_io = _gf_false; @@ -551,13 +580,19 @@ init(xlator_t *this) goto out; } + priv->anon_inode = GF_CALLOC(sizeof(unsigned char), child_count, + gf_afr_mt_char); + priv->child_up = GF_CALLOC(sizeof(unsigned char), child_count, gf_afr_mt_char); priv->child_latency = GF_MALLOC(sizeof(*priv->child_latency) * child_count, gf_afr_mt_child_latency_t); + priv->halo_child_up = GF_CALLOC(sizeof(unsigned char), child_count, + gf_afr_mt_char); - if (!priv->child_up || !priv->child_latency) { + if (!priv->child_up || !priv->child_latency || !priv->halo_child_up || + !priv->anon_inode) { ret = -ENOMEM; goto out; } @@ -1282,6 +1317,14 @@ struct volume_options options[] = { .tags = {"replicate"}, .description = "This option exists only for backward compatibility " "and configuring it doesn't have any effect"}, + {.key = {"use-anonymous-inode"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "no", + .op_version = {GD_OP_VERSION_8_0}, + .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE, + .tags = {"replicate"}, + .description = "Setting this option heals directory renames efficiently"}, + {.key = {NULL}}, }; diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h index 88456562610..d62f9a9caf2 100644 --- a/xlators/cluster/afr/src/afr.h +++ b/xlators/cluster/afr/src/afr.h @@ -42,6 +42,7 @@ #define AFR_LK_HEAL_DOM "afr.lock-heal.domain" #define AFR_HALO_MAX_LATENCY 99999 +#define AFR_ANON_DIR_PREFIX ".glusterfs-anonymous-inode" #define PFLAG_PENDING (1 << 0) #define PFLAG_SBRAIN (1 << 1) @@ -190,7 +191,9 @@ typedef struct _afr_private { struct list_head ta_waitq; struct list_head ta_onwireq; + unsigned char *anon_inode; unsigned char *child_up; + unsigned char *halo_child_up; int64_t *child_latency; unsigned char *local; @@ -274,10 +277,15 @@ typedef struct _afr_private { gf_boolean_t esh_granular; gf_boolean_t consistent_io; gf_boolean_t data_self_heal; /* on/off */ + gf_boolean_t use_anon_inode; /*For lock healing.*/ struct list_head saved_locks; struct list_head lk_healq; + + /*For anon-inode handling */ + char anon_inode_name[NAME_MAX + 1]; + char anon_gfid_str[UUID_SIZE + 1]; } afr_private_t; typedef enum { @@ -900,7 +908,7 @@ typedef struct _afr_local { gf_boolean_t uninherit_done; gf_boolean_t uninherit_value; - /* post-op hook */ + gf_boolean_t disable_delayed_post_op; } transaction; syncbarrier_t barrier; @@ -996,7 +1004,10 @@ afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, int event_generation); int -afr_inode_event_gen_reset(inode_t *inode, xlator_t *this); +__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this); + +int +afr_inode_need_refresh_set(inode_t *inode, xlator_t *this); int afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this, @@ -1267,8 +1278,8 @@ int afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice); int -afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this, - int *spb_choice); +afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this, + call_frame_t *frame, int *spb_subvol); int afr_get_child_index_from_name(xlator_t *this, char *name); @@ -1353,7 +1364,7 @@ int afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode); int -afr_fill_ta_loc(xlator_t *this, loc_t *loc); +afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop); int afr_ta_post_op_lock(xlator_t *this, loc_t *loc); @@ -1401,4 +1412,12 @@ afr_is_lock_mode_mandatory(dict_t *xdata); void afr_dom_lock_release(call_frame_t *frame); + +void +afr_fill_success_replies(afr_local_t *local, afr_private_t *priv, + unsigned char *replies); + +gf_boolean_t +afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name, + pid_t pid); #endif /* __AFR_H__ */ diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index 2231af647de..8ba0cc4c732 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -17,6 +17,7 @@ #include <glusterfs/quota-common-utils.h> #include <glusterfs/upcall-utils.h> #include "glusterfs/compat-errno.h" // for ENODATA on BSD +#include <glusterfs/common-utils.h> #include <sys/time.h> #include <libgen.h> @@ -43,15 +44,6 @@ dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, static int dht_rmdir_unlock(call_frame_t *frame, xlator_t *this); -char *xattrs_to_heal[] = {"user.", - POSIX_ACL_ACCESS_XATTR, - POSIX_ACL_DEFAULT_XATTR, - QUOTA_LIMIT_KEY, - QUOTA_LIMIT_OBJECTS_KEY, - GF_SELINUX_XATTR_KEY, - GF_XATTR_MDATA_KEY, - NULL}; - static const char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL}; /* Check the xdata to make sure EBADF has been set by client xlator */ @@ -84,6 +76,8 @@ dht_set_fixed_dir_stat(struct iatt *stat) static gf_boolean_t dht_match_xattr(const char *key) { + char **xattrs_to_heal = get_xattrs_to_heal(); + return gf_get_index_by_elem(xattrs_to_heal, (char *)key) >= 0; } @@ -388,7 +382,7 @@ out: /* Code to save hashed subvol on inode ctx as a mds subvol */ -static int +int dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol) { dht_inode_ctx_t *ctx = NULL; @@ -619,13 +613,14 @@ dht_discover_complete(xlator_t *this, call_frame_t *discover_frame) if (local->need_xattr_heal && !heal_path) { local->need_xattr_heal = 0; - ret = dht_dir_xattr_heal(this, local); - if (ret) - gf_msg(this->name, GF_LOG_ERROR, ret, + ret = dht_dir_xattr_heal(this, local, &op_errno); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DIR_XATTR_HEAL_FAILED, "xattr heal failed for " "directory gfid is %s ", gfid_local); + } } } @@ -695,6 +690,7 @@ dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int ret = -1; dht_conf_t *conf = 0; dht_layout_t *layout = NULL; + int32_t mds_heal_fresh_lookup = 0; GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, frame->local, out); @@ -702,6 +698,7 @@ dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, local = frame->local; conf = this->private; layout = local->selfheal.layout; + mds_heal_fresh_lookup = local->mds_heal_fresh_lookup; if (op_ret) { gf_msg_debug(this->name, op_ret, @@ -722,7 +719,7 @@ dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, layout); } out: - if (local && local->mds_heal_fresh_lookup) + if (mds_heal_fresh_lookup) DHT_STACK_DESTROY(frame); return 0; } @@ -1256,7 +1253,7 @@ err: to non hashed subvol */ int -dht_dir_xattr_heal(xlator_t *this, dht_local_t *local) +dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno) { dht_local_t *copy_local = NULL; call_frame_t *copy = NULL; @@ -1268,6 +1265,7 @@ dht_dir_xattr_heal(xlator_t *this, dht_local_t *local) "No gfid exists for path %s " "so healing xattr is not possible", local->loc.path); + *op_errno = EIO; goto out; } @@ -1281,6 +1279,7 @@ dht_dir_xattr_heal(xlator_t *this, dht_local_t *local) "Memory allocation failed " "for path %s gfid %s ", local->loc.path, gfid_local); + *op_errno = ENOMEM; DHT_STACK_DESTROY(copy); } else { copy_local->stbuf = local->stbuf; @@ -1295,6 +1294,7 @@ dht_dir_xattr_heal(xlator_t *this, dht_local_t *local) "Synctask creation failed to heal xattr " "for path %s gfid %s ", local->loc.path, gfid_local); + *op_errno = ENOMEM; DHT_STACK_DESTROY(copy); } } @@ -1435,15 +1435,31 @@ dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, dht_aggregate_xattr(local->xattr, xattr); } + if (__is_root_gfid(stbuf->ia_gfid)) { + ret = dht_dir_has_layout(xattr, conf->xattr_name); + if (ret >= 0) { + if (is_greater_time(local->prebuf.ia_ctime, + local->prebuf.ia_ctime_nsec, + stbuf->ia_ctime, stbuf->ia_ctime_nsec)) { + /* Choose source */ + local->prebuf.ia_gid = stbuf->ia_gid; + local->prebuf.ia_uid = stbuf->ia_uid; + + local->prebuf.ia_ctime = stbuf->ia_ctime; + local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec; + local->prebuf.ia_prot = stbuf->ia_prot; + } + } + } + if (local->stbuf.ia_type != IA_INVAL) { /* This is not the first subvol to respond * Compare values to see if attrs need to be healed */ - if (!__is_root_gfid(stbuf->ia_gfid) && - ((local->stbuf.ia_gid != stbuf->ia_gid) || - (local->stbuf.ia_uid != stbuf->ia_uid) || - (is_permission_different(&local->stbuf.ia_prot, - &stbuf->ia_prot)))) { + if ((local->stbuf.ia_gid != stbuf->ia_gid) || + (local->stbuf.ia_uid != stbuf->ia_uid) || + (is_permission_different(&local->stbuf.ia_prot, + &stbuf->ia_prot))) { local->need_attrheal = 1; } } @@ -1635,7 +1651,7 @@ dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, uint32_t vol_commit_hash = 0; xlator_t *subvol = NULL; int32_t check_mds = 0; - int errst = 0; + int errst = 0, i = 0; int32_t mds_xattr_val[1] = {0}; GF_VALIDATE_OR_GOTO("dht", frame, err); @@ -1702,6 +1718,14 @@ dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, local->need_lookup_everywhere = 1; } else if (IA_ISDIR(local->loc.inode->ia_type)) { + layout = local->layout; + for (i = 0; i < layout->cnt; i++) { + if (layout->list[i].xlator == prev) { + layout->list[i].err = op_errno; + break; + } + } + local->need_selfheal = 1; } } @@ -2137,31 +2161,18 @@ static int dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_t *dict) { int ret = 0; - xlator_t *this = NULL; - char *linktoskip_key = NULL; - - this = THIS; - GF_VALIDATE_OR_GOTO("dht", this, err); - - if (dht_is_tier_xlator(this)) - linktoskip_key = TIER_SKIP_NON_LINKTO_UNLINK; - else - linktoskip_key = DHT_SKIP_NON_LINKTO_UNLINK; - ret = dict_set_int32(dict, linktoskip_key, 1); + ret = dict_set_int32_sizen(dict, DHT_SKIP_NON_LINKTO_UNLINK, 1); if (ret) - goto err; + return -1; - ret = dict_set_int32(dict, DHT_SKIP_OPEN_FD_UNLINK, 1); + ret = dict_set_int32_sizen(dict, DHT_SKIP_OPEN_FD_UNLINK, 1); if (ret) - goto err; + return -1; return 0; - -err: - return -1; } static int32_t @@ -4290,6 +4301,8 @@ dht_find_local_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this, index = conf->local_subvols_cnt; uuid_list_copy = gf_strdup(uuid_list); + if (!uuid_list_copy) + goto unlock; for (uuid_str = strtok_r(uuid_list, " ", &saveptr); uuid_str; uuid_str = next_uuid_str) { @@ -4580,18 +4593,8 @@ dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, dict_del(xattr, conf->xattr_name); dict_del(xattr, conf->mds_xattr_key); - /* filter out following two xattrs that need not - * be visible on the mount point for geo-rep - - * trusted.tier.fix.layout.complete and - * trusted.tier.tier-dht.commithash - */ - dict_del(xattr, conf->commithash_xattr_name); - if (frame->root->pid >= 0 && dht_is_tier_xlator(this)) { - dict_del(xattr, GF_XATTR_TIER_LAYOUT_FIXED_KEY); - } - if (frame->root->pid >= 0) { GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr); GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr); @@ -5430,11 +5433,13 @@ dht_dir_common_set_remove_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc, int call_cnt = 0; dht_local_t *local = NULL; char gfid_local[GF_UUID_BUF_SIZE] = {0}; + char **xattrs_to_heal; conf = this->private; local = frame->local; call_cnt = conf->subvolume_cnt; local->flags = flags; + xattrs_to_heal = get_xattrs_to_heal(); if (!gf_uuid_is_null(local->gfid)) { gf_uuid_unparse(local->gfid, gfid_local); @@ -5867,22 +5872,7 @@ dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr, if (local->rebalance.target_node) { local->flags = forced_rebalance; - /* Flag to suggest its a tiering migration - * The reason for this dic key-value is that - * promotions and demotions are multithreaded - * so the original frame from gf_defrag_start() - * is not carried. A new frame will be created when - * we do syncop_setxattr(). This does not have the - * frame->root->pid of the original frame. So we pass - * this dic key-value when we do syncop_setxattr() to do - * data migration and set the frame->root->pid to - * GF_CLIENT_PID_TIER_DEFRAG in dht_setxattr() just before - * calling dht_start_rebalance_task() */ - tmp = dict_get(xattr, TIERING_MIGRATION_KEY); - if (tmp) - frame->root->pid = GF_CLIENT_PID_TIER_DEFRAG; - else - frame->root->pid = GF_CLIENT_PID_DEFRAG; + frame->root->pid = GF_CLIENT_PID_DEFRAG; ret = dht_start_rebalance_task(this, frame); if (!ret) @@ -6694,10 +6684,9 @@ dht_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, layout = local->layout; - /* We have seen crashes in while running "rm -rf" on tier volumes - when the layout was NULL on the hot tier. This will skip the - entries on the subvol without a layout, hence preventing the crash - but rmdir might fail with "directory not empty" errors*/ + /* This will skip the entries on the subvol without a layout, + * hence preventing the crash but rmdir might fail with + * "directory not empty" errors*/ if (layout == NULL) goto done; @@ -10824,23 +10813,17 @@ dht_notify(xlator_t *this, int event, void *data, ...) int had_heard_from_all = 0; int have_heard_from_all = 0; - struct timeval time = { - 0, - }; gf_defrag_info_t *defrag = NULL; dict_t *dict = NULL; gf_defrag_type cmd = 0; dict_t *output = NULL; va_list ap; - dht_methods_t *methods = NULL; struct gf_upcall *up_data = NULL; struct gf_upcall_cache_invalidation *up_ci = NULL; conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); - methods = &(conf->methods); - /* had all subvolumes reported status once till now? */ had_heard_from_all = 1; for (i = 0; i < conf->subvolume_cnt; i++) { @@ -10870,12 +10853,11 @@ dht_notify(xlator_t *this, int event, void *data, ...) break; } - gettimeofday(&time, NULL); LOCK(&conf->subvolume_lock); { conf->subvolume_status[cnt] = 1; conf->last_event[cnt] = event; - conf->subvol_up_time[cnt] = time.tv_sec; + conf->subvol_up_time[cnt] = gf_time(); } UNLOCK(&conf->subvolume_lock); @@ -10983,21 +10965,13 @@ dht_notify(xlator_t *this, int event, void *data, ...) if (defrag->is_exiting) goto unlock; if ((cmd == GF_DEFRAG_CMD_STATUS) || - (cmd == GF_DEFRAG_CMD_STATUS_TIER) || (cmd == GF_DEFRAG_CMD_DETACH_STATUS)) gf_defrag_status_get(conf, output); - else if (cmd == GF_DEFRAG_CMD_START_DETACH_TIER) - gf_defrag_start_detach_tier(defrag); else if (cmd == GF_DEFRAG_CMD_DETACH_START) defrag->cmd = GF_DEFRAG_CMD_DETACH_START; else if (cmd == GF_DEFRAG_CMD_STOP || - cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER || cmd == GF_DEFRAG_CMD_DETACH_STOP) gf_defrag_stop(conf, GF_DEFRAG_STATUS_STOPPED, output); - else if (cmd == GF_DEFRAG_CMD_PAUSE_TIER) - ret = gf_defrag_pause_tier(this, defrag); - else if (cmd == GF_DEFRAG_CMD_RESUME_TIER) - ret = gf_defrag_resume_tier(this, defrag); } unlock: UNLOCK(&defrag->lock); @@ -11072,15 +11046,13 @@ dht_notify(xlator_t *this, int event, void *data, ...) * thread has already started. */ if (conf->defrag && !run_defrag) { - if (methods->migration_needed(this)) { - run_defrag = 1; - ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start, - this, "dhtdg"); - if (ret) { - GF_FREE(conf->defrag); - conf->defrag = NULL; - kill(getpid(), SIGTERM); - } + run_defrag = 1; + ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start, + this, "dhtdg"); + if (ret) { + GF_FREE(conf->defrag); + conf->defrag = NULL; + kill(getpid(), SIGTERM); } } } @@ -11225,28 +11197,6 @@ out: return ret; } -int32_t -dht_migration_needed(xlator_t *this) -{ - gf_defrag_info_t *defrag = NULL; - dht_conf_t *conf = NULL; - int ret = 0; - - conf = this->private; - - GF_VALIDATE_OR_GOTO("dht", conf, out); - GF_VALIDATE_OR_GOTO("dht", conf->defrag, out); - - defrag = conf->defrag; - - if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && - (defrag->cmd != GF_DEFRAG_CMD_START_DETACH_TIER)) - ret = 1; - -out: - return ret; -} - /* This function should not be called more then once during a FOP handling path. It is valid only for for ops on files @@ -11281,14 +11231,6 @@ dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf, return 0; } -gf_boolean_t -dht_is_tier_xlator(xlator_t *this) -{ - if (strcmp(this->type, "cluster/tier") == 0) - return _gf_true; - return _gf_false; -} - int32_t dht_release(xlator_t *this, fd_t *fd) { @@ -11428,3 +11370,22 @@ dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key, FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata); return 0; } + +/* The job of this function is to check if all the xlators have updated + * error in the layout. */ +int +dht_dir_layout_error_check(xlator_t *this, inode_t *inode) +{ + dht_layout_t *layout = NULL; + int i = 0; + + layout = dht_layout_get(this, inode); + for (i = 0; i < layout->cnt; i++) { + if (layout->list[i].err == 0) { + return 0; + } + } + + /* Returning the first xlator error as all xlators have errors */ + return layout->list[0].err; +} diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index 871b7aed9b3..fe0dc3db34a 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -24,7 +24,6 @@ #define _DHT_H #define GF_XATTR_FIX_LAYOUT_KEY "distribute.fix.layout" -#define GF_XATTR_TIER_LAYOUT_FIXED_KEY "trusted.tier.fix.layout.complete" #define GF_XATTR_FILE_MIGRATE_KEY "trusted.distribute.migrate-data" #define DHT_MDS_STR "mds" #define GF_DHT_LOOKUP_UNHASHED_OFF 0 @@ -36,7 +35,6 @@ #define DHT_LAYOUT_HEAL_DOMAIN "dht.layout.heal" /* Namespace synchronization */ #define DHT_ENTRY_SYNC_DOMAIN "dht.entry.sync" -#define TIERING_MIGRATION_KEY "tiering.migration" #define DHT_LAYOUT_HASH_INVALID 1 #define MAX_REBAL_THREADS sysconf(_SC_NPROCESSORS_ONLN) @@ -52,10 +50,6 @@ #define DHT_DBG_HASHED_SUBVOL_PATTERN "dht.file.hashed-subvol.*" #define DHT_DBG_HASHED_SUBVOL_KEY "dht.file.hashed-subvol." -/* Array to hold custom xattr keys - */ -extern char *xattrs_to_heal[]; - /* Rebalance nodeuuid flags */ #define REBAL_NODEUUID_MINE 0x01 @@ -246,19 +240,6 @@ typedef gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem, dht_layout_t **ondisk); -typedef struct { - uint64_t blocks_used; - uint64_t pblocks_used; - uint64_t files_used; - uint64_t pfiles_used; - uint64_t unhashed_blocks_used; - uint64_t unhashed_pblocks_used; - uint64_t unhashed_files_used; - uint64_t unhashed_pfiles_used; - uint64_t unhashed_fsid; - uint64_t hashed_fsid; -} tier_statvfs_t; - struct dht_local { loc_t loc; loc_t loc2; @@ -276,7 +257,6 @@ struct dht_local { struct iatt preparent; struct iatt postparent; struct statvfs statvfs; - tier_statvfs_t tier_statvfs; fd_t *fd; inode_t *inode; dict_t *params; @@ -409,14 +389,7 @@ enum gf_defrag_type { GF_DEFRAG_CMD_STATUS = 1 + 2, GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3, GF_DEFRAG_CMD_START_FORCE = 1 + 4, - GF_DEFRAG_CMD_START_TIER = 1 + 5, - GF_DEFRAG_CMD_STATUS_TIER = 1 + 6, - GF_DEFRAG_CMD_START_DETACH_TIER = 1 + 7, - GF_DEFRAG_CMD_STOP_DETACH_TIER = 1 + 8, - GF_DEFRAG_CMD_PAUSE_TIER = 1 + 9, - GF_DEFRAG_CMD_RESUME_TIER = 1 + 10, GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11, - GF_DEFRAG_CMD_STOP_TIER = 1 + 12, GF_DEFRAG_CMD_DETACH_START = 1 + 13, GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14, GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15, @@ -467,75 +440,6 @@ struct dht_container { int local_subvol_index; }; -typedef enum tier_mode_ { - TIER_MODE_NONE = 0, - TIER_MODE_TEST, - TIER_MODE_WM -} tier_mode_t; - -typedef enum tier_pause_state_ { - TIER_RUNNING = 0, - TIER_REQUEST_PAUSE, - TIER_PAUSED -} tier_pause_state_t; - -/* This Structure is only used in tiering fixlayout */ -typedef struct gf_tier_fix_layout_arg { - xlator_t *this; - dict_t *fix_layout; - pthread_t thread_id; -} gf_tier_fix_layout_arg_t; - -typedef struct gf_tier_conf { - int is_tier; - int watermark_hi; - int watermark_low; - int watermark_last; - unsigned long block_size; - fsblkcnt_t blocks_total; - fsblkcnt_t blocks_used; - uint64_t max_migrate_bytes; - int max_migrate_files; - int query_limit; - tier_mode_t mode; - int percent_full; - /* These flags are only used for tier-compact */ - gf_boolean_t compact_active; - /* These 3 flags are set to true when the client changes the */ - /* compaction mode on the command line. */ - /* When they are set, the daemon will trigger compaction as */ - /* soon as possible to activate or deactivate compaction. */ - /* If in the middle of a compaction, then the switches take */ - /* effect on the next compaction, not the current one. */ - /* If the user switches it off, we want to avoid needless */ - /* compactions. */ - /* If the user switches it on, they want to compact as soon */ - /* as possible. */ - gf_boolean_t compact_mode_switched; - gf_boolean_t compact_mode_switched_hot; - gf_boolean_t compact_mode_switched_cold; - int tier_max_promote_size; - int tier_promote_frequency; - int tier_demote_frequency; - int tier_compact_hot_frequency; - int tier_compact_cold_frequency; - uint64_t st_last_promoted_size; - uint64_t st_last_demoted_size; - struct synctask *pause_synctask; - gf_timer_t *pause_timer; - pthread_mutex_t pause_mutex; - int promote_in_progress; - int demote_in_progress; - /* This Structure is only used in tiering fixlayout */ - gf_tier_fix_layout_arg_t tier_fix_layout_arg; - /* Indicates the index of the first queryfile picked - * in the last cycle of promote or demote */ - int32_t last_promote_qfile_index; - int32_t last_demote_qfile_index; - tier_pause_state_t pause_state; - char volname[GD_VOLUME_NAME_MAX + 1]; -} gf_tier_conf_t; - typedef struct nodeuuid_info { char info; /* Set to 1 is this is my node's uuid*/ uuid_t uuid; /* Store the nodeuuid as well for debugging*/ @@ -563,17 +467,10 @@ struct gf_defrag_info_ { int cmd; inode_t *root_inode; uuid_t node_uuid; - struct timeval start_time; + time_t start_time; uint32_t new_commit_hash; gf_defrag_status_t defrag_status; gf_defrag_pattern_list_t *defrag_pattern; - gf_tier_conf_t tier_conf; - - /*Data Tiering params for scanner*/ - uint64_t total_files_promoted; - uint64_t total_files_demoted; - int write_freq_threshold; - int read_freq_threshold; pthread_cond_t parallel_migration_cond; pthread_mutex_t dfq_mutex; @@ -609,7 +506,6 @@ typedef struct gf_defrag_info_ gf_defrag_info_t; struct dht_methods_s { int32_t (*migration_get_dst_subvol)(xlator_t *this, dht_local_t *local); int32_t (*migration_other)(xlator_t *this, gf_defrag_info_t *defrag); - int32_t (*migration_needed)(xlator_t *this); xlator_t *(*layout_search)(xlator_t *this, dht_layout_t *layout, const char *name); }; @@ -630,7 +526,7 @@ struct dht_conf { int subvolume_cnt; int32_t refresh_interval; gf_lock_t subvolume_lock; - struct timeval last_stat_fetch; + time_t last_stat_fetch; gf_lock_t layout_lock; dict_t *leaf_to_subvol; void *private; /* Can be used by wrapper xlators over @@ -752,6 +648,8 @@ struct dir_dfmeta { struct list_head **head; struct list_head **iterator; int *fetch_entries; + /* fds corresponding to local subvols only */ + fd_t **lfd; }; typedef struct dht_migrate_info { @@ -1238,24 +1136,6 @@ dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int gf_defrag_status_get(dht_conf_t *conf, dict_t *dict); -void -gf_defrag_set_pause_state(gf_tier_conf_t *tier_conf, tier_pause_state_t state); - -tier_pause_state_t -gf_defrag_get_pause_state(gf_tier_conf_t *tier_conf); - -int -gf_defrag_pause_tier(xlator_t *this, gf_defrag_info_t *defrag); - -tier_pause_state_t -gf_defrag_check_pause_tier(gf_tier_conf_t *defrag); - -int -gf_defrag_resume_tier(xlator_t *this, gf_defrag_info_t *defrag); - -int -gf_defrag_start_detach_tier(gf_defrag_info_t *defrag); - int gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output); @@ -1336,9 +1216,6 @@ dht_layout_missing_dirs(dht_layout_t *layout); int dht_refresh_layout(call_frame_t *frame); -gf_boolean_t -dht_is_tier_xlator(xlator_t *this); - int dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child, int32_t *op_errno); @@ -1451,7 +1328,7 @@ dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst, dict_t *src, int *uret, int *uflag); int -dht_dir_xattr_heal(xlator_t *this, dht_local_t *local); +dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno); int dht_common_mark_mdsxattr(call_frame_t *frame, int *errst, int flag); @@ -1499,4 +1376,9 @@ dht_create_lock(call_frame_t *frame, xlator_t *subvol); int dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local); +int +dht_dir_layout_error_check(xlator_t *this, inode_t *inode); + +int +dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol); #endif /* _DHT_H */ diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c index 27097ca2475..c0588828fdb 100644 --- a/xlators/cluster/dht/src/dht-diskusage.c +++ b/xlators/cluster/dht/src/dht-diskusage.c @@ -151,22 +151,18 @@ dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc) dht_conf_t *conf = NULL; call_frame_t *statfs_frame = NULL; dht_local_t *statfs_local = NULL; - struct timeval tv = { - 0, - }; loc_t tmp_loc = { 0, }; + time_t now; conf = this->private; - - gettimeofday(&tv, NULL); - + now = gf_time(); /* make it root gfid, should be enough to get the proper info back */ tmp_loc.gfid[15] = 1; - if (tv.tv_sec > (conf->refresh_interval + conf->last_stat_fetch.tv_sec)) { + if (now > (conf->refresh_interval + conf->last_stat_fetch)) { statfs_frame = copy_frame(frame); if (!statfs_frame) { goto err; @@ -198,7 +194,7 @@ dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc) statfs_local->params); } - conf->last_stat_fetch.tv_sec = tv.tv_sec; + conf->last_stat_fetch = now; } return 0; err: diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c index 023d4371c0e..3f2fe43d5f3 100644 --- a/xlators/cluster/dht/src/dht-helper.c +++ b/xlators/cluster/dht/src/dht-helper.c @@ -64,8 +64,8 @@ __dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst) ret = __fd_ctx_set(fd, this, value); if (ret < 0) { - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED, - "Failed to set fd ctx in fd=0x%p", fd); + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED, + "fd=0x%p", fd, NULL); GF_REF_PUT(fd_ctx); } out: @@ -96,8 +96,8 @@ dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst) /* Overwrite and hope for the best*/ fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst; UNLOCK(&fd->lock); - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE, - "Different dst found in the fd ctx"); + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE, + NULL); goto out; } @@ -383,9 +383,9 @@ dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame, &local->rebalance.flock, local->xattr_req); break; default: - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, - "Unknown FOP on fd (%p) on file %s @ %s", fd, - uuid_utoa(fd->inode->gfid), subvol->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p", + fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s", + subvol->name, NULL); break; } @@ -458,9 +458,9 @@ handle_err: break; default: - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, - "Unknown FOP on fd (%p) on file %s @ %s", fd, - uuid_utoa(fd->inode->gfid), subvol->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p", + fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s", + subvol->name, NULL); break; } @@ -513,10 +513,9 @@ dht_check_and_open_fd_on_subvol_task(void *data) fd, NULL, NULL); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED, - "Failed to open the fd" - " (%p, flags=0%o) on file %s @ %s", - fd, fd->flags, uuid_utoa(fd->inode->gfid), subvol->name); + gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED, + "fd=%p", fd, "flags=0%o", fd->flags, "gfid=%s", + uuid_utoa(fd->inode->gfid), "name=%s", subvol->name, NULL); /* This can happen if the cached subvol was updated in the * inode_ctx and the fd was opened on the new cached suvol * after this fop was wound on the old cached subvol. @@ -562,10 +561,8 @@ dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame) dht_check_and_open_fd_on_subvol_complete, frame, frame); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, 0, - "Failed to create synctask" - " to check and open fd=%p", - local->fd); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SYNCTASK_CREATE_FAILED, + "to-check-and-open fd=%p", local->fd, NULL); } return ret; @@ -674,9 +671,7 @@ dht_get_subvol_from_id(xlator_t *this, int client_id) ret = gf_asprintf(&sid, "%d", client_id); if (ret == -1) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, - "asprintf failed while " - "fetching subvol from the id"); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, NULL); goto out; } @@ -1336,9 +1331,9 @@ dht_migration_complete_check_task(void *data) * migrated by two different layers. Raise * a warning here. */ - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, - "%s: Found miginfo in the inode ctx", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid)); + gf_smsg( + this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s", + tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL); miginfo = (void *)(uintptr_t)tmp_miginfo; GF_REF_PUT(miginfo); @@ -1359,10 +1354,9 @@ dht_migration_complete_check_task(void *data) ret = syncop_lookup(this, &tmp_loc, &stbuf, 0, 0, 0); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED, - "%s: failed to lookup the file on %s", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), - this->name); + gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED, + "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), + "name=%s", this->name, NULL); local->op_errno = -ret; ret = -1; goto out; @@ -1370,18 +1364,15 @@ dht_migration_complete_check_task(void *data) dst_node = dht_subvol_get_cached(this, tmp_loc.inode); if (linkto_target && dst_node != linkto_target) { - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE, - "linkto target (%s) is " - "different from cached-subvol (%s). Treating %s as " - "destination subvol", - linkto_target->name, dst_node->name, dst_node->name); + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE, + "linkto_target_name=%s", linkto_target->name, "dst_name=%s", + dst_node->name, NULL); } if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, - "%s: gfid different on the target file on %s", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), - dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s", + tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), + "dst_name=%s", dst_node->name, NULL); ret = -1; local->op_errno = EIO; goto out; @@ -1463,12 +1454,10 @@ dht_migration_complete_check_task(void *data) (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd, NULL, NULL); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, -ret, - DHT_MSG_OPEN_FD_ON_DST_FAILED, - "failed" - " to open the fd" - " (%p, flags=0%o) on file %s @ %s", - iter_fd, iter_fd->flags, path, dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_OPEN_FD_ON_DST_FAILED, "id=%p", iter_fd, + "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s", + dst_node->name, NULL); open_failed = 1; local->op_errno = -ret; @@ -1622,9 +1611,9 @@ dht_rebalance_inprogress_task(void *data) * migrated by two different layers. Raise * a warning here. */ - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, - "%s: Found miginfo in the inode ctx", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid)); + gf_smsg( + this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s", + tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL); miginfo = (void *)(uintptr_t)tmp_miginfo; GF_REF_PUT(miginfo); } @@ -1633,17 +1622,16 @@ dht_rebalance_inprogress_task(void *data) } if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED, - "%s: failed to get the 'linkto' xattr", local->loc.path); + gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED, + "path=%s", local->loc.path, NULL); ret = -1; goto out; } dst_node = dht_linkfile_subvol(this, NULL, NULL, dict); if (!dst_node) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_NOT_FOUND, - "%s: failed to get the 'linkto' xattr from dict", - local->loc.path); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GET_XATTR_FAILED, + "path=%s", local->loc.path, NULL); ret = -1; goto out; } @@ -1660,20 +1648,17 @@ dht_rebalance_inprogress_task(void *data) /* lookup on dst */ ret = syncop_lookup(dst_node, &tmp_loc, &stbuf, NULL, NULL, NULL); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, -ret, - DHT_MSG_FILE_LOOKUP_ON_DST_FAILED, - "%s: failed to lookup the file on %s", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), - dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED, + "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), + "name=%s", dst_node->name, NULL); ret = -1; goto out; } if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, - "%s: gfid different on the target file on %s", - tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), - dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s", + tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), + "name=%s", dst_node->name, NULL); ret = -1; goto out; } @@ -1741,11 +1726,10 @@ dht_rebalance_inprogress_task(void *data) (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd, NULL, NULL); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, -ret, - DHT_MSG_OPEN_FD_ON_DST_FAILED, - "failed to send open " - "the fd (%p, flags=0%o) on file %s @ %s", - iter_fd, iter_fd->flags, path, dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_OPEN_FD_ON_DST_FAILED, "fd=%p", iter_fd, + "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s", + dst_node->name, NULL); ret = -1; open_failed = 1; } else { @@ -1777,9 +1761,8 @@ unlock: ret = dht_inode_ctx_set_mig_info(this, inode, src_node, dst_node); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED, - "%s: failed to set inode-ctx target file at %s", local->loc.path, - dst_node->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED, + "path=%s", local->loc.path, "name=%s", dst_node->name, NULL); goto out; } @@ -2001,10 +1984,9 @@ dht_heal_path(xlator_t *this, char *path, inode_table_t *itable) ret = syncop_lookup(this, &loc, &iatt, NULL, NULL, NULL); if (ret) { - gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED, - "Healing of path %s failed on subvolume %s for " - "directory %s", - path, this->name, bname); + gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED, + "path=%s", path, "subvolume=%s", this->name, "bname=%s", + bname, NULL); goto out; } @@ -2062,10 +2044,8 @@ dht_heal_full_path(void *data) ret = syncop_getxattr(source, &loc, &dict, GET_ANCESTRY_PATH_KEY, NULL, NULL); if (ret) { - gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED, - "Failed to get path from subvol %s. Aborting " - "directory healing.", - source->name); + gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_HEAL_ABORT, + "subvol=%s", source->name, NULL); goto out; } @@ -2103,6 +2083,7 @@ dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data) dht_local_t *local = NULL; xlator_t *this = NULL; int ret = -1; + int op_errno = 0; local = heal_frame->local; main_frame = local->main_frame; @@ -2112,10 +2093,12 @@ dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data) dht_set_fixed_dir_stat(&local->postparent); if (local->need_xattr_heal) { local->need_xattr_heal = 0; - ret = dht_dir_xattr_heal(this, local); - if (ret) - gf_msg(this->name, GF_LOG_ERROR, ret, DHT_MSG_DIR_XATTR_HEAL_FAILED, - "xattr heal failed for directory %s ", local->loc.path); + ret = dht_dir_xattr_heal(this, local, &op_errno); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, op_errno, + DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path, + NULL); + } } DHT_STACK_UNWIND(lookup, main_frame, 0, 0, local->inode, &local->stbuf, @@ -2203,8 +2186,8 @@ dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock, if (ret) { gf_uuid_unparse(inode->gfid, gfid); UNLOCK(&inode->lock); - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED, - "Failed to set lock_subvol in inode ctx for gfid %s", gfid); + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED, + "lock_subvol gfid=%s", gfid, NULL); goto post_unlock; } subvol = cached_subvol; @@ -2234,8 +2217,8 @@ dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret) inode = local->loc.inode ? local->loc.inode : local->fd->inode; } if (!inode) { - gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED, - "Found a NULL inode. Failed to unref the inode"); + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED, + NULL); goto out; } @@ -2261,11 +2244,8 @@ dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret) inode_unref(inode); } else { gf_uuid_unparse(inode->gfid, gfid); - gf_msg(this->name, GF_LOG_WARNING, 0, - DHT_MSG_LOCK_INODE_UNREF_FAILED, - "Unlock request failed for gfid %s." - "Failed to unref the inode", - gfid); + gf_smsg(this->name, GF_LOG_WARNING, 0, + DHT_MSG_LOCK_INODE_UNREF_FAILED, "gfid=%s", gfid, NULL); goto out; } default: @@ -2287,12 +2267,11 @@ dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst, int luret = -1; int luflag = -1; int i = 0; + char **xattrs_to_heal; if (!src || !dst) { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED, - "src or dst is NULL. Failed to set " - " dictionary value for path %s", - local->loc.path); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DST_NULL_SET_FAILED, + "path=%s", local->loc.path, NULL); return; } /* Check if any user xattr present in src dict and set @@ -2303,17 +2282,18 @@ dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst, and set it to dst dict, here index start from 1 because user xattr already checked in previous statement */ + + xattrs_to_heal = get_xattrs_to_heal(); + for (i = 1; xattrs_to_heal[i]; i++) { keyval = dict_get(src, xattrs_to_heal[i]); if (keyval) { luflag = 1; ret = dict_set(dst, xattrs_to_heal[i], keyval); if (ret) - gf_msg(this->name, GF_LOG_WARNING, ENOMEM, - DHT_MSG_DICT_SET_FAILED, - "Failed to set dictionary value:key = %s for " - "path %s", - xattrs_to_heal[i], local->loc.path); + gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, + DHT_MSG_DICT_SET_FAILED, "key=%s", xattrs_to_heal[i], + "path=%s", local->loc.path, NULL); keyval = NULL; } } diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c index eda2491e0ff..2f23ce90fbd 100644 --- a/xlators/cluster/dht/src/dht-inode-write.c +++ b/xlators/cluster/dht/src/dht-inode-write.c @@ -93,30 +93,28 @@ dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, /* Check if the rebalance phase1 is true */ if (IS_DHT_MIGRATION_PHASE1(postbuf)) { - if (!dht_is_tier_xlator(this)) { + if (!local->xattr_req) { + local->xattr_req = dict_new(); if (!local->xattr_req) { - local->xattr_req = dict_new(); - if (!local->xattr_req) { - gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, - "insufficient memory"); - local->op_errno = ENOMEM; - local->op_ret = -1; - goto out; - } - } - - ret = dict_set_uint32(local->xattr_req, - GF_PROTECT_FROM_EXTERNAL_WRITES, 1); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0, - "Failed to set key %s in dictionary", - GF_PROTECT_FROM_EXTERNAL_WRITES); + gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, + "insufficient memory"); local->op_errno = ENOMEM; local->op_ret = -1; goto out; } } + ret = dict_set_uint32(local->xattr_req, GF_PROTECT_FROM_EXTERNAL_WRITES, + 1); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0, + "Failed to set key %s in dictionary", + GF_PROTECT_FROM_EXTERNAL_WRITES); + local->op_errno = ENOMEM; + local->op_ret = -1; + goto out; + } + dht_iatt_merge(this, &local->stbuf, postbuf); dht_iatt_merge(this, &local->prebuf, prebuf); diff --git a/xlators/cluster/dht/src/dht-lock.c b/xlators/cluster/dht/src/dht-lock.c index f9bac4f97c8..638821ccee5 100644 --- a/xlators/cluster/dht/src/dht-lock.c +++ b/xlators/cluster/dht/src/dht-lock.c @@ -44,7 +44,8 @@ dht_log_lk_array(char *name, gf_loglevel_t log_level, dht_lock_t **lk_array, if (!lk_buf) goto out; - gf_msg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "%d. %s", i, lk_buf); + gf_smsg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "index=%d", i, + "lk_buf=%s", lk_buf, NULL); GF_FREE(lk_buf); } @@ -313,11 +314,9 @@ dht_unlock_entrylk_done(call_frame_t *frame, void *cookie, xlator_t *this, gfid); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_WARNING, op_errno, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "unlock failed on gfid: %s, stale lock might be left " - "in DHT_LAYOUT_HEAL_DOMAIN", - gfid); + gf_smsg(this->name, GF_LOG_WARNING, op_errno, + DHT_MSG_UNLOCK_GFID_FAILED, "gfid=%s", gfid, + "DHT_LAYOUT_HEAL_DOMAIN", NULL); } DHT_STACK_DESTROY(frame); @@ -339,9 +338,10 @@ dht_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, uuid_utoa_r(local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED, - "unlocking failed on %s:%s", - local->lock[0].ns.directory_ns.locks[lk_index]->xl->name, gfid); + gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED, + "name=%s", + local->lock[0].ns.directory_ns.locks[lk_index]->xl->name, + "gfid=%s", gfid, NULL); } else { local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0; } @@ -375,9 +375,9 @@ dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count, lock_frame = dht_lock_frame(frame); if (lock_frame == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED, - "cannot allocate a frame, not unlocking following " - "entrylks:"); + gf_smsg(frame->this->name, GF_LOG_WARNING, 0, + DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS, + NULL); dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count); goto done; @@ -385,9 +385,9 @@ dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count, ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk); if (ret < 0) { - gf_msg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED, - "storing locks in local failed, not unlocking " - "following entrylks:"); + gf_smsg(frame->this->name, GF_LOG_WARNING, 0, + DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK, + NULL); dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count); @@ -446,21 +446,17 @@ dht_unlock_entrylk_wrapper(call_frame_t *frame, dht_elock_wrap_t *entrylk) lock_frame = copy_frame(frame); if (lock_frame == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "mkdir (%s/%s) (path: %s): " - "copy frame failed", - pgfid, local->loc.name, local->loc.path); + gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM, + DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s", + local->loc.name, "path=%s", local->loc.path, NULL); goto done; } lock_local = dht_local_init(lock_frame, NULL, NULL, 0); if (lock_local == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "mkdir (%s/%s) (path: %s): " - "local creation failed", - pgfid, local->loc.name, local->loc.path); + gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM, + DHT_MSG_CREATE_FAILED, "local", "pgfid=%s", pgfid, "name=%s", + local->loc.name, "path=%s", local->loc.path, NULL); goto done; } @@ -700,9 +696,10 @@ dht_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, uuid_utoa_r(local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid, gfid); - gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED, - "unlocking failed on %s:%s", - local->lock[0].layout.my_layout.locks[lk_index]->xl->name, gfid); + gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED, + "name=%s", + local->lock[0].layout.my_layout.locks[lk_index]->xl->name, + "gfid=%s", gfid, NULL); } else { local->lock[0].layout.my_layout.locks[lk_index]->locked = 0; } @@ -727,11 +724,9 @@ dht_unlock_inodelk_done(call_frame_t *frame, void *cookie, xlator_t *this, gfid); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_WARNING, op_errno, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "unlock failed on gfid: %s, stale lock might be left " - "in DHT_LAYOUT_HEAL_DOMAIN", - gfid); + gf_smsg(this->name, GF_LOG_WARNING, op_errno, + DHT_MSG_UNLOCK_GFID_FAILED, "DHT_LAYOUT_HEAL_DOMAIN gfid=%s", + gfid, NULL); } DHT_STACK_DESTROY(frame); @@ -762,9 +757,9 @@ dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count, lock_frame = dht_lock_frame(frame); if (lock_frame == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED, - "cannot allocate a frame, not unlocking following " - "locks:"); + gf_smsg(frame->this->name, GF_LOG_WARNING, 0, + DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS, + NULL); dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count); goto done; @@ -772,9 +767,9 @@ dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count, ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk); if (ret < 0) { - gf_msg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED, - "storing locks in local failed, not unlocking " - "following locks:"); + gf_smsg(frame->this->name, GF_LOG_WARNING, 0, + DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK, + NULL); dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count); @@ -834,21 +829,17 @@ dht_unlock_inodelk_wrapper(call_frame_t *frame, dht_ilock_wrap_t *inodelk) lock_frame = copy_frame(frame); if (lock_frame == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "mkdir (%s/%s) (path: %s): " - "copy frame failed", - pgfid, local->loc.name, local->loc.path); + gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM, + DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s", + local->loc.name, "path=%s", local->loc.path, NULL); goto done; } lock_local = dht_local_init(lock_frame, NULL, NULL, 0); if (lock_local == NULL) { - gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, - DHT_MSG_PARENT_LAYOUT_CHANGED, - "mkdir (%s/%s) (path: %s): " - "local creation failed", - pgfid, local->loc.name, local->loc.path); + gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM, + DHT_MSG_CREATE_FAILED, "local", "gfid=%s", pgfid, "name=%s", + local->loc.name, "path=%s", local->loc.path, NULL); goto done; } @@ -1039,13 +1030,12 @@ dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, gfid); local->lock[0].layout.my_layout.op_ret = -1; local->lock[0].layout.my_layout.op_errno = op_errno; - gf_msg(this->name, GF_LOG_ERROR, op_errno, - DHT_MSG_INODELK_FAILED, - "inodelk failed on subvol %s. gfid:%s", - local->lock[0] - .layout.my_layout.locks[lk_index] - ->xl->name, - gfid); + gf_smsg(this->name, GF_LOG_ERROR, op_errno, + DHT_MSG_INODELK_FAILED, "subvol=%s", + local->lock[0] + .layout.my_layout.locks[lk_index] + ->xl->name, + "gfid=%s", gfid, NULL); goto cleanup; } break; @@ -1060,13 +1050,12 @@ dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, gfid); local->lock[0].layout.my_layout.op_ret = -1; local->lock[0].layout.my_layout.op_errno = op_errno; - gf_msg(this->name, GF_LOG_ERROR, op_errno, - DHT_MSG_INODELK_FAILED, - "inodelk failed on subvol %s. gfid:%s", - local->lock[0] - .layout.my_layout.locks[lk_index] - ->xl->name, - gfid); + gf_smsg(this->name, GF_LOG_ERROR, op_errno, + DHT_MSG_INODELK_FAILED, "subvol=%s", + local->lock[0] + .layout.my_layout.locks[lk_index] + ->xl->name, + "gfid=%s", gfid, NULL); goto cleanup; } break; @@ -1077,11 +1066,11 @@ dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, gfid); local->lock[0].layout.my_layout.op_ret = -1; local->lock[0].layout.my_layout.op_errno = op_errno; - gf_msg( + gf_smsg( this->name, GF_LOG_ERROR, op_errno, DHT_MSG_INODELK_FAILED, - "inodelk failed on subvol %s, gfid:%s", + "subvol=%s", local->lock[0].layout.my_layout.locks[lk_index]->xl->name, - gfid); + "gfid=%s", gfid, NULL); goto cleanup; } } @@ -1153,19 +1142,16 @@ dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count, lock_frame = dht_lock_frame(frame); if (lock_frame == NULL) { gf_uuid_unparse(tmp_local->loc.gfid, gfid); - gf_msg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED, - "memory allocation failed for lock_frame. gfid:%s" - " path:%s", - gfid, tmp_local->loc.path); + gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED, + "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL); goto out; } ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk); if (ret < 0) { gf_uuid_unparse(tmp_local->loc.gfid, gfid); - gf_msg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED, - "dht_local_lock_init failed, gfid: %s path:%s", gfid, - tmp_local->loc.path); + gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED, + "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL); goto out; } @@ -1246,11 +1232,10 @@ dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie, if (ret < 0) { local->op_ret = -1; local->op_errno = EIO; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, - DHT_MSG_ENTRYLK_ERROR, - "%s (%s/%s): " - "dht_blocking_entrylk failed after taking inodelk", - gf_fop_list[local->fop], pgfid, entrylk->locks[0]->basename); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, "fop=%s", + gf_fop_list[local->fop], "pgfid=%s", pgfid, "basename=%s", + entrylk->locks[0]->basename, NULL); goto err; } @@ -1310,10 +1295,9 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, ret = dht_build_parent_loc(this, &parent, loc, &op_errno); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED, - "gfid:%s (name:%s) (path: %s): " - "parent loc build failed", - loc->gfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED, + "gfid=%s", loc->gfid, "name=%s", loc->name, "path=%s", + loc->path, NULL); goto out; } gf_uuid_unparse(parent.gfid, pgfid); @@ -1322,10 +1306,10 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, inodelk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer); if (inodelk->locks == NULL) { local->op_errno = ENOMEM; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, DHT_MSG_NO_MEMORY, - "%s (%s/%s) (path: %s): " - "calloc failure", - gf_fop_list[local->fop], pgfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_CALLOC_FAILED, "fop=%s", gf_fop_list[local->fop], + "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path, + NULL); goto out; } @@ -1334,10 +1318,10 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, FAIL_ON_ANY_ERROR); if (inodelk->locks[0] == NULL) { local->op_errno = ENOMEM; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, DHT_MSG_NO_MEMORY, - "%s (%s/%s) (path: %s): " - "inodelk: lock allocation failed", - gf_fop_list[local->fop], pgfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_LOCK_ALLOC_FAILED, "inodelk-fop=%s", + gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s", + loc->name, "path=%s", loc->path, NULL); goto err; } inodelk->lk_count = count; @@ -1346,10 +1330,10 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, entrylk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer); if (entrylk->locks == NULL) { local->op_errno = ENOMEM; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, DHT_MSG_NO_MEMORY, - "%s (%s/%s) (path: %s): " - "entrylk: calloc failure", - gf_fop_list[local->fop], pgfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_CALLOC_FAILED, "entrylk-fop=%s", + gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s", + loc->name, "path=%s", loc->path, NULL); goto err; } @@ -1359,10 +1343,10 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, FAIL_ON_ANY_ERROR); if (entrylk->locks[0] == NULL) { local->op_errno = ENOMEM; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, DHT_MSG_NO_MEMORY, - "%s (%s/%s) (path: %s): " - "entrylk: lock allocation failed", - gf_fop_list[local->fop], pgfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_LOCK_ALLOC_FAILED, "entrylk-fop=%s", + gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s", + loc->name, "path=%s", loc->path, NULL); goto err; } @@ -1376,11 +1360,11 @@ dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol, dht_blocking_entrylk_after_inodelk); if (ret < 0) { local->op_errno = EIO; - gf_msg(this->name, GF_LOG_WARNING, local->op_errno, - DHT_MSG_INODELK_ERROR, - "%s (%s/%s) (path: %s): " - "dht_blocking_inodelk failed", - gf_fop_list[local->fop], pgfid, loc->name, loc->path); + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_BLOCK_INODELK_FAILED, "fop=%s", gf_fop_list[local->fop], + "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path, + NULL); + goto err; } diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h index 33f9832395b..e3c4471334a 100644 --- a/xlators/cluster/dht/src/dht-mem-types.h +++ b/xlators/cluster/dht/src/dht-mem-types.h @@ -30,10 +30,7 @@ enum gf_dht_mem_types_ { gf_dht_mt_container_t, gf_dht_mt_octx_t, gf_dht_mt_miginfo_t, - gf_tier_mt_bricklist_t, - gf_tier_mt_ipc_ctr_params_t, gf_dht_mt_fd_ctx_t, - gf_tier_mt_qfile_array_t, gf_dht_ret_cache_t, gf_dht_nodeuuids_t, gf_dht_mt_end diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h index d803e28fafc..601f8dad78b 100644 --- a/xlators/cluster/dht/src/dht-messages.h +++ b/xlators/cluster/dht/src/dht-messages.h @@ -38,12 +38,11 @@ GLFS_MSGID( DHT_MSG_REBALANCE_STATUS, DHT_MSG_REBALANCE_STOPPED, DHT_MSG_RENAME_FAILED, DHT_MSG_SETATTR_FAILED, DHT_MSG_SUBVOL_INSUFF_INODES, DHT_MSG_SUBVOL_INSUFF_SPACE, DHT_MSG_UNLINK_FAILED, - DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT, DHT_MSG_LOG_TIER_ERROR, - DHT_MSG_LOG_TIER_STATUS, DHT_MSG_GET_XATTR_FAILED, - DHT_MSG_FILE_LOOKUP_FAILED, DHT_MSG_OPEN_FD_FAILED, - DHT_MSG_SET_INODE_CTX_FAILED, DHT_MSG_UNLOCKING_FAILED, - DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO, DHT_MSG_CHUNK_SIZE_INFO, - DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR, + DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT, + DHT_MSG_GET_XATTR_FAILED, DHT_MSG_FILE_LOOKUP_FAILED, + DHT_MSG_OPEN_FD_FAILED, DHT_MSG_SET_INODE_CTX_FAILED, + DHT_MSG_UNLOCKING_FAILED, DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO, + DHT_MSG_CHUNK_SIZE_INFO, DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR, DHT_MSG_LAYOUT_SORT_FAILED, DHT_MSG_REGEX_INFO, DHT_MSG_FOPEN_FAILED, DHT_MSG_SET_HOSTNAME_FAILED, DHT_MSG_BRICK_ERROR, DHT_MSG_SYNCOP_FAILED, DHT_MSG_MIGRATE_INFO, DHT_MSG_SOCKET_ERROR, DHT_MSG_CREATE_FD_FAILED, @@ -69,8 +68,7 @@ GLFS_MSGID( DHT_MSG_INIT_LOCAL_SUBVOL_FAILED, DHT_MSG_SYS_CALL_GET_TIME_FAILED, DHT_MSG_NO_DISK_USAGE_STATUS, DHT_MSG_SUBVOL_DOWN_ERROR, DHT_MSG_REBAL_THROTTLE_INFO, DHT_MSG_COMMIT_HASH_INFO, - DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_LOG_IPC_TIER_ERROR, - DHT_MSG_TIER_PAUSED, DHT_MSG_TIER_RESUME, DHT_MSG_SETTLE_HASH_FAILED, + DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_SETTLE_HASH_FAILED, DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, DHT_MSG_FD_CTX_SET_FAILED, DHT_MSG_STALE_LOOKUP, DHT_MSG_PARENT_LAYOUT_CHANGED, DHT_MSG_LOCK_MIGRATION_FAILED, DHT_MSG_LOCK_INODE_UNREF_FAILED, @@ -96,15 +94,13 @@ GLFS_MSGID( DHT_MSG_UNLOCK_FILE_FAILED, DHT_MSG_REMOVE_XATTR_FAILED, DHT_MSG_DATA_MIGRATE_ABORT, DHT_MSG_DEFRAG_NULL, DHT_MSG_PARENT_NULL, DHT_MSG_GFID_NOT_PRESENT, DHT_MSG_CHILD_LOC_FAILED, - DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, - DHT_MSG_TIER_FIX_LAYOUT_STARTED, DHT_MSG_FIX_NOT_COMP, - DHT_MSG_REMOVE_TIER_FAILED, DHT_MSG_SUBVOL_DETER_FAILED, - DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID, DHT_MSG_SIZE_FILE, - DHT_MSG_GET_DATA_SIZE_FAILED, DHT_MSG_PTHREAD_JOIN_FAILED, - DHT_MSG_COUNTER_THREAD_CREATE_FAILED, DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, - DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE, DHT_MSG_ABORT_REBALANCE, - DHT_MSG_CREATE_TASK_REBAL_FAILED, DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, - DHT_MSG_MIG_TIER_PAUSED, DHT_MSG_ADD_CHOICES_ERROR, + DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, DHT_MSG_FIX_NOT_COMP, + DHT_MSG_SUBVOL_DETER_FAILED, DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID, + DHT_MSG_SIZE_FILE, DHT_MSG_GET_DATA_SIZE_FAILED, + DHT_MSG_PTHREAD_JOIN_FAILED, DHT_MSG_COUNTER_THREAD_CREATE_FAILED, + DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE, + DHT_MSG_ABORT_REBALANCE, DHT_MSG_CREATE_TASK_REBAL_FAILED, + DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, DHT_MSG_ADD_CHOICES_ERROR, DHT_MSG_GET_CHOICES_ERROR, DHT_MSG_PREPARE_STATUS_ERROR, DHT_MSG_SET_CHOICE_FAILED, DHT_MSG_SET_HASHED_SUBVOL_FAILED, DHT_MSG_XATTR_HEAL_NOT_POSS, DHT_MSG_LINKTO_FILE_FAILED, @@ -125,7 +121,14 @@ GLFS_MSGID( DHT_MSG_LINK_INODE_FAILED, DHT_MSG_SELFHEAL_FAILED, DHT_MSG_NO_MDS_SUBVOL, DHT_MSG_LIST_XATTRS_FAILED, DHT_MSG_RESET_INTER_XATTR_FAILED, DHT_MSG_MDS_DOWN_UNABLE_TO_SET, DHT_MSG_WIND_UNLOCK_FAILED, - DHT_MSG_COMMIT_HASH_FAILED); + DHT_MSG_COMMIT_HASH_FAILED, DHT_MSG_UNLOCK_GFID_FAILED, + DHT_MSG_UNLOCK_FOLLOW_ENTRYLK, DHT_MSG_COPY_FRAME_FAILED, + DHT_MSG_UNLOCK_FOLLOW_LOCKS, DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, + DHT_MSG_CALLOC_FAILED, DHT_MSG_LOCK_ALLOC_FAILED, + DHT_MSG_BLOCK_INODELK_FAILED, + DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK, + DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS, + DHT_MSG_DST_NULL_SET_FAILED); #define DHT_MSG_FD_CTX_SET_FAILED_STR "Failed to set fd ctx" #define DHT_MSG_INVALID_VALUE_STR "Different dst found in the fd ctx" @@ -173,7 +176,6 @@ GLFS_MSGID( "adding bricks" #define DHT_MSG_NEW_TARGET_FOUND_STR "New target found for file" #define DHT_MSG_INSUFF_MEMORY_STR "insufficient memory" -#define DHT_MSG_MIG_TIER_PAUSED_STR "Migrate file paused" #define DHT_MSG_SET_XATTR_FAILED_STR "failed to set xattr" #define DHT_MSG_SET_MODE_FAILED_STR "failed to set mode" #define DHT_MSG_FILE_EXISTS_IN_DEST_STR "file exists in destination" @@ -215,17 +217,14 @@ GLFS_MSGID( #define DHT_MSG_GFID_NOT_PRESENT_STR "gfid not present" #define DHT_MSG_CHILD_LOC_FAILED_STR "Child loc build failed" #define DHT_MSG_SET_LOOKUP_FAILED_STR "Failed to set lookup" -#define DHT_MSG_LOG_TIER_STATUS_STR "lookup to cold tier on attach heal failed" #define DHT_MSG_DIR_LOOKUP_FAILED_STR "lookup failed" #define DHT_MSG_DIR_REMOVED_STR "Dir renamed or removed. Skipping" #define DHT_MSG_READDIR_ERROR_STR "readdir failed, Aborting fix-layout" #define DHT_MSG_SETTLE_HASH_FAILED_STR "Settle hash failed" #define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED_STR "gf_defrag_process_dir failed" -#define DHT_MSG_TIER_FIX_LAYOUT_STARTED_STR "Tiering fix layout started" #define DHT_MSG_FIX_NOT_COMP_STR \ "Unable to retrieve fixlayout xattr. Assume background fix layout not " \ "complete" -#define DHT_MSG_REMOVE_TIER_FAILED_STR "Failed removing tier fix layout xattr" #define DHT_MSG_SUBVOL_DETER_FAILED_STR \ "local subvolume determination failed with error" #define DHT_MSG_LOCAL_SUBVOL_STR "local subvol" @@ -241,8 +240,6 @@ GLFS_MSGID( #define DHT_MSG_MIGRATION_INIT_QUEUE_FAILED_STR \ "Failed to initialise migration queue" #define DHT_MSG_REBALANCE_STOPPED_STR "Received stop command on rebalance" -#define DHT_MSG_TIER_RESUME_STR "Pause end. Resume tiering" -#define DHT_MSG_TIER_PAUSED_STR "Pause tiering" #define DHT_MSG_PAUSED_TIMEOUT_STR "Request pause timer timeout" #define DHT_MSG_WOKE_STR "woken" #define DHT_MSG_ABORT_REBALANCE_STR "Aborting rebalance" @@ -364,5 +361,26 @@ GLFS_MSGID( #define DHT_MSG_WIND_UNLOCK_FAILED_STR \ "Winding unlock failed: stale locks left on brick" #define DHT_MSG_COMMIT_HASH_FAILED_STR "Directory commit hash updaten failed" +#define DHT_MSG_LK_ARRAY_INFO_STR "lk info" +#define DHT_MSG_UNLOCK_GFID_FAILED_STR \ + "unlock failed on gfid: stale lock might be left" +#define DHT_MSG_UNLOCKING_FAILED_STR "unlocking failed" +#define DHT_MSG_UNLOCK_FOLLOW_ENTRYLK_STR "not unlocking following entrylks" +#define DHT_MSG_COPY_FRAME_FAILED_STR "copy frame failed" +#define DHT_MSG_UNLOCK_FOLLOW_LOCKS_STR "not unlocking following locks" +#define DHT_MSG_INODELK_FAILED_STR "inodelk failed on subvol" +#define DHT_MSG_LOCK_FRAME_FAILED_STR "memory allocation failed for lock_frame" +#define DHT_MSG_LOCAL_LOCK_INIT_FAILED_STR "dht_local_lock_init failed" +#define DHT_MSG_ENTRYLK_FAILED_AFT_INODELK_STR \ + "dht_blocking_entrylk failed after taking inodelk" +#define DHT_MSG_BLOCK_INODELK_FAILED_STR "dht_blocking_inodelk failed" +#define DHT_MSG_CALLOC_FAILED_STR "calloc failed" +#define DHT_MSG_LOCK_ALLOC_FAILED_STR "lock allocation failed" +#define DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS_STR \ + "cannot allocate a frame, not unlocking following entrylks" +#define DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK_STR \ + "storing locks in local failed, not unlocking following entrylks" +#define DHT_MSG_DST_NULL_SET_FAILED_STR \ + "src or dst is NULL, Failed to set dictionary value" #endif /* _DHT_MESSAGES_H_ */ diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c index 0d68ed53e08..8ba8082bd86 100644 --- a/xlators/cluster/dht/src/dht-rebalance.c +++ b/xlators/cluster/dht/src/dht-rebalance.c @@ -16,8 +16,8 @@ #include "glusterfs/compat-errno.h" // for ENODATA on BSD #define GF_DISK_SECTOR_SIZE 512 -#define DHT_REBALANCE_PID 4242 /* Change it if required */ -#define DHT_REBALANCE_BLKSIZE (1024 * 1024) /* 1 MB */ +#define DHT_REBALANCE_PID 4242 /* Change it if required */ +#define DHT_REBALANCE_BLKSIZE 1048576 /* 1 MB */ #define MAX_MIGRATE_QUEUE_COUNT 500 #define MIN_MIGRATE_QUEUE_COUNT 200 #define MAX_REBAL_TYPE_SIZE 16 @@ -45,7 +45,10 @@ gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt) if (meta) { for (i = 0; i < local_subvols_cnt; i++) { - gf_dirent_free(&meta->equeue[i]); + if (meta->equeue) + gf_dirent_free(&meta->equeue[i]); + if (meta->lfd && meta->lfd[i]) + fd_unref(meta->lfd[i]); } GF_FREE(meta->equeue); @@ -53,6 +56,7 @@ gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt) GF_FREE(meta->iterator); GF_FREE(meta->offset_var); GF_FREE(meta->fetch_entries); + GF_FREE(meta->lfd); GF_FREE(meta); } } @@ -84,26 +88,6 @@ dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret) return; } -static gf_boolean_t -dht_is_tier_command(int cmd) -{ - gf_boolean_t is_tier = _gf_false; - - switch (cmd) { - case GF_DEFRAG_CMD_START_TIER: - case GF_DEFRAG_CMD_STATUS_TIER: - case GF_DEFRAG_CMD_START_DETACH_TIER: - case GF_DEFRAG_CMD_STOP_DETACH_TIER: - case GF_DEFRAG_CMD_PAUSE_TIER: - case GF_DEFRAG_CMD_RESUME_TIER: - is_tier = _gf_true; - break; - default: - break; - } - return is_tier; -} - static int dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status) { @@ -112,8 +96,6 @@ dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status) char *tmpstr = NULL; char *ptr = NULL; char *suffix = "-dht"; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; int len = 0; eventtypes_t event = EVENT_LAST; @@ -132,21 +114,14 @@ dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status) break; } - if (dht_is_tier_command(cmd)) { - /* We should have the tier volume name*/ - conf = this->private; - defrag = conf->defrag; - volname = defrag->tier_conf.volname; - } else { - /* DHT volume */ - len = strlen(this->name) - strlen(suffix); - tmpstr = gf_strdup(this->name); - if (tmpstr) { - ptr = tmpstr + len; - if (!strcmp(ptr, suffix)) { - tmpstr[len] = '\0'; - volname = tmpstr; - } + /* DHT volume */ + len = strlen(this->name) - strlen(suffix); + tmpstr = gf_strdup(this->name); + if (tmpstr) { + ptr = tmpstr + len; + if (!strcmp(ptr, suffix)) { + tmpstr[len] = '\0'; + volname = tmpstr; } } @@ -172,75 +147,6 @@ dht_strip_out_acls(dict_t *dict) } } -static int -dht_write_with_holes(xlator_t *to, fd_t *fd, struct iovec *vec, int count, - int32_t size, off_t offset, struct iobref *iobref, - int *fop_errno) -{ - int i = 0; - int ret = -1; - int start_idx = 0; - int tmp_offset = 0; - int write_needed = 0; - int buf_len = 0; - int size_pending = 0; - char *buf = NULL; - - /* loop through each vector */ - for (i = 0; i < count; i++) { - buf = vec[i].iov_base; - buf_len = vec[i].iov_len; - - for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len; - start_idx += GF_DISK_SECTOR_SIZE) { - if (mem_0filled(buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) { - write_needed = 1; - continue; - } - - if (write_needed) { - ret = syncop_write( - to, fd, (buf + tmp_offset), (start_idx - tmp_offset), - (offset + tmp_offset), iobref, 0, NULL, NULL); - /* 'path' will be logged in calling function */ - if (ret < 0) { - gf_log(THIS->name, GF_LOG_WARNING, "failed to write (%s)", - strerror(-ret)); - *fop_errno = -ret; - ret = -1; - goto out; - } - - write_needed = 0; - } - tmp_offset = start_idx + GF_DISK_SECTOR_SIZE; - } - - if ((start_idx < buf_len) || write_needed) { - /* This means, last chunk is not yet written.. write it */ - ret = syncop_write(to, fd, (buf + tmp_offset), - (buf_len - tmp_offset), (offset + tmp_offset), - iobref, 0, NULL, NULL); - if (ret < 0) { - /* 'path' will be logged in calling function */ - gf_log(THIS->name, GF_LOG_WARNING, "failed to write (%s)", - strerror(-ret)); - *fop_errno = -ret; - ret = -1; - goto out; - } - } - - size_pending = (size - buf_len); - if (!size_pending) - break; - } - - ret = size; -out: - return ret; -} - /* return values: -1 : failure @@ -648,7 +554,7 @@ out: static int __dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf, fd_t **dst_fd, - int *fop_errno) + int *fop_errno, int file_has_holes) { int ret = -1; int ret2 = -1; @@ -703,26 +609,23 @@ __dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from, goto out; } - if (!!dht_is_tier_xlator(this)) { - xdata = dict_new(); - if (!xdata) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, - DHT_MSG_MIGRATE_FILE_FAILED, "%s: dict_new failed)", - loc->path); - goto out; - } + xdata = dict_new(); + if (!xdata) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: dict_new failed)", loc->path); + goto out; + } - ret = dict_set_int32(xdata, GF_CLEAN_WRITE_PROTECTION, 1); - if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, - "%s: failed to set dictionary value: key = %s ", loc->path, - GF_CLEAN_WRITE_PROTECTION); - goto out; - } + ret = dict_set_int32_sizen(xdata, GF_CLEAN_WRITE_PROTECTION, 1); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, + "%s: failed to set dictionary value: key = %s ", loc->path, + GF_CLEAN_WRITE_PROTECTION); + goto out; } ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL); @@ -817,7 +720,7 @@ __dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from, /* No need to bother about 0 byte size files */ if (stbuf->ia_size > 0) { - if (conf->use_fallocate) { + if (conf->use_fallocate && !file_has_holes) { ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL); if (ret < 0) { if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) { @@ -844,9 +747,7 @@ __dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from, goto out; } } - } - - if (!conf->use_fallocate) { + } else { ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL, NULL); if (ret < 0) { @@ -1097,22 +998,90 @@ __dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag, int ret = 0; int count = 0; off_t offset = 0; + off_t data_offset = 0; + off_t hole_offset = 0; struct iovec *vector = NULL; struct iobref *iobref = NULL; uint64_t total = 0; size_t read_size = 0; + size_t data_block_size = 0; dict_t *xdata = NULL; dht_conf_t *conf = NULL; conf = this->private; + /* if file size is '0', no need to enter this loop */ while (total < ia_size) { - read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) - ? DHT_REBALANCE_BLKSIZE - : (ia_size - total)); + /* This is a regular file - read it sequentially */ + if (!hole_exists) { + read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) + ? DHT_REBALANCE_BLKSIZE + : (ia_size - total)); + } else { + /* This is a sparse file - read only the data segments in the file + */ + + /* If the previous data block is fully copied, find the next data + * segment + * starting at the offset of the last read and written byte, */ + if (data_block_size <= 0) { + ret = syncop_seek(from, src, offset, GF_SEEK_DATA, NULL, + &data_offset); + if (ret) { + if (ret == -ENXIO) + ret = 0; /* No more data segments */ + else + *fop_errno = -ret; /* Error occurred */ + + break; + } + + /* If the position of the current data segment is greater than + * the position of the next hole, find the next hole in order to + * calculate the length of the new data segment */ + if (data_offset > hole_offset) { + /* Starting at the offset of the last data segment, find the + * next hole */ + ret = syncop_seek(from, src, data_offset, GF_SEEK_HOLE, + NULL, &hole_offset); + if (ret) { + /* If an error occurred here it's a real error because + * if the seek for a data segment was successful then + * necessarily another hole must exist (EOF is a hole) + */ + *fop_errno = -ret; + break; + } + + /* Calculate the total size of the current data block */ + data_block_size = hole_offset - data_offset; + } + } else { + /* There is still data in the current segment, move the + * data_offset to the position of the last written byte */ + data_offset = offset; + } + + /* Calculate how much data needs to be read and written. If the data + * segment's length is bigger than DHT_REBALANCE_BLKSIZE, read and + * write DHT_REBALANCE_BLKSIZE data length and the rest in the + * next iteration(s) */ + read_size = ((data_block_size > DHT_REBALANCE_BLKSIZE) + ? DHT_REBALANCE_BLKSIZE + : data_block_size); + + /* Calculate the remaining size of the data block - maybe there's no + * need to seek for data in the next iteration */ + data_block_size -= read_size; + + /* Set offset to the offset of the data segment so read and write + * will have the correct position */ + offset = data_offset; + } ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count, &iobref, NULL, NULL, NULL); + if (!ret || (ret < 0)) { if (!ret) { /* File was probably truncated*/ @@ -1124,57 +1093,42 @@ __dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag, break; } - if (hole_exists) { - ret = dht_write_with_holes(to, dst, vector, count, ret, offset, - iobref, fop_errno); - } else { - if (!conf->force_migration && !dht_is_tier_xlator(this)) { + if (!conf->force_migration) { + if (!xdata) { + xdata = dict_new(); if (!xdata) { - xdata = dict_new(); - if (!xdata) { - gf_msg("dht", GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "insufficient memory"); - ret = -1; - *fop_errno = ENOMEM; - break; - } + gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "insufficient memory"); + ret = -1; + *fop_errno = ENOMEM; + break; + } - /* Fail this write and abort rebalance if we - * detect a write from client since migration of - * this file started. This is done to avoid - * potential data corruption due to out of order - * writes from rebalance and client to the same - * region (as compared between src and dst - * files). See - * https://github.com/gluster/glusterfs/issues/308 - * for more details. - */ - ret = dict_set_int32(xdata, GF_AVOID_OVERWRITE, 1); - if (ret) { - gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM, - "failed to set dict"); - ret = -1; - *fop_errno = ENOMEM; - break; - } + /* Fail this write and abort rebalance if we + * detect a write from client since migration of + * this file started. This is done to avoid + * potential data corruption due to out of order + * writes from rebalance and client to the same + * region (as compared between src and dst + * files). See + * https://github.com/gluster/glusterfs/issues/308 + * for more details. + */ + ret = dict_set_int32_sizen(xdata, GF_AVOID_OVERWRITE, 1); + if (ret) { + gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM, + "failed to set dict"); + ret = -1; + *fop_errno = ENOMEM; + break; } } - ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL, - NULL, xdata, NULL); - if (ret < 0) { - *fop_errno = -ret; - } - } - - if ((defrag && defrag->cmd == GF_DEFRAG_CMD_START_TIER) && - (gf_defrag_get_pause_state(&defrag->tier_conf) != TIER_RUNNING)) { - gf_msg("tier", GF_LOG_INFO, 0, DHT_MSG_TIER_PAUSED, - "Migrate file paused"); - ret = -1; } + ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL, + NULL, xdata, NULL); if (ret < 0) { + *fop_errno = -ret; break; } @@ -1568,6 +1522,7 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, xlator_t *old_target = NULL; xlator_t *hashed_subvol = NULL; fd_t *linkto_fd = NULL; + dict_t *xdata = NULL; if (from == to) { gf_msg_debug(this->name, 0, @@ -1578,21 +1533,6 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, goto out; } - /* If defrag is NULL, it should be assumed that migration is triggered - * from client using the trusted.distribute.migrate-data virtual xattr - */ - defrag = conf->defrag; - - /* migration of files from clients is restricted to non-tiered clients - * for now */ - if (!defrag && dht_is_tier_xlator(this)) { - ret = ENOTSUP; - goto out; - } - - if (defrag && defrag->tier_conf.is_tier) - log_level = GF_LOG_TRACE; - gf_log(this->name, log_level, "%s: attempting to move from %s to %s", loc->path, from->name, to->name); @@ -1739,9 +1679,13 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, goto out; } + /* Try to preserve 'holes' while migrating data */ + if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE)) + file_has_holes = 1; + /* create the destination, with required modes/xattr */ ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd, - fop_errno); + fop_errno, file_has_holes); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Create dst failed" @@ -1785,8 +1729,8 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, * destination. We need to do update this only post migration * as in case of failure the linkto needs to point to the source * subvol */ - ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, - &dst_fd, fop_errno); + ret = __dht_rebalance_create_dst_file( + this, to, from, loc, &stbuf, &dst_fd, fop_errno, file_has_holes); if (ret) { gf_log(this->name, GF_LOG_ERROR, "Create dst failed" @@ -1873,9 +1817,6 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, ret = 0; goto out; } - /* Try to preserve 'holes' while migrating data */ - if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE)) - file_has_holes = 1; ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd, stbuf.ia_size, file_has_holes, @@ -1890,7 +1831,15 @@ dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, /* TODO: Sync the locks */ - ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, NULL, NULL); + xdata = dict_new(); + if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) { + gf_log(this->name, GF_LOG_ERROR, + "%s: failed to set last-fsync flag on " + "%s (%s)", + loc->path, to->name, strerror(ENOMEM)); + } + + ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, xdata, NULL); if (ret) { gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)", loc->path, to->name, strerror(-ret)); @@ -2333,14 +2282,12 @@ out: } } - if (!dht_is_tier_xlator(this)) { - lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, - NULL, NULL); - if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) { - gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0, - "%s: removexattr failed key %s", loc->path, - GF_PROTECT_FROM_EXTERNAL_WRITES); - } + lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, NULL, + NULL); + if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) { + gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0, + "%s: removexattr failed key %s", loc->path, + GF_PROTECT_FROM_EXTERNAL_WRITES); } if (dict) @@ -2353,11 +2300,15 @@ out: if (dst_fd) syncop_close(dst_fd); + if (src_fd) syncop_close(src_fd); if (linkto_fd) syncop_close(linkto_fd); + if (xdata) + dict_unref(xdata); + loc_wipe(&tmp_loc); loc_wipe(&parent_loc); @@ -2587,10 +2538,10 @@ out: * all hardlinks. */ -int +gf_boolean_t gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid) { - int ret = 0; + gf_boolean_t ret = _gf_false; int i = local_subvol_index; char *str = NULL; uint32_t hashval = 0; @@ -2612,12 +2563,11 @@ gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid) } str = uuid_utoa_r(gfid, buf); - ret = dht_hash_compute(this, 0, str, &hashval); - if (ret == 0) { + if (dht_hash_compute(this, 0, str, &hashval) == 0) { index = (hashval % entry->count); if (entry->elements[index].info == REBAL_NODEUUID_MINE) { /* Index matches this node's nodeuuid.*/ - ret = 1; + ret = _gf_true; goto out; } @@ -2630,12 +2580,12 @@ gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid) /* None of the bricks in the subvol are up. * CHILD_DOWN will kill the process soon */ - return 0; + return _gf_false; } if (entry->elements[index].info == REBAL_NODEUUID_MINE) { /* Index matches this node's nodeuuid.*/ - ret = 1; + ret = _gf_true; goto out; } } @@ -2684,6 +2634,7 @@ gf_defrag_migrate_single_file(void *opaque) struct iatt *iatt_ptr = NULL; gf_boolean_t update_skippedcount = _gf_true; int i = 0; + gf_boolean_t should_i_migrate = 0; rebal_entry = (struct dht_container *)opaque; if (!rebal_entry) { @@ -2738,11 +2689,29 @@ gf_defrag_migrate_single_file(void *opaque) goto out; } + should_i_migrate = gf_defrag_should_i_migrate( + this, rebal_entry->local_subvol_index, entry->d_stat.ia_gfid); + gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid); gf_uuid_copy(entry_loc.pargfid, loc->gfid); ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL); + + if (!should_i_migrate) { + /* this node isn't supposed to migrate the file. suppressing any + * potential error from lookup as this file is under migration by + * another node */ + if (ret) { + gf_msg_debug(this->name, -ret, + "Ignoring lookup failure: node isn't migrating %s", + entry_loc.path); + ret = 0; + } + gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path); + goto out; + } + if (ret) { gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, "Migrate file failed: %s lookup failed", entry_loc.path); @@ -2763,12 +2732,6 @@ gf_defrag_migrate_single_file(void *opaque) goto out; } - if (!gf_defrag_should_i_migrate(this, rebal_entry->local_subvol_index, - entry->d_stat.ia_gfid)) { - gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path); - goto out; - } - iatt_ptr = &iatt; hashed_subvol = dht_subvol_get_hashed(this, &entry_loc); @@ -2911,8 +2874,7 @@ gf_defrag_migrate_single_file(void *opaque) if (defrag->stats == _gf_true) { gettimeofday(&end, NULL); - elapsed = (end.tv_sec - start.tv_sec) * 1e6 + - (end.tv_usec - start.tv_usec); + elapsed = gf_tvdiff(&start, &end); gf_log(this->name, GF_LOG_INFO, "Migration of " "file:%s size:%" PRIu64 @@ -3091,9 +3053,9 @@ int static gf_defrag_get_entry(xlator_t *this, int i, dht_conf_t *conf, gf_defrag_info_t *defrag, fd_t *fd, dict_t *migrate_data, struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req, - int *should_commit_hash, int *perrno) + int *perrno) { - int ret = -1; + int ret = 0; char is_linkfile = 0; gf_dirent_t *df_entry = NULL; struct dht_container *tmp_container = NULL; @@ -3109,6 +3071,13 @@ int static gf_defrag_get_entry(xlator_t *this, int i, } if (dir_dfmeta->fetch_entries[i] == 1) { + if (!fd) { + dir_dfmeta->fetch_entries[i] = 0; + dir_dfmeta->offset_var[i].readdir_done = 1; + ret = 0; + goto out; + } + ret = syncop_readdirp(conf->local_subvols[i], fd, 131072, dir_dfmeta->offset_var[i].offset, &(dir_dfmeta->equeue[i]), xattr_req, NULL); @@ -3268,7 +3237,6 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, dict_t *migrate_data, int *perrno) { int ret = -1; - fd_t *fd = NULL; dht_conf_t *conf = NULL; gf_dirent_t entries; dict_t *xattr_req = NULL; @@ -3289,7 +3257,7 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, int dfc_index = 0; int throttle_up = 0; struct dir_dfmeta *dir_dfmeta = NULL; - int should_commit_hash = 1; + xlator_t *old_THIS = NULL; gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path); gettimeofday(&dir_start, NULL); @@ -3302,28 +3270,53 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, goto out; } - fd = fd_create(loc->inode, defrag->pid); - if (!fd) { - gf_log(this->name, GF_LOG_ERROR, "Failed to create fd"); + old_THIS = THIS; + THIS = this; + + dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer); + if (!dir_dfmeta) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL"); ret = -1; goto out; } - ret = syncop_opendir(this, loc, fd, NULL, NULL); - if (ret) { - gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_DATA_FAILED, - "Migrate data failed: Failed to open dir %s", loc->path); - *perrno = -ret; + dir_dfmeta->lfd = GF_CALLOC(local_subvols_cnt, sizeof(fd_t *), + gf_common_mt_pointer); + if (!dir_dfmeta->lfd) { + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY, + "for dir_dfmeta", NULL); ret = -1; + *perrno = ENOMEM; goto out; } - fd_bind(fd); - dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer); - if (!dir_dfmeta) { - gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL"); - ret = -1; - goto out; + for (i = 0; i < local_subvols_cnt; i++) { + dir_dfmeta->lfd[i] = fd_create(loc->inode, defrag->pid); + if (!dir_dfmeta->lfd[i]) { + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_FD_CREATE_FAILED, + NULL); + *perrno = ENOMEM; + ret = -1; + goto out; + } + + ret = syncop_opendir(conf->local_subvols[i], loc, dir_dfmeta->lfd[i], + NULL, NULL); + if (ret) { + fd_unref(dir_dfmeta->lfd[i]); + dir_dfmeta->lfd[i] = NULL; + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FAILED_TO_OPEN, + "dir: %s", loc->path, "subvol: %s", + conf->local_subvols[i]->name, NULL); + + if (conf->decommission_in_progress) { + *perrno = -ret; + ret = -1; + goto out; + } + } else { + fd_bind(dir_dfmeta->lfd[i]); + } } dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)), @@ -3358,6 +3351,7 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, ret = -1; goto out; } + ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this); if (ret) { gf_log(this->name, GF_LOG_ERROR, @@ -3370,7 +3364,8 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int), gf_common_mt_int); if (!dir_dfmeta->fetch_entries) { - gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->fetch_entries is NULL"); + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY, + "for dir_dfmeta->fetch_entries", NULL); ret = -1; goto out; } @@ -3440,8 +3435,13 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, ldfq_count <= MAX_MIGRATE_QUEUE_COUNT && !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) { ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf, - defrag, fd, migrate_data, dir_dfmeta, - xattr_req, &should_commit_hash, perrno); + defrag, dir_dfmeta->lfd[dfc_index], + migrate_data, dir_dfmeta, xattr_req, + perrno); + + if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED) { + goto out; + } if (ret) { gf_log(this->name, GF_LOG_WARNING, @@ -3481,27 +3481,19 @@ gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, } gettimeofday(&end, NULL); - elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 + - (end.tv_usec - dir_start.tv_usec); + elapsed = gf_tvdiff(&dir_start, &end); gf_log(this->name, GF_LOG_INFO, "Migration operation on dir %s took " "%.2f secs", loc->path, elapsed / 1e6); ret = 0; out: - + THIS = old_THIS; gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt); if (xattr_req) dict_unref(xattr_req); - if (fd) - fd_unref(fd); - - if (ret == 0 && should_commit_hash == 0) { - ret = 2; - } - /* It does not matter if it errored out - this number is * used to calculate rebalance estimated time to complete. * No locking required as dirs are processed by a single thread. @@ -3509,6 +3501,7 @@ out: defrag->num_dirs_processed++; return ret; } + int gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, dict_t *fix_layout) @@ -3523,7 +3516,6 @@ gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, * rebalance is complete. */ if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX || - defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { return 0; } @@ -3569,114 +3561,6 @@ gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, return 0; } -/* Function for doing a named lookup on file inodes during an attach tier - * So that a hardlink lookup heal i.e gfid to parent gfid lookup heal - * happens on pre-existing data. This is required so that the ctr database has - * hardlinks of all the exisitng file in the volume. CTR xlator on the - * brick/server side does db update/insert of the hardlink on a namelookup. - * Currently the namedlookup is done synchronous to the fixlayout that is - * triggered by attach tier. This is not performant, adding more time to - * fixlayout. The performant approach is record the hardlinks on a compressed - * datastore and then do the namelookup asynchronously later, giving the ctr db - * eventual consistency - * */ -int -gf_fix_layout_tier_attach_lookup(xlator_t *this, loc_t *parent_loc, - gf_dirent_t *file_dentry) -{ - int ret = -1; - dict_t *lookup_xdata = NULL; - dht_conf_t *conf = NULL; - loc_t file_loc = { - 0, - }; - struct iatt iatt = { - 0, - }; - - GF_VALIDATE_OR_GOTO("tier", this, out); - - GF_VALIDATE_OR_GOTO(this->name, parent_loc, out); - - GF_VALIDATE_OR_GOTO(this->name, file_dentry, out); - - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - if (!parent_loc->inode) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s parent is NULL", parent_loc->path, file_dentry->d_name); - goto out; - } - - conf = this->private; - - loc_wipe(&file_loc); - - if (gf_uuid_is_null(file_dentry->d_stat.ia_gfid)) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s gfid not present", parent_loc->path, file_dentry->d_name); - goto out; - } - - gf_uuid_copy(file_loc.gfid, file_dentry->d_stat.ia_gfid); - - if (gf_uuid_is_null(parent_loc->gfid)) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s" - " gfid not present", - parent_loc->path, file_dentry->d_name); - goto out; - } - - gf_uuid_copy(file_loc.pargfid, parent_loc->gfid); - - ret = dht_build_child_loc(this, &file_loc, parent_loc, file_dentry->d_name); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Child loc build failed"); - ret = -1; - goto out; - } - - lookup_xdata = dict_new(); - if (!lookup_xdata) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed creating lookup dict for %s", file_dentry->d_name); - goto out; - } - - ret = dict_set_int32(lookup_xdata, CTR_ATTACH_TIER_LOOKUP, 1); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed to set lookup flag"); - goto out; - } - - gf_uuid_copy(file_loc.parent->gfid, parent_loc->gfid); - - /* Sending lookup to cold tier only */ - ret = syncop_lookup(conf->subvolumes[0], &file_loc, &iatt, NULL, - lookup_xdata, NULL); - if (ret) { - /* If the file does not exist on the cold tier than it must */ - /* have been discovered on the hot tier. This is not an error. */ - gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS, - "%s lookup to cold tier on attach heal failed", file_loc.path); - goto out; - } - - ret = 0; - -out: - - loc_wipe(&file_loc); - - if (lookup_xdata) - dict_unref(lookup_xdata); - - return ret; -} - int gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, dict_t *fix_layout, dict_t *migrate_data) @@ -3696,7 +3580,6 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, }; inode_t *linked_inode = NULL, *inode = NULL; dht_conf_t *conf = NULL; - int should_commit_hash = 1; int perrno = 0; conf = this->private; @@ -3799,16 +3682,6 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; if (!IA_ISDIR(entry->d_stat.ia_type)) { - /* If its a fix layout during the attach - * tier operation do lookups on files - * on cold subvolume so that there is a - * CTR DB Lookup Heal triggered on existing - * data. - * */ - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - gf_fix_layout_tier_attach_lookup(this, loc, entry); - } - continue; } loc_wipe(&entry_loc); @@ -3825,8 +3698,6 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, goto out; } else { - should_commit_hash = 0; - continue; } } @@ -3889,7 +3760,6 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, ret = -1; goto out; } else { - should_commit_hash = 0; continue; } } @@ -3902,7 +3772,12 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout, migrate_data); - if (ret && ret != 2) { + if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED || + defrag->defrag_status == GF_DEFRAG_STATUS_FAILED) { + goto out; + } + + if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED, "Fix layout failed for %s", entry_loc.path); @@ -3933,6 +3808,17 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, */ ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL); + + /* In case of a race where the directory is deleted just before + * layout setxattr, the errors are updated in the layout structure. + * We can use this information to make a decision whether the directory + * is deleted entirely. + */ + if (ret == 0) { + ret = dht_dir_layout_error_check(this, loc->inode); + ret = -ret; + } + if (ret) { if (-ret == ENOENT || -ret == ESTALE) { gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED, @@ -3943,6 +3829,7 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, defrag->total_failures++; } ret = 0; + goto out; } else { gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED, "Setxattr failed for %s", loc->path); @@ -3957,11 +3844,10 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, } } - if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && - (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) { + if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) { ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno); - if (ret && (ret != 2)) { + if (ret) { if (perrno == ENOENT || perrno == ESTALE) { ret = 0; goto out; @@ -3977,18 +3863,13 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, if (conf->decommission_in_progress) { goto out; } - - should_commit_hash = 0; } - } else if (ret == 2) { - should_commit_hash = 0; } } gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path); - if (should_commit_hash && - gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) { + if (gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) { defrag->total_failures++; gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED, @@ -4012,245 +3893,34 @@ out: if (fd) fd_unref(fd); - if (ret == 0 && should_commit_hash == 0) { - ret = 2; - } - - return ret; -} - -/****************************************************************************** - * Tier background Fix layout functions - ******************************************************************************/ -/* This is the background tier fixlayout thread */ -void * -gf_tier_do_fix_layout(void *args) -{ - gf_tier_fix_layout_arg_t *tier_fix_layout_arg = args; - int ret = -1; - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - dict_t *dict = NULL; - loc_t loc = { - 0, - }; - struct iatt iatt = { - 0, - }; - struct iatt parent = { - 0, - }; - - GF_VALIDATE_OR_GOTO("tier", tier_fix_layout_arg, out); - GF_VALIDATE_OR_GOTO("tier", tier_fix_layout_arg->this, out); - this = tier_fix_layout_arg->this; - - conf = this->private; - GF_VALIDATE_OR_GOTO(this->name, conf, out); - - defrag = conf->defrag; - GF_VALIDATE_OR_GOTO(this->name, defrag, out); - GF_VALIDATE_OR_GOTO(this->name, defrag->root_inode, out); - - GF_VALIDATE_OR_GOTO(this->name, tier_fix_layout_arg->fix_layout, out); - - /* Get Root loc_t */ - dht_build_root_loc(defrag->root_inode, &loc); - ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED, - "Lookup on root failed."); - ret = -1; - goto out; - } - - /* Start the crawl */ - gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS, - "Tiering Fixlayout started"); - - ret = gf_defrag_fix_layout(this, defrag, &loc, - tier_fix_layout_arg->fix_layout, NULL); - if (ret && ret != 2) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED, - "Tiering fixlayout failed."); - ret = -1; - goto out; - } - - if (ret != 2 && - gf_defrag_settle_hash(this, defrag, &loc, - tier_fix_layout_arg->fix_layout) != 0) { - defrag->total_failures++; - ret = -1; - goto out; - } - - dict = dict_new(); - if (!dict) { - ret = -1; - goto out; - } - - ret = dict_set_str(dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, "yes"); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED, - "Failed to set dictionary value: key = %s", - GF_XATTR_TIER_LAYOUT_FIXED_KEY); - ret = -1; - goto out; - } - - /* Marking the completion of tiering fix layout via a xattr on root */ - ret = syncop_setxattr(this, &loc, dict, 0, NULL, NULL); - if (ret) { - gf_log(this->name, GF_LOG_ERROR, - "Failed to set tiering fix " - "layout completed xattr on %s", - loc.path); - ret = -1; - goto out; - } - - ret = 0; -out: - if (ret && defrag) - defrag->total_failures++; - - if (dict) - dict_unref(dict); - - return NULL; -} - -int -gf_tier_start_fix_layout(xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag, - dict_t *fix_layout) -{ - int ret = -1; - dict_t *tier_dict = NULL; - gf_tier_fix_layout_arg_t *tier_fix_layout_arg = NULL; - - tier_dict = dict_new(); - if (!tier_dict) { - gf_log("tier", GF_LOG_ERROR, - "Tier fix layout failed :" - "Creation of tier_dict failed"); - ret = -1; - goto out; - } - - /* Check if layout is fixed already */ - ret = syncop_getxattr(this, loc, &tier_dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, - NULL, NULL); - if (ret != 0) { - tier_fix_layout_arg = &defrag->tier_conf.tier_fix_layout_arg; - - /*Fill crawl arguments */ - tier_fix_layout_arg->this = this; - tier_fix_layout_arg->fix_layout = fix_layout; - - /* Spawn the fix layout thread so that its done in the - * background */ - ret = gf_thread_create(&tier_fix_layout_arg->thread_id, NULL, - gf_tier_do_fix_layout, tier_fix_layout_arg, - "tierfixl"); - if (ret) { - gf_log("tier", GF_LOG_ERROR, - "Thread creation failed. " - "Background fix layout for tiering will not " - "work."); - defrag->total_failures++; - goto out; - } - } - ret = 0; -out: - if (tier_dict) - dict_unref(tier_dict); - return ret; } -void -gf_tier_clear_fix_layout(xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag) -{ - int ret = -1; - dict_t *dict = NULL; - - GF_VALIDATE_OR_GOTO("tier", this, out); - GF_VALIDATE_OR_GOTO(this->name, loc, out); - GF_VALIDATE_OR_GOTO(this->name, defrag, out); - - /* Check if background fixlayout is completed. This is not - * multi-process safe i.e there is a possibility that by the time - * we move to remove the xattr there it might have been cleared by some - * other detach process from other node. We ignore the error if such - * a thing happens */ - ret = syncop_getxattr(this, loc, &dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, - NULL, NULL); - if (ret) { - /* Background fixlayout not complete - nothing to clear*/ - gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_LOG_TIER_STATUS, - "Unable to retrieve fixlayout xattr." - "Assume background fix layout not complete"); - goto out; - } - - ret = syncop_removexattr(this, loc, GF_XATTR_TIER_LAYOUT_FIXED_KEY, NULL, - NULL); - if (ret) { - gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_LOG_TIER_STATUS, - "Failed removing tier fix layout " - "xattr from %s", - loc->path); - goto out; - } - ret = 0; -out: - if (dict) - dict_unref(dict); -} - -void -gf_tier_wait_fix_lookup(gf_defrag_info_t *defrag) -{ - if (defrag->tier_conf.tier_fix_layout_arg.thread_id) { - pthread_join(defrag->tier_conf.tier_fix_layout_arg.thread_id, NULL); - } -} -/******************Tier background Fix layout functions END********************/ - int dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf, loc_t *loc) { dict_t *dict = NULL; - gf_defrag_info_t *defrag = NULL; uuid_t *uuid_ptr = NULL; int ret = -1; int i = 0; int j = 0; - defrag = conf->defrag; - - if (defrag->cmd != GF_DEFRAG_CMD_START_TIER) { - /* Find local subvolumes */ - ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, - NULL, NULL); - if (ret && (ret != -ENODATA)) { - gf_msg(this->name, GF_LOG_ERROR, -ret, 0, - "local " - "subvolume determination failed with error: %d", - -ret); - ret = -1; - goto out; - } - - if (!ret) - goto out; + /* Find local subvolumes */ + ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, NULL, + NULL); + if (ret && (ret != -ENODATA)) { + gf_msg(this->name, GF_LOG_ERROR, -ret, 0, + "local " + "subvolume determination failed with error: %d", + -ret); + ret = -1; + goto out; } + if (!ret) + goto out; + ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL, NULL, NULL); if (ret) { @@ -4341,9 +4011,6 @@ dht_file_counter_thread(void *args) struct timespec time_to_wait = { 0, }; - struct timeval now = { - 0, - }; uint64_t tmp_size = 0; if (!args) @@ -4353,9 +4020,8 @@ dht_file_counter_thread(void *args) dht_build_root_loc(defrag->root_inode, &root_loc); while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) { - gettimeofday(&now, NULL); - time_to_wait.tv_sec = now.tv_sec + 600; - time_to_wait.tv_nsec = 0; + timespec_now(&time_to_wait); + time_to_wait.tv_sec += 600; pthread_mutex_lock(&defrag->fc_mutex); pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex, @@ -4428,7 +4094,7 @@ gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread) goto out; } - ret = gf_thread_create(filecnt_thread, NULL, &dht_file_counter_thread, + ret = gf_thread_create(filecnt_thread, NULL, dht_file_counter_thread, (void *)defrag, "dhtfcnt"); if (ret) { @@ -4485,7 +4151,7 @@ gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag, /*Spawn Threads Here*/ while (index < thread_spawn_count) { - ret = gf_thread_create(&(tid[index]), NULL, &gf_defrag_task, + ret = gf_thread_create(&(tid[index]), NULL, gf_defrag_task, (void *)defrag, "dhtmig%d", (index + 1) & 0x3ff); if (ret != 0) { gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ", @@ -4559,7 +4225,6 @@ gf_defrag_start_crawl(void *data) dict_t *migrate_data = NULL; dict_t *status = NULL; glusterfs_ctx_t *ctx = NULL; - dht_methods_t *methods = NULL; call_frame_t *statfs_frame = NULL; xlator_t *old_THIS = NULL; int ret = -1; @@ -4575,7 +4240,6 @@ gf_defrag_start_crawl(void *data) int thread_index = 0; pthread_t *tid = NULL; pthread_t filecnt_thread; - gf_boolean_t is_tier_detach = _gf_false; gf_boolean_t fc_thread_started = _gf_false; this = data; @@ -4594,7 +4258,8 @@ gf_defrag_start_crawl(void *data) if (!defrag) goto exit; - gettimeofday(&defrag->start_time, NULL); + defrag->start_time = gf_time(); + dht_build_root_inode(this, &defrag->root_inode); if (!defrag->root_inode) goto out; @@ -4728,43 +4393,17 @@ gf_defrag_start_crawl(void *data) } } - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - /* Fix layout for attach tier */ - ret = gf_tier_start_fix_layout(this, &loc, defrag, fix_layout); - if (ret) { - goto out; - } - - methods = &(conf->methods); - - /* Calling tier_start of tier.c */ - methods->migration_other(this, defrag); - if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || - defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { - ret = dict_set_str(migrate_data, GF_XATTR_FILE_MIGRATE_KEY, - "force"); - if (ret) - goto out; - } - } else { - ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, - migrate_data); - if (ret && ret != 2) { - defrag->total_failures++; - ret = -1; - goto out; - } - - if (ret != 2 && - gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) { - defrag->total_failures++; - ret = -1; - goto out; - } + ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, migrate_data); + if (ret) { + defrag->total_failures++; + ret = -1; + goto out; + } - if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || - defrag->cmd == GF_DEFRAG_CMD_DETACH_START) - is_tier_detach = _gf_true; + if (gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) { + defrag->total_failures++; + ret = -1; + goto out; } gf_log("DHT", GF_LOG_INFO, "crawling file-system completed"); @@ -4778,19 +4417,6 @@ out: defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; } - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - /* Wait for the tier fixlayout to - * complete if its was started.*/ - gf_tier_wait_fix_lookup(defrag); - } - - if (is_tier_detach && ret == 0) { - /* If it was a detach remove the tier fix-layout - * xattr on root. Ignoring the failure, as nothing has to be - * done, logging is done in gf_tier_clear_fix_layout */ - gf_tier_clear_fix_layout(this, &loc, defrag); - } - gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index); if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) && @@ -4889,9 +4515,6 @@ gf_defrag_get_estimates_based_on_size(dht_conf_t *conf) uint64_t total_processed = 0; uint64_t tmp_count = 0; uint64_t time_to_complete = 0; - struct timeval now = { - 0, - }; double elapsed = 0; defrag = conf->defrag; @@ -4899,8 +4522,7 @@ gf_defrag_get_estimates_based_on_size(dht_conf_t *conf) if (!g_totalsize) goto out; - gettimeofday(&now, NULL); - elapsed = now.tv_sec - defrag->start_time.tv_sec; + elapsed = gf_time() - defrag->start_time; /* Don't calculate the estimates for the first 10 minutes. * It is unlikely to be accurate and estimates are not required @@ -4950,13 +4572,8 @@ gf_defrag_status_get(dht_conf_t *conf, dict_t *dict) uint64_t lookup = 0; uint64_t failures = 0; uint64_t skipped = 0; - uint64_t promoted = 0; - uint64_t demoted = 0; char *status = ""; double elapsed = 0; - struct timeval end = { - 0, - }; uint64_t time_to_complete = 0; uint64_t time_left = 0; gf_defrag_info_t *defrag = conf->defrag; @@ -4973,17 +4590,12 @@ gf_defrag_status_get(dht_conf_t *conf, dict_t *dict) lookup = defrag->num_files_lookedup; failures = defrag->total_failures; skipped = defrag->skipped; - promoted = defrag->total_files_promoted; - demoted = defrag->total_files_demoted; - gettimeofday(&end, NULL); - - elapsed = end.tv_sec - defrag->start_time.tv_sec; + elapsed = gf_time() - defrag->start_time; /* The rebalance is still in progress */ - if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && - (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED)) { + if (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) { time_to_complete = gf_defrag_get_estimates_based_on_size(conf); if (time_to_complete && (time_to_complete > elapsed)) @@ -4998,14 +4610,6 @@ gf_defrag_status_get(dht_conf_t *conf, dict_t *dict) if (!dict) goto log; - ret = dict_set_uint64(dict, "promoted", promoted); - if (ret) - gf_log(THIS->name, GF_LOG_WARNING, "failed to set promoted count"); - - ret = dict_set_uint64(dict, "demoted", demoted); - if (ret) - gf_log(THIS->name, GF_LOG_WARNING, "failed to set demoted count"); - ret = dict_set_uint64(dict, "files", files); if (ret) gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count"); @@ -5071,159 +4675,6 @@ out: return 0; } -void -gf_defrag_set_pause_state(gf_tier_conf_t *tier_conf, tier_pause_state_t state) -{ - pthread_mutex_lock(&tier_conf->pause_mutex); - tier_conf->pause_state = state; - pthread_mutex_unlock(&tier_conf->pause_mutex); -} - -tier_pause_state_t -gf_defrag_get_pause_state(gf_tier_conf_t *tier_conf) -{ - int state; - - pthread_mutex_lock(&tier_conf->pause_mutex); - state = tier_conf->pause_state; - pthread_mutex_unlock(&tier_conf->pause_mutex); - - return state; -} - -tier_pause_state_t -gf_defrag_check_pause_tier(gf_tier_conf_t *tier_conf) -{ - int woke = 0; - int state = -1; - - pthread_mutex_lock(&tier_conf->pause_mutex); - - if (tier_conf->pause_state == TIER_RUNNING) - goto out; - - if (tier_conf->pause_state == TIER_PAUSED) - goto out; - - if (tier_conf->promote_in_progress || tier_conf->demote_in_progress) - goto out; - - tier_conf->pause_state = TIER_PAUSED; - - if (tier_conf->pause_synctask) { - synctask_wake(tier_conf->pause_synctask); - tier_conf->pause_synctask = 0; - woke = 1; - } - - gf_msg("tier", GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, "woken %d", woke); - - gf_event(EVENT_TIER_PAUSE, "vol=%s", tier_conf->volname); -out: - state = tier_conf->pause_state; - - pthread_mutex_unlock(&tier_conf->pause_mutex); - - return state; -} - -void -gf_defrag_pause_tier_timeout(void *data) -{ - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - - this = (xlator_t *)data; - GF_VALIDATE_OR_GOTO("tier", this, out); - - conf = this->private; - GF_VALIDATE_OR_GOTO(this->name, conf, out); - - defrag = conf->defrag; - GF_VALIDATE_OR_GOTO(this->name, defrag, out); - - gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, - "Request pause timer timeout"); - - gf_defrag_check_pause_tier(&defrag->tier_conf); - -out: - return; -} - -int -gf_defrag_pause_tier(xlator_t *this, gf_defrag_info_t *defrag) -{ - int ret = 0; - struct timespec delta = { - 0, - }; - int delay = 2; - - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) - goto out; - - /* - * Set flag requesting to pause tiering. Wait 'delay' seconds for - * tiering to actually stop as indicated by the pause state - * before returning success or failure. - */ - gf_defrag_set_pause_state(&defrag->tier_conf, TIER_REQUEST_PAUSE); - - /* - * If migration is not underway, can pause immediately. - */ - gf_defrag_check_pause_tier(&defrag->tier_conf); - if (gf_defrag_get_pause_state(&defrag->tier_conf) == TIER_PAUSED) - goto out; - - gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, - "Request pause tier"); - - defrag->tier_conf.pause_synctask = synctask_get(); - delta.tv_sec = delay; - delta.tv_nsec = 0; - defrag->tier_conf.pause_timer = gf_timer_call_after( - this->ctx, delta, gf_defrag_pause_tier_timeout, this); - - synctask_yield(defrag->tier_conf.pause_synctask); - - if (gf_defrag_get_pause_state(&defrag->tier_conf) == TIER_PAUSED) - goto out; - - gf_defrag_set_pause_state(&defrag->tier_conf, TIER_RUNNING); - - ret = -1; -out: - - gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, - "Pause tiering ret=%d", ret); - - return ret; -} - -int -gf_defrag_resume_tier(xlator_t *this, gf_defrag_info_t *defrag) -{ - gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_RESUME, - "Pause end. Resume tiering"); - - gf_defrag_set_pause_state(&defrag->tier_conf, TIER_RUNNING); - - gf_event(EVENT_TIER_RESUME, "vol=%s", defrag->tier_conf.volname); - - return 0; -} - -int -gf_defrag_start_detach_tier(gf_defrag_info_t *defrag) -{ - defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER; - - return 0; -} - int gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output) { diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c index 08a05feefc5..3e24065227c 100644 --- a/xlators/cluster/dht/src/dht-selfheal.c +++ b/xlators/cluster/dht/src/dht-selfheal.c @@ -8,9 +8,7 @@ cases as published by the Free Software Foundation. */ -#include "dht-common.h" #include "dht-lock.h" -#include <glusterfs/glusterfs-acl.h> #define DHT_SET_LAYOUT_RANGE(layout, i, srt, chunk, path) \ do { \ @@ -35,7 +33,7 @@ } \ } while (0) -int +static int dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout, gf_boolean_t newdir, dht_selfheal_layout_t healer, dht_need_heal_t should_heal); @@ -523,7 +521,7 @@ out: return fixit; } -int +static int dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout, gf_boolean_t newdir, dht_selfheal_layout_t healer, dht_need_heal_t should_heal) @@ -614,7 +612,7 @@ err: return -1; } -int +static int dht_selfheal_dir_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xdata) { @@ -685,7 +683,7 @@ dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data) return ret; } -int +static int dht_selfheal_dir_xattr_persubvol(call_frame_t *frame, loc_t *loc, dht_layout_t *layout, int i, xlator_t *req_subvol) @@ -814,7 +812,7 @@ err: return 0; } -int +static int dht_fix_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout) { dht_local_t *local = NULL; @@ -863,7 +861,7 @@ out: return 0; } -int +static int dht_selfheal_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout) { dht_local_t *local = NULL; @@ -940,38 +938,6 @@ out: return 0; } -gf_boolean_t -dht_is_subvol_part_of_layout(dht_layout_t *layout, xlator_t *xlator) -{ - int i = 0; - gf_boolean_t ret = _gf_false; - - for (i = 0; i < layout->cnt; i++) { - if (!strcmp(layout->list[i].xlator->name, xlator->name)) { - ret = _gf_true; - break; - } - } - - return ret; -} - -int -dht_layout_index_from_conf(dht_layout_t *layout, xlator_t *xlator) -{ - int i = -1; - int j = 0; - - for (j = 0; j < layout->cnt; j++) { - if (!strcmp(layout->list[j].xlator->name, xlator->name)) { - i = j; - break; - } - } - - return i; -} - int dht_selfheal_dir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, struct iatt *statpre, @@ -1062,7 +1028,7 @@ dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf, return 0; } -int +static int dht_selfheal_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf, struct iatt *preparent, @@ -1114,86 +1080,7 @@ out: return 0; } -void -dht_selfheal_dir_mkdir_setacl(dict_t *xattr, dict_t *dict) -{ - data_t *acl_default = NULL; - data_t *acl_access = NULL; - xlator_t *this = NULL; - int ret = -1; - - GF_ASSERT(xattr); - GF_ASSERT(dict); - - this = THIS; - GF_ASSERT(this); - - acl_default = dict_get(xattr, POSIX_ACL_DEFAULT_XATTR); - - if (!acl_default) { - gf_msg_debug(this->name, 0, "ACL_DEFAULT xattr not present"); - goto cont; - } - ret = dict_set(dict, POSIX_ACL_DEFAULT_XATTR, acl_default); - if (ret) - gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, - "key=%s", POSIX_ACL_DEFAULT_XATTR, NULL); -cont: - acl_access = dict_get(xattr, POSIX_ACL_ACCESS_XATTR); - if (!acl_access) { - gf_msg_debug(this->name, 0, "ACL_ACCESS xattr not present"); - goto out; - } - ret = dict_set(dict, POSIX_ACL_ACCESS_XATTR, acl_access); - if (ret) - gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, - "key=%s", POSIX_ACL_ACCESS_XATTR, NULL); - -out: - return; -} - -void -dht_selfheal_dir_mkdir_setquota(dict_t *src, dict_t *dst) -{ - data_t *quota_limit_key = NULL; - data_t *quota_limit_obj_key = NULL; - xlator_t *this = NULL; - int ret = -1; - - GF_ASSERT(src); - GF_ASSERT(dst); - - this = THIS; - GF_ASSERT(this); - - quota_limit_key = dict_get(src, QUOTA_LIMIT_KEY); - if (!quota_limit_key) { - gf_msg_debug(this->name, 0, "QUOTA_LIMIT_KEY xattr not present"); - goto cont; - } - ret = dict_set(dst, QUOTA_LIMIT_KEY, quota_limit_key); - if (ret) - gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, - "key=%s", QUOTA_LIMIT_KEY, NULL); - -cont: - quota_limit_obj_key = dict_get(src, QUOTA_LIMIT_OBJECTS_KEY); - if (!quota_limit_obj_key) { - gf_msg_debug(this->name, 0, - "QUOTA_LIMIT_OBJECTS_KEY xattr not present"); - goto out; - } - ret = dict_set(dst, QUOTA_LIMIT_OBJECTS_KEY, quota_limit_obj_key); - if (ret) - gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, - "key=%s", QUOTA_LIMIT_OBJECTS_KEY, NULL); - -out: - return; -} - -int +static int dht_selfheal_dir_mkdir_lookup_done(call_frame_t *frame, xlator_t *this) { dht_local_t *local = NULL; @@ -1270,7 +1157,7 @@ err: return 0; } -int +static int dht_selfheal_dir_mkdir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf, @@ -1364,7 +1251,7 @@ err: return 0; } -int +static int dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata) @@ -1384,10 +1271,6 @@ dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie, local->call_cnt = conf->subvolume_cnt; if (op_ret < 0) { - /* We get this error when the directory entry was not created - * on a newky attached tier subvol. Hence proceed and do mkdir - * on the tier subvol. - */ if (op_errno == EINVAL) { local->call_cnt = 1; dht_selfheal_dir_mkdir_lookup_done(frame, this); @@ -1433,18 +1316,21 @@ err: return 0; } -int +static int dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout, int force) { int missing_dirs = 0; int i = 0; + int op_errno = 0; int ret = -1; dht_local_t *local = NULL; xlator_t *this = NULL; + dht_conf_t *conf = NULL; local = frame->local; this = frame->this; + conf = this->private; local->selfheal.force_mkdir = force; local->selfheal.hole_cnt = 0; @@ -1461,11 +1347,12 @@ dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout, if (!__is_root_gfid(local->stbuf.ia_gfid)) { if (local->need_xattr_heal) { local->need_xattr_heal = 0; - ret = dht_dir_xattr_heal(this, local); - if (ret) - gf_smsg(this->name, GF_LOG_ERROR, ret, + ret = dht_dir_xattr_heal(this, local, &op_errno); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path, "gfid=%s", local->gfid, NULL); + } } else { if (!gf_uuid_is_null(local->gfid)) gf_uuid_copy(loc->gfid, local->gfid); @@ -1483,15 +1370,44 @@ dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout, return 0; } - if (local->hashed_subvol == NULL) - local->hashed_subvol = dht_subvol_get_hashed(this, loc); + /* MDS xattr is populated only while DHT is having more than one + subvol.In case of graph switch while adding more dht subvols need to + consider hash subvol as a MDS to avoid MDS check failure at the time + of running fop on directory + */ + if (!dict_get(local->xattr, conf->mds_xattr_key) && + (conf->subvolume_cnt > 1)) { + if (local->hashed_subvol == NULL) { + local->hashed_subvol = dht_subvol_get_hashed(this, loc); + if (local->hashed_subvol == NULL) { + local->op_errno = EINVAL; + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", + loc->pargfid, "name=%s", loc->name, "path=%s", + loc->path, NULL); + goto err; + } + } + ret = dht_inode_ctx_mdsvol_set(local->inode, this, + local->hashed_subvol); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED, + "Failed to set hashed subvol for %s on inode vol is %s", + local->loc.path, + local->hashed_subvol ? local->hashed_subvol->name : "NULL"); + goto err; + } + } if (local->hashed_subvol == NULL) { - local->op_errno = EINVAL; - gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, - DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid, - "name=%s", loc->name, "path=%s", loc->path, NULL); - goto err; + local->hashed_subvol = dht_subvol_get_hashed(this, loc); + if (local->hashed_subvol == NULL) { + local->op_errno = EINVAL; + gf_smsg(this->name, GF_LOG_WARNING, local->op_errno, + DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid, + "name=%s", loc->name, "path=%s", loc->path, NULL); + goto err; + } } local->current = &local->lock[0]; @@ -1507,7 +1423,7 @@ err: return -1; } -int +static int dht_selfheal_layout_alloc_start(xlator_t *this, loc_t *loc, dht_layout_t *layout) { @@ -1616,8 +1532,6 @@ dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc, dht_layout_t *new_layout); void -dht_layout_entry_swap(dht_layout_t *layout, int i, int j); -void dht_layout_range_swap(dht_layout_t *layout, int i, int j); /* @@ -1626,7 +1540,7 @@ dht_layout_range_swap(dht_layout_t *layout, int i, int j); */ #define OV_ENTRY(x, y) table[x * new->cnt + y] -void +static void dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc, dht_layout_t *new, dht_layout_t *old) { @@ -1703,7 +1617,7 @@ dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc, } } -dht_layout_t * +static dht_layout_t * dht_fix_layout_of_directory(call_frame_t *frame, loc_t *loc, dht_layout_t *layout) { @@ -1806,7 +1720,7 @@ done: * Having to call this 2x for each entry in the layout is pretty horrible, but * that's what all of this layout-sorting nonsense gets us. */ -uint32_t +static uint32_t dht_get_chunks_from_xl(xlator_t *parent, xlator_t *child) { dht_conf_t *priv = parent->private; @@ -1924,7 +1838,7 @@ done: return; } -int +static int dht_selfheal_dir_getafix(call_frame_t *frame, loc_t *loc, dht_layout_t *layout) { dht_local_t *local = NULL; @@ -2056,9 +1970,18 @@ dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk, local->selfheal.dir_cbk = dir_cbk; local->selfheal.layout = dht_layout_ref(this, layout); - if (local->need_attrheal && !IA_ISINVAL(local->mds_stbuf.ia_type)) { - /*Use the one in the mds_stbuf*/ - local->stbuf = local->mds_stbuf; + if (local->need_attrheal) { + if (__is_root_gfid(local->stbuf.ia_gfid)) { + local->stbuf.ia_gid = local->prebuf.ia_gid; + local->stbuf.ia_uid = local->prebuf.ia_uid; + + local->stbuf.ia_ctime = local->prebuf.ia_ctime; + local->stbuf.ia_ctime_nsec = local->prebuf.ia_ctime_nsec; + local->stbuf.ia_prot = local->prebuf.ia_prot; + + } else if (!IA_ISINVAL(local->mds_stbuf.ia_type)) { + local->stbuf = local->mds_stbuf; + } } if (!__is_root_gfid(local->stbuf.ia_gfid)) { @@ -2260,6 +2183,15 @@ dht_dir_heal_xattrs(void *data) if (subvol == mds_subvol) continue; if (uret || uflag) { + /* Custom xattr heal is required - let posix handle it */ + ret = dict_set_int8(xdata, "sync_backend_xattrs", _gf_true); + if (ret) { + gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, + "path=%s", local->loc.path, "key=%s", + "sync_backend_xattrs", NULL); + goto out; + } + ret = syncop_setxattr(subvol, &local->loc, user_xattr, 0, xdata, NULL); if (ret) { @@ -2268,6 +2200,8 @@ dht_dir_heal_xattrs(void *data) DHT_MSG_DIR_XATTR_HEAL_FAILED, "set-user-xattr-failed path=%s", local->loc.path, "subvol=%s", subvol->name, "gfid=%s", gfid, NULL); + } else { + dict_del(xdata, "sync_backend_xattrs"); } } } @@ -2391,7 +2325,7 @@ dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data) } /* EXIT: dht_update_commit_hash_for_layout */ -int +static int dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata) @@ -2411,7 +2345,7 @@ dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie, return 0; } -int +static int dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this) { dht_local_t *local = NULL; @@ -2438,7 +2372,7 @@ dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this) return 0; } -int +static int dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, dict_t *xdata) @@ -2465,7 +2399,7 @@ dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie, return 0; } -int +static int dht_update_commit_hash_for_layout_resume(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata) diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c index d85b4d1ce13..bb72b0ffbb5 100644 --- a/xlators/cluster/dht/src/dht-shared.c +++ b/xlators/cluster/dht/src/dht-shared.c @@ -140,9 +140,9 @@ dht_priv_dump(xlator_t *this) } } - if (conf->last_stat_fetch.tv_sec) + if (conf->last_stat_fetch) gf_proc_dump_write("last_stat_fetch", "%s", - ctime(&conf->last_stat_fetch.tv_sec)); + ctime(&conf->last_stat_fetch)); UNLOCK(&conf->subvolume_lock); @@ -537,6 +537,8 @@ gf_defrag_pattern_list_fill(xlator_t *this, gf_defrag_info_t *defrag, pattern_str = strtok_r(data, ",", &tmp_str); while (pattern_str) { dup_str = gf_strdup(pattern_str); + if (!dup_str) + goto out; pattern_list = GF_CALLOC(1, sizeof(gf_defrag_pattern_list_t), 1); if (!pattern_list) { goto out; @@ -596,7 +598,6 @@ dht_init_methods(xlator_t *this) methods = &(conf->methods); methods->migration_get_dst_subvol = dht_migration_get_dst_subvol; - methods->migration_needed = dht_migration_needed; methods->migration_other = NULL; methods->layout_search = dht_layout_search; @@ -1045,84 +1046,6 @@ struct volume_options dht_options[] = { /* NUFA option */ {.key = {"local-volume-name"}, .type = GF_OPTION_TYPE_XLATOR}, - /* tier options */ - { - .key = {"tier-pause"}, - .type = GF_OPTION_TYPE_BOOL, - .default_value = "off", - }, - - { - .key = {"tier-promote-frequency"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "120", - }, - - { - .key = {"tier-demote-frequency"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "3600", - }, - - { - .key = {"write-freq-threshold"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "0", - }, - - { - .key = {"read-freq-threshold"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "0", - }, - { - .key = {"watermark-hi"}, - .type = GF_OPTION_TYPE_PERCENT, - .default_value = "90", - }, - { - .key = {"watermark-low"}, - .type = GF_OPTION_TYPE_PERCENT, - .default_value = "75", - }, - { - .key = {"tier-mode"}, - .type = GF_OPTION_TYPE_STR, - .default_value = "test", - }, - { - .key = {"tier-compact"}, - .type = GF_OPTION_TYPE_BOOL, - .default_value = "off", - }, - {.key = {"tier-hot-compact-frequency"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "604800", - .description = "Frequency to compact DBs on hot tier in system"}, - {.key = {"tier-cold-compact-frequency"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "604800", - .description = "Frequency to compact DBs on cold tier in system"}, - { - .key = {"tier-max-mb"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "4000", - }, - { - .key = {"tier-max-promote-file-size"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "0", - }, - { - .key = {"tier-max-files"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "10000", - }, - { - .key = {"tier-query-limit"}, - .type = GF_OPTION_TYPE_INT, - .default_value = "100", - }, /* switch option */ {.key = {"pattern.switch.case"}, .type = GF_OPTION_TYPE_ANY}, diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c index 59313639c45..3648a564840 100644 --- a/xlators/cluster/dht/src/nufa.c +++ b/xlators/cluster/dht/src/nufa.c @@ -595,7 +595,6 @@ nufa_init(xlator_t *this) dht_methods_t dht_methods = { .migration_get_dst_subvol = dht_migration_get_dst_subvol, - .migration_needed = dht_migration_needed, .layout_search = dht_layout_search, }; diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c index 9d712b359a0..703a30e2485 100644 --- a/xlators/cluster/ec/src/ec-combine.c +++ b/xlators/cluster/ec/src/ec-combine.c @@ -343,9 +343,8 @@ out: } static int32_t -ec_dict_data_concat(const char *fmt, ec_cbk_data_t *cbk, int32_t which, - char *key, char *new_key, const char *def, - gf_boolean_t global, ...) +ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key, + const char *def, gf_boolean_t global, const char *fmt, ...) { ec_t *ec = cbk->fop->xl->private; data_t *data[ec->nodes]; @@ -357,7 +356,7 @@ ec_dict_data_concat(const char *fmt, ec_cbk_data_t *cbk, int32_t which, ec_dict_list(data, cbk, which, key, global); - va_start(args, global); + va_start(args, fmt); err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args); va_end(args); @@ -730,14 +729,14 @@ ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg) if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) || (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) { - return ec_dict_data_concat("(<EC:%s> { })", data->cbk, data->which, key, - NULL, NULL, _gf_false, + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, _gf_false, "(<EC:%s> { })", data->cbk->fop->xl->name); } if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) { - return ec_dict_data_concat("{\n}", data->cbk, data->which, key, NULL, - NULL, _gf_false); + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, "{\n}"); } if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) { @@ -767,9 +766,9 @@ ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg) if (XATTR_IS_NODE_UUID(key)) { if (data->cbk->fop->int32) { /* List of node uuid is requested */ - return ec_dict_data_concat("{ }", data->cbk, data->which, key, + return ec_dict_data_concat(data->cbk, data->which, key, GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR, - _gf_true); + _gf_true, "{ }"); } else { return ec_dict_data_uuid(data->cbk, data->which, key); } diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c index ded34b81aa2..b955efd8c2d 100644 --- a/xlators/cluster/ec/src/ec-common.c +++ b/xlators/cluster/ec/src/ec-common.c @@ -230,7 +230,7 @@ ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx) int32_t ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good, - uintptr_t bad, dict_t *xdata) + uintptr_t bad, uint32_t pending, dict_t *xdata) { if (op_ret < 0) { gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL, @@ -316,17 +316,19 @@ ec_check_status(ec_fop_data_t *fop) } } - gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS, - "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, " - "remaining=%s, good=%s, bad=%s, %s)", - gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes, - ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), - ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), - ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes), - ec_bin(str4, sizeof(str4), fop->good, ec->nodes), - ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good), - ec->nodes), - ec_msg_str(fop)); + gf_msg( + fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS, + "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, " + "remaining=%s, good=%s, bad=%s," + "(Least significant bit represents first client/brick of subvol), %s)", + gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes, + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes), + ec_bin(str4, sizeof(str4), fop->good, ec->nodes), + ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good), + ec->nodes), + ec_msg_str(fop)); if (fop->use_fd) { if (fop->fd != NULL) { ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, @@ -614,10 +616,10 @@ ec_msg_str(ec_fop_data_t *fop) loc_t *loc2 = NULL; char gfid1[64] = {0}; char gfid2[64] = {0}; + ec_fop_data_t *parent = fop->parent; if (fop->errstr) return fop->errstr; - if (!fop->use_fd) { loc1 = &fop->loc[0]; loc2 = &fop->loc[1]; @@ -625,23 +627,45 @@ ec_msg_str(ec_fop_data_t *fop) if (fop->id == GF_FOP_RENAME) { gf_asprintf(&fop->errstr, "FOP : '%s' failed on '%s' and '%s' with gfids " - "%s and %s respectively", + "%s and %s respectively. Parent FOP: %s", ec_fop_name(fop->id), loc1->path, loc2->path, uuid_utoa_r(loc1->gfid, gfid1), - uuid_utoa_r(loc2->gfid, gfid2)); + uuid_utoa_r(loc2->gfid, gfid2), + parent ? ec_fop_name(parent->id) : "No Parent"); } else { - gf_asprintf(&fop->errstr, "FOP : '%s' failed on '%s' with gfid %s", - ec_fop_name(fop->id), loc1->path, - uuid_utoa_r(loc1->gfid, gfid1)); + gf_asprintf( + &fop->errstr, + "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), loc1->path, + uuid_utoa_r(loc1->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); } } else { - gf_asprintf(&fop->errstr, "FOP : '%s' failed on gfid %s", - ec_fop_name(fop->id), - uuid_utoa_r(fop->fd->inode->gfid, gfid1)); + gf_asprintf( + &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); } return fop->errstr; } +static void +ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need, + int32_t loglevel) +{ + ec_t *ec = fop->xl->private; + char str1[32], str2[32], str3[32]; + + gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT, + "Insufficient available children for this request: " + "Have : %d, Need : %u : Child UP : %s " + "Mask: %s, Healing : %s : %s ", + have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->healing, ec->nodes), + ec_msg_str(fop)); +} + static int32_t ec_child_select(ec_fop_data_t *fop) { @@ -699,11 +723,7 @@ ec_child_select(ec_fop_data_t *fop) ec_trace("SELECT", fop, ""); if ((num < fop->minimum) && (num < ec->fragments)) { - gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_CHILDS_INSUFFICIENT, - "Insufficient available children " - "for this request (have %d, need " - "%d). %s", - num, fop->minimum, ec_msg_str(fop)); + ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR); return 0; } @@ -711,11 +731,7 @@ ec_child_select(ec_fop_data_t *fop) (fop->locks[0].update[EC_DATA_TXN] || fop->locks[0].update[EC_METADATA_TXN])) { if (ec->quorum_count && (num < ec->quorum_count)) { - gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_CHILDS_INSUFFICIENT, - "Insufficient available children " - "for this request (have %d, need " - "%d). %s", - num, ec->quorum_count, ec_msg_str(fop)); + ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR); return 0; } } diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c index ef6b06fa4dd..f71dcfac293 100644 --- a/xlators/cluster/ec/src/ec-dir-read.c +++ b/xlators/cluster/ec/src/ec-dir-read.c @@ -386,9 +386,16 @@ ec_manager_readdir(ec_fop_data_t *fop, int32_t state) /* Return error if opendir has not been successfully called on * any subvolume. */ ctx = ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || (ctx->open == 0)) { - fop->error = EINVAL; + if (ctx == NULL) { + fop->error = ENOMEM; + } else if (ctx->open == 0) { + fop->error = EBADFD; + } + if (fop->error) { + gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error, + EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s", + ec_msg_str(fop)); return EC_STATE_REPORT; } diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c index 81f6add5bb0..7d991f04aac 100644 --- a/xlators/cluster/ec/src/ec-heal.c +++ b/xlators/cluster/ec/src/ec-heal.c @@ -70,6 +70,7 @@ struct ec_name_data { char *name; inode_t *parent; default_args_cbk_t *replies; + uint32_t heal_pending; }; static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL}; @@ -994,6 +995,7 @@ ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia, ret = -ENOTCONN; goto out; } + out: if (xattr) dict_unref(xattr); @@ -1172,6 +1174,7 @@ ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, dict_t *xdata = NULL; char *linkname = NULL; ec_config_t config; + /* There should be just one gfid key */ EC_REPLIES_ALLOC(replies, ec->nodes); if (gfid_db->count != 1) { @@ -1416,6 +1419,11 @@ __ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent, participants); + if (ret >= 0) { + /* If ec_create_name() succeeded we return 1 to indicate that a new + * file has been created and it will need to be healed. */ + ret = 1; + } out: cluster_replies_wipe(replies, ec->nodes); loc_wipe(&loc); @@ -1493,18 +1501,22 @@ ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name, name_on); - if (ret < 0) + if (ret < 0) { memset(name_on, 0, ec->nodes); + } else { + name_data->heal_pending += ret; + } for (i = 0; i < ec->nodes; i++) if (name_data->participants[i] && !name_on[i]) name_data->failed_on[i] = 1; + return 0; } int ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, - unsigned char *participants) + unsigned char *participants, uint32_t *pending) { int i = 0; int j = 0; @@ -1517,7 +1529,7 @@ ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, name_data.frame = frame; name_data.participants = participants; name_data.failed_on = alloca0(ec->nodes); - ; + name_data.heal_pending = 0; for (i = 0; i < ec->nodes; i++) { if (!participants[i]) @@ -1536,6 +1548,8 @@ ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, break; } } + *pending += name_data.heal_pending; + loc_wipe(&loc); return ret; } @@ -1543,7 +1557,7 @@ ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, int __ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, unsigned char *heal_on, unsigned char *sources, - unsigned char *healed_sinks) + unsigned char *healed_sinks, uint32_t *pending) { unsigned char *locked_on = NULL; unsigned char *output = NULL; @@ -1588,7 +1602,7 @@ unlock: if (sources[i] || healed_sinks[i]) participants[i] = 1; } - ret = ec_heal_names(frame, ec, inode, participants); + ret = ec_heal_names(frame, ec, inode, participants, pending); if (EC_COUNT(participants, ec->nodes) <= ec->fragments) goto out; @@ -1609,7 +1623,8 @@ out: int ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, - unsigned char *sources, unsigned char *healed_sinks) + unsigned char *sources, unsigned char *healed_sinks, + uint32_t *pending) { unsigned char *locked_on = NULL; unsigned char *up_subvols = NULL; @@ -1640,7 +1655,7 @@ ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, goto unlock; } ret = __ec_heal_entry(frame, ec, inode, locked_on, sources, - healed_sinks); + healed_sinks, pending); } unlock: cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, @@ -1961,14 +1976,14 @@ ec_manager_heal_block(ec_fop_data_t *fop, int32_t state) if (fop->cbks.heal) { fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0, (heal->good | heal->bad), heal->good, heal->bad, - NULL); + 0, NULL); } return EC_STATE_END; case -EC_STATE_REPORT: if (fop->cbks.heal) { fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1, - fop->error, 0, 0, 0, NULL); + fop->error, 0, 0, 0, 0, NULL); } return EC_STATE_END; @@ -2005,14 +2020,15 @@ out: if (fop != NULL) { ec_manager(fop, error); } else { - func(frame, heal, this, -1, error, 0, 0, 0, NULL); + func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL); } } int32_t ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, uintptr_t mask, - uintptr_t good, uintptr_t bad, dict_t *xdata) + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) { ec_heal_t *heal = cookie; @@ -2481,6 +2497,58 @@ out: return ret; } +int +ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode) +{ + int i = 0; + int ret = 0; + dict_t **xattr = NULL; + loc_t loc = {0}; + uint64_t dirty_xattr[EC_VERSION_SIZE] = {0}; + unsigned char *on = NULL; + default_args_cbk_t *replies = NULL; + dict_t *dict = NULL; + + /* Allocate the required memory */ + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + on = alloca0(ec->nodes); + EC_REPLIES_ALLOC(replies, ec->nodes); + xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer); + if (!xattr) { + ret = -ENOMEM; + goto out; + } + dict = dict_new(); + if (!dict) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict; + on[i] = 1; + } + ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + ret = -ENOMEM; + goto out; + } + PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame, + ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64, + xattr, NULL); +out: + if (dict) { + dict_unref(dict); + } + if (xattr) { + GF_FREE(xattr); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} + void ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) { @@ -2498,6 +2566,7 @@ ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) intptr_t mbad = 0; intptr_t good = 0; intptr_t bad = 0; + uint32_t pending = 0; ec_fop_data_t *fop = data; gf_boolean_t blocking = _gf_false; ec_heal_need_t need_heal = EC_HEAL_NONEED; @@ -2533,7 +2602,7 @@ ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) if (loc->name && strlen(loc->name)) { ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name, participants); - if (ret == 0) { + if (ret >= 0) { gf_msg_debug(this->name, 0, "%s: name heal " "successful on %" PRIXPTR, @@ -2551,23 +2620,34 @@ ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) /* Mount triggers heal only when it detects that it must need heal, shd * triggers heals periodically which need not be thorough*/ - if (ec->shd.iamshd) { + if (ec->shd.iamshd && (ret <= 0)) { ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false, &need_heal); - if (need_heal == EC_HEAL_NONEED) { + if (need_heal == EC_HEAL_PURGE_INDEX) { + gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL, + "Index entry needs to be purged for: %s ", + uuid_utoa(loc->gfid)); + /* We need to send zero-xattrop so that stale index entry could be + * removed. We need not take lock on this entry to do so as + * xattrop on a brick is atomic. */ + ec_heal_purge_stale_index(frame, ec, loc->inode); + goto out; + } else if (need_heal == EC_HEAL_NONEED) { gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL, "Heal is not required for : %s ", uuid_utoa(loc->gfid)); goto out; } } + sources = alloca0(ec->nodes); healed_sinks = alloca0(ec->nodes); if (IA_ISREG(loc->inode->ia_type)) { ret = ec_heal_data(frame, ec, blocking, loc->inode, sources, healed_sinks); } else if (IA_ISDIR(loc->inode->ia_type) && !partial) { - ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks); + ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks, + &pending); } else { ret = 0; memcpy(sources, participants, ec->nodes); @@ -2597,10 +2677,11 @@ out: if (fop->cbks.heal) { fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno, ec_char_array_to_mask(participants, ec->nodes), - mgood & good, mbad & bad, NULL); + mgood & good, mbad & bad, pending, NULL); } if (frame) STACK_DESTROY(frame->root); + return; } @@ -2648,7 +2729,7 @@ ec_heal_fail(ec_t *ec, ec_fop_data_t *fop) { if (fop->cbks.heal) { fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0, - 0, NULL); + 0, 0, NULL); } ec_fop_data_release(fop); } @@ -2835,7 +2916,7 @@ fail: if (fop) ec_fop_data_release(fop); if (func) - func(frame, data, this, -1, err, 0, 0, 0, NULL); + func(frame, data, this, -1, err, 0, 0, 0, 0, NULL); } int @@ -2964,6 +3045,13 @@ _need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources, goto out; } } + /* If lock count is 0, all dirty flags are 0 and all the + * versions are macthing then why are we here. It looks + * like something went wrong while removing the index entries + * after completing a successful heal or fop. In this case + * we need to remove this index entry to avoid triggering heal + * in a loop and causing lookups again and again*/ + *need_heal = EC_HEAL_PURGE_INDEX; } else { for (i = 0; i < ec->nodes; i++) { /* Since each lock can only increment the dirty diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c index 956e73c2088..5c1586bc9c5 100644 --- a/xlators/cluster/ec/src/ec-heald.c +++ b/xlators/cluster/ec/src/ec-heald.c @@ -62,7 +62,7 @@ __ec_shd_healer_wait(struct subvol_healer *healer) ec = healer->this->private; disabled_loop: - wait_till.tv_sec = time(NULL) + ec->shd.timeout; + wait_till.tv_sec = gf_time() + ec->shd.timeout; while (!healer->rerun) { ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till); @@ -156,19 +156,78 @@ ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name) return ret; } +static gf_boolean_t +ec_is_heal_completed(char *status) +{ + char *bad_pos = NULL; + char *zero_pos = NULL; + + if (!status) { + return _gf_false; + } + + /*Logic: + * Status will be of the form Good: <binary>, Bad: <binary> + * If heal completes, if we do strchr for '0' it should be present after + * 'Bad:' i.e. strRchr for ':' + * */ + + zero_pos = strchr(status, '0'); + bad_pos = strrchr(status, ':'); + if (!zero_pos || !bad_pos) { + /*malformed status*/ + return _gf_false; + } + + if (zero_pos > bad_pos) { + return _gf_true; + } + + return _gf_false; +} + int ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc, gf_boolean_t full) { + dict_t *xdata = NULL; + dict_t *dict = NULL; + uint32_t count; int32_t ret; + char *heal_status = NULL; + ec_t *ec = healer->this->private; + + GF_ATOMIC_INC(ec->stats.shd.attempted); + ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL, + &xdata); + if (ret == 0) { + if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) { + if (ec_is_heal_completed(heal_status)) { + GF_ATOMIC_INC(ec->stats.shd.completed); + } + } + } - ret = syncop_getxattr(healer->this, loc, NULL, EC_XATTR_HEAL, NULL, NULL); - if (!full && (ret >= 0) && (loc->inode->ia_type == IA_IFDIR)) { + if (!full && (loc->inode->ia_type == IA_IFDIR)) { /* If we have just healed a directory, it's possible that - * other index entries have appeared to be healed. We put a - * mark so that we can check it later and restart a scan - * without delay. */ - healer->rerun = _gf_true; + * other index entries have appeared to be healed. */ + if ((xdata != NULL) && + (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) && + (count > 0)) { + /* Force a rerun of the index healer. */ + gf_msg_debug(healer->this->name, 0, "%d more entries to heal", + count); + + healer->rerun = _gf_true; + } + } + + if (xdata != NULL) { + dict_unref(xdata); + } + + if (dict) { + dict_unref(dict); } return ret; diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c index a891ccd0952..dad5f4d7018 100644 --- a/xlators/cluster/ec/src/ec-inode-read.c +++ b/xlators/cluster/ec/src/ec-inode-read.c @@ -390,7 +390,8 @@ ec_manager_getxattr(ec_fop_data_t *fop, int32_t state) int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, int32_t op_ret, int32_t op_errno, uintptr_t mask, - uintptr_t good, uintptr_t bad, dict_t *xdata) + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) { fop_getxattr_cbk_t func = cookie; ec_t *ec = xl->private; @@ -398,6 +399,25 @@ ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, char *str; char bin1[65], bin2[65]; + /* We try to return the 'pending' information in xdata, but if this cannot + * be set, we will ignore it silently. We prefer to report the success or + * failure of the heal itself. */ + if (xdata == NULL) { + xdata = dict_new(); + } else { + dict_ref(xdata); + } + if (xdata != NULL) { + if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) { + /* dict_set_uint32() is marked as 'warn_unused_result' and gcc + * enforces to check the result in this case. However we don't + * really care if it succeeded or not. We'll just do the same. + * + * This empty 'if' avoids the warning, and it will be removed by + * the optimizer. */ + } + } + if (op_ret >= 0) { dict = dict_new(); if (dict == NULL) { @@ -431,11 +451,14 @@ ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, } out: - func(frame, NULL, xl, op_ret, op_errno, dict, NULL); + func(frame, NULL, xl, op_ret, op_errno, dict, xdata); if (dict != NULL) { dict_unref(dict); } + if (xdata != NULL) { + dict_unref(xdata); + } return 0; } diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c index 8e84977d2b3..601960d6154 100644 --- a/xlators/cluster/ec/src/ec-locks.c +++ b/xlators/cluster/ec/src/ec-locks.c @@ -24,9 +24,36 @@ ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask) ec_t *ec = fop->xl->private; ec_cbk_data_t *ans = NULL; ec_cbk_data_t *cbk = NULL; - uintptr_t locked = 0, notlocked = 0; + uintptr_t locked = 0; + int32_t good = 0; + int32_t eagain = 0; + int32_t estale = 0; int32_t error = -1; + /* There are some errors that we'll handle in an special way while trying + * to acquire a lock. + * + * EAGAIN: If it's found during a parallel non-blocking lock request, we + * consider that there's contention on the inode, so we consider + * the acquisition a failure and try again with a sequential + * blocking lock request. This will ensure that we get a lock on + * as many bricks as possible (ignoring EAGAIN here would cause + * unnecessary triggers of self-healing). + * + * If it's found during a sequential blocking lock request, it's + * considered an error. Lock will only succeed if there are + * enough other bricks locked. + * + * ESTALE: This can appear during parallel or sequential lock request if + * the inode has just been unlinked. We consider this error is + * not recoverable, but we also don't consider it as fatal. So, + * if it happens during parallel lock, we won't attempt a + * sequential one unless there are EAGAIN errors on other + * bricks (and are enough to form a quorum), but if we reach + * quorum counting the ESTALE bricks, we consider the whole + * result of the operation is ESTALE instead of EIO. + */ + list_for_each_entry(ans, &fop->cbk_list, list) { if (ans->op_ret >= 0) { @@ -34,24 +61,23 @@ ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask) error = EIO; } locked |= ans->mask; + good = ans->count; cbk = ans; - } else { - if (ans->op_errno == EAGAIN) { - switch (fop->uint32) { - case EC_LOCK_MODE_NONE: - case EC_LOCK_MODE_ALL: - /* Goal is to treat non-blocking lock as failure - * even if there is a single EAGAIN*/ - notlocked |= ans->mask; - break; - } - } + } else if (ans->op_errno == ESTALE) { + estale += ans->count; + } else if ((ans->op_errno == EAGAIN) && + (fop->uint32 != EC_LOCK_MODE_INC)) { + eagain += ans->count; } } if (error == -1) { - if (gf_bits_count(locked | notlocked) >= ec->fragments) { - if (notlocked == 0) { + /* If we have enough quorum with succeeded and EAGAIN answers, we + * ignore for now any ESTALE answer. If there are EAGAIN answers, + * we retry with a sequential blocking lock request if needed. + * Otherwise we succeed. */ + if ((good + eagain) >= ec->fragments) { + if (eagain == 0) { if (fop->answer == NULL) { fop->answer = cbk; } @@ -64,21 +90,28 @@ ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask) case EC_LOCK_MODE_NONE: error = EAGAIN; break; - case EC_LOCK_MODE_ALL: fop->uint32 = EC_LOCK_MODE_INC; break; - default: + /* This shouldn't happen because eagain cannot be > 0 + * when fop->uint32 is EC_LOCK_MODE_INC. */ error = EIO; break; } } } else { - if (fop->answer && fop->answer->op_ret < 0) + /* We have been unable to find enough candidates that will be able + * to take the lock. If we have quorum on some answer, we return + * it. Otherwise we check if ESTALE answers allow us to reach + * quorum. If so, we return ESTALE. */ + if (fop->answer && fop->answer->op_ret < 0) { error = fop->answer->op_errno; - else + } else if ((good + eagain + estale) >= ec->fragments) { + error = ESTALE; + } else { error = EIO; + } } } diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h index 7829b8c27b3..de9b89bb2c9 100644 --- a/xlators/cluster/ec/src/ec-types.h +++ b/xlators/cluster/ec/src/ec-types.h @@ -130,7 +130,12 @@ typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t); enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX }; -enum _ec_heal_need { EC_HEAL_NONEED, EC_HEAL_MAYBE, EC_HEAL_MUST }; +enum _ec_heal_need { + EC_HEAL_NONEED, + EC_HEAL_MAYBE, + EC_HEAL_MUST, + EC_HEAL_PURGE_INDEX +}; enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL }; @@ -186,10 +191,10 @@ struct _ec_inode { typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, int32_t, uintptr_t, uintptr_t, uintptr_t, - dict_t *); + uint32_t, dict_t *); typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, int32_t, uintptr_t, uintptr_t, uintptr_t, - dict_t *); + uint32_t, dict_t *); union _ec_cbk { fop_access_cbk_t access; @@ -621,6 +626,11 @@ struct _ec_statistics { requests. (Basically memory allocation errors). */ } stripe_cache; + struct { + gf_atomic_t attempted; /*Number of heals attempted on + files/directories*/ + gf_atomic_t completed; /*Number of heals complted on files/directories*/ + } shd; }; struct _ec { diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c index 66b4e634911..7344be4968d 100644 --- a/xlators/cluster/ec/src/ec.c +++ b/xlators/cluster/ec/src/ec.c @@ -325,13 +325,18 @@ ec_get_event_from_state(ec_t *ec) void ec_up(xlator_t *this, ec_t *ec) { + char str1[32], str2[32]; + if (ec->timer != NULL) { gf_timer_call_cancel(this->ctx, ec->timer); ec->timer = NULL; } ec->up = 1; - gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP, "Going UP"); + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP, + "Going UP : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name); } @@ -339,13 +344,18 @@ ec_up(xlator_t *this, ec_t *ec) void ec_down(xlator_t *this, ec_t *ec) { + char str1[32], str2[32]; + if (ec->timer != NULL) { gf_timer_call_cancel(this->ctx, ec->timer); ec->timer = NULL; } ec->up = 0; - gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN, "Going DOWN"); + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN, + "Going DOWN : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name); } @@ -700,6 +710,8 @@ ec_statistics_init(ec_t *ec) GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0); GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0); GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0); + GF_ATOMIC_INIT(ec->stats.shd.attempted, 0); + GF_ATOMIC_INIT(ec->stats.shd.completed, 0); } static int @@ -1569,6 +1581,10 @@ ec_dump_private(xlator_t *this) GF_ATOMIC_GET(ec->stats.stripe_cache.allocs)); gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC, GF_ATOMIC_GET(ec->stats.stripe_cache.errors)); + gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.attempted)); + gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.completed)); return 0; } diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h index 1b210d9adc1..6f6de6d5981 100644 --- a/xlators/cluster/ec/src/ec.h +++ b/xlators/cluster/ec/src/ec.h @@ -18,6 +18,7 @@ #define EC_XATTR_SIZE EC_XATTR_PREFIX "size" #define EC_XATTR_VERSION EC_XATTR_PREFIX "version" #define EC_XATTR_HEAL EC_XATTR_PREFIX "heal" +#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new" #define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty" #define EC_STRIPE_CACHE_MAX_SIZE 10 #define EC_VERSION_SIZE 2 diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c index ff993f7b5e5..d45655ef4c3 100644 --- a/xlators/debug/error-gen/src/error-gen.c +++ b/xlators/debug/error-gen/src/error-gen.c @@ -31,9 +31,9 @@ sys_error_t error_no_list[] = { [GF_FOP_LOOKUP] = {.error_no_count = 4, .error_no = {ENOENT, ENOTDIR, ENAMETOOLONG, EAGAIN}}, - [GF_FOP_STAT] = {.error_no_count = 7, - .error_no = {EACCES, EBADF, EFAULT, ENAMETOOLONG, ENOENT, - ENOMEM, ENOTDIR}}, + [GF_FOP_STAT] = {.error_no_count = 6, + .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOENT, ENOMEM, + ENOTDIR}}, [GF_FOP_READLINK] = {.error_no_count = 8, .error_no = {EACCES, EFAULT, EINVAL, EIO, ENAMETOOLONG, ENOENT, ENOMEM, ENOTDIR}}, @@ -79,21 +79,20 @@ sys_error_t error_no_list[] = { [GF_FOP_WRITE] = {.error_no_count = 7, .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG, ENOSPC, GF_ERROR_SHORT_WRITE}}, - [GF_FOP_STATFS] = {.error_no_count = 10, - .error_no = {EACCES, EBADF, EFAULT, EINTR, EIO, - ENAMETOOLONG, ENOENT, ENOMEM, ENOSYS, - ENOTDIR}}, + [GF_FOP_STATFS] = {.error_no_count = 9, + .error_no = {EACCES, EFAULT, EINTR, EIO, ENAMETOOLONG, + ENOENT, ENOMEM, ENOSYS, ENOTDIR}}, [GF_FOP_FLUSH] = {.error_no_count = 5, .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS, ENOENT}}, [GF_FOP_FSYNC] = {.error_no_count = 4, .error_no = {EBADF, EIO, EROFS, EINVAL}}, - [GF_FOP_SETXATTR] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}}, - [GF_FOP_GETXATTR] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}}, - [GF_FOP_REMOVEXATTR] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}}, + [GF_FOP_SETXATTR] = {.error_no_count = 3, + .error_no = {EACCES, EINTR, ENAMETOOLONG}}, + [GF_FOP_GETXATTR] = {.error_no_count = 3, + .error_no = {EACCES, ENAMETOOLONG, EINTR}}, + [GF_FOP_REMOVEXATTR] = {.error_no_count = 3, + .error_no = {EACCES, ENAMETOOLONG, EINTR}}, [GF_FOP_FSETXATTR] = {.error_no_count = 4, .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}}, [GF_FOP_FGETXATTR] = {.error_no_count = 4, @@ -125,26 +124,25 @@ sys_error_t error_no_list[] = { ENOENT}}, [GF_FOP_FXATTROP] = {.error_no_count = 4, .error_no = {EBADF, EIO, EROFS, EINVAL}}, - [GF_FOP_INODELK] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}}, + [GF_FOP_INODELK] = {.error_no_count = 3, + .error_no = {EACCES, EINTR, ENAMETOOLONG}}, [GF_FOP_FINODELK] = {.error_no_count = 4, .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}}, - [GF_FOP_ENTRYLK] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}}, + [GF_FOP_ENTRYLK] = {.error_no_count = 3, + .error_no = {EACCES, ENAMETOOLONG, EINTR}}, [GF_FOP_FENTRYLK] = {.error_no_count = 10, .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE, ENAMETOOLONG, ENFILE, ENODEV, ENOENT, ENOMEM}}, - [GF_FOP_SETATTR] = {.error_no_count = 11, + [GF_FOP_SETATTR] = {.error_no_count = 10, .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT, - ENOMEM, ENOTDIR, EPERM, EROFS, EBADF, - EIO}}, + ENOMEM, ENOTDIR, EPERM, EROFS, EIO}}, [GF_FOP_FSETATTR] = {.error_no_count = 11, .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT, ENOMEM, ENOTDIR, EPERM, EROFS, EBADF, EIO}}, - [GF_FOP_GETSPEC] = {.error_no_count = 4, - .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}}}; + [GF_FOP_GETSPEC] = {.error_no_count = 3, + .error_no = {EACCES, ENAMETOOLONG, EINTR}}}; int generate_rand_no(int op_no) @@ -1509,8 +1507,8 @@ init(xlator_t *this) this->private = pvt; - /* Give some seed value here */ - srand(time(NULL)); + /* Give some seed value here. */ + srand(gf_time()); ret = 0; out: diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c index 4e1f6e5af07..aa00c446e5a 100644 --- a/xlators/debug/io-stats/src/io-stats.c +++ b/xlators/debug/io-stats/src/io-stats.c @@ -135,7 +135,7 @@ struct ios_global_stats { gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE]; gf_atomic_t fop_hits[GF_FOP_MAXVALUE]; gf_atomic_t upcall_hits[GF_UPCALL_FLAGS_MAXVALUE]; - struct timeval started_at; + time_t started_at; struct ios_lat latency[GF_FOP_MAXVALUE]; uint64_t nr_opens; uint64_t max_nr_opens; @@ -292,9 +292,7 @@ is_fop_latency_started(call_frame_t *frame) begin = &frame->begin; \ end = &frame->end; \ \ - elapsed = ((end->tv_sec - begin->tv_sec) * 1e9 + \ - (end->tv_nsec - begin->tv_nsec)) / \ - 1000; \ + elapsed = gf_tsdiff(begin, end) / 1000.0; \ throughput = op_ret / elapsed; \ \ conf = this->private; \ @@ -678,10 +676,7 @@ ios_dump_throughput_stats(struct ios_stat_head *list_head, xlator_t *this, FILE *logfp, ios_stats_thru_t type) { struct ios_stat_list *entry = NULL; - struct timeval time = { - 0, - }; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; @@ -689,12 +684,9 @@ ios_dump_throughput_stats(struct ios_stat_head *list_head, xlator_t *this, { list_for_each_entry(entry, &list_head->iosstats->list, list) { - gf_time_fmt(timestr, sizeof timestr, - entry->iosstat->thru_counters[type].time.tv_sec, - gf_timefmt_FT); - snprintf(timestr + strlen(timestr), - sizeof timestr - strlen(timestr), ".%" GF_PRI_SUSECONDS, - time.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, + &entry->iosstat->thru_counters[type].time, + gf_timefmt_FT); ios_log(this, logfp, "%s \t %-10.2f \t %s", timestr, entry->value, entry->iosstat->filename); @@ -773,9 +765,8 @@ err: int io_stats_dump_global_to_json_logfp(xlator_t *this, - struct ios_global_stats *stats, - struct timeval *now, int interval, - FILE *logfp) + struct ios_global_stats *stats, time_t now, + int interval, FILE *logfp) { int i = 0; int j = 0; @@ -801,10 +792,7 @@ io_stats_dump_global_to_json_logfp(xlator_t *this, }; dict_t *xattr = NULL; - interval_sec = ((now->tv_sec * 1000000.0 + now->tv_usec) - - (stats->started_at.tv_sec * 1000000.0 + - stats->started_at.tv_usec)) / - 1000000.0; + interval_sec = (double)(now - stats->started_at); conf = this->private; @@ -956,8 +944,8 @@ io_stats_dump_global_to_json_logfp(xlator_t *this, } if (interval == -1) { - ios_log(this, logfp, "\"%s.%s.uptime\": %" PRId64 ",", key_prefix, - str_prefix, (uint64_t)(now->tv_sec - stats->started_at.tv_sec)); + ios_log(this, logfp, "\"%s.%s.uptime\": %" PRIu64 ",", key_prefix, + str_prefix, (uint64_t)(now - stats->started_at)); ios_log(this, logfp, "\"%s.%s.bytes_read\": " "%" GF_PRI_ATOMIC ",", @@ -1209,14 +1197,14 @@ out: int io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats, - struct timeval *now, int interval, FILE *logfp) + time_t now, int interval, FILE *logfp) { int i = 0; int per_line = 0; int index = 0; struct ios_stat_head *list_head = NULL; struct ios_conf *conf = NULL; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char str_header[128] = {0}; @@ -1232,8 +1220,8 @@ io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats, ios_log(this, logfp, "\n=== Cumulative stats ==="); else ios_log(this, logfp, "\n=== Interval %d stats ===", interval); - ios_log(this, logfp, " Duration : %" PRId64 " secs", - (uint64_t)(now->tv_sec - stats->started_at.tv_sec)); + ios_log(this, logfp, " Duration : %" PRIu64 " secs", + (uint64_t)(now - stats->started_at)); ios_log(this, logfp, " BytesRead : %" GF_PRI_ATOMIC, GF_ATOMIC_GET(stats->data_read)); ios_log(this, logfp, " BytesWritten : %" GF_PRI_ATOMIC "\n", @@ -1325,11 +1313,8 @@ io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats, if (interval == -1) { LOCK(&conf->lock); { - gf_time_fmt(timestr, sizeof timestr, - conf->cumulative.max_openfd_time.tv_sec, gf_timefmt_FT); - snprintf(timestr + strlen(timestr), - sizeof timestr - strlen(timestr), ".%" GF_PRI_SUSECONDS, - conf->cumulative.max_openfd_time.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, + &conf->cumulative.max_openfd_time, gf_timefmt_FT); ios_log(this, logfp, "Current open fd's: %" PRId64 " Max open fd's: %" PRId64 " time %s", @@ -1381,7 +1366,7 @@ io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats, int io_stats_dump_global_to_dict(xlator_t *this, struct ios_global_stats *stats, - struct timeval *now, int interval, dict_t *dict) + time_t now, int interval, dict_t *dict) { int ret = 0; char key[64] = {0}; @@ -1407,7 +1392,7 @@ io_stats_dump_global_to_dict(xlator_t *this, struct ios_global_stats *stats, interval); snprintf(key, sizeof(key), "%d-duration", interval); - sec = (uint64_t)(now->tv_sec - stats->started_at.tv_sec); + sec = now - stats->started_at; ret = dict_set_uint64(dict, key, sec); if (ret) { gf_log(this->name, GF_LOG_ERROR, @@ -1530,9 +1515,8 @@ out: } int -io_stats_dump_global(xlator_t *this, struct ios_global_stats *stats, - struct timeval *now, int interval, - struct ios_dump_args *args) +io_stats_dump_global(xlator_t *this, struct ios_global_stats *stats, time_t now, + int interval, struct ios_dump_args *args) { int ret = -1; @@ -1590,13 +1574,13 @@ ios_dump_args_init(struct ios_dump_args *args, ios_dump_type_t type, } static void -ios_global_stats_clear(struct ios_global_stats *stats, struct timeval *now) +ios_global_stats_clear(struct ios_global_stats *stats, time_t now) { GF_ASSERT(stats); GF_ASSERT(now); memset(stats, 0, sizeof(*stats)); - stats->started_at = *now; + stats->started_at = now; } int @@ -1607,7 +1591,7 @@ io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op, struct ios_global_stats cumulative = {}; struct ios_global_stats incremental = {}; int increment = 0; - struct timeval now; + time_t now = 0; GF_ASSERT(this); GF_ASSERT(args); @@ -1615,8 +1599,8 @@ io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op, GF_ASSERT(args->type < IOS_DUMP_TYPE_MAX); conf = this->private; + now = gf_time(); - gettimeofday(&now, NULL); LOCK(&conf->lock); { if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE) @@ -1629,17 +1613,17 @@ io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op, if (!is_peek) { increment = conf->increment++; - ios_global_stats_clear(&conf->incremental, &now); + ios_global_stats_clear(&conf->incremental, now); } } } UNLOCK(&conf->lock); if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE) - io_stats_dump_global(this, &cumulative, &now, -1, args); + io_stats_dump_global(this, &cumulative, now, -1, args); if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL) - io_stats_dump_global(this, &incremental, &now, increment, args); + io_stats_dump_global(this, &incremental, now, increment, args); return 0; } @@ -1649,9 +1633,8 @@ io_stats_dump_fd(xlator_t *this, struct ios_fd *iosfd) { struct ios_conf *conf = NULL; struct timeval now; - uint64_t sec = 0; - uint64_t usec = 0; int i = 0; + double usecs = 0; uint64_t data_read = 0; uint64_t data_written = 0; uint64_t block_count_read = 0; @@ -1666,23 +1649,15 @@ io_stats_dump_fd(xlator_t *this, struct ios_fd *iosfd) return 0; gettimeofday(&now, NULL); - - if (iosfd->opened_at.tv_usec > now.tv_usec) { - now.tv_usec += 1000000; - now.tv_usec--; - } - - sec = now.tv_sec - iosfd->opened_at.tv_sec; - usec = now.tv_usec - iosfd->opened_at.tv_usec; + usecs = gf_tvdiff(&iosfd->opened_at, &now); gf_log(this->name, GF_LOG_INFO, "--- fd stats ---"); if (iosfd->filename) gf_log(this->name, GF_LOG_INFO, " Filename : %s", iosfd->filename); - if (sec) - gf_log(this->name, GF_LOG_INFO, - " Lifetime : %" PRId64 "secs, %" PRId64 "usecs", sec, usec); + if (usecs) + gf_log(this->name, GF_LOG_INFO, " Lifetime : %lf secs", usecs); data_read = GF_ATOMIC_GET(iosfd->data_read); if (data_read) @@ -1785,9 +1760,7 @@ update_ios_latency(struct ios_conf *conf, call_frame_t *frame, begin = &frame->begin; end = &frame->end; - elapsed = ((end->tv_sec - begin->tv_sec) * 1e9 + - (end->tv_nsec - begin->tv_nsec)) / - 1000; + elapsed = gf_tsdiff(begin, end) / 1000.0; update_ios_latency_stats(&conf->cumulative, elapsed, op); update_ios_latency_stats(&conf->incremental, elapsed, op); @@ -1808,7 +1781,7 @@ io_stats_dump_stats_to_dict(xlator_t *this, dict_t *resp, struct ios_stat_list *entry = NULL; int ret = -1; ios_stats_thru_t index = IOS_STATS_THRU_MAX; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char *dict_timestr = NULL; @@ -1827,14 +1800,9 @@ io_stats_dump_stats_to_dict(xlator_t *this, dict_t *resp, ret = dict_set_uint64(resp, "max-open", conf->cumulative.max_nr_opens); - gf_time_fmt(timestr, sizeof timestr, - conf->cumulative.max_openfd_time.tv_sec, - gf_timefmt_FT); - if (conf->cumulative.max_openfd_time.tv_sec) - snprintf(timestr + strlen(timestr), - sizeof timestr - strlen(timestr), - ".%" GF_PRI_SUSECONDS, - conf->cumulative.max_openfd_time.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, + &conf->cumulative.max_openfd_time, + gf_timefmt_FT); dict_timestr = gf_strdup(timestr); if (!dict_timestr) @@ -3606,26 +3574,21 @@ ios_destroy_top_stats(struct ios_conf *conf) return; } -static int +static void io_stats_clear(struct ios_conf *conf) { - struct timeval now; - int ret = -1; + time_t now = 0; GF_ASSERT(conf); + now = gf_time(); - if (!gettimeofday(&now, NULL)) { - LOCK(&conf->lock); - { - ios_global_stats_clear(&conf->cumulative, &now); - ios_global_stats_clear(&conf->incremental, &now); - conf->increment = 0; - } - UNLOCK(&conf->lock); - ret = 0; + LOCK(&conf->lock); + { + ios_global_stats_clear(&conf->cumulative, now); + ios_global_stats_clear(&conf->incremental, now); + conf->increment = 0; } - - return ret; + UNLOCK(&conf->lock); } int32_t @@ -3866,7 +3829,7 @@ ios_conf_destroy(struct ios_conf *conf) _ios_destroy_dump_thread(conf); ios_destroy_sample_buf(conf->ios_sample_buf); LOCK_DESTROY(&conf->lock); - GF_FREE(conf->dnscache); + gf_dnscache_deinit(conf->dnscache); GF_FREE(conf); } @@ -3889,7 +3852,7 @@ ios_init_stats(struct ios_global_stats *stats) for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) GF_ATOMIC_INIT(stats->upcall_hits[i], 0); - gettimeofday(&stats->started_at, NULL); + stats->started_at = gf_time(); } int @@ -3978,11 +3941,14 @@ init(xlator_t *this) gf_log(this->name, GF_LOG_ERROR, "Out of memory."); goto out; } - ret = -1; GF_OPTION_INIT("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec, int32, out); conf->dnscache = gf_dnscache_init(conf->ios_dnscache_ttl_sec); + if (!conf->dnscache) { + ret = -1; + goto out; + } GF_OPTION_INIT("sys-log-level", sys_log_str, str, out); if (sys_log_str) { @@ -4133,12 +4099,9 @@ notify(xlator_t *this, int32_t event, void *data, ...) } if (GF_IOS_INFO_CLEAR == op) { - ret = io_stats_clear(this->private); - if (ret) - gf_log(this->name, GF_LOG_ERROR, - "Failed to clear info stats"); + io_stats_clear(this->private); - ret = dict_set_int32(output, "stats-cleared", ret ? 0 : 1); + ret = dict_set_int32(output, "stats-cleared", 1); if (ret) gf_log(this->name, GF_LOG_ERROR, "Failed to set stats-cleared" diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c index 3db2e263524..6ed0ca00342 100644 --- a/xlators/debug/trace/src/trace.c +++ b/xlators/debug/trace/src/trace.c @@ -22,13 +22,13 @@ static void trace_stat_to_str(struct iatt *buf, char *str, size_t len) { - char atime_buf[200] = { + char atime_buf[GF_TIMESTR_SIZE] = { 0, }; - char mtime_buf[200] = { + char mtime_buf[GF_TIMESTR_SIZE] = { 0, }; - char ctime_buf[200] = { + char ctime_buf[GF_TIMESTR_SIZE] = { 0, }; @@ -64,7 +64,7 @@ trace_stat_to_str(struct iatt *buf, char *str, size_t len) int dump_history_trace(circular_buffer_t *cb, void *data) { - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; @@ -72,9 +72,7 @@ dump_history_trace(circular_buffer_t *cb, void *data) gettimeofday () fails, it's safe to check tm and then dump the time at which the entry was added to the buffer */ - gf_time_fmt(timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_Ymd_T); - snprintf(timestr + strlen(timestr), 256 - strlen(timestr), - ".%" GF_PRI_SUSECONDS, cb->tv.tv_usec); + gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_Ymd_T); gf_proc_dump_write("TIME", "%s", timestr); gf_proc_dump_write("FOP", "%s\n", (char *)cb->data); @@ -2209,10 +2207,10 @@ int trace_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata) { - char actime_str[256] = { + char actime_str[GF_TIMESTR_SIZE] = { 0, }; - char modtime_str[256] = { + char modtime_str[GF_TIMESTR_SIZE] = { 0, }; trace_conf_t *conf = NULL; @@ -2278,10 +2276,10 @@ int trace_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata) { - char actime_str[256] = { + char actime_str[GF_TIMESTR_SIZE] = { 0, }; - char modtime_str[256] = { + char modtime_str[GF_TIMESTR_SIZE] = { 0, }; trace_conf_t *conf = NULL; diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am index 194634b003d..c57897f11ea 100644 --- a/xlators/features/Makefile.am +++ b/xlators/features/Makefile.am @@ -2,9 +2,13 @@ if BUILD_CLOUDSYNC CLOUDSYNC_DIR = cloudsync endif +if BUILD_METADISP + METADISP_DIR = metadisp +endif + SUBDIRS = locks quota read-only quiesce marker index barrier arbiter upcall \ compress changelog gfid-access snapview-client snapview-server trash \ shard bit-rot leases selinux sdfs namespace $(CLOUDSYNC_DIR) thin-arbiter \ - utime + utime $(METADISP_DIR) CLEANFILES = diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c index 34e20f9df11..5cef2ffa5e5 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c @@ -40,21 +40,21 @@ br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat) } void -br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, struct timeval *tv) +br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time) { if (!scrub_stat) return; pthread_mutex_lock(&scrub_stat->lock); { - scrub_stat->scrub_start_tv.tv_sec = tv->tv_sec; + scrub_stat->scrub_start_time = time; } pthread_mutex_unlock(&scrub_stat->lock); } void br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr, - struct timeval *tv) + time_t time) { int lst_size = 0; @@ -67,10 +67,10 @@ br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr, pthread_mutex_lock(&scrub_stat->lock); { - scrub_stat->scrub_end_tv.tv_sec = tv->tv_sec; + scrub_stat->scrub_end_time = time; - scrub_stat->scrub_duration = scrub_stat->scrub_end_tv.tv_sec - - scrub_stat->scrub_start_tv.tv_sec; + scrub_stat->scrub_duration = scrub_stat->scrub_end_time - + scrub_stat->scrub_start_time; snprintf(scrub_stat->last_scrub_time, lst_size, "%s", timestr); } diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h index 24128b90a66..f022aa831eb 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h @@ -15,20 +15,22 @@ #include <sys/time.h> #include <pthread.h> +#include <glusterfs/common-utils.h> + struct br_scrub_stats { - uint64_t scrubbed_files; /* Total number of scrubbed file */ + uint64_t scrubbed_files; /* Total number of scrubbed files. */ - uint64_t unsigned_files; /* Total number of unsigned file */ + uint64_t unsigned_files; /* Total number of unsigned files. */ - uint64_t scrub_duration; /* Duration of last scrub */ + uint64_t scrub_duration; /* Duration of last scrub. */ - char last_scrub_time[1024]; /*last scrub completion time */ + char last_scrub_time[GF_TIMESTR_SIZE]; /* Last scrub completion time. */ - struct timeval scrub_start_tv; /* Scrubbing starting time*/ + time_t scrub_start_time; /* Scrubbing starting time. */ - struct timeval scrub_end_tv; /* Scrubbing finishing time */ + time_t scrub_end_time; /* Scrubbing finishing time. */ - int8_t scrub_running; /* Scrub running or not */ + int8_t scrub_running; /* Whether scrub running or not. */ pthread_mutex_t lock; }; @@ -40,9 +42,9 @@ br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat); void br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat); void -br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, struct timeval *tv); +br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time); void br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr, - struct timeval *tv); + time_t time); #endif /* __BIT_ROT_SCRUB_STATUS_H__ */ diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c index d20ecc7cdbe..289dd53f610 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c @@ -601,25 +601,23 @@ br_fsscan_deactivate(xlator_t *this) static void br_scrubber_log_time(xlator_t *this, const char *sfx) { - char timestr[1024] = { - 0, - }; - struct timeval tv = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; br_private_t *priv = NULL; + time_t now = 0; + now = gf_time(); priv = this->private; - gettimeofday(&tv, NULL); - gf_time_fmt(timestr, sizeof(timestr), tv.tv_sec, gf_timefmt_FT); + gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT); if (strcasecmp(sfx, "started") == 0) { - br_update_scrub_start_time(&priv->scrub_stat, &tv); + br_update_scrub_start_time(&priv->scrub_stat, now); gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START, "Scrubbing %s at %s", sfx, timestr); } else { - br_update_scrub_finish_time(&priv->scrub_stat, timestr, &tv); + br_update_scrub_finish_time(&priv->scrub_stat, timestr, now); gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH, "Scrubbing %s at %s", sfx, timestr); } @@ -628,15 +626,13 @@ br_scrubber_log_time(xlator_t *this, const char *sfx) static void br_fsscanner_log_time(xlator_t *this, br_child_t *child, const char *sfx) { - char timestr[1024] = { - 0, - }; - struct timeval tv = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; + time_t now = 0; - gettimeofday(&tv, NULL); - gf_time_fmt(timestr, sizeof(timestr), tv.tv_sec, gf_timefmt_FT); + now = gf_time(); + gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT); if (strcasecmp(sfx, "started") == 0) { gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s", @@ -919,10 +915,7 @@ br_fsscan_schedule(xlator_t *this) { uint32_t timo = 0; br_private_t *priv = NULL; - struct timeval tv = { - 0, - }; - char timestr[1024] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; struct br_scrubber *fsscrub = NULL; @@ -933,8 +926,7 @@ br_fsscan_schedule(xlator_t *this) fsscrub = &priv->fsscrub; scrub_monitor = &priv->scrub_monitor; - (void)gettimeofday(&tv, NULL); - scrub_monitor->boot = tv.tv_sec; + scrub_monitor->boot = gf_time(); timo = br_fsscan_calculate_timeout(fsscrub->frequency); if (timo == 0) { @@ -975,12 +967,10 @@ int32_t br_fsscan_activate(xlator_t *this) { uint32_t timo = 0; - char timestr[1024] = { - 0, - }; - struct timeval now = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; + time_t now = 0; br_private_t *priv = NULL; struct br_scrubber *fsscrub = NULL; struct br_monitor *scrub_monitor = NULL; @@ -989,7 +979,7 @@ br_fsscan_activate(xlator_t *this) fsscrub = &priv->fsscrub; scrub_monitor = &priv->scrub_monitor; - (void)gettimeofday(&now, NULL); + now = gf_time(); timo = br_fsscan_calculate_timeout(fsscrub->frequency); if (timo == 0) { gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG, @@ -1003,7 +993,7 @@ br_fsscan_activate(xlator_t *this) } pthread_mutex_unlock(&scrub_monitor->donelock); - gf_time_fmt(timestr, sizeof(timestr), (now.tv_sec + timo), gf_timefmt_FT); + gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT); (void)gf_tw_mod_timer(priv->timer_wheel, scrub_monitor->timer, timo); _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING); @@ -1020,12 +1010,10 @@ br_fsscan_reschedule(xlator_t *this) { int32_t ret = 0; uint32_t timo = 0; - char timestr[1024] = { - 0, - }; - struct timeval now = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; + time_t now = 0; br_private_t *priv = NULL; struct br_scrubber *fsscrub = NULL; struct br_monitor *scrub_monitor = NULL; @@ -1037,7 +1025,7 @@ br_fsscan_reschedule(xlator_t *this) if (!fsscrub->frequency_reconf) return 0; - (void)gettimeofday(&now, NULL); + now = gf_time(); timo = br_fsscan_calculate_timeout(fsscrub->frequency); if (timo == 0) { gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG, @@ -1045,7 +1033,7 @@ br_fsscan_reschedule(xlator_t *this) return -1; } - gf_time_fmt(timestr, sizeof(timestr), (now.tv_sec + timo), gf_timefmt_FT); + gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT); pthread_mutex_lock(&scrub_monitor->donelock); { @@ -1073,23 +1061,19 @@ br_fsscan_ondemand(xlator_t *this) { int32_t ret = 0; uint32_t timo = 0; - char timestr[1024] = { - 0, - }; - struct timeval now = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; + time_t now = 0; br_private_t *priv = NULL; struct br_monitor *scrub_monitor = NULL; priv = this->private; scrub_monitor = &priv->scrub_monitor; - (void)gettimeofday(&now, NULL); - + now = gf_time(); timo = BR_SCRUB_ONDEMAND; - - gf_time_fmt(timestr, sizeof(timestr), (now.tv_sec + timo), gf_timefmt_FT); + gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT); pthread_mutex_lock(&scrub_monitor->donelock); { @@ -1799,7 +1783,7 @@ br_collect_bad_objects_of_child(xlator_t *this, br_child_t *child, dict_t *dict, tmp_count = total_count; for (j = 0; j < count; j++) { - len = snprintf(key, PATH_MAX, "quarantine-%d", j); + len = snprintf(key, sizeof(key), "quarantine-%d", j); ret = dict_get_strn(child_dict, key, len, &entry); if (ret) continue; @@ -1810,7 +1794,7 @@ br_collect_bad_objects_of_child(xlator_t *this, br_child_t *child, dict_t *dict, if ((len < 0) || (len >= PATH_MAX)) { continue; } - snprintf(main_key, PATH_MAX, "quarantine-%d", tmp_count); + snprintf(main_key, sizeof(main_key), "quarantine-%d", tmp_count); ret = dict_set_dynstr_with_alloc(dict, main_key, tmp); if (!ret) diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h index 8d2b7f051da..6c15a166f18 100644 --- a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h +++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h @@ -44,7 +44,8 @@ GLFS_MSGID(BITROT_STUB, BRS_MSG_NO_MEMORY, BRS_MSG_SET_EVENT_FAILED, BRS_MSG_NON_BITD_PID, BRS_MSG_SIGN_PREPARE_FAIL, BRS_MSG_USING_DEFAULT_THREAD_SIZE, BRS_MSG_ALLOC_MEM_FAILED, BRS_MSG_DICT_ALLOC_FAILED, BRS_MSG_CREATE_GF_DIRENT_FAILED, - BRS_MSG_ALLOC_FAILED, BRS_MSG_PATH_XATTR_GET_FAILED); + BRS_MSG_ALLOC_FAILED, BRS_MSG_PATH_XATTR_GET_FAILED, + BRS_MSG_VERSION_PREPARE_FAIL); #define BRS_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed" #define BRS_MSG_BAD_OBJ_THREAD_FAIL_STR "pthread_init failed" @@ -68,6 +69,8 @@ GLFS_MSGID(BITROT_STUB, BRS_MSG_NO_MEMORY, BRS_MSG_SET_EVENT_FAILED, "daemon. Unwinding the fop" #define BRS_MSG_SIGN_PREPARE_FAIL_STR \ "failed to prepare the signature. Unwinding the fop" +#define BRS_MSG_VERSION_PREPARE_FAIL_STR \ + "failed to prepare the version. Unwinding the fop" #define BRS_MSG_STUB_ALLOC_FAILED_STR "failed to allocate stub fop, Unwinding" #define BRS_MSG_BAD_OBJ_MARK_FAIL_STR "failed to mark object as bad" #define BRS_MSG_NON_SCRUB_BAD_OBJ_MARK_STR \ diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c index 605a5e4c3e4..447dd47ff41 100644 --- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c +++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c @@ -424,8 +424,8 @@ br_stub_prepare_version_request(xlator_t *this, dict_t *dict, priv = this->private; br_set_ongoingversion(obuf, oversion, priv->boot); - return dict_set_static_bin(dict, BITROT_CURRENT_VERSION_KEY, (void *)obuf, - sizeof(br_version_t)); + return dict_set_bin(dict, BITROT_CURRENT_VERSION_KEY, (void *)obuf, + sizeof(br_version_t)); } static int @@ -436,8 +436,7 @@ br_stub_prepare_signing_request(dict_t *dict, br_signature_t *sbuf, br_set_signature(sbuf, sign, signaturelen, &size); - return dict_set_static_bin(dict, BITROT_SIGNING_VERSION_KEY, (void *)sbuf, - size); + return dict_set_bin(dict, BITROT_SIGNING_VERSION_KEY, (void *)sbuf, size); } /** @@ -854,23 +853,27 @@ br_stub_perform_incversioning(xlator_t *this, call_frame_t *frame, op_errno = ENOMEM; dict = dict_new(); if (!dict) - goto done; + goto out; ret = br_stub_alloc_versions(&obuf, NULL, 0); - if (ret) - goto dealloc_dict; + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED, + "gfid=%s", uuid_utoa(fd->inode->gfid), NULL); + goto out; + } ret = br_stub_prepare_version_request(this, dict, obuf, writeback_version); - if (ret) - goto dealloc_versions; + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_VERSION_PREPARE_FAIL, + "gfid=%s", uuid_utoa(fd->inode->gfid), NULL); + br_stub_dealloc_versions(obuf); + goto out; + } ret = br_stub_fd_versioning( this, frame, stub, dict, fd, br_stub_fd_incversioning_cbk, writeback_version, BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE); - -dealloc_versions: - br_stub_dealloc_versions(obuf); -dealloc_dict: - dict_unref(dict); -done: +out: + if (dict) + dict_unref(dict); if (ret) { if (local) frame->local = NULL; @@ -1025,31 +1028,36 @@ static int br_stub_prepare_signature(xlator_t *this, dict_t *dict, inode_t *inode, br_isignature_t *sign, int *fakesuccess) { - int32_t ret = 0; + int32_t ret = -1; size_t signaturelen = 0; br_signature_t *sbuf = NULL; if (!br_is_signature_type_valid(sign->signaturetype)) - goto error_return; + goto out; signaturelen = sign->signaturelen; ret = br_stub_alloc_versions(NULL, &sbuf, signaturelen); - if (ret) - goto error_return; + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED, + "gfid=%s", uuid_utoa(inode->gfid), NULL); + ret = -1; + goto out; + } ret = br_stub_prepare_signing_request(dict, sbuf, sign, signaturelen); - if (ret) - goto dealloc_versions; + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SIGN_PREPARE_FAIL, + "gfid=%s", uuid_utoa(inode->gfid), NULL); + ret = -1; + br_stub_dealloc_versions(sbuf); + goto out; + } + /* At this point sbuf has been added to dict, so the memory will be freed + * when the data from the dict is destroyed + */ ret = br_stub_compare_sign_version(this, inode, sbuf, dict, fakesuccess); - if (ret) - goto dealloc_versions; - - return 0; - -dealloc_versions: - br_stub_dealloc_versions(sbuf); -error_return: - return -1; +out: + return ret; } static void diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c index 71fe1f032a0..e561997d858 100644 --- a/xlators/features/changelog/src/changelog-helpers.c +++ b/xlators/features/changelog/src/changelog-helpers.c @@ -242,8 +242,7 @@ changelog_write(int fd, char *buffer, size_t len) } int -htime_update(xlator_t *this, changelog_priv_t *priv, unsigned long ts, - char *buffer) +htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer) { char changelog_path[PATH_MAX + 1] = { 0, @@ -273,7 +272,7 @@ htime_update(xlator_t *this, changelog_priv_t *priv, unsigned long ts, goto out; } - len = snprintf(x_value, sizeof(x_value), "%lu:%d", ts, + len = snprintf(x_value, sizeof(x_value), "%ld:%d", ts, priv->rollover_count); if (len >= sizeof(x_value)) { ret = -1; @@ -382,8 +381,7 @@ out: } static int -changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, - unsigned long ts) +changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, time_t ts) { int ret = -1; int notify = 0; @@ -421,16 +419,14 @@ changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, priv->changelog_fd = -1; } - time_t time = (time_t)ts; - - /* Get GMT time */ - gmt = gmtime(&time); + /* Get GMT time. */ + gmt = gmtime(&ts); strftime(yyyymmdd, sizeof(yyyymmdd), "%Y/%m/%d", gmt); (void)snprintf(ofile, PATH_MAX, "%s/" CHANGELOG_FILE_NAME, priv->changelog_dir); - (void)snprintf(nfile, PATH_MAX, "%s/%s/" CHANGELOG_FILE_NAME ".%lu", + (void)snprintf(nfile, PATH_MAX, "%s/%s/" CHANGELOG_FILE_NAME ".%ld", priv->changelog_dir, yyyymmdd, ts); (void)snprintf(nfile_dir, PATH_MAX, "%s/%s", priv->changelog_dir, yyyymmdd); @@ -593,7 +589,7 @@ out: * returns -1 on failure or error */ int -htime_open(xlator_t *this, changelog_priv_t *priv, unsigned long ts) +htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts) { int ht_file_fd = -1; int ht_dir_fd = -1; @@ -723,7 +719,7 @@ out: * returns -1 on failure or error */ int -htime_create(xlator_t *this, changelog_priv_t *priv, unsigned long ts) +htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts) { int ht_file_fd = -1; int ht_dir_fd = -1; @@ -741,12 +737,12 @@ htime_create(xlator_t *this, changelog_priv_t *priv, unsigned long ts) int32_t len = 0; gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_NEW_HTIME_FILE, - "name=%lu", ts, NULL); + "name=%ld", ts, NULL); CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path); /* get the htime file name in ht_file_path */ - len = snprintf(ht_file_path, PATH_MAX, "%s/%s.%lu", ht_dir_path, + len = snprintf(ht_file_path, PATH_MAX, "%s/%s.%ld", ht_dir_path, HTIME_FILE_NAME, ts); if ((len < 0) || (len >= PATH_MAX)) { ret = -1; @@ -792,7 +788,7 @@ htime_create(xlator_t *this, changelog_priv_t *priv, unsigned long ts) goto out; } - (void)snprintf(ht_file_bname, sizeof(ht_file_bname), "%s.%lu", + (void)snprintf(ht_file_bname, sizeof(ht_file_bname), "%s.%ld", HTIME_FILE_NAME, ts); if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname, strlen(ht_file_bname), 0)) { @@ -963,8 +959,8 @@ out: } int -changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, - unsigned long ts, gf_boolean_t finale) +changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts, + gf_boolean_t finale) { int ret = -1; @@ -985,21 +981,12 @@ changelog_entry_length() return sizeof(changelog_log_data_t); } -int +void changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last) { - struct timeval tv = { - 0, - }; - cld->cld_type = CHANGELOG_TYPE_ROLLOVER; - - if (gettimeofday(&tv, NULL)) - return -1; - - cld->cld_roll_time = (unsigned long)tv.tv_sec; + cld->cld_roll_time = gf_time(); cld->cld_finale = is_last; - return 0; } int @@ -1274,7 +1261,7 @@ changelog_rollover(void *data) while (1) { (void)pthread_testcancel(); - tv.tv_sec = time(NULL) + priv->rollover_time; + tv.tv_sec = gf_time() + priv->rollover_time; tv.tv_nsec = 0; ret = 0; /* Reset ret to zero */ @@ -1355,12 +1342,7 @@ changelog_rollover(void *data) if (priv->explicit_rollover == _gf_true) sleep(1); - ret = changelog_fill_rollover_data(&cld, _gf_false); - if (ret) { - gf_smsg(this->name, GF_LOG_ERROR, 0, - CHANGELOG_MSG_ROLLOVER_DATA_FILL_FAILED, NULL); - continue; - } + changelog_fill_rollover_data(&cld, _gf_false); _mask_cancellation(); diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h index 0d06d98c9e1..38fa7590c32 100644 --- a/xlators/features/changelog/src/changelog-helpers.h +++ b/xlators/features/changelog/src/changelog-helpers.h @@ -31,7 +31,7 @@ */ typedef struct changelog_log_data { /* rollover related */ - unsigned long cld_roll_time; + time_t cld_roll_time; /* reopen changelog? */ gf_boolean_t cld_finale; @@ -97,12 +97,6 @@ struct changelog_encoder { typedef struct changelog_time_slice { /** - * just in case we need nanosecond granularity some day. - * field is unused as of now (maybe we'd need it later). - */ - struct timeval tv_start; - - /** * version of changelog file, incremented each time changes * rollover. */ @@ -423,11 +417,11 @@ changelog_local_t * changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid, int xtra_records, gf_boolean_t update_flag); int -changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, - unsigned long ts, gf_boolean_t finale); +changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts, + gf_boolean_t finale); int changelog_open_journal(xlator_t *this, changelog_priv_t *priv); -int +void changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last); int changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv, @@ -451,12 +445,11 @@ changelog_fsync_thread(void *data); int changelog_forget(xlator_t *this, inode_t *inode); int -htime_update(xlator_t *this, changelog_priv_t *priv, unsigned long ts, - char *buffer); +htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer); int -htime_open(xlator_t *this, changelog_priv_t *priv, unsigned long ts); +htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts); int -htime_create(xlator_t *this, changelog_priv_t *priv, unsigned long ts); +htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts); /* Geo-Rep snapshot dependency changes */ void diff --git a/xlators/features/changelog/src/changelog-messages.h b/xlators/features/changelog/src/changelog-messages.h index 4dd56b8ee97..cb0e16c85d8 100644 --- a/xlators/features/changelog/src/changelog-messages.h +++ b/xlators/features/changelog/src/changelog-messages.h @@ -59,12 +59,12 @@ GLFS_MSGID( CHANGELOG_MSG_NO_HTIME_CURRENT, CHANGELOG_MSG_HTIME_CURRENT, CHANGELOG_MSG_NEW_HTIME_FILE, CHANGELOG_MSG_MKDIR_ERROR, CHANGELOG_MSG_PATH_NOT_FOUND, CHANGELOG_MSG_XATTR_INIT_FAILED, - CHANGELOG_MSG_WROTE_TO_CSNAP, CHANGELOG_MSG_ROLLOVER_DATA_FILL_FAILED, + CHANGELOG_MSG_WROTE_TO_CSNAP, CHANGELOG_MSG_UNUSED_0, CHANGELOG_MSG_GET_BUFFER_FAILED, CHANGELOG_MSG_BARRIER_STATE_NOTIFY, CHANGELOG_MSG_BARRIER_DISABLED, CHANGELOG_MSG_BARRIER_ALREADY_DISABLED, CHANGELOG_MSG_BARRIER_ON_ERROR, CHANGELOG_MSG_BARRIER_ENABLE, CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, CHANGELOG_MSG_ERROR_IN_DICT_GET, - CHANGELOG_MSG_GET_TIME_FAILURE, CHANGELOG_MSG_HTIME_FETCH_FAILED, + CHANGELOG_MSG_UNUSED_1, CHANGELOG_MSG_UNUSED_2, CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS, CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED, CHANGELOG_MSG_BARRIER_TIMEOUT, CHANGELOG_MSG_TIMEOUT_ADD_FAILED, @@ -123,8 +123,6 @@ GLFS_MSGID( #define CHANGELOG_MSG_GET_TIME_OP_FAILED_STR "Problem rolling over changelog(s)" #define CHANGELOG_MSG_BARRIER_INFO_STR "Explicit wakeup on barrier notify" #define CHANGELOG_MSG_SELECT_FAILED_STR "pthread_cond_timedwait failed" -#define CHANGELOG_MSG_ROLLOVER_DATA_FILL_FAILED_STR \ - "failed to fill rollover data" #define CHANGELOG_MSG_INJECT_FSYNC_FAILED_STR "failed to inject fsync event" #define CHANGELOG_MSG_LOCAL_INIT_FAILED_STR \ "changelog local initialization failed" @@ -144,9 +142,7 @@ GLFS_MSGID( #define CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND_STR "barrier key not found" #define CHANGELOG_MSG_ERROR_IN_DICT_GET_STR \ "Something went wrong in dict_get_str_boolean" -#define CHANGELOG_MSG_GET_TIME_FAILURE_STR "gettimeofday() failure" #define CHANGELOG_MSG_DIR_OPTIONS_NOT_SET_STR "changelog-dir option is not set" -#define CHANGELOG_MSG_HTIME_FETCH_FAILED_STR "unable to fetch htime" #define CHANGELOG_MSG_FREEUP_FAILED_STR "could not cleanup bootstrapper" #define CHANGELOG_MSG_CHILD_MISCONFIGURED_STR \ "translator needs a single subvolume" diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c index 75d5ae267fd..6a6e5af859e 100644 --- a/xlators/features/changelog/src/changelog.c +++ b/xlators/features/changelog/src/changelog.c @@ -2036,20 +2036,20 @@ notify(xlator_t *this, int event, void *data, ...) priv->notify_down = _gf_true; } UNLOCK(&priv->lock); - list_for_each_entry_safe(listener, next, &priv->rpc->listeners, - list) - { - if (listener->trans) { - rpc_transport_unref(listener->trans); + if (priv->rpc) { + list_for_each_entry_safe(listener, next, + &priv->rpc->listeners, list) + { + if (listener->trans) { + rpc_transport_unref(listener->trans); + } } + rpcsvc_destroy(priv->rpc); + priv->rpc = NULL; } CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX); sys_unlink(sockfile); - if (priv->rpc) { - rpcsvc_destroy(priv->rpc); - priv->rpc = NULL; - } if (!cleanup_notify) default_notify(this, GF_EVENT_PARENT_DOWN, data); } @@ -2252,23 +2252,11 @@ static int changelog_init(xlator_t *this, changelog_priv_t *priv) { int i = 0; - int ret = -1; - struct timeval tv = { - 0, - }; + int ret = 0; changelog_log_data_t cld = { 0, }; - ret = gettimeofday(&tv, NULL); - if (ret) { - gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_GET_TIME_FAILURE, - NULL); - goto out; - } - - priv->slice.tv_start = tv; - priv->maps[CHANGELOG_TYPE_DATA] = "D "; priv->maps[CHANGELOG_TYPE_METADATA] = "M "; priv->maps[CHANGELOG_TYPE_METADATA_XATTR] = "M "; @@ -2287,9 +2275,7 @@ changelog_init(xlator_t *this, changelog_priv_t *priv) * in case there was an encoding change. so... things are kept * simple here. */ - ret = changelog_fill_rollover_data(&cld, _gf_false); - if (ret) - goto out; + changelog_fill_rollover_data(&cld, _gf_false); ret = htime_open(this, priv, cld.cld_roll_time); /* call htime open with cld's rollover_time */ @@ -2470,9 +2456,6 @@ reconfigure(xlator_t *this, dict_t *options) char csnap_dir[PATH_MAX] = { 0, }; - struct timeval tv = { - 0, - }; uint32_t timeout = 0; priv = this->private; @@ -2564,9 +2547,7 @@ reconfigure(xlator_t *this, dict_t *options) out); if (active_now || active_earlier) { - ret = changelog_fill_rollover_data(&cld, !active_now); - if (ret) - goto out; + changelog_fill_rollover_data(&cld, !active_now); slice = &priv->slice; @@ -2585,13 +2566,7 @@ reconfigure(xlator_t *this, dict_t *options) if (!active_earlier) { gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_RECONFIGURE, NULL); - if (gettimeofday(&tv, NULL)) { - gf_smsg(this->name, GF_LOG_ERROR, 0, - CHANGELOG_MSG_HTIME_FETCH_FAILED, NULL); - ret = -1; - goto out; - } - htime_create(this, priv, tv.tv_sec); + htime_create(this, priv, gf_time()); } ret = changelog_spawn_helper_threads(this, priv); } @@ -2852,6 +2827,7 @@ fini(xlator_t *this) if (priv->active || priv->rpc_active) { /* terminate RPC server/threads */ changelog_cleanup_rpc(this, priv); + GF_FREE(priv->ev_dispatcher); } /* call barrier_disable to cancel timer */ if (priv->barrier_enabled) diff --git a/xlators/features/cloudsync/src/Makefile.am b/xlators/features/cloudsync/src/Makefile.am index 0c3966c968b..e2a277e372b 100644 --- a/xlators/features/cloudsync/src/Makefile.am +++ b/xlators/features/cloudsync/src/Makefile.am @@ -21,9 +21,9 @@ cloudsync_la_SOURCES = $(cloudsync_sources) $(cloudsynccommon_sources) nodist_cloudsync_la_SOURCES = cloudsync-autogen-fops.c cloudsync-autogen-fops.h BUILT_SOURCES = cloudsync-autogen-fops.h -cloudsync_la_LDFLAGS = $(LIB_DL) -module $(GF_XLATOR_DEFAULT_LDFLAGS) +cloudsync_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) -cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la +cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(LIB_DL) AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \ -DCS_PLUGINDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins\" diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c index 7680260988b..23c3599825a 100644 --- a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c +++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c @@ -237,7 +237,7 @@ aws_form_request(char *resource, char **date, char *reqtype, char *bucketid, int date_len = -1; int res_len = -1; - ctime = time(NULL); + ctime = gf_time(); gtime = gmtime(&ctime); date_len = strftime(httpdate, sizeof(httpdate), diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c index 4ece7ff6fc8..4abb2c73ce5 100644 --- a/xlators/features/index/src/index.c +++ b/xlators/features/index/src/index.c @@ -2104,7 +2104,7 @@ index_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req) worker_enqueue(this, stub); return 0; normal: - ret = dict_get_str(xattr_req, "link-count", &flag); + ret = dict_get_str_sizen(xattr_req, "link-count", &flag); if ((ret == 0) && (strcmp(flag, GF_XATTROP_INDEX_COUNT) == 0)) { STACK_WIND(frame, index_lookup_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, loc, xattr_req); @@ -2592,7 +2592,7 @@ notify(xlator_t *this, int event, void *data, ...) if ((event == GF_EVENT_PARENT_DOWN) && victim->cleanup_starting) { stub_cnt = GF_ATOMIC_GET(priv->stub_cnt); - clock_gettime(CLOCK_REALTIME, &sleep_till); + timespec_now_realtime(&sleep_till); sleep_till.tv_sec += 1; /* Wait for draining stub from queue before notify PARENT_DOWN */ diff --git a/xlators/features/leases/src/leases-internal.c b/xlators/features/leases/src/leases-internal.c index 67fdd53cee2..56dee244281 100644 --- a/xlators/features/leases/src/leases-internal.c +++ b/xlators/features/leases/src/leases-internal.c @@ -897,7 +897,7 @@ __recall_lease(xlator_t *this, lease_inode_ctx_t *lease_ctx) } priv = this->private; - recall_time = time(NULL); + recall_time = gf_time(); list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list, lease_id_list) { @@ -1367,7 +1367,7 @@ expired_recall_cleanup(void *data) gf_msg_debug(this->name, 0, "Started the expired_recall_cleanup thread"); while (1) { - time_now = time(NULL); + time_now = gf_time(); pthread_mutex_lock(&priv->mutex); { if (priv->fini) { diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c index 116aed68690..ab1eac68a53 100644 --- a/xlators/features/locks/src/clear.c +++ b/xlators/features/locks/src/clear.c @@ -181,9 +181,9 @@ clrlk_clear_posixlk(xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args, if (plock->blocked) { bcount++; pl_trace_out(this, plock->frame, NULL, NULL, F_SETLKW, - &plock->user_flock, -1, EAGAIN, NULL); + &plock->user_flock, -1, EINTR, NULL); - STACK_UNWIND_STRICT(lk, plock->frame, -1, EAGAIN, + STACK_UNWIND_STRICT(lk, plock->frame, -1, EINTR, &plock->user_flock, NULL); } else { diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c index c78d5372032..a2c6be93e03 100644 --- a/xlators/features/locks/src/common.c +++ b/xlators/features/locks/src/common.c @@ -460,11 +460,16 @@ pl_inode_get(xlator_t *this, inode_t *inode, pl_local_t *local) INIT_LIST_HEAD(&pl_inode->blocked_calls); INIT_LIST_HEAD(&pl_inode->metalk_list); INIT_LIST_HEAD(&pl_inode->queued_locks); + INIT_LIST_HEAD(&pl_inode->waiting); gf_uuid_copy(pl_inode->gfid, inode->gfid); pl_inode->check_mlock_info = _gf_true; pl_inode->mlock_enforced = _gf_false; + /* -2 means never looked up. -1 means something went wrong and link + * tracking is disabled. */ + pl_inode->links = -2; + ret = __inode_ctx_put(inode, this, (uint64_t)(long)(pl_inode)); if (ret) { pthread_mutex_destroy(&pl_inode->mutex); @@ -600,13 +605,11 @@ static void __insert_lock(pl_inode_t *pl_inode, posix_lock_t *lock) { if (lock->blocked) - gettimeofday(&lock->blkd_time, NULL); + lock->blkd_time = gf_time(); else - gettimeofday(&lock->granted_time, NULL); + lock->granted_time = gf_time(); list_add_tail(&lock->list, &pl_inode->ext_list); - - return; } /* Return true if the locks overlap, false otherwise */ @@ -962,7 +965,7 @@ grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode) struct list_head granted_list; posix_lock_t *tmp = NULL; posix_lock_t *lock = NULL; - + pl_local_t *local = NULL; INIT_LIST_HEAD(&granted_list); pthread_mutex_lock(&pl_inode->mutex); @@ -977,9 +980,9 @@ grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode) pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock, 0, 0, NULL); - - STACK_UNWIND_STRICT(lk, lock->frame, 0, 0, &lock->user_flock, NULL); - + local = lock->frame->local; + PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0, + &lock->user_flock, NULL); __destroy_lock(lock); } @@ -999,6 +1002,7 @@ pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode, struct list_head granted_list; posix_lock_t *tmp = NULL; posix_lock_t *lock = NULL; + pl_local_t *local = NULL; int ret = -1; @@ -1026,9 +1030,9 @@ pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode, pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock, 0, 0, NULL); - - STACK_UNWIND_STRICT(lk, lock->frame, 0, 0, &lock->user_flock, NULL); - + local = lock->frame->local; + PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0, + &lock->user_flock, NULL); __destroy_lock(lock); } @@ -1289,3 +1293,299 @@ pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client) } return _gf_true; } + +static int32_t +pl_inode_from_loc(loc_t *loc, inode_t **pinode) +{ + inode_t *inode = NULL; + int32_t error = 0; + + if (loc->inode != NULL) { + inode = inode_ref(loc->inode); + goto done; + } + + if (loc->parent == NULL) { + error = EINVAL; + goto done; + } + + if (!gf_uuid_is_null(loc->gfid)) { + inode = inode_find(loc->parent->table, loc->gfid); + if (inode != NULL) { + goto done; + } + } + + if (loc->name == NULL) { + error = EINVAL; + goto done; + } + + inode = inode_grep(loc->parent->table, loc->parent, loc->name); + if (inode == NULL) { + /* We haven't found any inode. This means that the file doesn't exist + * or that even if it exists, we don't have any knowledge about it, so + * we don't have locks on it either, which is fine for our purposes. */ + goto done; + } + +done: + *pinode = inode; + + return error; +} + +static gf_boolean_t +pl_inode_has_owners(xlator_t *xl, client_t *client, pl_inode_t *pl_inode, + struct timespec *now, struct list_head *contend) +{ + pl_dom_list_t *dom; + pl_inode_lock_t *lock; + gf_boolean_t has_owners = _gf_false; + + list_for_each_entry(dom, &pl_inode->dom_list, inode_list) + { + list_for_each_entry(lock, &dom->inodelk_list, list) + { + /* If the lock belongs to the same client, we assume it's related + * to the same operation, so we allow the removal to continue. */ + if (lock->client == client) { + continue; + } + /* If the lock belongs to an internal process, we don't block the + * removal. */ + if (lock->client_pid < 0) { + continue; + } + if (contend == NULL) { + return _gf_true; + } + has_owners = _gf_true; + inodelk_contention_notify_check(xl, lock, now, contend); + } + } + + return has_owners; +} + +int32_t +pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc, + pl_inode_t **ppl_inode, struct list_head *contend) +{ + struct timespec now; + inode_t *inode; + pl_inode_t *pl_inode; + int32_t error; + + pl_inode = NULL; + + error = pl_inode_from_loc(loc, &inode); + if ((error != 0) || (inode == NULL)) { + goto done; + } + + pl_inode = pl_inode_get(xl, inode, NULL); + if (pl_inode == NULL) { + inode_unref(inode); + error = ENOMEM; + goto done; + } + + /* pl_inode_from_loc() already increments ref count for inode, so + * we only assign here our reference. */ + pl_inode->inode = inode; + + timespec_now(&now); + + pthread_mutex_lock(&pl_inode->mutex); + + if (pl_inode->removed) { + error = ESTALE; + goto unlock; + } + + if (pl_inode_has_owners(xl, frame->root->client, pl_inode, &now, contend)) { + error = -1; + /* We skip the unlock here because the caller must create a stub when + * we return -1 and do a call to pl_inode_remove_complete(), which + * assumes the lock is still acquired and will release it once + * everything else is prepared. */ + goto done; + } + + pl_inode->is_locked = _gf_true; + pl_inode->remove_running++; + +unlock: + pthread_mutex_unlock(&pl_inode->mutex); + +done: + *ppl_inode = pl_inode; + + return error; +} + +int32_t +pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub, + struct list_head *contend) +{ + pl_inode_lock_t *lock; + int32_t error = -1; + + if (stub != NULL) { + list_add_tail(&stub->list, &pl_inode->waiting); + pl_inode->is_locked = _gf_true; + } else { + error = ENOMEM; + + while (!list_empty(contend)) { + lock = list_first_entry(contend, pl_inode_lock_t, list); + list_del_init(&lock->list); + __pl_inodelk_unref(lock); + } + } + + pthread_mutex_unlock(&pl_inode->mutex); + + if (error < 0) { + inodelk_contention_notify(xl, contend); + } + + inode_unref(pl_inode->inode); + + return error; +} + +void +pl_inode_remove_wake(struct list_head *list) +{ + call_stub_t *stub; + + while (!list_empty(list)) { + stub = list_first_entry(list, call_stub_t, list); + list_del_init(&stub->list); + + call_resume(stub); + } +} + +void +pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error) +{ + struct list_head contend, granted; + struct timespec now; + pl_dom_list_t *dom; + + if (pl_inode == NULL) { + return; + } + + INIT_LIST_HEAD(&contend); + INIT_LIST_HEAD(&granted); + timespec_now(&now); + + pthread_mutex_lock(&pl_inode->mutex); + + if (error == 0) { + if (pl_inode->links >= 0) { + pl_inode->links--; + } + if (pl_inode->links == 0) { + pl_inode->removed = _gf_true; + } + } + + pl_inode->remove_running--; + + if ((pl_inode->remove_running == 0) && list_empty(&pl_inode->waiting)) { + pl_inode->is_locked = _gf_false; + + list_for_each_entry(dom, &pl_inode->dom_list, inode_list) + { + __grant_blocked_inode_locks(xl, pl_inode, &granted, dom, &now, + &contend); + } + } + + pthread_mutex_unlock(&pl_inode->mutex); + + unwind_granted_inodes(xl, pl_inode, &granted); + + inodelk_contention_notify(xl, &contend); + + inode_unref(pl_inode->inode); +} + +void +pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode, + struct list_head *list) +{ + call_stub_t *stub, *tmp; + + if (!pl_inode->is_locked) { + return; + } + + list_for_each_entry_safe(stub, tmp, &pl_inode->waiting, list) + { + if (!pl_inode_has_owners(xl, stub->frame->root->client, pl_inode, NULL, + NULL)) { + list_move_tail(&stub->list, list); + } + } +} + +/* This function determines if an inodelk attempt can be done now or it needs + * to wait. + * + * Possible return values: + * < 0: An error occurred. Currently only -ESTALE can be returned if the + * inode has been deleted previously by unlink/rmdir/rename + * = 0: The lock can be attempted. + * > 0: The lock needs to wait because a conflicting remove operation is + * ongoing. + */ +int32_t +pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock) +{ + pl_dom_list_t *dom; + pl_inode_lock_t *ilock; + + /* If the inode has been deleted, we won't allow any lock. */ + if (pl_inode->removed) { + return -ESTALE; + } + + /* We only synchronize with locks made for regular operations coming from + * the user. Locks done for internal purposes are hard to control and could + * lead to long delays or deadlocks quite easily. */ + if (lock->client_pid < 0) { + return 0; + } + if (!pl_inode->is_locked) { + return 0; + } + if (pl_inode->remove_running > 0) { + return 1; + } + + list_for_each_entry(dom, &pl_inode->dom_list, inode_list) + { + list_for_each_entry(ilock, &dom->inodelk_list, list) + { + /* If a lock from the same client is already granted, we allow this + * one to continue. This is necessary to prevent deadlocks when + * multiple locks are taken for the same operation. + * + * On the other side it's unlikely that the same client sends + * completely unrelated locks for the same inode. + */ + if (ilock->client == lock->client) { + return 0; + } + } + } + + return 1; +} diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h index 0916c299e84..281223bf3b8 100644 --- a/xlators/features/locks/src/common.h +++ b/xlators/features/locks/src/common.h @@ -105,6 +105,15 @@ void __pl_inodelk_unref(pl_inode_lock_t *lock); void +__grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, + struct list_head *granted, pl_dom_list_t *dom, + struct timespec *now, struct list_head *contend); + +void +unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode, + struct list_head *granted); + +void grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom, struct timespec *now, struct list_head *contend); @@ -204,6 +213,16 @@ pl_metalock_is_active(pl_inode_t *pl_inode); void __pl_queue_lock(pl_inode_t *pl_inode, posix_lock_t *reqlock); +void +inodelk_contention_notify_check(xlator_t *xl, pl_inode_lock_t *lock, + struct timespec *now, + struct list_head *contend); + +void +entrylk_contention_notify_check(xlator_t *xl, pl_entry_lock_t *lock, + struct timespec *now, + struct list_head *contend); + gf_boolean_t pl_does_monkey_want_stuck_lock(); @@ -218,4 +237,26 @@ pl_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd); gf_boolean_t pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client); + +int32_t +pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc, + pl_inode_t **ppl_inode, struct list_head *contend); + +int32_t +pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub, + struct list_head *contend); + +void +pl_inode_remove_wake(struct list_head *list); + +void +pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error); + +void +pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode, + struct list_head *list); + +int32_t +pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock); + #endif /* __COMMON_H__ */ diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c index 0911659b437..fd772c850dd 100644 --- a/xlators/features/locks/src/entrylk.c +++ b/xlators/features/locks/src/entrylk.c @@ -121,7 +121,6 @@ __stale_entrylk(xlator_t *this, pl_entry_lock_t *candidate_lock, pl_entry_lock_t *requested_lock, time_t *lock_age_sec) { posix_locks_private_t *priv = NULL; - struct timeval curr; priv = this->private; @@ -129,8 +128,7 @@ __stale_entrylk(xlator_t *this, pl_entry_lock_t *candidate_lock, * chance? Or just the locks we are attempting to acquire? */ if (names_conflict(candidate_lock->basename, requested_lock->basename)) { - gettimeofday(&curr, NULL); - *lock_age_sec = curr.tv_sec - candidate_lock->granted_time.tv_sec; + *lock_age_sec = gf_time() - candidate_lock->granted_time; if (*lock_age_sec > priv->revocation_secs) return _gf_true; } @@ -204,9 +202,9 @@ out: return revoke_lock; } -static gf_boolean_t -__entrylk_needs_contention_notify(xlator_t *this, pl_entry_lock_t *lock, - struct timespec *now) +void +entrylk_contention_notify_check(xlator_t *this, pl_entry_lock_t *lock, + struct timespec *now, struct list_head *contend) { posix_locks_private_t *priv; int64_t elapsed; @@ -216,7 +214,7 @@ __entrylk_needs_contention_notify(xlator_t *this, pl_entry_lock_t *lock, /* If this lock is in a list, it means that we are about to send a * notification for it, so no need to do anything else. */ if (!list_empty(&lock->contend)) { - return _gf_false; + return; } elapsed = now->tv_sec; @@ -225,7 +223,7 @@ __entrylk_needs_contention_notify(xlator_t *this, pl_entry_lock_t *lock, elapsed--; } if (elapsed < priv->notify_contention_delay) { - return _gf_false; + return; } /* All contention notifications will be sent outside of the locked @@ -238,7 +236,7 @@ __entrylk_needs_contention_notify(xlator_t *this, pl_entry_lock_t *lock, lock->contention_time = *now; - return _gf_true; + list_add_tail(&lock->contend, contend); } void @@ -332,9 +330,7 @@ __entrylk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_entry_lock_t *lock, break; } } - if (__entrylk_needs_contention_notify(this, tmp, now)) { - list_add_tail(&tmp->contend, contend); - } + entrylk_contention_notify_check(this, tmp, now, contend); } } @@ -546,14 +542,10 @@ static int __lock_blocked_add(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom, pl_entry_lock_t *lock, int nonblock) { - struct timeval now; - if (nonblock) goto out; - gettimeofday(&now, NULL); - - lock->blkd_time = now; + lock->blkd_time = gf_time(); list_add_tail(&lock->blocked_locks, &dom->blocked_entrylks); gf_msg_trace(this->name, 0, "Blocking lock: {pinode=%p, basename=%s}", @@ -614,7 +606,7 @@ __lock_entrylk(xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock, } __pl_entrylk_ref(lock); - gettimeofday(&lock->granted_time, NULL); + lock->granted_time = gf_time(); list_add(&lock->domain_list, &dom->entrylk_list); ret = 0; @@ -697,10 +689,9 @@ __grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode, bl_ret = __lock_entrylk(bl->this, pl_inode, bl, 0, dom, now, contend); if (bl_ret == 0) { - list_add(&bl->blocked_locks, granted); + list_add_tail(&bl->blocked_locks, granted); } } - return; } /* Grants locks if possible which are blocked on a lock */ diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c index e0e3b8f1f2d..d4e51d6e0a1 100644 --- a/xlators/features/locks/src/inodelk.c +++ b/xlators/features/locks/src/inodelk.c @@ -140,15 +140,13 @@ __stale_inodelk(xlator_t *this, pl_inode_lock_t *candidate_lock, pl_inode_lock_t *requested_lock, time_t *lock_age_sec) { posix_locks_private_t *priv = NULL; - struct timeval curr; priv = this->private; /* Question: Should we just prune them all given the * chance? Or just the locks we are attempting to acquire? */ if (inodelk_conflict(candidate_lock, requested_lock)) { - gettimeofday(&curr, NULL); - *lock_age_sec = curr.tv_sec - candidate_lock->granted_time.tv_sec; + *lock_age_sec = gf_time() - candidate_lock->granted_time; if (*lock_age_sec > priv->revocation_secs) return _gf_true; } @@ -229,9 +227,9 @@ out: return revoke_lock; } -static gf_boolean_t -__inodelk_needs_contention_notify(xlator_t *this, pl_inode_lock_t *lock, - struct timespec *now) +void +inodelk_contention_notify_check(xlator_t *this, pl_inode_lock_t *lock, + struct timespec *now, struct list_head *contend) { posix_locks_private_t *priv; int64_t elapsed; @@ -241,7 +239,7 @@ __inodelk_needs_contention_notify(xlator_t *this, pl_inode_lock_t *lock, /* If this lock is in a list, it means that we are about to send a * notification for it, so no need to do anything else. */ if (!list_empty(&lock->contend)) { - return _gf_false; + return; } elapsed = now->tv_sec; @@ -250,7 +248,7 @@ __inodelk_needs_contention_notify(xlator_t *this, pl_inode_lock_t *lock, elapsed--; } if (elapsed < priv->notify_contention_delay) { - return _gf_false; + return; } /* All contention notifications will be sent outside of the locked @@ -263,7 +261,7 @@ __inodelk_needs_contention_notify(xlator_t *this, pl_inode_lock_t *lock, lock->contention_time = *now; - return _gf_true; + list_add_tail(&lock->contend, contend); } void @@ -351,9 +349,7 @@ __inodelk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock, break; } } - if (__inodelk_needs_contention_notify(this, l, now)) { - list_add_tail(&l->contend, contend); - } + inodelk_contention_notify_check(this, l, now, contend); } } @@ -399,15 +395,11 @@ static int __lock_blocked_add(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock, int can_block) { - struct timeval now; - if (can_block == 0) { goto out; } - gettimeofday(&now, NULL); - - lock->blkd_time = now; + lock->blkd_time = gf_time(); list_add_tail(&lock->blocked_locks, &dom->blocked_inodelks); gf_msg_trace(this->name, 0, @@ -433,12 +425,17 @@ __lock_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock, struct list_head *contend) { pl_inode_lock_t *conf = NULL; - int ret = -EINVAL; + int ret; - conf = __inodelk_grantable(this, dom, lock, now, contend); - if (conf) { - ret = __lock_blocked_add(this, dom, lock, can_block); - goto out; + ret = pl_inode_remove_inodelk(pl_inode, lock); + if (ret < 0) { + return ret; + } + if (ret == 0) { + conf = __inodelk_grantable(this, dom, lock, now, contend); + } + if ((ret > 0) || (conf != NULL)) { + return __lock_blocked_add(this, dom, lock, can_block); } /* To prevent blocked locks starvation, check if there are any blocked @@ -460,17 +457,13 @@ __lock_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock, "starvation"); } - ret = __lock_blocked_add(this, dom, lock, can_block); - goto out; + return __lock_blocked_add(this, dom, lock, can_block); } __pl_inodelk_ref(lock); - gettimeofday(&lock->granted_time, NULL); + lock->granted_time = gf_time(); list_add(&lock->list, &dom->inodelk_list); - ret = 0; - -out: - return ret; + return 0; } /* Return true if the two inodelks have exactly same lock boundaries */ @@ -527,12 +520,11 @@ out: return conf; } -static void +void __grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, struct list_head *granted, pl_dom_list_t *dom, struct timespec *now, struct list_head *contend) { - int bl_ret = 0; pl_inode_lock_t *bl = NULL; pl_inode_lock_t *tmp = NULL; @@ -545,52 +537,48 @@ __grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, { list_del_init(&bl->blocked_locks); - bl_ret = __lock_inodelk(this, pl_inode, bl, 1, dom, now, contend); + bl->status = __lock_inodelk(this, pl_inode, bl, 1, dom, now, contend); - if (bl_ret == 0) { - list_add(&bl->blocked_locks, granted); + if (bl->status != -EAGAIN) { + list_add_tail(&bl->blocked_locks, granted); } } - return; } -/* Grant all inodelks blocked on a lock */ void -grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, - pl_dom_list_t *dom, struct timespec *now, - struct list_head *contend) +unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode, + struct list_head *granted) { - struct list_head granted; pl_inode_lock_t *lock; pl_inode_lock_t *tmp; + int32_t op_ret; + int32_t op_errno; - INIT_LIST_HEAD(&granted); - - pthread_mutex_lock(&pl_inode->mutex); - { - __grant_blocked_inode_locks(this, pl_inode, &granted, dom, now, - contend); - } - pthread_mutex_unlock(&pl_inode->mutex); - - list_for_each_entry_safe(lock, tmp, &granted, blocked_locks) + list_for_each_entry_safe(lock, tmp, granted, blocked_locks) { - gf_log(this->name, GF_LOG_TRACE, - "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => Granted", - lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid, - lkowner_utoa(&lock->owner), lock->user_flock.l_start, - lock->user_flock.l_len); - + if (lock->status == 0) { + op_ret = 0; + op_errno = 0; + gf_log(this->name, GF_LOG_TRACE, + "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 + " => Granted", + lock->fl_type == F_UNLCK ? "Unlock" : "Lock", + lock->client_pid, lkowner_utoa(&lock->owner), + lock->user_flock.l_start, lock->user_flock.l_len); + } else { + op_ret = -1; + op_errno = -lock->status; + } pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock, - 0, 0, lock->volume); + op_ret, op_errno, lock->volume); - STACK_UNWIND_STRICT(inodelk, lock->frame, 0, 0, NULL); + STACK_UNWIND_STRICT(inodelk, lock->frame, op_ret, op_errno, NULL); lock->frame = NULL; } pthread_mutex_lock(&pl_inode->mutex); { - list_for_each_entry_safe(lock, tmp, &granted, blocked_locks) + list_for_each_entry_safe(lock, tmp, granted, blocked_locks) { list_del_init(&lock->blocked_locks); __pl_inodelk_unref(lock); @@ -599,6 +587,26 @@ grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, pthread_mutex_unlock(&pl_inode->mutex); } +/* Grant all inodelks blocked on a lock */ +void +grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode, + pl_dom_list_t *dom, struct timespec *now, + struct list_head *contend) +{ + struct list_head granted; + + INIT_LIST_HEAD(&granted); + + pthread_mutex_lock(&pl_inode->mutex); + { + __grant_blocked_inode_locks(this, pl_inode, &granted, dom, now, + contend); + } + pthread_mutex_unlock(&pl_inode->mutex); + + unwind_granted_inodes(this, pl_inode, &granted); +} + static void pl_inodelk_log_cleanup(pl_inode_lock_t *lock) { @@ -660,7 +668,7 @@ pl_inodelk_client_cleanup(xlator_t *this, pl_ctx_t *ctx) * and blocked lists, then this means that a parallel * unlock on another inodelk (L2 say) may have 'granted' * L1 and added it to 'granted' list in - * __grant_blocked_node_locks() (although using the + * __grant_blocked_inode_locks() (although using the * 'blocked_locks' member). In that case, the cleanup * codepath must try and grant other overlapping * blocked inodelks from other clients, now that L1 is @@ -745,6 +753,7 @@ pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode, gf_boolean_t need_inode_unref = _gf_false; struct list_head *pcontend = NULL; struct list_head contend; + struct list_head wake; struct timespec now = {}; short fl_type; @@ -796,6 +805,8 @@ pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode, timespec_now(&now); } + INIT_LIST_HEAD(&wake); + if (ctx) pthread_mutex_lock(&ctx->lock); pthread_mutex_lock(&pl_inode->mutex); @@ -818,18 +829,17 @@ pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode, lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid, lkowner_utoa(&lock->owner), lock->user_flock.l_start, lock->user_flock.l_len); - if (can_block) + if (can_block) { unref = _gf_false; - /* For all but the case where a non-blocking - * lock attempt fails, the extra ref taken at - * the start of this function must be negated. - */ - else - need_inode_unref = _gf_true; + } } - - if (ctx && (!ret || can_block)) + /* For all but the case where a non-blocking lock attempt fails + * with -EAGAIN, the extra ref taken at the start of this function + * must be negated. */ + need_inode_unref = (ret != 0) && ((ret != -EAGAIN) || !can_block); + if (ctx && !need_inode_unref) { list_add_tail(&lock->client_list, &ctx->inodelk_lockers); + } } else { /* Irrespective of whether unlock succeeds or not, * the extra inode ref that was done at the start of @@ -847,6 +857,8 @@ pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode, list_del_init(&retlock->client_list); __pl_inodelk_unref(retlock); + pl_inode_remove_unlocked(this, pl_inode, &wake); + ret = 0; } out: @@ -857,6 +869,8 @@ pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode, if (ctx) pthread_mutex_unlock(&ctx->lock); + pl_inode_remove_wake(&wake); + /* The following (extra) unref corresponds to the ref that * was done at the time the lock was granted. */ @@ -1037,10 +1051,14 @@ pl_common_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, inode); if (ret < 0) { - if ((can_block) && (F_UNLCK != lock_type)) { - goto out; + if (ret == -EAGAIN) { + if (can_block && (F_UNLCK != lock_type)) { + goto out; + } + gf_log(this->name, GF_LOG_TRACE, "returning EAGAIN"); + } else { + gf_log(this->name, GF_LOG_TRACE, "returning %d", ret); } - gf_log(this->name, GF_LOG_TRACE, "returning EAGAIN"); op_errno = -ret; goto unwind; } diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h index 3305350afb1..c868eb494a2 100644 --- a/xlators/features/locks/src/locks.h +++ b/xlators/features/locks/src/locks.h @@ -43,9 +43,8 @@ struct __posix_lock { fd_t *fd; call_frame_t *frame; - struct timeval blkd_time; /*time at which lock was queued into blkd list*/ - struct timeval - granted_time; /*time at which lock was queued into active list*/ + time_t blkd_time; /* time at which lock was queued into blkd list */ + time_t granted_time; /* time at which lock was queued into active list */ /* These two together serve to uniquely identify each process across nodes */ @@ -85,9 +84,9 @@ struct __pl_inode_lock { call_frame_t *frame; - struct timeval blkd_time; /*time at which lock was queued into blkd list*/ - struct timeval - granted_time; /*time at which lock was queued into active list*/ + time_t blkd_time; /* time at which lock was queued into blkd list */ + time_t granted_time; /* time at which lock was queued into active list */ + /*last time at which lock contention was detected and notified*/ struct timespec contention_time; @@ -102,6 +101,9 @@ struct __pl_inode_lock { struct list_head client_list; /* list of all locks from a client */ short fl_type; + + int32_t status; /* Error code when we try to grant a lock in blocked + state */ }; typedef struct __pl_inode_lock pl_inode_lock_t; @@ -136,9 +138,9 @@ struct __entry_lock { const char *basename; - struct timeval blkd_time; /*time at which lock was queued into blkd list*/ - struct timeval - granted_time; /*time at which lock was queued into active list*/ + time_t blkd_time; /* time at which lock was queued into blkd list */ + time_t granted_time; /* time at which lock was queued into active list */ + /*last time at which lock contention was detected and notified*/ struct timespec contention_time; @@ -164,13 +166,14 @@ struct __pl_inode { struct list_head rw_list; /* list of waiting r/w requests */ struct list_head reservelk_list; /* list of reservelks */ struct list_head blocked_reservelks; /* list of blocked reservelks */ - struct list_head - blocked_calls; /* List of blocked lock calls while a reserve is held*/ - struct list_head metalk_list; /* Meta lock list */ - /* This is to store the incoming lock - requests while meta lock is enabled */ - struct list_head queued_locks; - int mandatory; /* if mandatory locking is enabled */ + struct list_head blocked_calls; /* List of blocked lock calls while a + reserve is held*/ + struct list_head metalk_list; /* Meta lock list */ + struct list_head queued_locks; /* This is to store the incoming lock + requests while meta lock is enabled */ + struct list_head waiting; /* List of pending fops waiting to unlink/rmdir + the inode. */ + int mandatory; /* if mandatory locking is enabled */ inode_t *refkeeper; /* hold refs on an inode while locks are held to prevent pruning */ @@ -197,7 +200,13 @@ struct __pl_inode { */ int fop_wind_count; pthread_cond_t check_fop_wind_count; + gf_boolean_t track_fop_wind_count; + + int32_t links; /* Number of hard links the inode has. */ + uint32_t remove_running; /* Number of remove operations running. */ + gf_boolean_t is_locked; /* Regular locks will be blocked. */ + gf_boolean_t removed; /* The inode has been deleted. */ }; typedef struct __pl_inode pl_inode_t; diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c index af530aafd84..cf0ae4c57dd 100644 --- a/xlators/features/locks/src/posix.c +++ b/xlators/features/locks/src/posix.c @@ -148,6 +148,29 @@ fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **); } \ } while (0) +#define PL_INODE_REMOVE(_fop, _frame, _xl, _loc1, _loc2, _cont, _cbk, \ + _args...) \ + ({ \ + struct list_head contend; \ + pl_inode_t *__pl_inode; \ + call_stub_t *__stub; \ + int32_t __error; \ + INIT_LIST_HEAD(&contend); \ + __error = pl_inode_remove_prepare(_xl, _frame, _loc2 ? _loc2 : _loc1, \ + &__pl_inode, &contend); \ + if (__error < 0) { \ + __stub = fop_##_fop##_stub(_frame, _cont, ##_args); \ + __error = pl_inode_remove_complete(_xl, __pl_inode, __stub, \ + &contend); \ + } else if (__error == 0) { \ + PL_LOCAL_GET_REQUESTS(_frame, _xl, xdata, ((fd_t *)NULL), _loc1, \ + _loc2); \ + STACK_WIND_COOKIE(_frame, _cbk, __pl_inode, FIRST_CHILD(_xl), \ + FIRST_CHILD(_xl)->fops->_fop, ##_args); \ + } \ + __error; \ + }) + gf_boolean_t pl_has_xdata_requests(dict_t *xdata) { @@ -471,6 +494,9 @@ pl_inodelk_xattr_fill_multiple(dict_t *this, char *key, data_t *value, char *save_ptr = NULL; tmp_key = gf_strdup(key); + if (!tmp_key) + return -1; + strtok_r(tmp_key, ":", &save_ptr); if (!*save_ptr) { if (tmp_key) @@ -2962,11 +2988,85 @@ out: return ret; } +static int32_t +pl_request_link_count(dict_t **pxdata) +{ + dict_t *xdata; + + xdata = *pxdata; + if (xdata == NULL) { + xdata = dict_new(); + if (xdata == NULL) { + return ENOMEM; + } + } else { + dict_ref(xdata); + } + + if (dict_set_uint32(xdata, GET_LINK_COUNT, 0) != 0) { + dict_unref(xdata); + return ENOMEM; + } + + *pxdata = xdata; + + return 0; +} + +static int32_t +pl_check_link_count(dict_t *xdata) +{ + int32_t count; + + /* In case we are unable to read the link count from xdata, we take a + * conservative approach and return -2, which will prevent the inode from + * being considered deleted. In fact it will cause link tracking for this + * inode to be disabled completely to avoid races. */ + + if (xdata == NULL) { + return -2; + } + + if (dict_get_int32(xdata, GET_LINK_COUNT, &count) != 0) { + return -2; + } + + return count; +} + int32_t pl_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata, struct iatt *postparent) { + pl_inode_t *pl_inode; + + if (op_ret >= 0) { + pl_inode = pl_inode_get(this, inode, NULL); + if (pl_inode == NULL) { + PL_STACK_UNWIND(lookup, xdata, frame, -1, ENOMEM, NULL, NULL, NULL, + NULL); + return 0; + } + + pthread_mutex_lock(&pl_inode->mutex); + + /* We only update the link count if we previously didn't know it. + * Doing it always can lead to races since lookup is not executed + * atomically most of the times. */ + if (pl_inode->links == -2) { + pl_inode->links = pl_check_link_count(xdata); + if (buf->ia_type == IA_IFDIR) { + /* Directories have at least 2 links. To avoid special handling + * for directories, we simply decrement the value here to make + * them equivalent to regular files. */ + pl_inode->links--; + } + } + + pthread_mutex_unlock(&pl_inode->mutex); + } + PL_STACK_UNWIND(lookup, xdata, frame, op_ret, op_errno, inode, buf, xdata, postparent); return 0; @@ -2975,9 +3075,17 @@ pl_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t pl_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) { - PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL); - STACK_WIND(frame, pl_lookup_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->lookup, loc, xdata); + int32_t error; + + error = pl_request_link_count(&xdata); + if (error == 0) { + PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL); + STACK_WIND(frame, pl_lookup_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->lookup, loc, xdata); + dict_unref(xdata); + } else { + STACK_UNWIND_STRICT(lookup, frame, -1, error, NULL, NULL, NULL, NULL); + } return 0; } @@ -3502,10 +3610,10 @@ pl_dump_lock(char *str, int size, struct gf_flock *flock, gf_lkowner_t *owner, time_t *blkd_time, gf_boolean_t active) { char *type_str = NULL; - char granted[256] = { + char granted[GF_TIMESTR_SIZE] = { 0, }; - char blocked[256] = { + char blocked[GF_TIMESTR_SIZE] = { 0, }; @@ -3556,10 +3664,10 @@ __dump_entrylks(pl_inode_t *pl_inode) { pl_dom_list_t *dom = NULL; pl_entry_lock_t *lock = NULL; - char blocked[256] = { + char blocked[GF_TIMESTR_SIZE] = { 0, }; - char granted[256] = { + char granted[GF_TIMESTR_SIZE] = { 0, }; int count = 0; @@ -3579,10 +3687,10 @@ __dump_entrylks(pl_inode_t *pl_inode) list_for_each_entry(lock, &dom->entrylk_list, domain_list) { - gf_time_fmt(granted, sizeof(granted), lock->granted_time.tv_sec, + gf_time_fmt(granted, sizeof(granted), lock->granted_time, gf_timefmt_FT); gf_proc_dump_build_key(key, k, "entrylk[%d](ACTIVE)", count); - if (lock->blkd_time.tv_sec == 0) { + if (lock->blkd_time == 0) { snprintf(tmp, sizeof(tmp), ENTRY_GRNTD_FMT, lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK", @@ -3590,7 +3698,7 @@ __dump_entrylks(pl_inode_t *pl_inode) lkowner_utoa(&lock->owner), lock->client, lock->connection_id, granted); } else { - gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time.tv_sec, + gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time, gf_timefmt_FT); snprintf(tmp, sizeof(tmp), ENTRY_BLKD_GRNTD_FMT, lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" @@ -3607,7 +3715,7 @@ __dump_entrylks(pl_inode_t *pl_inode) list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks) { - gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time.tv_sec, + gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time, gf_timefmt_FT); gf_proc_dump_build_key(key, k, "entrylk[%d](BLOCKED)", count); @@ -3659,9 +3767,8 @@ __dump_inodelks(pl_inode_t *pl_inode) SET_FLOCK_PID(&lock->user_flock, lock); pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner, - lock->client, lock->connection_id, - &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec, - _gf_true); + lock->client, lock->connection_id, &lock->granted_time, + &lock->blkd_time, _gf_true); gf_proc_dump_write(key, "%s", tmp); count++; @@ -3673,8 +3780,8 @@ __dump_inodelks(pl_inode_t *pl_inode) count); SET_FLOCK_PID(&lock->user_flock, lock); pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner, - lock->client, lock->connection_id, 0, - &lock->blkd_time.tv_sec, _gf_false); + lock->client, lock->connection_id, 0, &lock->blkd_time, + _gf_false); gf_proc_dump_write(key, "%s", tmp); count++; @@ -3707,9 +3814,8 @@ __dump_posixlks(pl_inode_t *pl_inode) gf_proc_dump_build_key(key, "posixlk", "posixlk[%d](%s)", count, lock->blocked ? "BLOCKED" : "ACTIVE"); pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner, - lock->client, lock->client_uid, &lock->granted_time.tv_sec, - &lock->blkd_time.tv_sec, - (lock->blocked) ? _gf_false : _gf_true); + lock->client, lock->client_uid, &lock->granted_time, + &lock->blkd_time, (lock->blocked) ? _gf_false : _gf_true); gf_proc_dump_write(key, "%s", tmp); count++; @@ -3793,6 +3899,10 @@ unlock: gf_proc_dump_write("posixlk-count", "%d", count); __dump_posixlks(pl_inode); } + + gf_proc_dump_write("links", "%d", pl_inode->links); + gf_proc_dump_write("removes_pending", "%u", pl_inode->remove_running); + gf_proc_dump_write("removed", "%u", pl_inode->removed); } pthread_mutex_unlock(&pl_inode->mutex); @@ -4104,6 +4214,10 @@ fini(xlator_t *this) if (!priv) return; this->private = NULL; + if (this->local_pool) { + mem_pool_destroy(this->local_pool); + this->local_pool = NULL; + } GF_FREE(priv->brickname); GF_FREE(priv); @@ -4134,8 +4248,11 @@ pl_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, struct iatt *postoldparent, struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata) { + pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0); + PL_STACK_UNWIND(rename, xdata, frame, op_ret, op_errno, buf, preoldparent, postoldparent, prenewparent, postnewparent, xdata); + return 0; } @@ -4143,10 +4260,15 @@ int32_t pl_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata) { - PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), oldloc, newloc); + int32_t error; + + error = PL_INODE_REMOVE(rename, frame, this, oldloc, newloc, pl_rename, + pl_rename_cbk, oldloc, newloc, xdata); + if (error > 0) { + STACK_UNWIND_STRICT(rename, frame, -1, error, NULL, NULL, NULL, NULL, + NULL, NULL); + } - STACK_WIND(frame, pl_rename_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata); return 0; } @@ -4270,8 +4392,11 @@ pl_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { + pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0); + PL_STACK_UNWIND(unlink, xdata, frame, op_ret, op_errno, preparent, postparent, xdata); + return 0; } @@ -4279,9 +4404,14 @@ int32_t pl_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, dict_t *xdata) { - PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL); - STACK_WIND(frame, pl_unlink_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata); + int32_t error; + + error = PL_INODE_REMOVE(unlink, frame, this, loc, NULL, pl_unlink, + pl_unlink_cbk, loc, xflag, xdata); + if (error > 0) { + STACK_UNWIND_STRICT(unlink, frame, -1, error, NULL, NULL, NULL); + } + return 0; } @@ -4348,8 +4478,11 @@ pl_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { + pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0); + PL_STACK_UNWIND_FOR_CLIENT(rmdir, xdata, frame, op_ret, op_errno, preparent, postparent, xdata); + return 0; } @@ -4357,9 +4490,14 @@ int pl_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, dict_t *xdata) { - PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL); - STACK_WIND(frame, pl_rmdir_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata); + int32_t error; + + error = PL_INODE_REMOVE(rmdir, frame, this, loc, NULL, pl_rmdir, + pl_rmdir_cbk, loc, xflags, xdata); + if (error > 0) { + STACK_UNWIND_STRICT(rmdir, frame, -1, error, NULL, NULL, NULL); + } + return 0; } @@ -4389,6 +4527,19 @@ pl_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, inode_t *inode, struct iatt *buf, struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { + pl_inode_t *pl_inode = (pl_inode_t *)cookie; + + if (op_ret >= 0) { + pthread_mutex_lock(&pl_inode->mutex); + + /* TODO: can happen pl_inode->links == 0 ? */ + if (pl_inode->links >= 0) { + pl_inode->links++; + } + + pthread_mutex_unlock(&pl_inode->mutex); + } + PL_STACK_UNWIND_FOR_CLIENT(link, xdata, frame, op_ret, op_errno, inode, buf, preparent, postparent, xdata); return 0; @@ -4398,9 +4549,18 @@ int pl_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata) { + pl_inode_t *pl_inode; + + pl_inode = pl_inode_get(this, oldloc->inode, NULL); + if (pl_inode == NULL) { + STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, + NULL); + return 0; + } + PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), oldloc, newloc); - STACK_WIND(frame, pl_link_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata); + STACK_WIND_COOKIE(frame, pl_link_cbk, pl_inode, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata); return 0; } diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c index 4176ab6a8f1..3de2ea1c92c 100644 --- a/xlators/features/marker/src/marker-quota.c +++ b/xlators/features/marker/src/marker-quota.c @@ -1713,21 +1713,17 @@ mq_initiate_quota_task(void *opaque) } out: - if (dirty) { - if (ret < 0) { - /* On failure clear dirty status flag. - * In the next lookup inspect_directory_xattr - * can set the status flag and fix the - * dirty directory. - * Do the same if the dir was dirty before - * txn - */ - ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx); - if (ret == 0) - mq_set_ctx_dirty_status(parent_ctx, _gf_false); - } else { - ret = mq_mark_dirty(this, &parent_loc, 0); - } + if ((dirty) && (ret < 0)) { + /* On failure clear dirty status flag. + * In the next lookup inspect_directory_xattr + * can set the status flag and fix the + * dirty directory. + * Do the same if the dir was dirty before + * txn + */ + ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx); + if (ret == 0) + mq_set_ctx_dirty_status(parent_ctx, _gf_false); } if (locked) diff --git a/xlators/features/metadisp/Makefile.am b/xlators/features/metadisp/Makefile.am new file mode 100644 index 00000000000..a985f42a877 --- /dev/null +++ b/xlators/features/metadisp/Makefile.am @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = diff --git a/xlators/features/metadisp/src/Makefile.am b/xlators/features/metadisp/src/Makefile.am new file mode 100644 index 00000000000..1520ad8c424 --- /dev/null +++ b/xlators/features/metadisp/src/Makefile.am @@ -0,0 +1,38 @@ +noinst_PYTHON = gen-fops.py + +EXTRA_DIST = fops-tmpl.c + +xlator_LTLIBRARIES = metadisp.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features + +nodist_metadisp_la_SOURCES = fops.c + +BUILT_SOURCES = fops.c + +metadisp_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) + +metadisp_la_SOURCES = metadisp.c \ + metadisp-unlink.c \ + metadisp-stat.c \ + metadisp-lookup.c \ + metadisp-readdir.c \ + metadisp-create.c \ + metadisp-open.c \ + metadisp-fsync.c \ + metadisp-setattr.c \ + backend.c + +metadisp_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la + +noinst_HEADERS = metadisp.h metadisp-fops.h + +AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ + -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src + +AM_CFLAGS = -Wall $(GF_CFLAGS) + +fops.c: fops-tmpl.c $(top_srcdir)/libglusterfs/src/generator.py gen-fops.py + PYTHONPATH=$(top_srcdir)/libglusterfs/src \ + $(PYTHON) $(srcdir)/gen-fops.py $(srcdir)/fops-tmpl.c > $@ + +CLEANFILES = $(nodist_metadisp_la_SOURCES) diff --git a/xlators/features/metadisp/src/backend.c b/xlators/features/metadisp/src/backend.c new file mode 100644 index 00000000000..ee2c25bfaa7 --- /dev/null +++ b/xlators/features/metadisp/src/backend.c @@ -0,0 +1,45 @@ +#define GFID_STR_LEN 37 + +#include "metadisp.h" + +/* + * backend.c + * + * functions responsible for converting user-facing paths to backend-style + * "/$GFID" paths. + */ + +int32_t +build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc) +{ + static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + char gfid_buf[GFID_STR_LEN + 1] = { + 0, + }; + char *path = NULL; + + GF_VALIDATE_OR_GOTO("metadisp", src_loc, out); + GF_VALIDATE_OR_GOTO("metadisp", dst_loc, out); + + loc_copy(dst_loc, src_loc); + memcpy(dst_loc->pargfid, root, sizeof(root)); + GF_FREE((char *)dst_loc->path); // we are overwriting path so nuke + // whatever loc_copy gave us + + uuid_utoa_r(gfid, gfid_buf); + + path = GF_CALLOC(GFID_STR_LEN + 1, sizeof(char), + gf_common_mt_char); // freed via loc_wipe + + path[0] = '/'; + strncpy(path + 1, gfid_buf, GFID_STR_LEN); + path[GFID_STR_LEN] = 0; + dst_loc->path = path; + if (src_loc->name) + dst_loc->name = strrchr(dst_loc->path, '/'); + if (dst_loc->name) + dst_loc->name++; + return 0; +out: + return -1; +} diff --git a/xlators/features/metadisp/src/fops-tmpl.c b/xlators/features/metadisp/src/fops-tmpl.c new file mode 100644 index 00000000000..4385b7dd5b7 --- /dev/null +++ b/xlators/features/metadisp/src/fops-tmpl.c @@ -0,0 +1,10 @@ +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include <glusterfs/xlator.h> +#include "metadisp.h" +#include "metadisp-fops.h" + +#pragma generate diff --git a/xlators/features/metadisp/src/gen-fops.py b/xlators/features/metadisp/src/gen-fops.py new file mode 100644 index 00000000000..8b5e120fdec --- /dev/null +++ b/xlators/features/metadisp/src/gen-fops.py @@ -0,0 +1,160 @@ +#!/usr/bin/python + +import sys +from generator import fop_subs, generate + +FN_METADATA_CHILD_GENERIC = """ +int32_t +metadisp_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + METADISP_TRACE("@NAME@ metadata"); + STACK_WIND (frame, default_@NAME@_cbk, + METADATA_CHILD(this), METADATA_CHILD(this)->fops->@NAME@, + @SHORT_ARGS@); + return 0; +} +""" + +FN_GENERIC_TEMPLATE = """ +int32_t +metadisp_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + METADISP_TRACE("@NAME@ generic"); + STACK_WIND (frame, default_@NAME@_cbk, + DATA_CHILD(this), DATA_CHILD(this)->fops->@NAME@, + @SHORT_ARGS@); + return 0; +} +""" + +FN_DATAFD_TEMPLATE = """ +int32_t +metadisp_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + METADISP_TRACE("@NAME@ datafd"); + xlator_t *child = NULL; + child = DATA_CHILD(this); + STACK_WIND (frame, default_@NAME@_cbk, + child, child->fops->@NAME@, + @SHORT_ARGS@); + return 0; +} +""" + +FN_DATALOC_TEMPLATE = """ +int32_t +metadisp_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + METADISP_TRACE("@NAME@ dataloc"); + loc_t backend_loc = { + 0, + }; + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + xlator_t *child = NULL; + child = DATA_CHILD(this); + STACK_WIND (frame, default_@NAME@_cbk, + child, child->fops->@NAME@, + @SHORT_ARGS@); + return 0; + +unwind: + STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL); + return 0; +} +""" + +FOPS_LINE_TEMPLATE = "\t.@NAME@ = metadisp_@NAME@," + +skipped = [ + "readdir", + "readdirp", + "lookup", + "fsync", + "stat", + "open", + "create", + "unlink", + "setattr", + # TODO: implement "inodelk", +] + + +def gen_fops(): + done = skipped + + # + # these are fops that wind to the DATA_CHILD + # + # NOTE: re-written in order from google doc: + # https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q + for name in [ + "writev", + "readv", + "ftruncate", + "zerofill", + "discard", + "seek", + "fstat", + ]: + done = done + [name] + print(generate(FN_DATAFD_TEMPLATE, name, fop_subs)) + + for name in ["truncate"]: + done = done + [name] + print(generate(FN_DATALOC_TEMPLATE, name, fop_subs)) + + # these are fops that operate solely on dentries, folders, + # or extended attributes. Therefore, they must always + # wind to METADATA_CHILD and should never perform + # any path rewriting + # + # NOTE: re-written in order from google doc: + # https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q + for name in [ + "mkdir", + "symlink", + "link", + "rename", + "mknod", + "opendir", + # "readdir, # special-cased + # "readdirp, # special-cased + "fsyncdir", + # "setattr", # special-cased + "readlink", + "fentrylk", + "access", + # TODO: these wind to both, + # data for backend-attributes and metadata for the rest + "xattrop", + "setxattr", + "getxattr", + "removexattr", + "fgetxattr", + "fsetxattr", + "fremovexattr", + ]: + + done = done + [name] + print(generate(FN_METADATA_CHILD_GENERIC, name, fop_subs)) + + print("struct xlator_fops fops = {") + for name in done: + print(generate(FOPS_LINE_TEMPLATE, name, fop_subs)) + + print("};") + + +for l in open(sys.argv[1], "r").readlines(): + if l.find("#pragma generate") != -1: + print("/* BEGIN GENERATED CODE - DO NOT MODIFY */") + gen_fops() + print("/* END GENERATED CODE */") + else: + print(l[:-1]) diff --git a/xlators/features/metadisp/src/metadisp-create.c b/xlators/features/metadisp/src/metadisp-create.c new file mode 100644 index 00000000000..f8c9798dd59 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-create.c @@ -0,0 +1,101 @@ +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +/** + * Create, like stat, is a two-step process. We send a create + * to the METADATA_CHILD, then send another create to the DATA_CHILD. + * + * We do the metadata child first to ensure that the ACLs are enforced. + */ + +int32_t +metadisp_create_dentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, + inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, + dict_t *xdata) +{ + STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf, + preparent, postparent, xdata); + return 0; +} + +int32_t +metadisp_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, + dict_t *xdata) +{ + // create the backend data inode + STACK_WIND(frame, metadisp_create_dentry_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->create, loc, flags, mode, umask, fd, + xdata); + return 0; +} + +int32_t +metadisp_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode, + struct iatt *buf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + METADISP_TRACE("%d %d", op_ret, op_errno); + call_stub_t *stub = cookie; + if (op_ret != 0) { + STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf, + preparent, postparent, xdata); + return 0; + } + + if (stub == NULL) { + goto unwind; + } + + if (stub->poison) { + call_stub_destroy(stub); + return 0; + } + + call_resume(stub); + return 0; + +unwind: + STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL, + NULL); + return 0; +} + +int32_t +metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) +{ + METADISP_TRACE("."); + + loc_t backend_loc = { + 0, + }; + call_stub_t *stub = NULL; + uuid_t *gfid_req = NULL; + + RESOLVE_GFID_REQ(xdata, gfid_req, out); + + if (build_backend_loc(*gfid_req, loc, &backend_loc)) { + goto unwind; + } + + frame->local = loc; + + stub = fop_create_stub(frame, metadisp_create_resume, &backend_loc, flags, + mode, umask, fd, xdata); + + STACK_WIND_COOKIE(frame, metadisp_create_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->create, loc, flags, mode, + umask, fd, xdata); + return 0; + +unwind: + STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL, + NULL); + return 0; +out: + return -1; +} diff --git a/xlators/features/metadisp/src/metadisp-fops.h b/xlators/features/metadisp/src/metadisp-fops.h new file mode 100644 index 00000000000..56dd427cf34 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-fops.h @@ -0,0 +1,51 @@ +#ifndef GF_METADISP_FOPS_H_ +#define GF_METADISP_FOPS_H_ + +#include <glusterfs/xlator.h> +#include <glusterfs/dict.h> +#include <glusterfs/glusterfs.h> + +#include <sys/types.h> + +/* fops in here are defined in their own file. Every other fop is just defined + * inline of fops.c */ + +int +metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t off, dict_t *xdata); + +int +metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t off, dict_t *dict); + +int +metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata); + +int +metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata); + +int +metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + fd_t *fd, dict_t *xdata); + +int +metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata); + +int +metadisp_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, + loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata); + +int +metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, + dict_t *xdata); + +int +metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, + dict_t *xdata); + +int +metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata); + +#endif diff --git a/xlators/features/metadisp/src/metadisp-fsync.c b/xlators/features/metadisp/src/metadisp-fsync.c new file mode 100644 index 00000000000..2e46fa84eac --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-fsync.c @@ -0,0 +1,54 @@ + +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +int32_t +metadisp_fsync_resume(call_frame_t *frame, xlator_t *this, fd_t *fd, + int32_t flags, dict_t *xdata) +{ + STACK_WIND(frame, default_fsync_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->fsync, fd, flags, xdata); + return 0; +} + +int32_t +metadisp_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + call_stub_t *stub = NULL; + if (cookie) { + stub = cookie; + } + + if (op_ret != 0) { + goto unwind; + } + + if (stub->poison) { + call_stub_destroy(stub); + stub = NULL; + return 0; + } + + call_resume(stub); + return 0; + +unwind: + if (stub) { + call_stub_destroy(stub); + } + STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata); + return 0; +} + +int32_t +metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, + dict_t *xdata) +{ + call_stub_t *stub = NULL; + stub = fop_fsync_stub(frame, metadisp_fsync_resume, fd, flags, xdata); + STACK_WIND_COOKIE(frame, metadisp_fsync_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->fsync, fd, flags, xdata); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-lookup.c b/xlators/features/metadisp/src/metadisp-lookup.c new file mode 100644 index 00000000000..27d90c9f746 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-lookup.c @@ -0,0 +1,90 @@ +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +/** + * Lookup, like stat, is a two-step process for grabbing the metadata details + * as well as the data details. + */ + +int32_t +metadisp_backend_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, + struct iatt *postparent) +{ + METADISP_TRACE("backend_lookup_cbk"); + if (op_errno == ENOENT) { + op_errno = ENODATA; + op_ret = -1; + } + STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata, + postparent); + return 0; +} + +int32_t +metadisp_backend_lookup_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + dict_t *xdata) +{ + METADISP_TRACE("backend_lookup_resume"); + loc_t backend_loc = { + 0, + }; + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + + STACK_WIND(frame, metadisp_backend_lookup_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->lookup, &backend_loc, xdata); + return 0; + +unwind: + STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL); + return 0; +} + +int32_t +metadisp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent) +{ + METADISP_TRACE("%d %d", op_ret, op_errno); + call_stub_t *stub = NULL; + stub = cookie; + + if (op_ret != 0) { + goto unwind; + } + + if (!IA_ISREG(buf->ia_type)) { + goto unwind; + } else if (!stub) { + op_errno = EINVAL; + goto unwind; + } + + METADISP_TRACE("resuming stub"); + + // memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t)); + call_resume(stub); + return 0; +unwind: + METADISP_TRACE("unwinding %d %d", op_ret, op_errno); + STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata, + postparent); + if (stub) { + call_stub_destroy(stub); + } + return 0; +} + +int32_t +metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) +{ + METADISP_TRACE("lookup"); + call_stub_t *stub = NULL; + stub = fop_lookup_stub(frame, metadisp_backend_lookup_resume, loc, xdata); + STACK_WIND_COOKIE(frame, metadisp_lookup_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->lookup, loc, xdata); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-open.c b/xlators/features/metadisp/src/metadisp-open.c new file mode 100644 index 00000000000..64814afe636 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-open.c @@ -0,0 +1,70 @@ +#include <glusterfs/call-stub.h> +#include "metadisp.h" + +int32_t +metadisp_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) +{ + METADISP_TRACE("got open results %d %d", op_ret, op_errno); + + call_stub_t *stub = NULL; + if (cookie) { + stub = cookie; + } + + if (op_ret != 0) { + goto unwind; + } + + if (!stub) { + goto unwind; + } + + if (stub->poison) { + call_stub_destroy(stub); + stub = NULL; + return 0; + } + + call_resume(stub); + return 0; + +unwind: + if (stub) { + call_stub_destroy(stub); + } + STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata); + return 0; +} + +int32_t +metadisp_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + int32_t flags, fd_t *fd, dict_t *xdata) +{ + STACK_WIND_COOKIE(frame, metadisp_open_cbk, NULL, DATA_CHILD(this), + DATA_CHILD(this)->fops->open, loc, flags, fd, xdata); + return 0; +} + +int32_t +metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + fd_t *fd, dict_t *xdata) +{ + call_stub_t *stub = NULL; + loc_t backend_loc = { + 0, + }; + + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + + stub = fop_open_stub(frame, metadisp_open_resume, &backend_loc, flags, fd, + xdata); + STACK_WIND_COOKIE(frame, metadisp_open_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->open, loc, flags, fd, xdata); + return 0; +unwind: + STACK_UNWIND_STRICT(open, frame, -1, EINVAL, NULL, NULL); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-readdir.c b/xlators/features/metadisp/src/metadisp-readdir.c new file mode 100644 index 00000000000..5f840b1e88f --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-readdir.c @@ -0,0 +1,65 @@ +#include "metadisp.h" + +/** + * With a change to the posix xlator, readdir and readdirp are shockingly + * simple. + * + * The issue with separating the backend data of the files + * with the metadata is that readdirs must now read from multiple sources + * to coalesce the directory entries. + * + * The way we do this is to tell the METADATA_CHILD that when it's + * running readdirp, each file entry should have a stat wound to + * 'stat-source-of-truth'. + * + * see metadisp_stat for how it handles winds _from_posix. + */ + +int32_t +metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t off, dict_t *xdata) +{ + METADISP_TRACE("."); + /* + * Always use readdirp, even if the original was readdir. Why? Because NFS. + * There are multiple translations between Gluster, UNIX, and NFS stat + * structures in that path. One of them uses the type etc. from the stat + * structure, which is only filled in by readdirp. If we use readdir, the + * entries do actually go all the way back to the client and are visible in + * getdents, but then the readdir throws them away because of the + * uninitialized type. + */ + GF_UNUSED int32_t ret; + if (!xdata) { + xdata = dict_new(); + } + + // ret = dict_set_int32 (xdata, "list-xattr", 1); + + // I'm my own source of truth! + ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this); + + STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata); + + return 0; +} + +int32_t +metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t off, dict_t *xdata) +{ + METADISP_TRACE("."); + if (!xdata) { + xdata = dict_new(); + } + GF_UNUSED int32_t ret; + // ret = dict_set_int32 (xdata, "list-xattr", 1); + + // I'm my own source of truth! + ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this); + + STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-setattr.c b/xlators/features/metadisp/src/metadisp-setattr.c new file mode 100644 index 00000000000..6991cf644f3 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-setattr.c @@ -0,0 +1,90 @@ +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +int32_t +metadisp_backend_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, + struct iatt *statpre, struct iatt *statpost, + dict_t *xdata) + +{ + METADISP_TRACE("backend_setattr_cbk"); + if (op_errno == ENOENT) { + op_errno = ENODATA; + op_ret = -1; + } + STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost, + xdata); + return 0; +} + +int32_t +metadisp_backend_setattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, + dict_t *xdata) + +{ + METADISP_TRACE("backend_setattr_resume"); + loc_t backend_loc = { + 0, + }; + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + + STACK_WIND(frame, metadisp_backend_setattr_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->setattr, &backend_loc, stbuf, valid, + xdata); + return 0; + +unwind: + STACK_UNWIND_STRICT(setattr, frame, -1, EINVAL, NULL, NULL, NULL); + return 0; +} + +int32_t +metadisp_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *statpre, + struct iatt *statpost, dict_t *xdata) +{ + METADISP_TRACE("%d %d", op_ret, op_errno); + call_stub_t *stub = NULL; + stub = cookie; + + if (op_ret != 0) { + goto unwind; + } + + if (!IA_ISREG(statpost->ia_type)) { + goto unwind; + } else if (!stub) { + op_errno = EINVAL; + goto unwind; + } + + METADISP_TRACE("resuming stub"); + call_resume(stub); + return 0; +unwind: + METADISP_TRACE("unwinding %d %d", op_ret, op_errno); + STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost, + xdata); + if (stub) { + call_stub_destroy(stub); + } + return 0; +} + +int32_t +metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + METADISP_TRACE("setattr"); + call_stub_t *stub = NULL; + stub = fop_setattr_stub(frame, metadisp_backend_setattr_resume, loc, stbuf, + valid, xdata); + STACK_WIND_COOKIE(frame, metadisp_setattr_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->setattr, loc, stbuf, valid, + xdata); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-stat.c b/xlators/features/metadisp/src/metadisp-stat.c new file mode 100644 index 00000000000..b06d0dbcddd --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-stat.c @@ -0,0 +1,124 @@ +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +/** + * The stat flow in METADISP is complicated because we must + * do ensure a few things: + * 1. stat, on the path within the metadata layer, + * MUST get the backend FD of the data layer. + * --- we wind to the metadata layer, then the data layer. + * + * 2. the metadata layer MUST be able to ask the data + * layer for stat information. + * --- this is 'syncop-internal-from-posix' + * + * 3. when the metadata exists BUT the data is missing, + * we MUST mark the backend file as bad and heal it. + */ + +int32_t +metadisp_stat_backend_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *buf, + dict_t *xdata) +{ + METADISP_TRACE("got backend stat results %d %d", op_ret, op_errno); + if (op_errno == ENOENT) { + STACK_UNWIND_STRICT(open, frame, -1, ENODATA, NULL, NULL); + return 0; + } + STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata); + return 0; +} + +int32_t +metadisp_stat_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + dict_t *xdata) +{ + METADISP_TRACE("winding stat to path %s", loc->path); + if (gf_uuid_is_null(loc->gfid)) { + METADISP_TRACE("bad object, sending EUCLEAN"); + STACK_UNWIND_STRICT(open, frame, -1, EUCLEAN, NULL, NULL); + return 0; + } + + STACK_WIND(frame, metadisp_stat_backend_cbk, SECOND_CHILD(this), + SECOND_CHILD(this)->fops->stat, loc, xdata); + return 0; +} + +int32_t +metadisp_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *buf, + dict_t *xdata) +{ + call_stub_t *stub = NULL; + + METADISP_TRACE("got stat results %d %d", op_ret, op_errno); + + if (cookie) { + stub = cookie; + } + + if (op_ret != 0) { + goto unwind; + } + + // only use the stub for the files + if (!IA_ISREG(buf->ia_type)) { + goto unwind; + } + + if (stub->poison) { + call_stub_destroy(stub); + stub = NULL; + return 0; + } + + call_resume(stub); + return 0; + +unwind: + if (stub) { + call_stub_destroy(stub); + } + STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata); + return 0; +} + +int32_t +metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) +{ + call_stub_t *stub = NULL; + int32_t ret = 0; + loc_t backend_loc = { + 0, + }; + METADISP_FILTER_ROOT(stat, loc, xdata); + + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + + if (dict_get_int32(xdata, "syncop-internal-from-posix", &ret) == 0) { + // if we've just been sent a stat from posix, then we know + // that we must send down a stat for a file to the second child. + // + // that means we can skip the stat for the first child and just + // send to the data disk. + METADISP_TRACE("got syncop-internal-from-posix"); + STACK_WIND(frame, default_stat_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->stat, &backend_loc, xdata); + return 0; + } + + // we do not know if the request is for a file, folder, etc. wind + // to first child to find out. + stub = fop_stat_stub(frame, metadisp_stat_resume, &backend_loc, xdata); + METADISP_TRACE("winding stat to first child %s", loc->path); + STACK_WIND_COOKIE(frame, metadisp_stat_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->stat, loc, xdata); + return 0; +unwind: + STACK_UNWIND_STRICT(stat, frame, -1, EINVAL, NULL, NULL); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp-unlink.c b/xlators/features/metadisp/src/metadisp-unlink.c new file mode 100644 index 00000000000..1f6a8eb35ce --- /dev/null +++ b/xlators/features/metadisp/src/metadisp-unlink.c @@ -0,0 +1,160 @@ + +#include "metadisp.h" +#include <glusterfs/call-stub.h> + +/** + * The unlink flow in metadisp is complicated because we must + * do ensure that UNLINK causes both the metadata objects + * to get removed and the data objects to get removed. + */ + +int32_t +metadisp_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, + int xflag, dict_t *xdata) +{ + METADISP_TRACE("winding backend unlink to path %s", loc->path); + STACK_WIND(frame, default_unlink_cbk, DATA_CHILD(this), + DATA_CHILD(this)->fops->unlink, loc, xflag, xdata); + return 0; +} + +int32_t +metadisp_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + METADISP_TRACE(". %d %d", op_ret, op_errno); + + int ret = 0; + call_stub_t *stub = NULL; + int nlink = 0; + + if (cookie) { + stub = cookie; + } + + if (op_ret != 0) { + goto unwind; + } + + if (stub->poison) { + call_stub_destroy(stub); + stub = NULL; + return 0; + } + + ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink); + if (ret != 0) { + op_errno = EINVAL; + op_ret = -1; + goto unwind; + } + METADISP_TRACE("frontend hardlink count %d %d", ret, nlink); + if (nlink > 1) { + goto unwind; + } + + call_resume(stub); + return 0; + +unwind: + if (stub) { + call_stub_destroy(stub); + } + STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent, + xdata); + return 0; +} + +int32_t +metadisp_unlink_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, + struct iatt *postparent) +{ + call_stub_t *stub = NULL; + + if (cookie) { + stub = cookie; + } + + if (op_ret != 0) { + goto unwind; + } + + // fail fast on empty gfid so we don't loop forever + if (gf_uuid_is_null(buf->ia_gfid)) { + op_ret = -1; + op_errno = ENODATA; + goto unwind; + } + + // fill gfid since the stub is incomplete + memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t)); + memcpy(stub->args.loc.pargfid, postparent->ia_gfid, sizeof(uuid_t)); + + if (stub->poison) { + call_stub_destroy(stub); + stub = NULL; + return 0; + } + + call_resume(stub); + return 0; + +unwind: + if (stub) { + call_stub_destroy(stub); + } + STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL); + return 0; +} + +int32_t +metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag, + dict_t *xdata) +{ + call_stub_t *stub = NULL; + loc_t backend_loc = { + 0, + }; + + if (gf_uuid_is_null(loc->gfid)) { + METADISP_TRACE("winding lookup for unlink to path %s", loc->path); + + // loop back to ourselves after a lookup + stub = fop_unlink_stub(frame, metadisp_unlink, loc, xflag, xdata); + STACK_WIND_COOKIE(frame, metadisp_unlink_lookup_cbk, stub, + METADATA_CHILD(this), + METADATA_CHILD(this)->fops->lookup, loc, xdata); + return 0; + } + + if (build_backend_loc(loc->gfid, loc, &backend_loc)) { + goto unwind; + } + + // + // ensure we get the link count on the unlink response, so we can + // account for hardlinks before winding to the backend. + // NOTE: + // multiple xlators use GF_REQUEST_LINK_COUNT_XDATA. confirmation + // is needed to ensure that multiple requests will work in the same + // xlator stack. + // + if (!xdata) { + xdata = dict_new(); + } + dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1); + + METADISP_TRACE("winding frontend unlink to path %s", loc->path); + stub = fop_unlink_stub(frame, metadisp_unlink_resume, &backend_loc, xflag, + xdata); + + STACK_WIND_COOKIE(frame, metadisp_unlink_cbk, stub, METADATA_CHILD(this), + METADATA_CHILD(this)->fops->unlink, loc, xflag, xdata); + return 0; +unwind: + STACK_UNWIND_STRICT(unlink, frame, -1, EINVAL, NULL, NULL, NULL); + return 0; +} diff --git a/xlators/features/metadisp/src/metadisp.c b/xlators/features/metadisp/src/metadisp.c new file mode 100644 index 00000000000..3c8f150cebc --- /dev/null +++ b/xlators/features/metadisp/src/metadisp.c @@ -0,0 +1,46 @@ +#include <glusterfs/call-stub.h> + +#include "metadisp.h" +#include "metadisp-fops.h" + +int32_t +init(xlator_t *this) +{ + if (!this->children) { + gf_log(this->name, GF_LOG_ERROR, + "not configured with children. exiting"); + return -1; + } + + if (!this->parents) { + gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile "); + } + + return 0; +} + +void +fini(xlator_t *this) +{ + return; +} + +/* defined in fops.c */ +struct xlator_fops fops; + +struct xlator_cbks cbks = {}; + +struct volume_options options[] = { + {.key = {NULL}}, +}; + +xlator_api_t xlator_api = { + .init = init, + .fini = fini, + .fops = &fops, + .cbks = &cbks, + .options = options, + .op_version = {1}, + .identifier = "metadisp", + .category = GF_EXPERIMENTAL, +}; diff --git a/xlators/features/metadisp/src/metadisp.h b/xlators/features/metadisp/src/metadisp.h new file mode 100644 index 00000000000..c8fd7a13c04 --- /dev/null +++ b/xlators/features/metadisp/src/metadisp.h @@ -0,0 +1,45 @@ +/* + Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ +#ifndef GF_METADISP_H_ +#define GF_METADISP_H_ + +#include <glusterfs/glusterfs.h> +#include <glusterfs/logging.h> +#include <glusterfs/dict.h> +#include <glusterfs/xlator.h> +#include <glusterfs/defaults.h> + +#define METADATA_CHILD(_this) FIRST_CHILD(_this) +#define DATA_CHILD(_this) SECOND_CHILD(_this) + +int32_t +build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc); + +#define METADISP_TRACE(_args...) gf_log("metadisp", GF_LOG_INFO, _args) + +#define METADISP_FILTER_ROOT(_op, _args...) \ + if (strcmp(loc->path, "/") == 0) { \ + STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this), \ + METADATA_CHILD(this)->fops->_op, _args); \ + return 0; \ + } + +#define METADISP_FILTER_ROOT_BY_GFID(_op, _gfid, _args...) \ + if (__is_root_gfid(_gfid)) { \ + STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this), \ + METADATA_CHILD(this)->fops->_op, _args); \ + return 0; \ + } + +#define RESOLVE_GFID_REQ(_dict, _dest, _lbl) \ + VALIDATE_OR_GOTO(dict_get_ptr(_dict, "gfid-req", (void **)&_dest) == 0, \ + _lbl) + +#endif /* __TEMPLATE_H__ */ diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c index 1a4c2e30dd6..480d64ade27 100644 --- a/xlators/features/quota/src/quota-enforcer-client.c +++ b/xlators/features/quota/src/quota-enforcer-client.c @@ -32,12 +32,6 @@ #include <malloc.h> #endif -#ifdef HAVE_MALLOC_STATS -#ifdef DEBUG -#include <mcheck.h> -#endif -#endif - #include "quota.h" #include "quota-messages.h" @@ -362,16 +356,28 @@ quota_enforcer_notify(struct rpc_clnt *rpc, void *mydata, { xlator_t *this = NULL; int ret = 0; + quota_priv_t *priv = NULL; this = mydata; - + priv = this->private; switch (event) { case RPC_CLNT_CONNECT: { + pthread_mutex_lock(&priv->conn_mutex); + { + priv->conn_status = _gf_true; + } + pthread_mutex_unlock(&priv->conn_mutex); gf_msg_trace(this->name, 0, "got RPC_CLNT_CONNECT"); break; } case RPC_CLNT_DISCONNECT: { + pthread_mutex_lock(&priv->conn_mutex); + { + priv->conn_status = _gf_false; + pthread_cond_signal(&priv->conn_cond); + } + pthread_mutex_unlock(&priv->conn_mutex); gf_msg_trace(this->name, 0, "got RPC_CLNT_DISCONNECT"); break; } diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c index 01b01b5f01b..18df9ae6d19 100644 --- a/xlators/features/quota/src/quota.c +++ b/xlators/features/quota/src/quota.c @@ -586,9 +586,6 @@ quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, quota_meta_t size = { 0, }; - struct timeval tv = { - 0, - }; local = frame->local; @@ -626,13 +623,12 @@ quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, * loop of validation and checking * limit when timeout is zero. */ - gettimeofday(&tv, NULL); LOCK(&ctx->lock); { ctx->size = size.size; + ctx->validate_time = gf_time(); ctx->file_count = size.file_count; ctx->dir_count = size.dir_count; - memcpy(&ctx->tv, &tv, sizeof(struct timeval)); } UNLOCK(&ctx->lock); @@ -644,27 +640,10 @@ unwind: return 0; } -static uint64_t -quota_time_elapsed(struct timeval *now, struct timeval *then) +static inline gf_boolean_t +quota_timeout(time_t t, uint32_t timeout) { - return (now->tv_sec - then->tv_sec); -} - -int32_t -quota_timeout(struct timeval *tv, int32_t timeout) -{ - struct timeval now = { - 0, - }; - int32_t timed_out = 0; - - gettimeofday(&now, NULL); - - if (quota_time_elapsed(&now, tv) >= timeout) { - timed_out = 1; - } - - return timed_out; + return (gf_time() - t) >= timeout; } /* Return: 1 if new entry added @@ -1128,7 +1107,7 @@ quota_check_object_limit(call_frame_t *frame, quota_inode_ctx_t *ctx, timeout = priv->hard_timeout; } - if (!just_validated && quota_timeout(&ctx->tv, timeout)) { + if (!just_validated && quota_timeout(ctx->validate_time, timeout)) { need_validate = 1; } else if ((object_aggr_count) > ctx->object_hard_lim) { hard_limit_exceeded = 1; @@ -1195,7 +1174,7 @@ quota_check_size_limit(call_frame_t *frame, quota_inode_ctx_t *ctx, timeout = priv->hard_timeout; } - if (!just_validated && quota_timeout(&ctx->tv, timeout)) { + if (!just_validated && quota_timeout(ctx->validate_time, timeout)) { need_validate = 1; } else if (wouldbe_size >= ctx->hard_lim) { hard_limit_exceeded = 1; @@ -4314,9 +4293,6 @@ quota_statfs_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, quota_meta_t size = { 0, }; - struct timeval tv = { - 0, - }; local = frame->local; @@ -4348,13 +4324,12 @@ quota_statfs_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, op_errno = EINVAL; } - gettimeofday(&tv, NULL); LOCK(&ctx->lock); { ctx->size = size.size; + ctx->validate_time = gf_time(); ctx->file_count = size.file_count; ctx->dir_count = size.dir_count; - memcpy(&ctx->tv, &tv, sizeof(struct timeval)); } UNLOCK(&ctx->lock); @@ -4873,7 +4848,7 @@ off: void quota_log_helper(char **usage_str, int64_t cur_size, inode_t *inode, - char **path, struct timeval *cur_time) + char **path, time_t *cur_time) { xlator_t *this = THIS; @@ -4892,7 +4867,7 @@ quota_log_helper(char **usage_str, int64_t cur_size, inode_t *inode, if (!(*path)) *path = uuid_utoa(inode->gfid); - gettimeofday(cur_time, NULL); + *cur_time = gf_time(); } /* Logs if @@ -4903,9 +4878,7 @@ void quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode, int64_t delta) { - struct timeval cur_time = { - 0, - }; + time_t cur_time = 0; char *usage_str = NULL; char *path = NULL; int64_t cur_size = 0; @@ -4931,12 +4904,12 @@ quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode, "path=%s", usage_str, priv->volume_uuid, path); - ctx->prev_log = cur_time; + ctx->prev_log_time = cur_time; } /* Usage is above soft limit */ else if (cur_size > ctx->soft_lim && - quota_timeout(&ctx->prev_log, priv->log_timeout)) { + quota_timeout(ctx->prev_log_time, priv->log_timeout)) { quota_log_helper(&usage_str, cur_size, inode, &path, &cur_time); gf_msg(this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT, @@ -4947,7 +4920,7 @@ quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode, "path=%s", usage_str, priv->volume_uuid, path); - ctx->prev_log = cur_time; + ctx->prev_log_time = cur_time; } if (path) @@ -5008,6 +4981,43 @@ quota_forget(xlator_t *this, inode_t *inode) return 0; } +int +notify(xlator_t *this, int event, void *data, ...) +{ + quota_priv_t *priv = NULL; + int ret = 0; + rpc_clnt_t *rpc = NULL; + gf_boolean_t conn_status = _gf_true; + xlator_t *victim = data; + + priv = this->private; + if (!priv || !priv->is_quota_on) + goto out; + + if (event == GF_EVENT_PARENT_DOWN) { + rpc = priv->rpc_clnt; + if (rpc) { + rpc_clnt_disable(rpc); + pthread_mutex_lock(&priv->conn_mutex); + { + conn_status = priv->conn_status; + while (conn_status) { + (void)pthread_cond_wait(&priv->conn_cond, + &priv->conn_mutex); + conn_status = priv->conn_status; + } + } + pthread_mutex_unlock(&priv->conn_mutex); + gf_log(this->name, GF_LOG_INFO, + "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name); + } + } + +out: + ret = default_notify(this, event, data); + return ret; +} + int32_t init(xlator_t *this) { @@ -5050,6 +5060,10 @@ init(xlator_t *this) goto err; } + pthread_mutex_init(&priv->conn_mutex, NULL); + pthread_cond_init(&priv->conn_cond, NULL); + priv->conn_status = _gf_false; + if (priv->is_quota_on) { rpc = quota_enforcer_init(this, this->options); if (rpc == NULL) { @@ -5143,9 +5157,9 @@ quota_priv_dump(xlator_t *this) if (ret) goto out; else { - gf_proc_dump_write("soft-timeout", "%d", priv->soft_timeout); - gf_proc_dump_write("hard-timeout", "%d", priv->hard_timeout); - gf_proc_dump_write("alert-time", "%d", priv->log_timeout); + gf_proc_dump_write("soft-timeout", "%u", priv->soft_timeout); + gf_proc_dump_write("hard-timeout", "%u", priv->hard_timeout); + gf_proc_dump_write("alert-time", "%u", priv->log_timeout); gf_proc_dump_write("quota-on", "%d", priv->is_quota_on); gf_proc_dump_write("statfs", "%d", priv->consider_statfs); gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid); @@ -5163,20 +5177,22 @@ fini(xlator_t *this) { quota_priv_t *priv = NULL; rpc_clnt_t *rpc = NULL; - int i = 0, cnt = 0; priv = this->private; if (!priv) return; rpc = priv->rpc_clnt; priv->rpc_clnt = NULL; - this->private = NULL; if (rpc) { - cnt = GF_ATOMIC_GET(rpc->refcount); - for (i = 0; i < cnt; i++) - rpc_clnt_unref(rpc); + rpc_clnt_connection_cleanup(&rpc->conn); + rpc_clnt_unref(rpc); } + + this->private = NULL; LOCK_DESTROY(&priv->lock); + pthread_mutex_destroy(&priv->conn_mutex); + pthread_cond_destroy(&priv->conn_cond); + GF_FREE(priv); if (this->local_pool) { mem_pool_destroy(this->local_pool); @@ -5308,6 +5324,7 @@ struct volume_options options[] = { xlator_api_t xlator_api = { .init = init, .fini = fini, + .notify = notify, .reconfigure = reconfigure, .mem_acct_init = mem_acct_init, .op_version = {1}, /* Present from the initial version */ diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h index 00012c22f46..0395d78c9ef 100644 --- a/xlators/features/quota/src/quota.h +++ b/xlators/features/quota/src/quota.h @@ -153,8 +153,8 @@ struct quota_inode_ctx { int64_t object_soft_lim; struct iatt buf; struct list_head parents; - struct timeval tv; - struct timeval prev_log; + time_t validate_time; + time_t prev_log_time; gf_boolean_t ancestry_built; gf_lock_t lock; }; @@ -199,6 +199,7 @@ struct quota_local { typedef struct quota_local quota_local_t; struct quota_priv { + /* FIXME: consider time_t for timeouts. */ uint32_t soft_timeout; uint32_t hard_timeout; uint32_t log_timeout; @@ -214,6 +215,9 @@ struct quota_priv { char *volume_uuid; uint64_t validation_count; int32_t quotad_conn_status; + pthread_mutex_t conn_mutex; + pthread_cond_t conn_cond; + gf_boolean_t conn_status; }; typedef struct quota_priv quota_priv_t; diff --git a/xlators/features/read-only/src/worm-helper.c b/xlators/features/read-only/src/worm-helper.c index 25fbd4aa748..df45f2a940b 100644 --- a/xlators/features/read-only/src/worm-helper.c +++ b/xlators/features/read-only/src/worm-helper.c @@ -41,7 +41,7 @@ worm_init_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr) GF_VALIDATE_OR_GOTO("worm", this, out); GF_VALIDATE_OR_GOTO(this->name, file_ptr, out); - start_time = time(NULL); + start_time = gf_time(); dict = dict_new(); if (!dict) { gf_log(this->name, GF_LOG_ERROR, "Error creating the dict"); @@ -94,7 +94,7 @@ worm_set_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr, if (ret) goto out; stbuf->ia_mtime = stpre.ia_mtime; - stbuf->ia_atime = time(NULL) + retention_state->ret_period; + stbuf->ia_atime = gf_time() + retention_state->ret_period; if (fop_with_fd) ret = syncop_fsetattr(this, (fd_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME, @@ -286,6 +286,7 @@ gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd, { int op_errno = EROFS; int ret = -1; + time_t now = 0; uint64_t com_period = 0; uint64_t start_time = 0; dict_t *dict = NULL; @@ -337,8 +338,10 @@ gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd, goto out; } - if (ret == -1 && (time(NULL) - start_time) >= com_period) { - if ((time(NULL) - stbuf.ia_mtime) >= com_period) { + now = gf_time(); + + if (ret == -1 && (now - start_time) >= com_period) { + if ((now - stbuf.ia_mtime) >= com_period) { ret = worm_set_state(this, fop_with_fd, file_ptr, &reten_state, &stbuf); if (ret) { @@ -352,10 +355,10 @@ gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd, op_errno = 0; goto out; } - } else if (ret == -1 && (time(NULL) - start_time) < com_period) { + } else if (ret == -1 && (now - start_time) < com_period) { op_errno = 0; goto out; - } else if (reten_state.retain && ((time(NULL) >= stbuf.ia_atime))) { + } else if (reten_state.retain && ((now >= stbuf.ia_atime))) { gf_worm_state_lookup(this, fop_with_fd, file_ptr, &reten_state, &stbuf); } if (reten_state.worm && !reten_state.retain && priv->worm_files_deletable && diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c index 46078c1a96e..1cc5526d5cd 100644 --- a/xlators/features/read-only/src/worm.c +++ b/xlators/features/read-only/src/worm.c @@ -440,8 +440,6 @@ worm_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, { int ret = 0; read_only_priv_t *priv = NULL; - dict_t *dict = NULL; - // In case of an error exit because fd can be NULL and this would // cause an segfault when performing fsetxattr . We explicitly // unwind to avoid future problems @@ -452,24 +450,12 @@ worm_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, priv = this->private; GF_ASSERT(priv); if (priv->worm_file) { - dict = dict_new(); - if (!dict) { - gf_log(this->name, GF_LOG_ERROR, - "Error creating the " - "dict"); - goto out; - } - ret = dict_set_int8(dict, "trusted.worm_file", 1); + ret = fd_ctx_set(fd, this, 1); if (ret) { gf_log(this->name, GF_LOG_ERROR, - "Error in setting " - "the dict"); - goto out; - } - ret = syncop_fsetxattr(this, fd, dict, 0, NULL, NULL); - if (ret) { - gf_log(this->name, GF_LOG_ERROR, "Error setting xattr"); - goto out; + "Failed to set the fd ctx " + "for gfid:%s . Worm feature may not work for the gfid", + uuid_utoa(inode->gfid)); } ret = worm_init_state(this, _gf_true, fd); if (ret) { @@ -480,8 +466,6 @@ worm_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, out: STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf, preparent, postparent, xdata); - if (dict) - dict_unref(dict); return ret; } @@ -617,7 +601,62 @@ struct xlator_fops fops = { .lk = ro_lk, }; -struct xlator_cbks cbks; +int32_t +worm_release(xlator_t *this, fd_t *fd) +{ + dict_t *dict = NULL; + int ret = -1; + dict = dict_new(); + uint64_t value = 0; + loc_t loc = { + 0, + }; + read_only_priv_t *priv = NULL; + priv = this->private; + + if (priv->worm_file) { + if (!dict) { + gf_log(this->name, GF_LOG_ERROR, "Error creating the dict"); + goto out; + } + + ret = fd_ctx_get(fd, this, &value); + if (ret) { + gf_log(this->name, GF_LOG_DEBUG, "Failed to get the fd ctx"); + } + if (!value) { + goto out; + } + + ret = dict_set_int8(dict, "trusted.worm_file", 1); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "Error in setting " + "the dict"); + goto out; + } + + loc.inode = inode_ref(fd->inode); + gf_uuid_copy(loc.gfid, fd->inode->gfid); + ret = syncop_setxattr(this, &loc, dict, 0, NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "Error setting xattr"); + goto out; + } + + gf_worm_state_transition(this, _gf_false, &loc, GF_FOP_WRITE); + } + +out: + loc_wipe(&loc); + if (dict) + dict_unref(dict); + return 0; +} + +struct xlator_cbks cbks = { + .release = worm_release, +}; struct volume_options options[] = { {.key = {"worm"}, diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c index c59e244429a..e5f93063943 100644 --- a/xlators/features/shard/src/shard.c +++ b/xlators/features/shard/src/shard.c @@ -513,6 +513,9 @@ shard_local_wipe(shard_local_t *local) loc_wipe(&local->int_entrylk.loc); loc_wipe(&local->newloc); + if (local->name) + GF_FREE(local->name); + if (local->int_entrylk.basename) GF_FREE(local->int_entrylk.basename); if (local->fd) @@ -1001,6 +1004,10 @@ shard_initiate_evicted_inode_fsync(xlator_t *this, inode_t *inode) } int +shard_common_inode_write_post_lookup_shards_handler(call_frame_t *frame, + xlator_t *this); + +int shard_common_resolve_shards(call_frame_t *frame, xlator_t *this, shard_post_resolve_fop_handler_t post_res_handler) { @@ -1017,21 +1024,47 @@ shard_common_resolve_shards(call_frame_t *frame, xlator_t *this, inode_t *fsync_inode = NULL; shard_priv_t *priv = NULL; shard_local_t *local = NULL; + uint64_t resolve_count = 0; priv = this->private; local = frame->local; local->call_count = 0; shard_idx_iter = local->first_block; res_inode = local->resolver_base_inode; + + if ((local->op_ret < 0) || (local->resolve_not)) + goto out; + + /* If this prealloc FOP is for fresh file creation, then the size of the + * file will be 0. Then there will be no shards associated with this file. + * So we can skip the lookup process for the shards which do not exists + * and directly issue mknod to crete shards. + * + * In case the prealloc fop is to extend the preallocated file to bigger + * size then just lookup and populate inodes of existing shards and + * update the create count + */ + if (local->fop == GF_FOP_FALLOCATE) { + if (!local->prebuf.ia_size) { + local->inode_list[0] = inode_ref(res_inode); + local->create_count = local->last_block; + shard_common_inode_write_post_lookup_shards_handler(frame, this); + return 0; + } + if (local->prebuf.ia_size < local->total_size) + local->create_count = local->last_block - + ((local->prebuf.ia_size - 1) / + local->block_size); + } + + resolve_count = local->last_block - local->create_count; + if (res_inode) gf_uuid_copy(gfid, res_inode->gfid); else gf_uuid_copy(gfid, local->base_gfid); - if ((local->op_ret < 0) || (local->resolve_not)) - goto out; - - while (shard_idx_iter <= local->last_block) { + while (shard_idx_iter <= resolve_count) { i++; if (shard_idx_iter == 0) { local->inode_list[i] = inode_ref(res_inode); @@ -1659,26 +1692,24 @@ err: } int -shard_lookup_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno, inode_t *inode, - struct iatt *buf, dict_t *xdata, - struct iatt *postparent) +shard_set_iattr_invoke_post_handler(call_frame_t *frame, xlator_t *this, + inode_t *inode, int32_t op_ret, + int32_t op_errno, struct iatt *buf, + dict_t *xdata) { int ret = -1; int32_t mask = SHARD_INODE_WRITE_MASK; - shard_local_t *local = NULL; + shard_local_t *local = frame->local; shard_inode_ctx_t ctx = { 0, }; - local = frame->local; - if (op_ret < 0) { gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_BASE_FILE_LOOKUP_FAILED, "Lookup on base file" " failed : %s", - loc_gfid_utoa(&(local->loc))); + uuid_utoa(inode->gfid)); local->op_ret = op_ret; local->op_errno = op_errno; goto unwind; @@ -1712,18 +1743,57 @@ unwind: } int -shard_lookup_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc, - shard_post_fop_handler_t handler) +shard_fstat_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *buf, + dict_t *xdata) +{ + shard_local_t *local = frame->local; + + shard_set_iattr_invoke_post_handler(frame, this, local->fd->inode, op_ret, + op_errno, buf, xdata); + return 0; +} + +int +shard_lookup_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, + struct iatt *postparent) +{ + /* In case of op_ret < 0, inode passed to this function will be NULL + ex: in case of op_errno = ENOENT. So refer prefilled inode data + which is part of local. + Note: Reassigning/overriding the inode passed to this cbk with inode + which is part of *struct shard_local_t* won't cause any issue as + both inodes have same reference/address as of the inode passed */ + inode = ((shard_local_t *)frame->local)->loc.inode; + + shard_set_iattr_invoke_post_handler(frame, this, inode, op_ret, op_errno, + buf, xdata); + return 0; +} + +/* This function decides whether to make file based lookup or + * fd based lookup (fstat) depending on the 3rd and 4th arg. + * If fd != NULL and loc == NULL then call is for fstat + * If fd == NULL and loc != NULL then call is for file based + * lookup. Please pass args based on the requirement. + */ +int +shard_refresh_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc, + fd_t *fd, shard_post_fop_handler_t handler) { int ret = -1; + inode_t *inode = NULL; shard_local_t *local = NULL; dict_t *xattr_req = NULL; gf_boolean_t need_refresh = _gf_false; local = frame->local; local->handler = handler; + inode = fd ? fd->inode : loc->inode; - ret = shard_inode_ctx_fill_iatt_from_cache(loc->inode, this, &local->prebuf, + ret = shard_inode_ctx_fill_iatt_from_cache(inode, this, &local->prebuf, &need_refresh); /* By this time, inode ctx should have been created either in create, * mknod, readdirp or lookup. If not it is a bug! @@ -1732,7 +1802,7 @@ shard_lookup_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc, gf_msg_debug(this->name, 0, "Skipping lookup on base file: %s" "Serving prebuf off the inode ctx cache", - uuid_utoa(loc->gfid)); + uuid_utoa(inode->gfid)); goto out; } @@ -1743,10 +1813,14 @@ shard_lookup_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc, goto out; } - SHARD_MD_READ_FOP_INIT_REQ_DICT(this, xattr_req, loc->gfid, local, out); + SHARD_MD_READ_FOP_INIT_REQ_DICT(this, xattr_req, inode->gfid, local, out); - STACK_WIND(frame, shard_lookup_base_file_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->lookup, loc, xattr_req); + if (fd) + STACK_WIND(frame, shard_fstat_base_file_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fstat, fd, xattr_req); + else + STACK_WIND(frame, shard_lookup_base_file_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->lookup, loc, xattr_req); dict_unref(xattr_req); return 0; @@ -2015,8 +2089,8 @@ shard_truncate_last_shard(call_frame_t *frame, xlator_t *this, inode_t *inode) */ if (!inode) { gf_msg_debug(this->name, 0, - "Last shard to be truncated absent in backend: %d of " - "gfid %s. Directly proceeding to update file size", + "Last shard to be truncated absent in backend: %" PRIu64 + " of gfid %s. Directly proceeding to update file size", local->first_block, uuid_utoa(local->loc.inode->gfid)); shard_update_file_size(frame, this, NULL, &local->loc, shard_post_update_size_truncate_handler); @@ -2399,7 +2473,7 @@ shard_common_lookup_shards(call_frame_t *frame, xlator_t *this, inode_t *inode, int count = 0; int call_count = 0; int32_t shard_idx_iter = 0; - int last_block = 0; + int lookup_count = 0; char path[PATH_MAX] = { 0, }; @@ -2419,7 +2493,7 @@ shard_common_lookup_shards(call_frame_t *frame, xlator_t *this, inode_t *inode, local = frame->local; count = call_count = local->call_count; shard_idx_iter = local->first_block; - last_block = local->last_block; + lookup_count = local->last_block - local->create_count; local->pls_fop_handler = handler; if (local->lookup_shards_barriered) local->barrier.waitfor = local->call_count; @@ -2429,7 +2503,7 @@ shard_common_lookup_shards(call_frame_t *frame, xlator_t *this, inode_t *inode, else gf_uuid_copy(gfid, local->base_gfid); - while (shard_idx_iter <= last_block) { + while (shard_idx_iter <= lookup_count) { if (local->inode_list[i]) { i++; shard_idx_iter++; @@ -2574,6 +2648,7 @@ shard_truncate_begin(call_frame_t *frame, xlator_t *this) local->block_size); local->num_blocks = local->last_block - local->first_block + 1; + GF_ASSERT(local->num_blocks > 0); local->resolver_base_inode = (local->fop == GF_FOP_TRUNCATE) ? local->loc.inode : local->fd->inode; @@ -2723,8 +2798,8 @@ shard_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, local->resolver_base_inode = loc->inode; GF_ATOMIC_INIT(local->delta_blocks, 0); - shard_lookup_base_file(frame, this, &local->loc, - shard_post_lookup_truncate_handler); + shard_refresh_base_file(frame, this, &local->loc, NULL, + shard_post_lookup_truncate_handler); return 0; err: @@ -2779,8 +2854,8 @@ shard_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, local->resolver_base_inode = fd->inode; GF_ATOMIC_INIT(local->delta_blocks, 0); - shard_lookup_base_file(frame, this, &local->loc, - shard_post_lookup_truncate_handler); + shard_refresh_base_file(frame, this, NULL, fd, + shard_post_lookup_truncate_handler); return 0; err: shard_common_failure_unwind(GF_FOP_FTRUNCATE, frame, -1, ENOMEM); @@ -2924,8 +2999,8 @@ shard_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, if (!local->xattr_req) goto err; - shard_lookup_base_file(frame, this, &local->loc, - shard_post_lookup_link_handler); + shard_refresh_base_file(frame, this, &local->loc, NULL, + shard_post_lookup_link_handler); return 0; err: shard_common_failure_unwind(GF_FOP_LINK, frame, -1, ENOMEM); @@ -2939,13 +3014,20 @@ int shard_post_lookup_shards_unlink_handler(call_frame_t *frame, xlator_t *this) { shard_local_t *local = NULL; + uuid_t gfid = { + 0, + }; local = frame->local; + if (local->resolver_base_inode) + gf_uuid_copy(gfid, local->resolver_base_inode->gfid); + else + gf_uuid_copy(gfid, local->base_gfid); + if ((local->op_ret < 0) && (local->op_errno != ENOENT)) { gf_msg(this->name, GF_LOG_ERROR, local->op_errno, SHARD_MSG_FOP_FAILED, - "failed to delete shards of %s", - uuid_utoa(local->resolver_base_inode->gfid)); + "failed to delete shards of %s", uuid_utoa(gfid)); return 0; } local->op_ret = 0; @@ -4247,8 +4329,8 @@ shard_post_inodelk_fop_handler(call_frame_t *frame, xlator_t *this) switch (local->fop) { case GF_FOP_UNLINK: case GF_FOP_RENAME: - shard_lookup_base_file(frame, this, &local->int_inodelk.loc, - shard_post_lookup_base_shard_rm_handler); + shard_refresh_base_file(frame, this, &local->int_inodelk.loc, NULL, + shard_post_lookup_base_shard_rm_handler); break; default: gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP, @@ -4503,8 +4585,8 @@ shard_rename_src_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (local->block_size) { local->tmp_loc.inode = inode_new(this->itable); gf_uuid_copy(local->tmp_loc.gfid, (local->loc.inode)->gfid); - shard_lookup_base_file(frame, this, &local->tmp_loc, - shard_post_rename_lookup_handler); + shard_refresh_base_file(frame, this, &local->tmp_loc, NULL, + shard_post_rename_lookup_handler); } else { shard_rename_cbk(frame, this); } @@ -5143,6 +5225,7 @@ shard_post_lookup_readv_handler(call_frame_t *frame, xlator_t *this) local->block_size); local->num_blocks = local->last_block - local->first_block + 1; + GF_ASSERT(local->num_blocks > 0); local->resolver_base_inode = local->loc.inode; local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *), @@ -5239,8 +5322,8 @@ shard_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, local->loc.inode = inode_ref(fd->inode); gf_uuid_copy(local->loc.gfid, fd->inode->gfid); - shard_lookup_base_file(frame, this, &local->loc, - shard_post_lookup_readv_handler); + shard_refresh_base_file(frame, this, NULL, fd, + shard_post_lookup_readv_handler); return 0; err: shard_common_failure_unwind(GF_FOP_READ, frame, -1, ENOMEM); @@ -5603,6 +5686,8 @@ shard_common_inode_write_post_resolve_handler(call_frame_t *frame, shard_common_lookup_shards( frame, this, local->resolver_base_inode, shard_common_inode_write_post_lookup_shards_handler); + } else if (local->create_count) { + shard_common_inode_write_post_lookup_shards_handler(frame, this); } else { shard_common_inode_write_do(frame, this); } @@ -5633,6 +5718,7 @@ shard_common_inode_write_post_lookup_handler(call_frame_t *frame, local->last_block = get_highest_block(local->offset, local->total_size, local->block_size); local->num_blocks = local->last_block - local->first_block + 1; + GF_ASSERT(local->num_blocks > 0); local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *), gf_shard_mt_inode_list); if (!local->inode_list) { @@ -5641,9 +5727,9 @@ shard_common_inode_write_post_lookup_handler(call_frame_t *frame, } gf_msg_trace(this->name, 0, - "%s: gfid=%s first_block=%" PRIu32 + "%s: gfid=%s first_block=%" PRIu64 " " - "last_block=%" PRIu32 " num_blocks=%" PRIu32 " offset=%" PRId64 + "last_block=%" PRIu64 " num_blocks=%" PRIu64 " offset=%" PRId64 " total_size=%zu flags=%" PRId32 "", gf_fop_list[local->fop], uuid_utoa(local->resolver_base_inode->gfid), @@ -6038,8 +6124,8 @@ shard_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, local->loc.inode = inode_ref(fd->inode); gf_uuid_copy(local->loc.gfid, fd->inode->gfid); - shard_lookup_base_file(frame, this, &local->loc, - shard_post_lookup_fsync_handler); + shard_refresh_base_file(frame, this, NULL, fd, + shard_post_lookup_fsync_handler); return 0; err: shard_common_failure_unwind(GF_FOP_FSYNC, frame, -1, ENOMEM); @@ -6231,48 +6317,210 @@ shard_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, } int32_t -shard_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, - const char *name, dict_t *xdata) +shard_modify_and_set_iatt_in_dict(dict_t *xdata, shard_local_t *local, + char *key) { - int op_errno = EINVAL; + int ret = 0; + struct iatt *tmpbuf = NULL; + struct iatt *stbuf = NULL; + data_t *data = NULL; - if (frame->root->pid != GF_CLIENT_PID_GSYNCD) { - GF_IF_NATIVE_XATTR_GOTO(SHARD_XATTR_PREFIX "*", name, op_errno, out); + if (!xdata) + return 0; + + data = dict_get(xdata, key); + if (!data) + return 0; + + tmpbuf = data_to_iatt(data, key); + stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char); + if (stbuf == NULL) { + local->op_ret = -1; + local->op_errno = ENOMEM; + goto err; } + *stbuf = *tmpbuf; + stbuf->ia_size = local->prebuf.ia_size; + stbuf->ia_blocks = local->prebuf.ia_blocks; + ret = dict_set_iatt(xdata, key, stbuf, false); + if (ret < 0) { + local->op_ret = -1; + local->op_errno = ENOMEM; + goto err; + } + return 0; - if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) { - dict_del(xdata, GF_XATTR_SHARD_BLOCK_SIZE); - dict_del(xdata, GF_XATTR_SHARD_FILE_SIZE); +err: + GF_FREE(stbuf); + return -1; +} + +int32_t +shard_common_remove_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + int ret = -1; + shard_local_t *local = NULL; + + local = frame->local; + + if (op_ret < 0) { + local->op_ret = op_ret; + local->op_errno = op_errno; + goto err; } - STACK_WIND_TAIL(frame, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->removexattr, loc, name, xdata); + ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT); + if (ret < 0) + goto err; + + ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT); + if (ret < 0) + goto err; + + if (local->fd) + SHARD_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno, + xdata); + else + SHARD_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno, + xdata); return 0; -out: - shard_common_failure_unwind(GF_FOP_REMOVEXATTR, frame, -1, op_errno); + +err: + shard_common_failure_unwind(local->fop, frame, local->op_ret, + local->op_errno); return 0; } int32_t -shard_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, - const char *name, dict_t *xdata) +shard_post_lookup_remove_xattr_handler(call_frame_t *frame, xlator_t *this) { - int op_errno = EINVAL; + shard_local_t *local = NULL; + + local = frame->local; + if (local->op_ret < 0) { + shard_common_failure_unwind(local->fop, frame, local->op_ret, + local->op_errno); + return 0; + } + + if (local->fd) + STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fremovexattr, local->fd, + local->name, local->xattr_req); + else + STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->removexattr, &local->loc, + local->name, local->xattr_req); + return 0; +} + +int32_t +shard_common_remove_xattr(call_frame_t *frame, xlator_t *this, + glusterfs_fop_t fop, loc_t *loc, fd_t *fd, + const char *name, dict_t *xdata) +{ + int ret = -1; + int op_errno = ENOMEM; + uint64_t block_size = 0; + shard_local_t *local = NULL; + inode_t *inode = loc ? loc->inode : fd->inode; + + if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) { + if (loc) + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->removexattr, loc, name, + xdata); + else + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fremovexattr, fd, name, + xdata); + return 0; + } + + /* If shard's special xattrs are attempted to be removed, + * fail the fop with EPERM (except if the client is gsyncd). + */ if (frame->root->pid != GF_CLIENT_PID_GSYNCD) { - GF_IF_NATIVE_XATTR_GOTO(SHARD_XATTR_PREFIX "*", name, op_errno, out); + GF_IF_NATIVE_XATTR_GOTO(SHARD_XATTR_PREFIX "*", name, op_errno, err); } + /* Repeat the same check for bulk-removexattr */ if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) { dict_del(xdata, GF_XATTR_SHARD_BLOCK_SIZE); dict_del(xdata, GF_XATTR_SHARD_FILE_SIZE); } - STACK_WIND_TAIL(frame, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata); + ret = shard_inode_ctx_get_block_size(inode, this, &block_size); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED, + "Failed to get block size from inode ctx of %s", + uuid_utoa(inode->gfid)); + goto err; + } + + if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) { + if (loc) + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->removexattr, loc, name, + xdata); + else + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fremovexattr, fd, name, + xdata); + return 0; + } + + local = mem_get0(this->local_pool); + if (!local) + goto err; + + frame->local = local; + local->fop = fop; + if (loc) { + if (loc_copy(&local->loc, loc) != 0) + goto err; + } + + if (fd) { + local->fd = fd_ref(fd); + local->loc.inode = inode_ref(fd->inode); + gf_uuid_copy(local->loc.gfid, fd->inode->gfid); + } + + if (name) { + local->name = gf_strdup(name); + if (!local->name) + goto err; + } + + if (xdata) + local->xattr_req = dict_ref(xdata); + + shard_refresh_base_file(frame, this, loc, fd, + shard_post_lookup_remove_xattr_handler); return 0; -out: - shard_common_failure_unwind(GF_FOP_FREMOVEXATTR, frame, -1, op_errno); +err: + shard_common_failure_unwind(fop, frame, -1, op_errno); + return 0; +} + +int32_t +shard_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) +{ + shard_common_remove_xattr(frame, this, GF_FOP_REMOVEXATTR, loc, NULL, name, + xdata); + return 0; +} + +int32_t +shard_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, + const char *name, dict_t *xdata) +{ + shard_common_remove_xattr(frame, this, GF_FOP_FREMOVEXATTR, NULL, fd, name, + xdata); return 0; } @@ -6353,38 +6601,164 @@ out: } int32_t -shard_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict, - int32_t flags, dict_t *xdata) +shard_common_set_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - int op_errno = EINVAL; + int ret = -1; + shard_local_t *local = NULL; - if (frame->root->pid != GF_CLIENT_PID_GSYNCD) { - GF_IF_INTERNAL_XATTR_GOTO(SHARD_XATTR_PREFIX "*", dict, op_errno, out); + local = frame->local; + + if (op_ret < 0) { + local->op_ret = op_ret; + local->op_errno = op_errno; + goto err; } - STACK_WIND_TAIL(frame, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata); + ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT); + if (ret < 0) + goto err; + + ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT); + if (ret < 0) + goto err; + + if (local->fd) + SHARD_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno, + xdata); + else + SHARD_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno, + xdata); return 0; -out: - shard_common_failure_unwind(GF_FOP_FSETXATTR, frame, -1, op_errno); + +err: + shard_common_failure_unwind(local->fop, frame, local->op_ret, + local->op_errno); return 0; } int32_t -shard_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, - int32_t flags, dict_t *xdata) +shard_post_lookup_set_xattr_handler(call_frame_t *frame, xlator_t *this) { - int op_errno = EINVAL; + shard_local_t *local = NULL; + + local = frame->local; + + if (local->op_ret < 0) { + shard_common_failure_unwind(local->fop, frame, local->op_ret, + local->op_errno); + return 0; + } + if (local->fd) + STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fsetxattr, local->fd, + local->xattr_req, local->flags, local->xattr_rsp); + else + STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->setxattr, &local->loc, + local->xattr_req, local->flags, local->xattr_rsp); + return 0; +} + +int32_t +shard_common_set_xattr(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop, + loc_t *loc, fd_t *fd, dict_t *dict, int32_t flags, + dict_t *xdata) +{ + int ret = -1; + int op_errno = ENOMEM; + uint64_t block_size = 0; + shard_local_t *local = NULL; + inode_t *inode = loc ? loc->inode : fd->inode; + + if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) { + if (loc) + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, + xdata); + else + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, + xdata); + return 0; + } + + /* Sharded or not, if shard's special xattrs are attempted to be set, + * fail the fop with EPERM (except if the client is gsyncd. + */ if (frame->root->pid != GF_CLIENT_PID_GSYNCD) { - GF_IF_INTERNAL_XATTR_GOTO(SHARD_XATTR_PREFIX "*", dict, op_errno, out); + GF_IF_INTERNAL_XATTR_GOTO(SHARD_XATTR_PREFIX "*", dict, op_errno, err); } - STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, - loc, dict, flags, xdata); + ret = shard_inode_ctx_get_block_size(inode, this, &block_size); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED, + "Failed to get block size from inode ctx of %s", + uuid_utoa(inode->gfid)); + goto err; + } + + if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) { + if (loc) + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, + xdata); + else + STACK_WIND_TAIL(frame, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, + xdata); + return 0; + } + + local = mem_get0(this->local_pool); + if (!local) + goto err; + + frame->local = local; + local->fop = fop; + if (loc) { + if (loc_copy(&local->loc, loc) != 0) + goto err; + } + + if (fd) { + local->fd = fd_ref(fd); + local->loc.inode = inode_ref(fd->inode); + gf_uuid_copy(local->loc.gfid, fd->inode->gfid); + } + local->flags = flags; + /* Reusing local->xattr_req and local->xattr_rsp to store the setxattr dict + * and the xdata dict + */ + if (dict) + local->xattr_req = dict_ref(dict); + if (xdata) + local->xattr_rsp = dict_ref(xdata); + + shard_refresh_base_file(frame, this, loc, fd, + shard_post_lookup_set_xattr_handler); return 0; -out: - shard_common_failure_unwind(GF_FOP_SETXATTR, frame, -1, op_errno); +err: + shard_common_failure_unwind(fop, frame, -1, op_errno); + return 0; +} + +int32_t +shard_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict, + int32_t flags, dict_t *xdata) +{ + shard_common_set_xattr(frame, this, GF_FOP_FSETXATTR, NULL, fd, dict, flags, + xdata); + return 0; +} + +int32_t +shard_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, + int32_t flags, dict_t *xdata) +{ + shard_common_set_xattr(frame, this, GF_FOP_SETXATTR, loc, NULL, dict, flags, + xdata); return 0; } @@ -6647,8 +7021,8 @@ shard_common_inode_write_begin(call_frame_t *frame, xlator_t *this, local->loc.inode = inode_ref(fd->inode); gf_uuid_copy(local->loc.gfid, fd->inode->gfid); - shard_lookup_base_file(frame, this, &local->loc, - shard_common_inode_write_post_lookup_handler); + shard_refresh_base_file(frame, this, NULL, fd, + shard_common_inode_write_post_lookup_handler); return 0; out: shard_common_failure_unwind(fop, frame, -1, ENOMEM); diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h index 04abd62c21c..4fe181b64d5 100644 --- a/xlators/features/shard/src/shard.h +++ b/xlators/features/shard/src/shard.h @@ -254,9 +254,9 @@ typedef int32_t (*shard_post_update_size_fop_handler_t)(call_frame_t *frame, typedef struct shard_local { int op_ret; int op_errno; - int first_block; - int last_block; - int num_blocks; + uint64_t first_block; + uint64_t last_block; + uint64_t num_blocks; int call_count; int eexist_count; int create_count; @@ -318,6 +318,7 @@ typedef struct shard_local { uint32_t deletion_rate; gf_boolean_t cleanup_required; uuid_t base_gfid; + char *name; } shard_local_t; typedef struct shard_inode_ctx { diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c index f44b11c6872..7d09cba3e9c 100644 --- a/xlators/features/trash/src/trash.c +++ b/xlators/features/trash/src/trash.c @@ -212,11 +212,11 @@ void append_time_stamp(char *name, size_t name_size) { int i; - char timestr[64] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; - gf_time_fmt(timestr, sizeof(timestr), time(NULL), gf_timefmt_F_HMS); + gf_time_fmt(timestr, sizeof(timestr), gf_time(), gf_timefmt_F_HMS); /* removing white spaces in timestamp */ for (i = 0; i < strlen(timestr); i++) { diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c index 978825f6b56..c641bd6f432 100644 --- a/xlators/features/upcall/src/upcall-internal.c +++ b/xlators/features/upcall/src/upcall-internal.c @@ -316,7 +316,7 @@ upcall_reaper_thread(void *data) priv = this->private; GF_ASSERT(priv); - time_now = time(NULL); + time_now = gf_time(); while (!priv->fini) { list_for_each_entry_safe(inode_ctx, tmp, &priv->inode_ctx_list, inode_ctx_list) @@ -344,7 +344,7 @@ upcall_reaper_thread(void *data) /* don't do a very busy loop */ timeout = get_cache_invalidation_timeout(this); sleep(timeout / 2); - time_now = time(NULL); + time_now = gf_time(); } return NULL; @@ -533,7 +533,7 @@ upcall_cache_invalidate(call_frame_t *frame, xlator_t *this, client_t *client, goto out; } - time_now = time(NULL); + time_now = gf_time(); pthread_mutex_lock(&up_inode_ctx->client_list_lock); { list_for_each_entry_safe(up_client_entry, tmp, @@ -670,13 +670,13 @@ upcall_cache_forget(xlator_t *this, inode_t *inode, return; } - time_now = time(NULL); + time_now = gf_time(); pthread_mutex_lock(&up_inode_ctx->client_list_lock); { list_for_each_entry_safe(up_client_entry, tmp, &up_inode_ctx->client_list, client_list) { - /* Set the access time to time(NULL) + /* Set the access time to gf_time() * to send notify */ up_client_entry->access_time = time_now; diff --git a/xlators/features/utime/src/utime.c b/xlators/features/utime/src/utime.c index 3b8dadd3191..2acc63e6a05 100644 --- a/xlators/features/utime/src/utime.c +++ b/xlators/features/utime/src/utime.c @@ -147,6 +147,7 @@ gf_utime_set_mdata_setxattr_cbk(call_frame_t *frame, void *cookie, } frame->local = NULL; call_resume(stub); + STACK_DESTROY(frame->root); return 0; } @@ -162,6 +163,7 @@ gf_utime_set_mdata_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, loc_t loc = { 0, }; + call_frame_t *new_frame = NULL; if (!op_ret && dict_get(xdata, GF_XATTR_MDATA_KEY) == NULL) { dict = dict_new(); @@ -181,19 +183,32 @@ gf_utime_set_mdata_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, "dict set of key for set-ctime-mdata failed"); goto err; } - frame->local = fop_lookup_cbk_stub(frame, default_lookup_cbk, op_ret, - op_errno, inode, stbuf, xdata, - postparent); - if (!frame->local) { + new_frame = copy_frame(frame); + if (!new_frame) { + op_errno = ENOMEM; + goto stub_err; + } + + new_frame->local = fop_lookup_cbk_stub(frame, default_lookup_cbk, + op_ret, op_errno, inode, stbuf, + xdata, postparent); + if (!new_frame->local) { gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY, "lookup_cbk stub allocation failed"); + op_errno = ENOMEM; + STACK_DESTROY(new_frame->root); goto stub_err; } loc.inode = inode_ref(inode); gf_uuid_copy(loc.gfid, stbuf->ia_gfid); - STACK_WIND(frame, gf_utime_set_mdata_setxattr_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->setxattr, &loc, dict, 0, NULL); + + new_frame->root->uid = 0; + new_frame->root->gid = 0; + new_frame->root->pid = GF_CLIENT_PID_SET_UTIME; + STACK_WIND(new_frame, gf_utime_set_mdata_setxattr_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, &loc, + dict, 0, NULL); dict_unref(dict); inode_unref(loc.inode); diff --git a/xlators/meta/src/meta-helpers.c b/xlators/meta/src/meta-helpers.c index 8b3d7b2f2f2..cb54f547468 100644 --- a/xlators/meta/src/meta-helpers.c +++ b/xlators/meta/src/meta-helpers.c @@ -182,14 +182,15 @@ meta_uuid_copy(uuid_t dst, uuid_t src) } static void -default_meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type) +default_meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type, + gf_boolean_t is_tunable) { struct timeval tv = {}; iatt->ia_type = type; switch (type) { case IA_IFDIR: - iatt->ia_prot = ia_prot_from_st_mode(0755); + iatt->ia_prot = ia_prot_from_st_mode(0555); iatt->ia_nlink = 2; break; case IA_IFLNK: @@ -197,7 +198,7 @@ default_meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type) iatt->ia_nlink = 1; break; default: - iatt->ia_prot = ia_prot_from_st_mode(0644); + iatt->ia_prot = ia_prot_from_st_mode(is_tunable ? 0644 : 0444); iatt->ia_nlink = 1; break; } @@ -225,7 +226,7 @@ meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type) return; if (!ops->iatt_fill) - default_meta_iatt_fill(iatt, inode, type); + default_meta_iatt_fill(iatt, inode, type, !!ops->file_write); else ops->iatt_fill(THIS, inode, iatt); return; diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index eaa61c435e5..685beb42d27 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -25,13 +25,14 @@ glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \ glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \ glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c \ glusterd-reset-brick.c glusterd-shd-svc.c glusterd-shd-svc-helper.c \ - glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c glusterd-ganesha.c + glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c glusterd-ganesha.c \ + $(CONTRIBDIR)/mount/mntent.c glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ $(top_builddir)/libglusterd/src/libglusterd.la \ $(top_builddir)/rpc/xdr/src/libgfxdr.la \ $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ - $(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS) $(LIB_DL) + $(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS) $(LIB_DL) $(GF_XLATOR_MGNT_LIBADD) noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \ glusterd-sm.h glusterd-store.h glusterd-mem-types.h \ @@ -46,7 +47,8 @@ noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \ glusterd-scrub-svc.h glusterd-server-quorum.h glusterd-errno.h \ glusterd-shd-svc.h glusterd-shd-svc-helper.h \ glusterd-gfproxyd-svc.h glusterd-gfproxyd-svc-helper.h \ - $(CONTRIBDIR)/userspace-rcu/rculist-extra.h + $(CONTRIBDIR)/userspace-rcu/rculist-extra.h \ + $(CONTRIBDIR)/mount/mntent_compat.h AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \ diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c index 5e5421ed2dc..e56cd0e6c74 100644 --- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c @@ -21,7 +21,6 @@ #include "glusterd-messages.h" #include "glusterd-server-quorum.h" #include <glusterfs/run.h> -#include "glusterd-volgen.h" #include <glusterfs/syscall.h> #include <sys/signal.h> @@ -183,6 +182,9 @@ gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo, { int ret = -1; int replica_nodes = 0; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); switch (volinfo->type) { case GF_CLUSTER_TYPE_NONE: @@ -191,8 +193,8 @@ gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo, "replica count (%d) option given for non replicate " "volume %s", replica_count, volinfo->volname); - gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_REPLICA, "%s", - err_str); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ARGUMENT, + err_str, NULL); goto out; case GF_CLUSTER_TYPE_REPLICATE: @@ -203,8 +205,8 @@ gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo, "than volume %s's replica count (%d)", replica_count, volinfo->volname, volinfo->replica_count); - gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY, - "%s", err_str); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, + GD_MSG_INVALID_ARGUMENT, err_str, NULL); goto out; } if (replica_count == volinfo->replica_count) { @@ -218,8 +220,8 @@ gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo, "(or %dxN)", brick_count, volinfo->dist_leaf_count, volinfo->dist_leaf_count); - gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, - GD_MSG_INVALID_ENTRY, "%s", err_str); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, + GD_MSG_INVALID_ARGUMENT, err_str, NULL); goto out; } ret = 1; @@ -234,6 +236,8 @@ gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo, "need %d(xN) bricks for reducing replica " "count of the volume from %d to %d", replica_nodes, volinfo->replica_count, replica_count); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, + GD_MSG_INVALID_ARGUMENT, err_str, NULL); goto out; } break; @@ -283,6 +287,7 @@ __glusterd_handle_add_brick(rpcsvc_request_t *req) // failed to decode msg; req->rpc_err = GARBAGE_ARGS; snprintf(err_str, sizeof(err_str), "Garbage args received"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } @@ -510,6 +515,8 @@ subvol_matcher_verify(int *subvols, glusterd_volinfo_t *volinfo, char *err_str, int i = 0; int ret = 0; int count = volinfo->replica_count - replica_count; + xlator_t *this = THIS; + GF_ASSERT(this); if (replica_count && subvols) { for (i = 0; i < volinfo->subvol_count; i++) { @@ -519,6 +526,8 @@ subvol_matcher_verify(int *subvols, glusterd_volinfo_t *volinfo, char *err_str, "Remove exactly %d" " brick(s) from each subvolume.", count); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL); break; } } @@ -532,6 +541,8 @@ subvol_matcher_verify(int *subvols, glusterd_volinfo_t *volinfo, char *err_str, ret = -1; snprintf(err_str, err_len, "Bricks not from same subvol for %s", vol_type); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL); break; } } while (++i < volinfo->subvol_count); @@ -556,6 +567,9 @@ glusterd_remove_brick_validate_arbiters(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *brickinfo = NULL; glusterd_brickinfo_t *last = NULL; char *arbiter_array = NULL; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); if (volinfo->type != GF_CLUSTER_TYPE_REPLICATE) goto out; @@ -574,6 +588,8 @@ glusterd_remove_brick_validate_arbiters(glusterd_volinfo_t *volinfo, "Remove arbiter " "brick(s) only when converting from " "arbiter to replica 2 subvolume."); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL); ret = -1; goto out; } @@ -598,6 +614,8 @@ glusterd_remove_brick_validate_arbiters(glusterd_volinfo_t *volinfo, "Removed bricks " "must contain arbiter when converting" " to plain distribute."); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL); ret = -1; break; } @@ -621,6 +639,7 @@ __glusterd_handle_remove_brick(rpcsvc_request_t *req) char key[64] = ""; int keylen; int i = 1; + glusterd_conf_t *conf = NULL; glusterd_volinfo_t *volinfo = NULL; glusterd_brickinfo_t *brickinfo = NULL; glusterd_brickinfo_t **brickinfo_list = NULL; @@ -639,12 +658,15 @@ __glusterd_handle_remove_brick(rpcsvc_request_t *req) GF_ASSERT(req); this = THIS; GF_ASSERT(this); + conf = this->private; + GF_ASSERT(conf); ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; snprintf(err_str, sizeof(err_str), "Received garbage args"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } @@ -835,7 +857,17 @@ __glusterd_handle_remove_brick(rpcsvc_request_t *req) if (ret) goto out; - ret = glusterd_op_begin_synctask(req, GD_OP_REMOVE_BRICK, dict); + if (conf->op_version < GD_OP_VERSION_8_0) { + gf_msg_debug(this->name, 0, + "The cluster is operating at " + "version less than %d. remove-brick operation" + "falling back to syncop framework.", + GD_OP_VERSION_8_0); + ret = glusterd_op_begin_synctask(req, GD_OP_REMOVE_BRICK, dict); + } else { + ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_REMOVE_BRICK, + dict); + } out: if (ret) { @@ -991,6 +1023,7 @@ glusterd_op_perform_add_bricks(glusterd_volinfo_t *volinfo, int32_t count, xlator_t *this = NULL; glusterd_conf_t *conf = NULL; gf_boolean_t is_valid_add_brick = _gf_false; + gf_boolean_t restart_shd = _gf_false; struct statvfs brickstat = { 0, }; @@ -1147,6 +1180,15 @@ glusterd_op_perform_add_bricks(glusterd_volinfo_t *volinfo, int32_t count, if (glusterd_is_volume_replicate(volinfo)) { if (replica_count && conf->op_version >= GD_OP_VERSION_3_7_10) { is_valid_add_brick = _gf_true; + if (volinfo->status == GLUSTERD_STATUS_STARTED) { + ret = volinfo->shd.svc.stop(&(volinfo->shd.svc), SIGTERM); + if (ret) { + gf_msg("glusterd", GF_LOG_ERROR, 0, + GD_MSG_GLUSTER_SERVICES_STOP_FAIL, + "Failed to stop shd for %s.", volinfo->volname); + } + restart_shd = _gf_true; + } ret = generate_dummy_client_volfiles(volinfo); if (ret) { gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, @@ -1221,6 +1263,14 @@ generate_volfiles: out: GF_FREE(free_ptr1); GF_FREE(free_ptr2); + if (restart_shd) { + if (volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo, + PROC_START_NO_WAIT)) { + gf_msg("glusterd", GF_LOG_CRITICAL, 0, + GD_MSG_GLUSTER_SERVICE_START_FAIL, + "Failed to start shd for %s.", volinfo->volname); + } + } gf_msg_debug("glusterd", 0, "Returning %d", ret); return ret; @@ -1309,14 +1359,14 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "Unable to get volume name"); goto out; } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "Unable to find volume: %s", volname); goto out; } @@ -1328,13 +1378,7 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"), &replica_count); if (ret) { - gf_msg_debug(THIS->name, 0, "Unable to get replica count"); - } - - ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"), - &arbiter_count); - if (ret) { - gf_msg_debug(THIS->name, 0, "No arbiter count present in the dict"); + gf_msg_debug(this->name, 0, "Unable to get replica count"); } if (replica_count > 0) { @@ -1348,18 +1392,20 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) } } - if (glusterd_is_volume_replicate(volinfo)) { + glusterd_add_peers_to_auth_list(volname); + + if (replica_count && glusterd_is_volume_replicate(volinfo)) { /* Do not allow add-brick for stopped volumes when replica-count * is being increased. */ - if (conf->op_version >= GD_OP_VERSION_3_7_10 && replica_count && - GLUSTERD_STATUS_STOPPED == volinfo->status) { + if (GLUSTERD_STATUS_STOPPED == volinfo->status && + conf->op_version >= GD_OP_VERSION_3_7_10) { ret = -1; snprintf(msg, sizeof(msg), " Volume must not be in" " stopped state when replica-count needs to " " be increased."); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", msg); *op_errstr = gf_strdup(msg); goto out; @@ -1367,25 +1413,31 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) /* op-version check for replica 2 to arbiter conversion. If we * don't have this check, an older peer added as arbiter brick * will not have the arbiter xlator in its volfile. */ - if ((conf->op_version < GD_OP_VERSION_3_8_0) && (arbiter_count == 1) && - (replica_count == 3)) { - ret = -1; - snprintf(msg, sizeof(msg), - "Cluster op-version must " - "be >= 30800 to add arbiter brick to a " - "replica 2 volume."); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", - msg); - *op_errstr = gf_strdup(msg); - goto out; + if ((replica_count == 3) && (conf->op_version < GD_OP_VERSION_3_8_0)) { + ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"), + &arbiter_count); + if (ret) { + gf_msg_debug(this->name, 0, + "No arbiter count present in the dict"); + } else if (arbiter_count == 1) { + ret = -1; + snprintf(msg, sizeof(msg), + "Cluster op-version must " + "be >= 30800 to add arbiter brick to a " + "replica 2 volume."); + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", + msg); + *op_errstr = gf_strdup(msg); + goto out; + } } /* Do not allow increasing replica count for arbiter volumes. */ - if (replica_count && volinfo->arbiter_count) { + if (volinfo->arbiter_count) { ret = -1; snprintf(msg, sizeof(msg), "Increasing replica count " "for arbiter volumes is not supported."); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", msg); *op_errstr = gf_strdup(msg); goto out; @@ -1394,6 +1446,43 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) is_force = dict_get_str_boolean(dict, "force", _gf_false); + /* Check brick order if the volume type is replicate or disperse. If + * force at the end of command not given then check brick order. + * doing this check at the originator node is sufficient. + */ + + if (!is_force && is_origin_glusterd(dict)) { + ret = 0; + if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) { + gf_msg_debug(this->name, 0, + "Replicate cluster type " + "found. Checking brick order."); + if (replica_count) + ret = glusterd_check_brick_order(dict, msg, volinfo->type, + &volname, &bricks, &count, + replica_count); + else + ret = glusterd_check_brick_order(dict, msg, volinfo->type, + &volname, &bricks, &count, + volinfo->replica_count); + } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) { + gf_msg_debug(this->name, 0, + "Disperse cluster type" + " found. Checking brick order."); + ret = glusterd_check_brick_order(dict, msg, volinfo->type, &volname, + &bricks, &count, + volinfo->disperse_count); + } + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER, + "Not adding brick because of " + "bad brick order. %s", + msg); + *op_errstr = gf_strdup(msg); + goto out; + } + } + if (volinfo->replica_count < replica_count && !is_force) { cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list) { @@ -1410,7 +1499,7 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) if (len < 0) { strcpy(msg, "<error>"); } - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s", msg); *op_errstr = gf_strdup(msg); goto out; @@ -1442,24 +1531,40 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) "Volume name %s rebalance is in " "progress. Please retry after completion", volname); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER, "%s", msg); + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER, "%s", msg); *op_errstr = gf_strdup(msg); ret = -1; goto out; } - ret = dict_get_int32n(dict, "count", SLEN("count"), &count); - if (ret) { - gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, - "Unable to get count"); - goto out; + if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) { + snprintf(msg, sizeof(msg), + "Volume %s has %" PRIu64 + " snapshots. " + "Changing the volume configuration will not effect snapshots." + "But the snapshot brick mount should be intact to " + "make them function.", + volname, volinfo->snap_count); + gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg); + msg[0] = '\0'; + } + + if (!count) { + ret = dict_get_int32n(dict, "count", SLEN("count"), &count); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get count"); + goto out; + } } - ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks); - if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, - "Unable to get bricks"); - goto out; + if (!bricks) { + ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get bricks"); + goto out; + } } if (bricks) { @@ -1478,7 +1583,7 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) "brick path %s is " "too long", brick); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRKPATH_TOO_LONG, "%s", + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRKPATH_TOO_LONG, "%s", msg); *op_errstr = gf_strdup(msg); @@ -1489,7 +1594,7 @@ glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict) ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true, NULL); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND, + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND, "Add-brick: Unable" " to get brickinfo"); goto out; @@ -1559,7 +1664,7 @@ out: GF_FREE(str_ret); GF_FREE(all_bricks); - gf_msg_debug(THIS->name, 0, "Returning %d", ret); + gf_msg_debug(this->name, 0, "Returning %d", ret); return ret; } @@ -1583,6 +1688,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, }; glusterd_conf_t *priv = THIS->private; int pid = -1; + xlator_t *this = THIS; + GF_ASSERT(this); /* Check whether all the nodes of the bricks to be removed are * up, if not fail the operation */ @@ -1591,6 +1698,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, ret = dict_get_strn(dict, key, keylen, &brick); if (ret) { snprintf(msg, sizeof(msg), "Unable to get %s", key); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "key=%s", key, NULL); *errstr = gf_strdup(msg); goto out; } @@ -1602,6 +1711,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "Incorrect brick " "%s for volume %s", brick, volinfo->volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INCORRECT_BRICK, + "Brick=%s, Volume=%s", brick, volinfo->volname, NULL); *errstr = gf_strdup(msg); goto out; } @@ -1614,6 +1725,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "is not decommissioned. " "Use start or force option", brick); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_DECOM, + "Use 'start' or 'force' option, Brick=%s", brick, NULL); *errstr = gf_strdup(msg); ret = -1; goto out; @@ -1640,6 +1753,10 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "brick %s. Use force option to " "remove the offline brick", brick); + gf_smsg( + this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_STOPPED, + "Use 'force' option to remove the offline brick, Brick=%s", + brick, NULL); *errstr = gf_strdup(msg); ret = -1; goto out; @@ -1650,6 +1767,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "Found dead " "brick %s", brick); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_DEAD, + "Brick=%s", brick, NULL); *errstr = gf_strdup(msg); ret = -1; goto out; @@ -1667,6 +1786,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "Host node of the " "brick %s is not in cluster", brick); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRICK_HOST_NOT_FOUND, "Brick=%s", brick, NULL); *errstr = gf_strdup(msg); ret = -1; goto out; @@ -1677,6 +1798,8 @@ glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count, "Host node of the " "brick %s is down", brick); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_HOST_DOWN, + "Brick=%s", brick, NULL); *errstr = gf_strdup(msg); ret = -1; goto out; @@ -1756,6 +1879,7 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) errstr = gf_strdup( "Deleting all the bricks of the " "volume is not allowed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DELETE, NULL); ret = -1; goto out; } @@ -1764,6 +1888,8 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) switch (cmd) { case GF_OP_CMD_NONE: errstr = gf_strdup("no remove-brick command issued"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NO_REMOVE_CMD, + NULL); goto out; case GF_OP_CMD_STATUS: @@ -1837,6 +1963,21 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) goto out; } + if (volinfo->snap_count > 0 || + !cds_list_empty(&volinfo->snap_volumes)) { + snprintf(msg, sizeof(msg), + "Volume %s has %" PRIu64 + " snapshots. " + "Changing the volume configuration will not effect " + "snapshots." + "But the snapshot brick mount should be intact to " + "make them function.", + volname, volinfo->snap_count); + gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", + msg); + msg[0] = '\0'; + } + ret = glusterd_remove_brick_validate_bricks( cmd, brick_count, dict, volinfo, &errstr, GF_DEFRAG_CMD_NONE); if (ret) @@ -1873,6 +2014,8 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) errstr = gf_strdup( "use 'force' option as migration " "is in progress"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_PROG, + "Use 'force' option", NULL); goto out; } @@ -1880,6 +2023,8 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) errstr = gf_strdup( "use 'force' option as migration " "has failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_FAIL, + "Use 'force' option", NULL); goto out; } @@ -1890,6 +2035,11 @@ glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr) "use 'force' option as migration " "of some files might have been skipped or " "has failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, + GD_MSG_MIGRATION_FAIL, + "Use 'force' option, some files might have been " + "skipped", + NULL); goto out; } } @@ -2084,6 +2234,119 @@ out: } int +glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr) +{ + int ret = 0; + char *volname = NULL; + + ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); + + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get volume name"); + goto out; + } + ret = glusterd_replace_old_auth_allow_list(volname); +out: + return ret; +} + +int +glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr) +{ + int ret = 0; + char *volname = NULL; + + ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); + + if (ret) { + gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get volume name"); + goto out; + } + ret = glusterd_replace_old_auth_allow_list(volname); +out: + return ret; +} + +int +glusterd_set_rebalance_id_for_remove_brick(dict_t *req_dict, dict_t *rsp_dict) +{ + int ret = -1; + char *volname = NULL; + glusterd_volinfo_t *volinfo = NULL; + char msg[2048] = {0}; + char *task_id_str = NULL; + xlator_t *this = NULL; + int32_t cmd = 0; + + this = THIS; + GF_ASSERT(this); + + GF_ASSERT(rsp_dict); + GF_ASSERT(req_dict); + + ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname); + if (ret) { + gf_msg_debug(this->name, 0, "volname not found"); + goto out; + } + + ret = glusterd_volinfo_find(volname, &volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, + "Unable to allocate memory"); + goto out; + } + + ret = dict_get_int32n(rsp_dict, "command", SLEN("command"), &cmd); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get command"); + goto out; + } + + /* remove brick task id is generted in glusterd_op_stage_remove_brick(), + * but rsp_dict is unavailable there. So copying it to rsp_dict from + * req_dict here. */ + + if (is_origin_glusterd(rsp_dict)) { + ret = dict_get_strn(req_dict, GF_REMOVE_BRICK_TID_KEY, + SLEN(GF_REMOVE_BRICK_TID_KEY), &task_id_str); + if (ret) { + snprintf(msg, sizeof(msg), "Missing rebalance id for remove-brick"); + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_REBALANCE_ID_MISSING, + "%s", msg); + ret = 0; + } else { + gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id); + + ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, + rsp_dict, GF_REMOVE_BRICK_TID_KEY, + SLEN(GF_REMOVE_BRICK_TID_KEY)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, + GD_MSG_REMOVE_BRICK_ID_SET_FAIL, + "Failed to set remove-brick-id"); + goto out; + } + } + } + if (!gf_uuid_is_null(volinfo->rebal.rebalance_id) && + GD_OP_REMOVE_BRICK == volinfo->rebal.op) { + ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, rsp_dict, + GF_REMOVE_BRICK_TID_KEY, + SLEN(GF_REMOVE_BRICK_TID_KEY)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Failed to set task-id for volume %s", volname); + goto out; + } + } +out: + return ret; +} +int glusterd_op_remove_brick(dict_t *dict, char **op_errstr) { int ret = -1; @@ -2397,7 +2660,7 @@ out: GF_FREE(brick_tmpstr); if (bricks_dict) dict_unref(bricks_dict); - + gf_msg_debug(this->name, 0, "returning %d ", ret); return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c index 09f0a35dc45..5c01f0c70b6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c @@ -26,12 +26,17 @@ glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout, xlator_t *this = THIS; glusterd_svc_t *svc = NULL; - if (!this) + if (!this) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED, + NULL); goto out; + } options = dict_new(); - if (!options) + if (!options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } svc = glusterd_conn_get_svc_object(conn); if (!svc) { @@ -46,8 +51,11 @@ glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout, ret = dict_set_int32n(options, "transport.socket.ignore-enoent", SLEN("transport.socket.ignore-enoent"), 1); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=transport.socket.ignore-enoent", NULL); goto out; + } /* @options is free'd by rpc_transport when destroyed */ rpc = rpc_clnt_new(options, this, (char *)svc->name, 16); @@ -61,9 +69,10 @@ glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout, goto out; ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; - else + } else ret = 0; conn->frame_timeout = frame_timeout; diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c index cf567fa4172..f08bd6cebee 100644 --- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c +++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c @@ -421,6 +421,35 @@ check_host_list(void) } int +gd_ganesha_send_dbus(char *volname, char *value) +{ + runner_t runner = { + 0, + }; + int ret = -1; + runinit(&runner); + + GF_VALIDATE_OR_GOTO("glusterd-ganesha", volname, out); + GF_VALIDATE_OR_GOTO("glusterd-ganesha", value, out); + + ret = 0; + if (check_host_list()) { + /* Check whether ganesha is running on this node */ + if (manage_service("status")) { + gf_msg("glusterd-ganesha", GF_LOG_WARNING, 0, + GD_MSG_GANESHA_NOT_RUNNING, + "Export failed, NFS-Ganesha is not running"); + } else { + runner_add_args(&runner, GANESHA_PREFIX "/dbus-send.sh", CONFDIR, + value, volname, NULL); + ret = runner_run(&runner); + } + } +out: + return ret; +} + +int manage_export_config(char *volname, char *value, char **op_errstr) { runner_t runner = { @@ -447,9 +476,6 @@ int ganesha_manage_export(dict_t *dict, char *value, gf_boolean_t update_cache_invalidation, char **op_errstr) { - runner_t runner = { - 0, - }; int ret = -1; glusterd_volinfo_t *volinfo = NULL; dict_t *vol_opts = NULL; @@ -458,7 +484,6 @@ ganesha_manage_export(dict_t *dict, char *value, glusterd_conf_t *priv = NULL; gf_boolean_t option = _gf_false; - runinit(&runner); this = THIS; GF_ASSERT(this); priv = this->private; @@ -538,26 +563,13 @@ ganesha_manage_export(dict_t *dict, char *value, goto out; } } - - if (check_host_list()) { - /* Check whether ganesha is running on this node */ - if (manage_service("status")) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GANESHA_NOT_RUNNING, - "Export failed, NFS-Ganesha is not running"); - } else { - runner_add_args(&runner, GANESHA_PREFIX "/dbus-send.sh", CONFDIR, - value, volname, NULL); - ret = runner_run(&runner); - if (ret) { - gf_asprintf(op_errstr, - "Dynamic export" - " addition/deletion failed." - " Please see log file for details"); - goto out; - } - } + ret = gd_ganesha_send_dbus(volname, value); + if (ret) { + gf_asprintf(op_errstr, + "Dynamic export addition/deletion failed." + " Please see log file for details"); + goto out; } - if (update_cache_invalidation) { vol_opts = volinfo->dict; ret = dict_set_dynstr_with_alloc(vol_opts, @@ -617,8 +629,9 @@ tear_down_cluster(gf_boolean_t run_teardown) goto out; } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; snprintf(path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name); ret = sys_lstat(path, &st); if (ret == -1) { @@ -649,7 +662,6 @@ tear_down_cluster(gf_boolean_t run_teardown) gf_msg_debug(THIS->name, 0, "%s %s", ret ? "Failed to remove" : "Removed", entry->d_name); - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } ret = sys_closedir(dir); diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c index 76b7684538f..bf062c87060 100644 --- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c +++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c @@ -115,13 +115,18 @@ __glusterd_handle_sys_exec(rpcsvc_request_t *req) ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + snprintf(err_str, sizeof(err_str), "Garbage args received"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } if (cli_req.dict.dict_len) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict); @@ -142,13 +147,18 @@ __glusterd_handle_sys_exec(rpcsvc_request_t *req) snprintf(err_str, sizeof(err_str), "Failed to get " "the uuid of local glusterd"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL, + NULL); ret = -1; goto out; } ret = dict_set_dynstr(dict, "host-uuid", host_uuid); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=host-uuid", NULL); goto out; + } } ret = glusterd_op_begin_synctask(req, cli_op, dict); @@ -188,13 +198,18 @@ __glusterd_handle_copy_file(rpcsvc_request_t *req) ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + snprintf(err_str, sizeof(err_str), "Garbage args received"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } if (cli_req.dict.dict_len) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict); @@ -215,6 +230,8 @@ __glusterd_handle_copy_file(rpcsvc_request_t *req) snprintf(err_str, sizeof(err_str), "Failed to get " "the uuid of local glusterd"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL, + NULL); ret = -1; goto out; } @@ -267,13 +284,18 @@ __glusterd_handle_gsync_set(rpcsvc_request_t *req) ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + snprintf(err_str, sizeof(err_str), "Garbage args received"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } if (cli_req.dict.dict_len) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict); @@ -294,6 +316,8 @@ __glusterd_handle_gsync_set(rpcsvc_request_t *req) snprintf(err_str, sizeof(err_str), "Failed to get " "the uuid of local glusterd"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL, + NULL); ret = -1; goto out; } @@ -2251,6 +2275,9 @@ glusterd_op_verify_gsync_running(glusterd_volinfo_t *volinfo, char *slave, "Volume %s needs to be started " "before " GEOREP " start", volinfo->volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEO_REP_START_FAILED, + "Volume is not in a started state, Volname=%s", + volinfo->volname, NULL); goto out; } @@ -2555,6 +2582,7 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) len = snprintf(abs_filename, sizeof(abs_filename), "%s/%s", priv->workdir, filename); if ((len < 0) || (len >= sizeof(abs_filename))) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -2567,6 +2595,9 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) if (len < 0) { strcpy(errmsg, "<error>"); } + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL, + "Realpath=%s, Reason=%s", priv->workdir, strerror(errno), + NULL); *op_errstr = gf_strdup(errmsg); ret = -1; goto out; @@ -2577,6 +2608,8 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) "Failed to get " "realpath of %s: %s", filename, strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL, + "Filename=%s, Reason=%s", filename, strerror(errno), NULL); *op_errstr = gf_strdup(errmsg); ret = -1; goto out; @@ -2586,6 +2619,7 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) will succeed for /var/lib/glusterd_bad */ len = snprintf(workdir, sizeof(workdir), "%s/", realpath_workdir); if ((len < 0) || (len >= sizeof(workdir))) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -2599,6 +2633,8 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) if (len < 0) { strcpy(errmsg, "<error>"); } + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg, + NULL); *op_errstr = gf_strdup(errmsg); ret = -1; goto out; @@ -2613,6 +2649,8 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) if (len < 0) { strcpy(errmsg, "<error>"); } + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg, + NULL); *op_errstr = gf_strdup(errmsg); goto out; } @@ -2621,9 +2659,9 @@ glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr) snprintf(errmsg, sizeof(errmsg), "Source file" " is not a regular file."); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg, + NULL); *op_errstr = gf_strdup(errmsg); - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, "%s", - errmsg); ret = -1; goto out; } @@ -2842,8 +2880,11 @@ glusterd_verify_slave(char *volname, char *slave_url, char *slave_vol, */ if (strstr(slave_url, "@")) { slave_url_buf = gf_strdup(slave_url); - if (!slave_url_buf) + if (!slave_url_buf) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED, + "Slave_url=%s", slave_url, NULL); goto out; + } slave_user = strtok_r(slave_url_buf, "@", &save_ptr); slave_ip = strtok_r(NULL, "@", &save_ptr); diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c index b01fd4da24b..a0bfea41f0f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c @@ -310,7 +310,7 @@ glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags) } runinit(&runner); - if (this->ctx->cmd_args.valgrind) { + if (this->ctx->cmd_args.vgtool != _gf_none) { len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s", svc->proc.logdir, svc->proc.logfile); if ((len < 0) || (len >= PATH_MAX)) { @@ -318,8 +318,13 @@ glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags) goto out; } - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", NULL); + if (this->ctx->cmd_args.vgtool == _gf_memcheck) + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + else + runner_add_args(&runner, "valgrind", "--tool=drd", NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 175e73f5ae4..1b21c40596d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -140,6 +140,7 @@ glusterd_handle_friend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname, ctx->req = req; if (!dict) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -147,9 +148,11 @@ glusterd_handle_friend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname, ret = dict_unserialize(friend_req->vols.vols_val, friend_req->vols.vols_len, &dict); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; - else + } else dict->extra_stdfree = friend_req->vols.vols_val; ctx->vols = dict; @@ -386,82 +389,129 @@ glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes, keylen = snprintf(key, sizeof(key), "volume%d.name", count); ret = dict_set_strn(volumes, key, keylen, volinfo->volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.type", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.status", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->status); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.brick_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->brick_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.dist_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->dist_leaf_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->stripe_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.replica_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->replica_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->disperse_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->redundancy_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->arbiter_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.transport", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->transport_type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->thin_arbiter_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id)); - if (!volume_id_str) + if (!volume_id_str) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.volume_id", count); ret = dict_set_dynstrn(volumes, key, keylen, volume_id_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.rebalance", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->rebal.defrag_cmd); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.snap_count", count); ret = dict_set_int32n(volumes, key, keylen, volinfo->snap_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list) { @@ -474,23 +524,33 @@ glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes, len = snprintf(brick, sizeof(brick), "%s:%s", brickinfo->hostname, brickinfo->path); if ((len < 0) || (len >= sizeof(brick))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } buf = gf_strdup(brick); keylen = snprintf(key, sizeof(key), "volume%d.brick%d", count, i); ret = dict_set_dynstrn(volumes, key, keylen, buf); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "volume%d.brick%d.uuid", count, i); snprintf(brick_uuid, sizeof(brick_uuid), "%s", uuid_utoa(brickinfo->uuid)); buf = gf_strdup(brick_uuid); - if (!buf) + if (!buf) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "brick_uuid=%s", brick_uuid, NULL); goto out; + } ret = dict_set_dynstrn(volumes, key, keylen, buf); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } i++; } @@ -500,6 +560,7 @@ glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes, len = snprintf(ta_brick, sizeof(ta_brick), "%s:%s", ta_brickinfo->hostname, ta_brickinfo->path); if ((len < 0) || (len >= sizeof(ta_brick))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -507,16 +568,23 @@ glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes, keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick", count); ret = dict_set_dynstrn(volumes, key, keylen, buf); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } ret = glusterd_add_arbiter_info_to_bricks(volinfo, volumes, count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, NULL); goto out; + } dict = volinfo->dict; if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = 0; goto out; } @@ -812,11 +880,14 @@ glusterd_req_ctx_create(rpcsvc_request_t *rpc_req, int op, uuid_t uuid, gf_msg_debug(this->name, 0, "Received op from uuid %s", str); dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } req_ctx = GF_CALLOC(1, sizeof(*req_ctx), mem_type); if (!req_ctx) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; } @@ -824,8 +895,8 @@ glusterd_req_ctx_create(rpcsvc_request_t *rpc_req, int op, uuid_t uuid, req_ctx->op = op; ret = dict_unserialize(buf_val, buf_len, &dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } @@ -1399,7 +1470,7 @@ __glusterd_handle_cli_get_volume(rpcsvc_request_t *req) goto out; } - gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_GET_VOL_REQ_RCVD, + gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_GET_VOL_REQ_RCVD, "Received get vol req"); if (cli_req.dict.dict_len) { @@ -1601,6 +1672,8 @@ __glusterd_handle_cli_uuid_get(rpcsvc_request_t *req) if (cli_req.dict.dict_len) { dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); ret = -1; goto out; } @@ -1623,6 +1696,7 @@ __glusterd_handle_cli_uuid_get(rpcsvc_request_t *req) rsp_dict = dict_new(); if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -1639,9 +1713,8 @@ __glusterd_handle_cli_uuid_get(rpcsvc_request_t *req) ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize " - "dictionary."); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } ret = 0; @@ -1694,8 +1767,10 @@ __glusterd_handle_cli_list_volume(rpcsvc_request_t *req) GF_ASSERT(priv); dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } cds_list_for_each_entry(volinfo, &priv->volumes, vol_list) { @@ -1707,8 +1782,11 @@ __glusterd_handle_cli_list_volume(rpcsvc_request_t *req) } ret = dict_set_int32n(dict, "count", SLEN("count"), count); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; + } ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val, &rsp.dict.dict_len); @@ -1790,6 +1868,8 @@ __glusterd_handle_ganesha_cmd(rpcsvc_request_t *req) /* Unserialize the dictionary */ dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); ret = -1; goto out; } @@ -2158,9 +2238,8 @@ glusterd_fsm_log_send_resp(rpcsvc_request_t *req, int op_ret, char *op_errstr, ret = dict_allocate_and_serialize(dict, &rsp.fsm_log.fsm_log_val, &rsp.fsm_log.fsm_log_len); if (ret < 0) { - gf_msg("glusterd", GF_LOG_ERROR, 0, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); return ret; } } @@ -2206,6 +2285,7 @@ __glusterd_handle_fsm_log(rpcsvc_request_t *req) dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -2432,8 +2512,8 @@ glusterd_op_stage_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); return ret; } @@ -2472,9 +2552,8 @@ glusterd_op_commit_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } } @@ -2715,12 +2794,18 @@ __glusterd_handle_friend_update(rpcsvc_request_t *req) } ret = dict_get_int32n(dict, "count", SLEN("count"), &count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); goto out; + } ret = dict_get_int32n(dict, "op", SLEN("op"), &op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=op", NULL); goto out; + } if (GD_FRIEND_UPDATE_DEL == op) { (void)glusterd_handle_friend_update_delete(dict); @@ -2979,8 +3064,11 @@ __glusterd_handle_cli_profile_volume(rpcsvc_request_t *req) if (cli_req.dict.dict_len > 0) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict); } @@ -3207,6 +3295,7 @@ __glusterd_handle_umount(rpcsvc_request_t *req) /* check if it is allowed to umount path */ path = gf_strdup(umnt_req.path); if (!path) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL); rsp.op_errno = ENOMEM; goto out; } @@ -3414,12 +3503,16 @@ glusterd_friend_rpc_create(xlator_t *this, glusterd_peerinfo_t *peerinfo, char *af = NULL; peerctx = GF_CALLOC(1, sizeof(*peerctx), gf_gld_mt_peerctx_t); - if (!peerctx) + if (!peerctx) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } options = dict_new(); - if (!options) + if (!options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } if (args) peerctx->args = *args; @@ -3513,6 +3606,7 @@ glusterd_friend_add(const char *hoststr, int port, *friend = glusterd_peerinfo_new(state, uuid, hoststr, port); if (*friend == NULL) { ret = -1; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADD_FAIL, NULL); goto out; } @@ -4090,13 +4184,15 @@ glusterd_list_friends(rpcsvc_request_t *req, dict_t *dict, int32_t flags) }; int keylen; - priv = THIS->private; + xlator_t *this = THIS; + GF_ASSERT(this); + + priv = this->private; GF_ASSERT(priv); friends = dict_new(); if (!friends) { - gf_msg(THIS->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, - "Out of Memory"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; } @@ -4122,24 +4218,36 @@ unlock: keylen = snprintf(key, sizeof(key), "friend%d.uuid", count); uuid_utoa_r(MY_UUID, my_uuid_str); ret = dict_set_strn(friends, key, keylen, my_uuid_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.hostname", count); ret = dict_set_nstrn(friends, key, keylen, "localhost", SLEN("localhost")); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.connected", count); ret = dict_set_int32n(friends, key, keylen, 1); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } ret = dict_set_int32n(friends, "count", SLEN("count"), count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; + } ret = dict_allocate_and_serialize(friends, &rsp.friends.friends_val, &rsp.friends.friends_len); @@ -4311,8 +4419,11 @@ __glusterd_handle_status_volume(rpcsvc_request_t *req) if (cli_req.dict.dict_len > 0) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict); if (ret < 0) { @@ -4580,6 +4691,7 @@ __glusterd_handle_barrier(rpcsvc_request_t *req) dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -5114,12 +5226,17 @@ glusterd_print_gsync_status_by_vol(FILE *fp, glusterd_volinfo_t *volinfo) 0, }; + xlator_t *this = THIS; + GF_ASSERT(this); + GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out); GF_VALIDATE_OR_GOTO(THIS->name, fp, out); gsync_rsp_dict = dict_new(); - if (!gsync_rsp_dict) + if (!gsync_rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = gethostname(my_hostname, sizeof(my_hostname)); if (ret) { @@ -5146,7 +5263,7 @@ glusterd_print_snapinfo_by_vol(FILE *fp, glusterd_volinfo_t *volinfo, glusterd_volinfo_t *tmp_vol = NULL; glusterd_snap_t *snapinfo = NULL; int snapcount = 0; - char timestr[64] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; char snap_status_str[STATUS_STRLEN] = { @@ -5264,16 +5381,25 @@ glusterd_print_client_details(FILE *fp, dict_t *dict, ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"), brickinfo->path); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=brick-name", NULL); goto out; + } ret = dict_set_int32n(dict, "cmd", SLEN("cmd"), GF_CLI_STATUS_CLIENTS); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cmd", NULL); goto out; + } ret = dict_set_strn(dict, "volname", SLEN("volname"), volinfo->volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=volname", NULL); goto out; + } ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val, &brick_req->input.input_len); @@ -5467,7 +5593,7 @@ glusterd_get_state(rpcsvc_request_t *req, dict_t *dict) ret = dict_get_strn(dict, "filename", SLEN("filename"), &tmp_str); if (ret) { - now = time(NULL); + now = gf_time(); strftime(timestamp, sizeof(timestamp), "%Y%m%d_%H%M%S", localtime(&now)); gf_asprintf(&filename, "%s_%s", "glusterd_state", timestamp); @@ -5911,14 +6037,27 @@ get_brickinfo_from_brickid(char *brickid, glusterd_brickinfo_t **brickinfo) uuid_t volid = {0}; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); + brickid_dup = gf_strdup(brickid); - if (!brickid_dup) + if (!brickid_dup) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "brick_id=%s", brickid, NULL); goto out; + } volid_str = brickid_dup; brick = strchr(brickid_dup, ':'); - if (!volid_str || !brick) + if (!volid_str) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); goto out; + } + + if (!brick) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); + goto out; + } *brick = '\0'; brick++; diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index 7cb70fcb4e2..d96e35503dd 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -111,6 +111,8 @@ get_snap_volname_and_volinfo(const char *volpath, char **volname, volfile_token = strtok_r(NULL, "/", &save_ptr); *volname = gf_strdup(volfile_token); if (NULL == *volname) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "Volname=%s", volfile_token, NULL); ret = -1; goto out; } @@ -236,6 +238,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -256,6 +259,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -271,6 +275,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -292,6 +297,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -312,6 +318,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -366,6 +373,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -386,6 +394,7 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (volid_ptr) { volid_ptr = strchr(volid_ptr, '/'); if (!volid_ptr) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } @@ -402,6 +411,8 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, /* Split the volume name */ vol = strtok_r(dup_volname, ".", &save_ptr); if (!vol) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL, + "Volume name=%s", dup_volname, NULL); ret = -1; goto out; } @@ -446,18 +457,25 @@ build_volfile_path(char *volume_id, char *path, size_t path_len, if (ret) { dup_volname = gf_strdup(volid_ptr); if (!dup_volname) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "Volume name=%s", volid_ptr, NULL); ret = -1; goto out; } /* Split the volume name */ vol = strtok_r(dup_volname, ".", &save_ptr); if (!vol) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL, + "Volume name=%s", dup_volname, NULL); ret = -1; goto out; } ret = glusterd_volinfo_find(vol, &volinfo); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, + NULL); goto out; + } } gotvolinfo: @@ -466,8 +484,10 @@ gotvolinfo: ret = snprintf(path, path_len, "%s/%s/%s.vol", path_prefix, volinfo->volname, volid_ptr); - if (ret == -1) + if (ret == -1) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = sys_stat(path, &stbuf); @@ -522,12 +542,14 @@ glusterd_get_args_from_dict(gf_getspec_req *args, peer_info_t *peerinfo, GF_ASSERT(peerinfo); if (!args->xdata.xdata_len) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); ret = 0; goto out; } dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -561,6 +583,8 @@ glusterd_get_args_from_dict(gf_getspec_req *args, peer_info_t *peerinfo, } *brick_name = gf_strdup(name); if (*brick_name == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "Brick_name=%s", name, NULL); ret = -1; goto out; } @@ -976,6 +1000,7 @@ __server_getspec(rpcsvc_request_t *req) dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -ENOMEM; goto fail; } @@ -1037,10 +1062,11 @@ __server_getspec(rpcsvc_request_t *req) } RCU_READ_UNLOCK; if (peer_cnt) { - ret = dict_set_str(dict, GLUSTERD_BRICK_SERVERS, peer_hosts); - if (ret) { + op_ret = dict_set_str(dict, GLUSTERD_BRICK_SERVERS, peer_hosts); + if (op_ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "failed to set peer_host in dict"); + ret = op_ret; goto fail; } } @@ -1050,9 +1076,8 @@ __server_getspec(rpcsvc_request_t *req) ret = dict_allocate_and_serialize(dict, &rsp.xdata.xdata_val, &rsp.xdata.xdata_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict to request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto fail; } } @@ -1073,6 +1098,7 @@ __server_getspec(rpcsvc_request_t *req) } ret = file_len = stbuf.st_size; } else { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL); op_errno = ENOENT; goto fail; } @@ -1080,6 +1106,7 @@ __server_getspec(rpcsvc_request_t *req) if (file_len) { rsp.spec = CALLOC(file_len + 1, sizeof(char)); if (!rsp.spec) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; op_errno = ENOMEM; goto fail; @@ -1158,13 +1185,17 @@ __server_event_notify(rpcsvc_request_t *req) (xdrproc_t)xdr_gf_event_notify_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto fail; } if (args.dict.dict_len) { dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); return ret; + } ret = dict_unserialize(args.dict.dict_val, args.dict.dict_len, &dict); if (ret) { gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, @@ -1357,6 +1388,7 @@ __glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req) if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } @@ -1370,8 +1402,10 @@ __glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req) } dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_set_int32(dict, GD_OP_VERSION_KEY, conf->op_version); if (ret) { @@ -1457,6 +1491,7 @@ __glusterd_mgmt_hndsk_versions_ack(rpcsvc_request_t *req) if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } @@ -1529,22 +1564,25 @@ __server_get_volume_info(rpcsvc_request_t *req) char *volume_id_str = NULL; int32_t flags = 0; + xlator_t *this = THIS; + GF_ASSERT(this); + ret = xdr_to_generic(req->msg[0], &vol_info_req, (xdrproc_t)xdr_gf_get_volume_info_req); if (ret < 0) { /* failed to decode msg */ req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } - gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VOL_INFO_REQ_RECVD, - "Received get volume info req"); + gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_INFO_REQ_RECVD, NULL); if (vol_info_req.dict.dict_len) { /* Unserialize the dictionary */ dict = dict_new(); if (!dict) { - gf_msg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_NO_MEMORY, - "Out of Memory"); + gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); op_errno = ENOMEM; ret = -1; goto out; @@ -1553,9 +1591,8 @@ __server_get_volume_info(rpcsvc_request_t *req) ret = dict_unserialize(vol_info_req.dict.dict_val, vol_info_req.dict.dict_len, &dict); if (ret < 0) { - gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to " - "unserialize req-buffer to dictionary"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); op_errno = -ret; ret = -1; goto out; @@ -1566,8 +1603,8 @@ __server_get_volume_info(rpcsvc_request_t *req) ret = dict_get_int32(dict, "flags", &flags); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED, - "failed to get flags"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=flags", NULL); op_errno = -ret; ret = -1; goto out; @@ -1575,13 +1612,15 @@ __server_get_volume_info(rpcsvc_request_t *req) if (!flags) { /* Nothing to query about. Just return success */ - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_NO_FLAG_SET, "No flags set"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_FLAG_SET, NULL); ret = 0; goto out; } ret = dict_get_str(dict, "volname", &volname); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); op_errno = EINVAL; ret = -1; goto out; @@ -1589,6 +1628,8 @@ __server_get_volume_info(rpcsvc_request_t *req) ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, + "Volname=%s", volname, NULL); op_errno = EINVAL; ret = -1; goto out; @@ -1597,6 +1638,8 @@ __server_get_volume_info(rpcsvc_request_t *req) if (flags & (int32_t)GF_GET_VOLUME_UUID) { volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id)); if (!volume_id_str) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + NULL); op_errno = ENOMEM; ret = -1; goto out; @@ -1604,8 +1647,8 @@ __server_get_volume_info(rpcsvc_request_t *req) dict_rsp = dict_new(); if (!dict_rsp) { - gf_msg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_NO_MEMORY, - "Out of Memory"); + gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); op_errno = ENOMEM; GF_FREE(volume_id_str); ret = -1; @@ -1613,6 +1656,8 @@ __server_get_volume_info(rpcsvc_request_t *req) } ret = dict_set_dynstr(dict_rsp, "volume_id", volume_id_str); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=volume_id", NULL); op_errno = -ret; ret = -1; goto out; @@ -1621,6 +1666,8 @@ __server_get_volume_info(rpcsvc_request_t *req) ret = dict_allocate_and_serialize(dict_rsp, &vol_info_rsp.dict.dict_val, &vol_info_rsp.dict.dict_len); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); op_errno = -ret; ret = -1; goto out; @@ -1686,6 +1733,8 @@ __server_get_snap_info(rpcsvc_request_t *req) if (snap_info_req.dict.dict_len) { dict = dict_new(); if (!dict) { + gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); op_errno = ENOMEM; ret = -1; goto out; @@ -1716,6 +1765,8 @@ __server_get_snap_info(rpcsvc_request_t *req) dict_rsp = dict_new(); if (!dict_rsp) { + gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); op_errno = ENOMEM; ret = -1; goto out; @@ -1908,22 +1959,45 @@ gd_validate_peer_op_version(xlator_t *this, glusterd_peerinfo_t *peerinfo, int32_t peer_min_op_version = 0; int32_t peer_max_op_version = 0; - if (!dict || !this || !peerinfo) + if (!dict) { + gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } + + if (!this) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED, + NULL); + goto out; + } + + if (!peerinfo) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); + goto out; + } conf = this->private; ret = dict_get_int32(dict, GD_OP_VERSION_KEY, &peer_op_version); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GD_OP_VERSION_KEY, NULL); goto out; + } ret = dict_get_int32(dict, GD_MAX_OP_VERSION_KEY, &peer_max_op_version); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GD_MAX_OP_VERSION_KEY, NULL); goto out; + } ret = dict_get_int32(dict, GD_MIN_OP_VERSION_KEY, &peer_min_op_version); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GD_MIN_OP_VERSION_KEY, NULL); goto out; + } ret = -1; /* Check if peer can support our op_version */ @@ -2189,14 +2263,20 @@ glusterd_mgmt_handshake(xlator_t *this, glusterd_peerctx_t *peerctx) int ret = -1; frame = create_frame(this, this->ctx->pool); - if (!frame) + if (!frame) { + gf_smsg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto out; + } frame->local = peerctx; req_dict = dict_new(); - if (!req_dict) + if (!req_dict) { + gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = dict_set_dynstr(req_dict, GD_PEER_ID_KEY, gf_strdup(uuid_utoa(MY_UUID))); @@ -2463,12 +2543,17 @@ glusterd_peer_dump_version(xlator_t *this, struct rpc_clnt *rpc, int ret = -1; frame = create_frame(this, this->ctx->pool); - if (!frame) + if (!frame) { + gf_smsg(this->name, GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto out; + } frame->local = peerctx; - if (!peerctx) + if (!peerctx) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } RCU_READ_LOCK; diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c index 511a102d016..61c0f1c946f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-hooks.c +++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c @@ -87,21 +87,24 @@ glusterd_hooks_create_hooks_directory(char *basedir) glusterd_conf_t *priv = NULL; int32_t len = 0; - priv = THIS->private; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); + priv = this->private; snprintf(path, sizeof(path), "%s/hooks", basedir); ret = mkdir_p(path, 0755, _gf_true); if (ret) { - gf_msg(THIS->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED, - "Unable to create %s", path); + gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED, + "Path=%s", path, NULL); goto out; } GLUSTERD_GET_HOOKS_DIR(version_dir, GLUSTERD_HOOK_VER, priv); ret = mkdir_p(version_dir, 0755, _gf_true); if (ret) { - gf_msg(THIS->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED, - "Unable to create %s", version_dir); + gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED, + "Directory=%s", version_dir, NULL); goto out; } @@ -112,13 +115,14 @@ glusterd_hooks_create_hooks_directory(char *basedir) len = snprintf(path, sizeof(path), "%s/%s", version_dir, cmd_subdir); if ((len < 0) || (len >= sizeof(path))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } ret = mkdir_p(path, 0755, _gf_true); if (ret) { - gf_msg(THIS->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED, - "Unable to create %s", path); + gf_smsg(this->name, GF_LOG_CRITICAL, errno, + GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL); goto out; } @@ -126,13 +130,15 @@ glusterd_hooks_create_hooks_directory(char *basedir) len = snprintf(path, sizeof(path), "%s/%s/%s", version_dir, cmd_subdir, type_subdir[type]); if ((len < 0) || (len >= sizeof(path))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, + NULL); ret = -1; goto out; } ret = mkdir_p(path, 0755, _gf_true); if (ret) { - gf_msg(THIS->name, GF_LOG_CRITICAL, errno, - GD_MSG_CREATE_DIR_FAILED, "Unable to create %s", path); + gf_smsg(this->name, GF_LOG_CRITICAL, errno, + GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL); goto out; } } @@ -200,20 +206,31 @@ glusterd_hooks_set_volume_args(dict_t *dict, runner_t *runner) int i = 0; int count = 0; int ret = -1; + int flag = 0; char query[1024] = { 0, }; char *key = NULL; char *value = NULL; + char *inet_family = NULL; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); ret = dict_get_int32(dict, "count", &count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); goto out; + } /* This will not happen unless op_ctx * is corrupted*/ - if (!count) + if (!count) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, "count", + NULL); goto out; + } runner_add_arg(runner, "-o"); for (i = 1; ret == 0; i++) { @@ -228,9 +245,23 @@ glusterd_hooks_set_volume_args(dict_t *dict, runner_t *runner) continue; runner_argprintf(runner, "%s=%s", key, value); + if ((strncmp(key, "cluster.enable-shared-storage", + SLEN("cluster.enable-shared-storage")) == 0 || + strncmp(key, "enable-shared-storage", + SLEN("enable-shared-storage")) == 0) && + strncmp(value, "enable", SLEN("enable")) == 0) + flag = 1; } glusterd_hooks_add_custom_args(dict, runner); + if (flag == 1) { + ret = dict_get_str_sizen(this->options, "transport.address-family", + &inet_family); + if (!ret) { + runner_argprintf(runner, "transport.address-family=%s", + inet_family); + } + } ret = 0; out: @@ -357,27 +388,31 @@ glusterd_hooks_run_hooks(char *hooks_path, glusterd_op_t op, dict_t *op_ctx, lines = GF_CALLOC(1, N * sizeof(*lines), gf_gld_mt_charptr); if (!lines) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } ret = -1; line_count = 0; - GF_SKIP_IRRELEVANT_ENTRIES(entry, hookdir, scratch); - while (entry) { + + while ((entry = sys_readdir(hookdir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; if (line_count == N - 1) { N *= 2; lines = GF_REALLOC(lines, N * sizeof(char *)); - if (!lines) + if (!lines) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } } if (glusterd_is_hook_enabled(entry->d_name)) { lines[line_count] = gf_strdup(entry->d_name); line_count++; } - - GF_SKIP_IRRELEVANT_ENTRIES(entry, hookdir, scratch); } lines[line_count] = NULL; @@ -461,31 +496,40 @@ glusterd_hooks_stub_init(glusterd_hooks_stub_t **stub, char *scriptdir, int ret = -1; glusterd_hooks_stub_t *hooks_stub = NULL; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); GF_ASSERT(stub); if (!stub) goto out; hooks_stub = GF_CALLOC(1, sizeof(*hooks_stub), gf_gld_mt_hooks_stub_t); - if (!hooks_stub) + if (!hooks_stub) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } CDS_INIT_LIST_HEAD(&hooks_stub->all_hooks); hooks_stub->op = op; hooks_stub->scriptdir = gf_strdup(scriptdir); - if (!hooks_stub->scriptdir) + if (!hooks_stub->scriptdir) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "scriptdir=%s", scriptdir, NULL); goto out; + } hooks_stub->op_ctx = dict_copy_with_ref(op_ctx, hooks_stub->op_ctx); - if (!hooks_stub->op_ctx) + if (!hooks_stub->op_ctx) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_COPY_FAIL, NULL); goto out; + } *stub = hooks_stub; ret = 0; out: if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_POST_HOOK_STUB_INIT_FAIL, - "Failed to initialize " - "post hooks stub"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_HOOK_STUB_INIT_FAIL, + NULL); glusterd_hooks_stub_cleanup(hooks_stub); } @@ -547,12 +591,20 @@ glusterd_hooks_priv_init(glusterd_hooks_private_t **new) int ret = -1; glusterd_hooks_private_t *hooks_priv = NULL; - if (!new) + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); + + if (!new) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } hooks_priv = GF_CALLOC(1, sizeof(*hooks_priv), gf_gld_mt_hooks_priv_t); - if (!hooks_priv) + if (!hooks_priv) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } pthread_mutex_init(&hooks_priv->mutex, NULL); pthread_cond_init(&hooks_priv->cond, NULL); diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c index a48923e26e1..34abf35cb00 100644 --- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c @@ -43,6 +43,7 @@ __glusterd_handle_log_rotate(rpcsvc_request_t *req) if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } @@ -75,7 +76,7 @@ __glusterd_handle_log_rotate(rpcsvc_request_t *req) "for volume %s", volname); - ret = dict_set_uint64(dict, "rotate-key", (uint64_t)time(NULL)); + ret = dict_set_uint64(dict, "rotate-key", (uint64_t)gf_time()); if (ret) goto out; @@ -138,6 +139,8 @@ glusterd_op_stage_log_rotate(dict_t *dict, char **op_errstr) /* If no brick is specified, do log-rotate for all the bricks in the volume */ if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=brick", NULL); ret = 0; goto out; } @@ -204,8 +207,11 @@ glusterd_op_log_rotate(dict_t *dict) ret = dict_get_str(dict, "brick", &brick); /* If no brick is specified, do log-rotate for all the bricks in the volume */ - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=brick", NULL); goto cont; + } ret = glusterd_brickinfo_new_from_brick(brick, &tmpbrkinfo, _gf_false, NULL); diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h index 17052cee263..d7257e1a7b5 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h +++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h @@ -27,6 +27,7 @@ typedef enum gf_gld_mem_types_ { gf_gld_mt_mop_stage_req_t, gf_gld_mt_probe_ctx_t, gf_gld_mt_glusterd_volinfo_t, + gf_gld_mt_volinfo_dict_data_t, gf_gld_mt_glusterd_brickinfo_t, gf_gld_mt_peer_hostname_t, gf_gld_mt_defrag_info, diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h index f75ba905372..3a1e600fb03 100644 --- a/xlators/mgmt/glusterd/src/glusterd-messages.h +++ b/xlators/mgmt/glusterd/src/glusterd-messages.h @@ -46,7 +46,7 @@ GLFS_MSGID( GD_MSG_SNAP_STATUS_FAIL, GD_MSG_SNAP_INIT_FAIL, GD_MSG_VOLINFO_SET_FAIL, GD_MSG_VOLINFO_GET_FAIL, GD_MSG_BRICK_CREATION_FAIL, GD_MSG_BRICK_GET_INFO_FAIL, GD_MSG_BRICK_NEW_INFO_FAIL, GD_MSG_LVS_FAIL, - GD_MSG_SETXATTR_FAIL, GD_MSG_UMOUNTING_SNAP_BRICK, GD_MSG_OP_UNSUPPORTED, + GD_MSG_SET_XATTR_FAIL, GD_MSG_UMOUNTING_SNAP_BRICK, GD_MSG_OP_UNSUPPORTED, GD_MSG_SNAP_NOT_FOUND, GD_MSG_FS_LABEL_UPDATE_FAIL, GD_MSG_LVM_MOUNT_FAILED, GD_MSG_DICT_SET_FAILED, GD_MSG_CANONICALIZE_FAIL, GD_MSG_DICT_GET_FAILED, GD_MSG_SNAP_INFO_FAIL, GD_MSG_SNAP_VOL_CONFIG_FAIL, @@ -78,7 +78,7 @@ GLFS_MSGID( GD_MSG_COMMIT_OP_FAIL, GD_MSG_PEER_LIST_CREATE_FAIL, GD_MSG_BRICK_OP_FAIL, GD_MSG_OPINFO_SET_FAIL, GD_MSG_OP_EVENT_UNLOCK_FAIL, GD_MSG_MGMTV3_OP_RESP_FAIL, GD_MSG_PEER_NOT_FOUND, GD_MSG_REQ_DECODE_FAIL, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, GD_MSG_ALREADY_STOPPED, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, GD_MSG_ALREADY_STOPPED, GD_MSG_PRE_VALD_RESP_FAIL, GD_MSG_SVC_GET_FAIL, GD_MSG_VOLFILE_NOT_FOUND, GD_MSG_OP_EVENT_LOCK_FAIL, GD_MSG_NON_STRIPE_VOL, GD_MSG_SNAPD_OBJ_GET_FAIL, GD_MSG_QUOTA_DISABLED, GD_MSG_CACHE_MINMAX_SIZE_INVALID, @@ -116,7 +116,7 @@ GLFS_MSGID( GD_MSG_PARSE_BRICKINFO_FAIL, GD_MSG_VERS_STORE_FAIL, GD_MSG_HEADER_ADD_FAIL, GD_MSG_QUOTA_CONF_WRITE_FAIL, GD_MSG_QUOTA_CONF_CORRUPT, GD_MSG_FORK_FAIL, GD_MSG_CKSUM_COMPUTE_FAIL, GD_MSG_VERS_CKSUM_STORE_FAIL, - GD_MSG_GETXATTR_FAIL, GD_MSG_CONVERSION_FAILED, GD_MSG_VOL_NOT_DISTRIBUTE, + GD_MSG_GET_XATTR_FAIL, GD_MSG_CONVERSION_FAILED, GD_MSG_VOL_NOT_DISTRIBUTE, GD_MSG_VOL_STOPPED, GD_MSG_OPCTX_GET_FAIL, GD_MSG_TASKID_GEN_FAIL, GD_MSG_REBALANCE_ID_MISSING, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME, GD_MSG_DEFRAG_STATUS_UPDATE_FAIL, GD_MSG_UUID_GEN_STORE_FAIL, @@ -302,6 +302,150 @@ GLFS_MSGID( GD_MSG_SHD_OBJ_GET_FAIL, GD_MSG_SVC_ATTACH_FAIL, GD_MSG_ATTACH_INFO, GD_MSG_DETACH_INFO, GD_MSG_SVC_DETACH_FAIL, GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL, GD_MSG_CLUSTER_RC_ENABLE, - GD_MSG_NFS_GANESHA_DISABLED, GD_MSG_GANESHA_NOT_RUNNING); + GD_MSG_NFS_GANESHA_DISABLED, GD_MSG_GANESHA_NOT_RUNNING, GD_MSG_SNAP_WARN, + GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, GD_MSG_REMOVE_ARBITER_BRICK, + GD_MSG_BRICK_NOT_DECOM, GD_MSG_BRICK_STOPPED, GD_MSG_BRICK_DEAD, + GD_MSG_BRICK_HOST_NOT_FOUND, GD_MSG_BRICK_HOST_DOWN, GD_MSG_BRICK_DELETE, + GD_MSG_BRICK_NO_REMOVE_CMD, GD_MSG_MIGRATION_PROG, GD_MSG_MIGRATION_FAIL, + GD_MSG_COPY_FAIL, GD_MSG_REALPATH_GET_FAIL, + GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, GD_MSG_STRCHR_FAIL, GD_MSG_SPLIT_FAIL, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, GD_MSG_VOL_SHD_NOT_COMP, + GD_MSG_BITROT_NOT_ENABLED, GD_MSG_CREATE_BRICK_DIR_FAILED, + GD_MSG_CREATE_GLUSTER_DIR_FAILED, GD_MSG_BRICK_CREATE_MNTPNT, + GD_MSG_BRICK_CREATE_ROOT, GD_MSG_SET_XATTR_BRICK_FAIL, + GD_MSG_REMOVE_XATTR_FAIL, GD_MSG_XLATOR_NOT_DEFINED, + GD_MSG_BRICK_NOT_RUNNING, GD_MSG_INCORRECT_BRICK, GD_MSG_UUID_GET_FAIL, + GD_MSG_INVALID_ARGUMENT, GD_MSG_FRAME_CREATE_FAIL, + GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED, GD_MSG_VOL_STOP_ARGS_GET_FAILED, + GD_MSG_LSTAT_FAIL, GD_MSG_VOLUME_NOT_IMPORTED, + GD_MSG_ADD_BRICK_MNT_INFO_FAIL, GD_MSG_GET_MNT_ENTRY_INFO_FAIL, + GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, GD_MSG_POST_COMMIT_OP_FAIL, + GD_MSG_POST_COMMIT_FROM_UUID_REJCT, GD_MSG_POST_COMMIT_REQ_SEND_FAIL); + +#define GD_MSG_INVALID_ENTRY_STR "Invalid data entry" +#define GD_MSG_INVALID_ARGUMENT_STR \ + "Invalid arguments have been given to function" +#define GD_MSG_GARBAGE_ARGS_STR "Garbage args received" +#define GD_MSG_BRICK_SUBVOL_VERIFY_FAIL_STR "Brick's subvol verification fail" +#define GD_MSG_REMOVE_ARBITER_BRICK_STR "Failed to remove arbiter bricks" +#define GD_MSG_DICT_GET_FAILED_STR "Dict get failed" +#define GD_MSG_DICT_SET_FAILED_STR "Dict set failed" +#define GD_MSG_BRICK_NOT_FOUND_STR "Brick not found in volume" +#define GD_MSG_BRICK_NOT_DECOM_STR "Brick is not decommissoned" +#define GD_MSG_BRICK_STOPPED_STR "Found stopped brick" +#define GD_MSG_BRICK_DEAD_STR "Found dead brick" +#define GD_MSG_BRICK_HOST_NOT_FOUND_STR \ + "Host node of the brick is not a part of cluster" +#define GD_MSG_BRICK_HOST_DOWN_STR "Host node of the brick is down" +#define GD_MSG_BRICK_DELETE_STR \ + "Deleting all the bricks of the volume is not allowed" +#define GD_MSG_BRICK_NO_REMOVE_CMD_STR "No remove-brick command issued" +#define GD_MSG_INCORRECT_BRICK_STR "Incorrect brick for volume" +#define GD_MSG_MIGRATION_PROG_STR "Migration is in progress" +#define GD_MSG_MIGRATION_FAIL_STR "Migration has failed" +#define GD_MSG_XLATOR_NOT_DEFINED_STR "Xlator not defined" +#define GD_MSG_DICT_CREATE_FAIL_STR "Failed to create dictionary" +#define GD_MSG_COPY_FAIL_STR "Failed to copy" +#define GD_MSG_UUID_GET_FAIL_STR "Failed to get the uuid of local glusterd" +#define GD_MSG_GEO_REP_START_FAILED_STR "Georep start failed for volume" +#define GD_MSG_REALPATH_GET_FAIL_STR "Failed to get realpath" +#define GD_MSG_FILE_NOT_FOUND_STR "File not found in directory" +#define GD_MSG_SRC_FILE_ERROR_STR "Error in source file" +#define GD_MSG_DICT_UNSERIALIZE_FAIL_STR "Failed to unserialize dict" +#define GD_MSG_VOL_ID_SET_FAIL_STR "Failed to set volume id" +#define GD_MSG_ARBITER_BRICK_SET_INFO_FAIL_STR \ + "Failed to add arbiter info to brick" +#define GD_MSG_NO_MEMORY_STR "Out of memory" +#define GD_MSG_GLUSTERD_UMOUNT_FAIL_STR "Failed to unmount path" +#define GD_MSG_PEER_ADD_FAIL_STR "Failed to add new peer" +#define GD_MSG_BRICK_GET_INFO_FAIL_STR "Failed to get brick info" +#define GD_MSG_STRCHR_FAIL_STR "Failed to get the character" +#define GD_MSG_SPLIT_FAIL_STR "Failed to split" +#define GD_MSG_VOLINFO_GET_FAIL_STR "Failed to get volinfo" +#define GD_MSG_PEER_NOT_FOUND_STR "Failed to find peer info" +#define GD_MSG_DICT_COPY_FAIL_STR "Failed to copy values from dictionary" +#define GD_MSG_ALLOC_AND_COPY_UUID_FAIL_STR \ + "Failed to allocate memory or copy uuid" +#define GD_MSG_VOL_NOT_FOUND_STR "Volume not found" +#define GD_MSG_PEER_DISCONNECTED_STR "Peer is disconnected" +#define GD_MSG_QUOTA_GET_STAT_FAIL_STR "Failed to get quota status" +#define GD_MSG_SNAP_STATUS_FAIL_STR "Failed to get status of snapd" +#define GD_MSG_VALIDATE_FAILED_STR "Failed to validate volume" +#define GD_MSG_VOL_NOT_STARTED_STR "Volume is not started" +#define GD_MSG_VOL_SHD_NOT_COMP_STR "Volume is not Self-heal compatible" +#define GD_MSG_SELF_HEALD_DISABLED_STR "Self-heal daemon is disabled" +#define GD_MSG_NFS_GANESHA_DISABLED_STR "NFS server is disabled" +#define GD_MSG_QUOTA_DISABLED_STR "Quota is disabled" +#define GD_MSG_BITROT_NOT_RUNNING_STR "Bitrot is not enabled" +#define GD_MSG_BITROT_NOT_ENABLED_STR "Volume does not have bitrot enabled" +#define GD_MSG_SNAPD_NOT_RUNNING_STR "Snapd is not enabled" +#define GD_MSG_STRDUP_FAILED_STR "Strdup operation failed" +#define GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL_STR \ + "Failed to get quorum cluster counts" +#define GD_MSG_GLUSTER_SERVICE_START_FAIL_STR "Failed to start glusterd service" +#define GD_MSG_PEER_ADDRESS_GET_FAIL_STR "Failed to get the address of peer" +#define GD_MSG_INVALID_SLAVE_STR "Volume is not a slave volume" +#define GD_MSG_BRICK_NOT_RUNNING_STR "One or more bricks are not running" +#define GD_MSG_BRK_MNTPATH_GET_FAIL_STR "Failed to get brick mount device" +#define GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED_STR \ + "Snapshot is supported only for thin provisioned LV." +#define GD_MSG_SNAP_DEVICE_NAME_GET_FAIL_STR \ + "Failed to copy snapshot device name" +#define GD_MSG_SNAP_NOT_FOUND_STR "Snapshot does not exist" +#define GD_MSG_CREATE_BRICK_DIR_FAILED_STR "Failed to create brick directory" +#define GD_MSG_LSTAT_FAIL_STR "Lstat operation failed" +#define GD_MSG_DIR_OP_FAILED_STR \ + "The provided path is already present. It is not a directory" +#define GD_MSG_BRICK_CREATION_FAIL_STR \ + "Brick isn't allowed to be created inside glusterd's working directory." +#define GD_MSG_BRICK_CREATE_ROOT_STR \ + "The brick is being created in the root partition. It is recommended " \ + "that you don't use the system's root partition for storage backend." +#define GD_MSG_BRICK_CREATE_MNTPNT_STR \ + "The brick is a mount point. Please create a sub-directory under the " \ + "mount point and use that as the brick directory." +#define GD_MSG_CREATE_GLUSTER_DIR_FAILED_STR \ + "Failed to create glusterfs directory" +#define GD_MSG_VOLINFO_IMPORT_FAIL_STR "Volume is not yet imported" +#define GD_MSG_BRICK_SET_INFO_FAIL_STR \ + "Failed to add brick mount details to dict" +#define GD_MSG_SET_XATTR_BRICK_FAIL_STR \ + "Glusterfs is not supported on brick. Setting extended attribute failed" +#define GD_MSG_SET_XATTR_FAIL_STR "Failed to set extended attribute" +#define GD_MSG_REMOVE_XATTR_FAIL_STR "Failed to remove extended attribute" +#define GD_MSG_XLATOR_SET_OPT_FAIL_STR "Failed to set xlator type" +#define GD_MSG_XLATOR_LINK_FAIL_STR \ + "Failed to do the link of xlator with children" +#define GD_MSG_READ_ERROR_STR "Failed to read directory" +#define GD_MSG_INCOMPATIBLE_VALUE_STR "Incompatible transport type" +#define GD_MSG_VOL_STOP_ARGS_GET_FAILED_STR "Failed to get volume stop args" +#define GD_MSG_FRAME_CREATE_FAIL_STR "Failed to create frame" +#define GD_MSG_VOLUME_NOT_IMPORTED_STR "Volume has not been imported" +#define GD_MSG_ADD_BRICK_MNT_INFO_FAIL_STR \ + "Failed to add brick mount details to dict" +#define GD_MSG_GET_MNT_ENTRY_INFO_FAIL_STR "Failed to get mount entry details" +#define GD_MSG_BRICKPATH_ROOT_GET_FAIL_STR "failed to get brick root details" +#define GD_MSG_VOL_INFO_REQ_RECVD_STR "Received get volume info req" +#define GD_MSG_NO_FLAG_SET_STR "No flags set" +#define GD_MSG_CREATE_DIR_FAILED_STR "Failed to create directory" +#define GD_MSG_POST_HOOK_STUB_INIT_FAIL_STR \ + "Failed to initialize post hooks stub" +#define GD_MSG_FILE_OP_FAILED_STR "File operation failed" +#define GD_MSG_INODE_SIZE_GET_FAIL_STR "Failed to get inode size" +#define GD_MSG_CMD_EXEC_FAIL_STR "Command execution failed" +#define GD_MSG_XLATOR_CREATE_FAIL_STR "Failed to create xlator" +#define GD_MSG_CLRCLK_VOL_REQ_RCVD_STR "Received clear-locks request for volume" +#define GD_MSG_BRK_PORT_NUM_GET_FAIL_STR \ + "Couldn't get port number of local bricks" +#define GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL_STR \ + "Creating mount directory for clear-locks failed" +#define GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL_STR \ + "Failed to mount clear-locks maintenance client" +#define GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL_STR \ + "Failed to unmount clear-locks mount point" +#define GD_MSG_CLRCLK_SND_CMD_FAIL_STR "Failed to send command for clear-locks" +#define GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL_STR \ + "Failed to allocate memory or get serialized length of dict" +#define GD_MSG_GET_XATTR_FAIL_STR "Failed to get extended attribute" #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c index ef8b4c38571..1069688a89d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c @@ -165,6 +165,7 @@ glusterd_handle_mgmt_v3_lock_fn(rpcsvc_request_t *req) ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t); if (!ctx) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -174,6 +175,7 @@ glusterd_handle_mgmt_v3_lock_fn(rpcsvc_request_t *req) ctx->dict = dict_new(); if (!ctx->dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -181,8 +183,8 @@ glusterd_handle_mgmt_v3_lock_fn(rpcsvc_request_t *req) ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len, &ctx->dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } @@ -264,8 +266,8 @@ glusterd_mgmt_v3_pre_validate_send_resp(rpcsvc_request_t *req, int32_t op, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -315,20 +317,21 @@ glusterd_handle_pre_validate_fn(rpcsvc_request_t *req) } dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } rsp_dict = dict_new(); if (!rsp_dict) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, - "Failed to get new dictionary"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); return -1; } @@ -391,8 +394,8 @@ glusterd_mgmt_v3_brick_op_send_resp(rpcsvc_request_t *req, int32_t op, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -441,20 +444,21 @@ glusterd_handle_brick_op_fn(rpcsvc_request_t *req) } dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } rsp_dict = dict_new(); if (!rsp_dict) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, - "Failed to get new dictionary"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); return -1; } @@ -518,8 +522,8 @@ glusterd_mgmt_v3_commit_send_resp(rpcsvc_request_t *req, int32_t op, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -569,20 +573,21 @@ glusterd_handle_commit_fn(rpcsvc_request_t *req) } dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } rsp_dict = dict_new(); if (!rsp_dict) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, - "Failed to get new dictionary"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); return -1; } @@ -621,6 +626,136 @@ out: } static int +glusterd_mgmt_v3_post_commit_send_resp(rpcsvc_request_t *req, int32_t op, + int32_t status, char *op_errstr, + uint32_t op_errno, dict_t *rsp_dict) +{ + gd1_mgmt_v3_post_commit_rsp rsp = { + {0}, + }; + int ret = -1; + xlator_t *this = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(req); + + rsp.op_ret = status; + glusterd_get_uuid(&rsp.uuid); + rsp.op = op; + rsp.op_errno = op_errno; + if (op_errstr) + rsp.op_errstr = op_errstr; + else + rsp.op_errstr = ""; + + ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, + &rsp.dict.dict_len); + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); + goto out; + } + + ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL, + (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp); + + GF_FREE(rsp.dict.dict_val); +out: + gf_msg_debug(this->name, 0, "Responded to post commit, ret: %d", ret); + return ret; +} + +static int +glusterd_handle_post_commit_fn(rpcsvc_request_t *req) +{ + int32_t ret = -1; + gd1_mgmt_v3_post_commit_req op_req = { + {0}, + }; + xlator_t *this = NULL; + char *op_errstr = NULL; + dict_t *dict = NULL; + dict_t *rsp_dict = NULL; + uint32_t op_errno = 0; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(req); + + ret = xdr_to_generic(req->msg[0], &op_req, + (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, + "Failed to decode post commit " + "request received from peer"); + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) { + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND, + "%s doesn't " + "belong to the cluster. Ignoring request.", + uuid_utoa(op_req.uuid)); + ret = -1; + goto out; + } + + dict = dict_new(); + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); + goto out; + } + + ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict); + if (ret) { + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); + goto out; + } + + rsp_dict = dict_new(); + if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); + return -1; + } + + ret = gd_mgmt_v3_post_commit_fn(op_req.op, dict, &op_errstr, &op_errno, + rsp_dict); + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "post commit failed on operation %s", gd_op_list[op_req.op]); + } + + ret = glusterd_mgmt_v3_post_commit_send_resp(req, op_req.op, ret, op_errstr, + op_errno, rsp_dict); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL, + "Failed to send post commit " + "response for operation %s", + gd_op_list[op_req.op]); + goto out; + } + +out: + if (op_errstr && (strcmp(op_errstr, ""))) + GF_FREE(op_errstr); + + free(op_req.dict.dict_val); + + if (dict) + dict_unref(dict); + + if (rsp_dict) + dict_unref(rsp_dict); + + /* Return 0 from handler to avoid double deletion of req obj */ + return 0; +} + +static int glusterd_mgmt_v3_post_validate_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status, char *op_errstr, dict_t *rsp_dict) @@ -646,8 +781,8 @@ glusterd_mgmt_v3_post_validate_send_resp(rpcsvc_request_t *req, int32_t op, ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to get serialized length of dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -696,20 +831,21 @@ glusterd_handle_post_validate_fn(rpcsvc_request_t *req) } dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } rsp_dict = dict_new(); if (!rsp_dict) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, - "Failed to get new dictionary"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); return -1; } @@ -867,6 +1003,7 @@ glusterd_handle_mgmt_v3_unlock_fn(rpcsvc_request_t *req) ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t); if (!ctx) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -876,6 +1013,7 @@ glusterd_handle_mgmt_v3_unlock_fn(rpcsvc_request_t *req) ctx->dict = dict_new(); if (!ctx->dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -883,8 +1021,8 @@ glusterd_handle_mgmt_v3_unlock_fn(rpcsvc_request_t *req) ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len, &ctx->dict); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize the dictionary"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); goto out; } @@ -955,6 +1093,12 @@ glusterd_handle_commit(rpcsvc_request_t *req) } static int +glusterd_handle_post_commit(rpcsvc_request_t *req) +{ + return glusterd_big_locked_handler(req, glusterd_handle_post_commit_fn); +} + +static int glusterd_handle_post_validate(rpcsvc_request_t *req) { return glusterd_big_locked_handler(req, glusterd_handle_post_validate_fn); @@ -978,6 +1122,9 @@ static rpcsvc_actor_t gd_svc_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = { GLUSTERD_MGMT_V3_BRICK_OP, DRC_NA, 0}, [GLUSTERD_MGMT_V3_COMMIT] = {"COMMIT", glusterd_handle_commit, NULL, GLUSTERD_MGMT_V3_COMMIT, DRC_NA, 0}, + [GLUSTERD_MGMT_V3_POST_COMMIT] = {"POST_COMMIT", + glusterd_handle_post_commit, NULL, + GLUSTERD_MGMT_V3_POST_COMMIT, DRC_NA, 0}, [GLUSTERD_MGMT_V3_POST_VALIDATE] = {"POST_VAL", glusterd_handle_post_validate, NULL, GLUSTERD_MGMT_V3_POST_VALIDATE, DRC_NA, diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c index bf9b5a870a0..bca7221062b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c @@ -86,6 +86,11 @@ gd_mgmt_v3_collate_errors(struct syncargs *args, int op_ret, int op_errno, peer_str, err_string); break; } + case GLUSTERD_MGMT_V3_POST_COMMIT: { + snprintf(op_err, sizeof(op_err), "Post commit failed on %s. %s", + peer_str, err_string); + break; + } case GLUSTERD_MGMT_V3_POST_VALIDATE: { snprintf(op_err, sizeof(op_err), "Post Validation failed on %s. %s", peer_str, @@ -187,6 +192,16 @@ gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, goto out; } break; + case GD_OP_REMOVE_BRICK: + ret = glusterd_op_stage_remove_brick(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, + GD_MSG_PRE_VALIDATION_FAIL, + "Remove brick prevalidation failed."); + goto out; + } + break; + case GD_OP_RESET_BRICK: ret = glusterd_reset_brick_prevalidate(dict, op_errstr, rsp_dict); if (ret) { @@ -337,6 +352,15 @@ gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, } break; } + case GD_OP_REMOVE_BRICK: { + ret = glusterd_op_remove_brick(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL, + "Remove-brick commit failed."); + goto out; + } + break; + } case GD_OP_RESET_BRICK: { ret = glusterd_op_reset_brick(dict, rsp_dict); if (ret) { @@ -386,6 +410,47 @@ out: } int32_t +gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, + uint32_t *op_errno, dict_t *rsp_dict) +{ + int32_t ret = -1; + xlator_t *this = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(dict); + GF_ASSERT(op_errstr); + GF_VALIDATE_OR_GOTO(this->name, op_errno, out); + GF_ASSERT(rsp_dict); + + switch (op) { + case GD_OP_ADD_BRICK: + ret = glusterd_post_commit_add_brick(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "Add-brick post commit failed."); + goto out; + } + break; + case GD_OP_REPLACE_BRICK: + ret = glusterd_post_commit_replace_brick(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "Replace-brick post commit failed."); + goto out; + } + break; + default: + break; + } + + ret = 0; +out: + gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret); + return ret; +} + +int32_t gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict, char **op_errstr, dict_t *rsp_dict) { @@ -582,15 +647,21 @@ gd_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx, glusterd_peerinfo_t *peerinfo, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_LOCK, @@ -759,6 +830,7 @@ glusterd_pre_validate_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp) goto out; } case GD_OP_STOP_VOLUME: + case GD_OP_REMOVE_BRICK: case GD_OP_PROFILE_VOLUME: case GD_OP_DEFRAG_BRICK_VOLUME: case GD_OP_REBALANCE: @@ -897,15 +969,21 @@ gd_mgmt_v3_pre_validate_req(glusterd_op_t op, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request( peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, @@ -948,7 +1026,7 @@ glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict, } if (op == GD_OP_PROFILE_VOLUME || op == GD_OP_STOP_VOLUME || - op == GD_OP_REBALANCE) { + op == GD_OP_REBALANCE || op == GD_OP_REMOVE_BRICK) { ret = glusterd_validate_quorum(this, op, req_dict, op_errstr); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET, @@ -1076,6 +1154,7 @@ glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict, case GD_OP_START_VOLUME: case GD_OP_STOP_VOLUME: case GD_OP_ADD_BRICK: + case GD_OP_REMOVE_BRICK: case GD_OP_DEFRAG_BRICK_VOLUME: case GD_OP_REPLACE_BRICK: case GD_OP_RESET_BRICK: @@ -1258,15 +1337,21 @@ gd_mgmt_v3_brick_op_req(glusterd_op_t op, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_BRICK_OP, @@ -1515,15 +1600,21 @@ gd_mgmt_v3_commit_req(glusterd_op_t op, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_COMMIT, @@ -1559,12 +1650,25 @@ glusterd_mgmt_v3_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, GF_ASSERT(op_errstr); GF_VALIDATE_OR_GOTO(this->name, op_errno, out); - if (op == GD_OP_REBALANCE || op == GD_OP_DEFRAG_BRICK_VOLUME) { - ret = glusterd_set_rebalance_id_in_rsp_dict(req_dict, op_ctx); - if (ret) { - gf_log(this->name, GF_LOG_WARNING, - "Failed to set rebalance id in dict."); - } + switch (op) { + case GD_OP_REBALANCE: + case GD_OP_DEFRAG_BRICK_VOLUME: + + ret = glusterd_set_rebalance_id_in_rsp_dict(req_dict, op_ctx); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "Failed to set rebalance id in dict."); + } + break; + case GD_OP_REMOVE_BRICK: + ret = glusterd_set_rebalance_id_for_remove_brick(req_dict, op_ctx); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "Failed to set rebalance id for remove-brick in dict."); + } + break; + default: + break; } rsp_dict = dict_new(); if (!rsp_dict) { @@ -1662,6 +1766,274 @@ out: } int32_t +gd_mgmt_v3_post_commit_cbk_fn(struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + int32_t ret = -1; + struct syncargs *args = NULL; + gd1_mgmt_v3_post_commit_rsp rsp = { + {0}, + }; + call_frame_t *frame = NULL; + int32_t op_ret = -1; + int32_t op_errno = -1; + dict_t *rsp_dict = NULL; + xlator_t *this = NULL; + uuid_t *peerid = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(req); + GF_ASSERT(myframe); + + frame = myframe; + args = frame->local; + peerid = frame->cookie; + frame->local = NULL; + frame->cookie = NULL; + + if (-1 == req->rpc_status) { + op_errno = ENOTCONN; + goto out; + } + + GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL); + + ret = xdr_to_generic(*iov, &rsp, + (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp); + if (ret < 0) + goto out; + + if (rsp.dict.dict_len) { + /* Unserialize the dictionary */ + rsp_dict = dict_new(); + + ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict); + if (ret < 0) { + free(rsp.dict.dict_val); + goto out; + } else { + rsp_dict->extra_stdfree = rsp.dict.dict_val; + } + } + + gf_uuid_copy(args->uuid, rsp.uuid); + pthread_mutex_lock(&args->lock_dict); + { + ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict); + } + pthread_mutex_unlock(&args->lock_dict); + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s", + "Failed to aggregate response from " + " node/brick"); + if (!rsp.op_ret) + op_ret = ret; + else { + op_ret = rsp.op_ret; + op_errno = rsp.op_errno; + } + } else { + op_ret = rsp.op_ret; + op_errno = rsp.op_errno; + } + +out: + if (rsp_dict) + dict_unref(rsp_dict); + + gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr, + GLUSTERD_MGMT_V3_POST_COMMIT, *peerid, rsp.uuid); + GF_FREE(peerid); + + if (rsp.op_errstr) + free(rsp.op_errstr); + + /* req->rpc_status set to -1 means, STACK_DESTROY will be called from + * the caller function. + */ + if (req->rpc_status != -1) + STACK_DESTROY(frame->root); + synctask_barrier_wake(args); + return 0; +} + +int32_t +gd_mgmt_v3_post_commit_cbk(struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + return glusterd_big_locked_cbk(req, iov, count, myframe, + gd_mgmt_v3_post_commit_cbk_fn); +} + +int +gd_mgmt_v3_post_commit_req(glusterd_op_t op, dict_t *op_ctx, + glusterd_peerinfo_t *peerinfo, struct syncargs *args, + uuid_t my_uuid, uuid_t recv_uuid) +{ + int32_t ret = -1; + gd1_mgmt_v3_post_commit_req req = { + {0}, + }; + xlator_t *this = NULL; + uuid_t *peerid = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(op_ctx); + GF_ASSERT(peerinfo); + GF_ASSERT(args); + + ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, + &req.dict.dict_len); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); + goto out; + } + + gf_uuid_copy(req.uuid, my_uuid); + req.op = op; + + GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); + goto out; + } + + ret = gd_syncop_submit_request( + peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, + GLUSTERD_MGMT_V3_POST_COMMIT, gd_mgmt_v3_post_commit_cbk, + (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req); +out: + GF_FREE(req.dict.dict_val); + gf_msg_trace(this->name, 0, "Returning %d", ret); + return ret; +} + +int +glusterd_mgmt_v3_post_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, + char **op_errstr, uint32_t *op_errno, + uint32_t txn_generation) +{ + int32_t ret = -1; + int32_t peer_cnt = 0; + dict_t *rsp_dict = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + struct syncargs args = {0}; + uuid_t peer_uuid = {0}; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + + this = THIS; + GF_ASSERT(this); + conf = this->private; + GF_ASSERT(conf); + + GF_ASSERT(op_ctx); + GF_ASSERT(req_dict); + GF_ASSERT(op_errstr); + GF_VALIDATE_OR_GOTO(this->name, op_errno, out); + + rsp_dict = dict_new(); + if (!rsp_dict) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, + "Failed to create response dictionary"); + goto out; + } + + /* Post commit on local node */ + ret = gd_mgmt_v3_post_commit_fn(op, req_dict, op_errstr, op_errno, + rsp_dict); + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "Post commit failed for " + "operation %s on local node", + gd_op_list[op]); + + if (*op_errstr == NULL) { + ret = gf_asprintf(op_errstr, + "Post commit failed " + "on localhost. Please " + "check log file for details."); + if (ret == -1) + *op_errstr = NULL; + + ret = -1; + } + goto out; + } + + ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s", + "Failed to aggregate response from " + " node/brick"); + goto out; + } + + dict_unref(rsp_dict); + rsp_dict = NULL; + + /* Sending post commit req to other nodes in the cluster */ + gd_syncargs_init(&args, op_ctx); + ret = synctask_barrier_init((&args)); + if (ret) + goto out; + peer_cnt = 0; + + RCU_READ_LOCK; + cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list) + { + /* Only send requests to peers who were available before the + * transaction started + */ + if (peerinfo->generation > txn_generation) + continue; + if (!peerinfo->connected) + continue; + + if (op != GD_OP_SYNC_VOLUME && + peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) + continue; + + gd_mgmt_v3_post_commit_req(op, req_dict, peerinfo, &args, MY_UUID, + peer_uuid); + peer_cnt++; + } + RCU_READ_UNLOCK; + + if (0 == peer_cnt) { + ret = 0; + goto out; + } + + gd_synctask_barrier_wait((&args), peer_cnt); + + if (args.op_ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "Post commit failed on peers"); + + if (args.errstr) + *op_errstr = gf_strdup(args.errstr); + } + + ret = args.op_ret; + *op_errno = args.op_errno; + + gf_msg_debug(this->name, 0, + "Sent post commit req for %s to %d " + "peers. Returning %d", + gd_op_list[op], peer_cnt, ret); +out: + glusterd_op_modify_op_ctx(op, op_ctx); + return ret; +} + +int32_t gd_mgmt_v3_post_validate_cbk_fn(struct rpc_req *req, struct iovec *iov, int count, void *myframe) { @@ -1751,16 +2123,22 @@ gd_mgmt_v3_post_validate_req(glusterd_op_t op, int32_t op_ret, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; req.op_ret = op_ret; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request( peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, @@ -1967,15 +2345,21 @@ gd_mgmt_v3_unlock(glusterd_op_t op, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); req.op = op; GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL); goto out; + } ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_UNLOCK, @@ -2338,6 +2722,15 @@ glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op, goto out; } + /* POST COMMIT OP PHASE */ + ret = glusterd_mgmt_v3_post_commit(op, dict, req_dict, &op_errstr, + &op_errno, txn_generation); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL, + "Post commit Op Failed"); + goto out; + } + /* POST-COMMIT VALIDATE PHASE */ /* As of now, post_validate is not trying to cleanup any failed commands. So as of now, I am sending 0 (op_ret as 0). diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h index 71f793d0397..27dd1849519 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h @@ -28,6 +28,10 @@ gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, uint32_t *op_errno, dict_t *rsp_dict); int32_t +gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, + uint32_t *op_errno, dict_t *rsp_dict); + +int32_t gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict, char **op_errstr, dict_t *rsp_dict); @@ -84,4 +88,10 @@ glusterd_reset_brick_prevalidate(dict_t *dict, char **op_errstr, dict_t *rsp_dict); int glusterd_op_reset_brick(dict_t *dict, dict_t *rsp_dict); + +int +glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr); + +int +glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr); #endif /* _GLUSTERD_MGMT_H_ */ diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c index 9c4b2fb18cc..645d845ee76 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c +++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c @@ -81,6 +81,7 @@ parse_mount_pattern_desc(gf_mount_spec_t *mspec, char *pdesc) mspec->patterns = GF_CALLOC(mspec->len, sizeof(*mspec->patterns), gf_gld_mt_mount_pattern); if (!mspec->patterns) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -261,8 +262,11 @@ make_georep_mountspec(gf_mount_spec_t *mspec, const char *volnames, char *user, int ret = 0; vols = gf_strdup((char *)volnames); - if (!vols) + if (!vols) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "Volume name=%s", volnames, NULL); goto out; + } for (vc = 1, p = vols; *p; p++) { if (*p == ',') @@ -270,8 +274,10 @@ make_georep_mountspec(gf_mount_spec_t *mspec, const char *volnames, char *user, } siz = strlen(volnames) + vc * SLEN("volfile-id="); meetspec = GF_CALLOC(1, siz + 1, gf_gld_mt_georep_meet_spec); - if (!meetspec) + if (!meetspec) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } for (p = vols;;) { vol = strtok_r(p, ",", &savetok); diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 14915b3fc17..c537fc33a85 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -106,6 +106,7 @@ glusterd_txn_opinfo_dict_init() priv->glusterd_txn_opinfo = dict_new(); if (!priv->glusterd_txn_opinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -178,8 +179,10 @@ glusterd_generate_txn_id(dict_t *dict, uuid_t **txn_id) GF_ASSERT(dict); *txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); - if (!*txn_id) + if (!*txn_id) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } if (priv->op_version < GD_OP_VERSION_3_6_0) gf_uuid_copy(**txn_id, priv->global_txn_id); @@ -541,8 +544,11 @@ glusterd_brick_op_build_payload(glusterd_op_t op, case GD_OP_STOP_VOLUME: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_TERMINATE; brick_req->name = brickinfo->path; glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPING); @@ -551,8 +557,11 @@ glusterd_brick_op_build_payload(glusterd_op_t op, brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_XLATOR_INFO; brick_req->name = brickinfo->path; @@ -561,45 +570,69 @@ glusterd_brick_op_build_payload(glusterd_op_t op, case GD_OP_HEAL_VOLUME: { brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_XLATOR_OP; brick_req->name = ""; ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"), (int32_t *)&heal_op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=heal-op", NULL); goto out; + } ret = dict_set_int32n(dict, "xl-op", SLEN("xl-op"), heal_op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=xl-op", NULL); goto out; + } } break; case GD_OP_STATUS_VOLUME: { brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_STATUS; brick_req->name = ""; ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"), brickinfo->path); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=brick-name", NULL); goto out; + } } break; case GD_OP_REBALANCE: case GD_OP_DEFRAG_BRICK_VOLUME: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG; ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; + } ret = glusterd_volinfo_find(volname, &volinfo); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_VOLINFO_GET_FAIL, "Volume=%s", volname, NULL); goto out; + } snprintf(name, sizeof(name), "%s-dht", volname); brick_req->name = gf_strdup(name); @@ -608,8 +641,11 @@ glusterd_brick_op_build_payload(glusterd_op_t op, case GD_OP_BARRIER: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_BRICK_BARRIER; brick_req->name = brickinfo->path; break; @@ -623,8 +659,11 @@ glusterd_brick_op_build_payload(glusterd_op_t op, brick_req->dict.dict_val = NULL; ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val, &brick_req->input.input_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } *req = brick_req; ret = 0; @@ -646,13 +685,19 @@ glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req, GF_ASSERT(op < GD_OP_MAX); GF_ASSERT(op > GD_OP_NONE); GF_ASSERT(req); + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); switch (op) { case GD_OP_PROFILE_VOLUME: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_NODE_PROFILE; brick_req->name = ""; @@ -662,8 +707,11 @@ glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req, case GD_OP_STATUS_VOLUME: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_NODE_STATUS; brick_req->name = ""; @@ -674,14 +722,20 @@ glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req, case GD_OP_SCRUB_ONDEMAND: brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t); - if (!brick_req) + if (!brick_req) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, + NULL); goto out; + } brick_req->op = GLUSTERD_NODE_BITROT; ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; + } brick_req->name = gf_strdup(volname); break; @@ -694,8 +748,11 @@ glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req, ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val, &brick_req->input.input_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } *req = brick_req; ret = 0; @@ -703,7 +760,7 @@ glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req, out: if (ret && brick_req) GF_FREE(brick_req); - gf_msg_debug(THIS->name, 0, "Returning %d", ret); + gf_msg_debug(this->name, 0, "Returning %d", ret); return ret; } @@ -719,12 +776,14 @@ glusterd_validate_quorum_options(xlator_t *this, char *fullkey, char *value, goto out; key = strchr(fullkey, '.'); if (key == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL); ret = -1; goto out; } key++; opt = xlator_volume_option_get(this, key); if (!opt) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL); ret = -1; goto out; } @@ -988,8 +1047,8 @@ glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr) if (check_op_version) { ret = dict_get_uint32(dict, "new-op-version", &new_op_version); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get new_op_version"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=new-op-version", NULL); goto out; } @@ -1043,8 +1102,8 @@ glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr) ret = dict_get_str_sizen(dict, "volname", &volname); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get volume name"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; } @@ -1069,14 +1128,19 @@ glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr) } val_dict = dict_new(); - if (!val_dict) + if (!val_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } for (count = 1; ret != 1; count++) { keystr_len = sprintf(keystr, "key%d", count); ret = dict_get_strn(dict, keystr, keystr_len, &key); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", keystr, NULL); break; + } keystr_len = sprintf(keystr, "value%d", count); ret = dict_get_strn(dict, keystr, keystr_len, &value); @@ -1603,12 +1667,17 @@ glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr) 0, }; glusterd_volinfo_t *volinfo = NULL; + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname); if (ret) { snprintf(msg, sizeof(msg), "hostname couldn't be " "retrieved from msg"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=hostname", NULL); *op_errstr = gf_strdup(msg); goto out; } @@ -1623,6 +1692,8 @@ glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr) "Volume %s " "does not exist", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_FOUND, + "Volume=%s", volname, NULL); *op_errstr = gf_strdup(msg); goto out; } @@ -1635,6 +1706,8 @@ glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr) RCU_READ_UNLOCK; ret = -1; snprintf(msg, sizeof(msg), "%s, is not a friend", hostname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, + "Peer_name=%s", hostname, NULL); *op_errstr = gf_strdup(msg); goto out; @@ -1645,6 +1718,8 @@ glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr) "%s, is not connected at " "the moment", hostname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_DISCONNECTED, + "Peer_name=%s", hostname, NULL); *op_errstr = gf_strdup(msg); goto out; } @@ -1685,8 +1760,11 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) GF_ASSERT(priv); ret = dict_get_uint32(dict, "cmd", &cmd); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=cmd", NULL); goto out; + } if (cmd & GF_CLI_STATUS_ALL) goto out; @@ -1697,6 +1775,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "The cluster is operating at " "version 1. Getting the status of quotad is not " "allowed in this state."); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_GET_STAT_FAIL, + msg, NULL); ret = -1; goto out; } @@ -1708,6 +1788,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "version less than %d. Getting the " "status of snapd is not allowed in this state.", GD_OP_VERSION_3_6_0); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAP_STATUS_FAIL, msg, + NULL); ret = -1; goto out; } @@ -1722,17 +1804,23 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, + "Volume=%s", volname, NULL); ret = -1; goto out; } ret = glusterd_validate_volume_id(dict, volinfo); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VALIDATE_FAILED, NULL); goto out; + } ret = glusterd_is_volume_started(volinfo); if (!ret) { snprintf(msg, sizeof(msg), "Volume %s is not started", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_STARTED, + "Volume=%s", volname, NULL); ret = -1; goto out; } @@ -1746,12 +1834,16 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) ret = -1; snprintf(msg, sizeof(msg), "Volume %s is not Self-heal compatible", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_SHD_NOT_COMP, + "Volume=%s", volname, NULL); goto out; } if (!shd_enabled) { ret = -1; snprintf(msg, sizeof(msg), "Self-heal Daemon is disabled for volume %s", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SELF_HEALD_DISABLED, + "Volume=%s", volname, NULL); goto out; } #ifdef BUILD_GNFS @@ -1762,6 +1854,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) ret = -1; snprintf(msg, sizeof(msg), "NFS server is disabled for volume %s", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_NFS_GANESHA_DISABLED, "Volume=%s", volname, NULL); goto out; } #endif @@ -1772,6 +1866,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "Volume %s does not have " "quota enabled", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_DISABLED, + "Volume=%s", volname, NULL); goto out; } } else if ((cmd & GF_CLI_STATUS_BITD) != 0) { @@ -1781,6 +1877,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "Volume %s does not have " "bitrot enabled", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED, + "Volume=%s", volname, NULL); goto out; } } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) { @@ -1791,6 +1889,10 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "bitrot enabled. Scrubber will be enabled " "automatically if bitrot is enabled", volname); + gf_smsg( + this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED, + "Scrubber will be enabled automatically if bitrot is enabled", + "Volume=%s", volname, NULL); goto out; } } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) { @@ -1800,12 +1902,17 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "Volume %s does not have " "uss enabled", volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAPD_NOT_RUNNING, + "Volume=%s", volname, NULL); goto out; } } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) { ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=brick", NULL); goto out; + } ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo, _gf_false); @@ -1814,6 +1921,8 @@ glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr) "No brick %s in" " volume %s", brick, volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_FOUND, + "Brick=%s, Volume=%s", brick, volname, NULL); ret = -1; goto out; } @@ -2100,8 +2209,10 @@ glusterd_op_reset_all_volume_options(xlator_t *this, dict_t *dict) ret = -1; dup_opt = dict_new(); - if (!dup_opt) + if (!dup_opt) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } if (!all) { dict_copy(conf->opts, dup_opt); dict_del(dup_opt, key); @@ -2112,8 +2223,11 @@ glusterd_op_reset_all_volume_options(xlator_t *this, dict_t *dict) ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL); goto out; + } ret = glusterd_store_options(this, dup_opt); if (ret) @@ -2124,9 +2238,11 @@ glusterd_op_reset_all_volume_options(xlator_t *this, dict_t *dict) ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION, SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL); goto out; - else + } else next_version = NULL; if (!all) { @@ -2410,8 +2526,11 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, conf = this->private; ret = dict_get_strn(dict, "key1", SLEN("key1"), &key); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=key1", NULL); goto out; + } ret = dict_get_strn(dict, "value1", SLEN("value1"), &value); if (ret) { @@ -2530,12 +2649,17 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, } ret = -1; dup_opt = dict_new(); - if (!dup_opt) + if (!dup_opt) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } dict_copy(conf->opts, dup_opt); ret = dict_set_str(dup_opt, key, value); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = glusterd_get_next_global_opt_version_str(conf->opts, &next_version); if (ret) @@ -2543,8 +2667,11 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL); goto out; + } ret = glusterd_store_options(this, dup_opt); if (ret) @@ -2555,9 +2682,11 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION, SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL); goto out; - else + } else next_version = NULL; dup_value = gf_strdup(value); @@ -2565,9 +2694,11 @@ glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict, goto out; ret = dict_set_dynstr(conf->opts, key, dup_value); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; - else + } else dup_value = NULL; /* Protect the allocation from GF_FREE */ out: @@ -3002,6 +3133,8 @@ glusterd_op_sync_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) snprintf(msg, sizeof(msg), "hostname couldn't be " "retrieved from msg"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=hostname", NULL); *op_errstr = gf_strdup(msg); goto out; } @@ -3026,6 +3159,7 @@ glusterd_op_sync_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) if (!rsp_dict) { // this should happen only on source + gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_INVALID_ARGUMENT, NULL); ret = 0; goto out; } @@ -3582,27 +3716,30 @@ glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) } ret = dict_set_int32n(rsp_dict, "type", SLEN("type"), volinfo->type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=type", NULL); goto out; + } ret = dict_set_int32n(rsp_dict, "brick-index-max", SLEN("brick-index-max"), brick_index); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Error setting brick-index-max to dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=brick-index-max", NULL); goto out; } ret = dict_set_int32n(rsp_dict, "other-count", SLEN("other-count"), other_count); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Error setting other-count to dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=other-count", NULL); goto out; } ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), node_count); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Error setting node count to dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; } @@ -4079,8 +4216,10 @@ glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr) this = THIS; GF_ASSERT(this); - if (!dict || !volname) + if (!dict || !volname) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { @@ -5838,13 +5977,8 @@ glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr, static void glusterd_wait_for_blockers(glusterd_conf_t *priv) { - uint64_t blockers = GF_ATOMIC_GET(priv->blockers); - - while (blockers) { - synclock_unlock(&priv->big_lock); - sleep(1); - blockers = GF_ATOMIC_GET(priv->blockers); - synclock_lock(&priv->big_lock); + while (GF_ATOMIC_GET(priv->blockers)) { + synccond_wait(&priv->cond_blockers, &priv->big_lock); } } @@ -6479,6 +6613,10 @@ _select_hxlators_for_full_self_heal(xlator_t *this, glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *brickinfo = NULL; int hxl_children = 0; uuid_t candidate = {0}; + int brick_index = 0; + glusterd_peerinfo_t *peerinfo = NULL; + int delta = 0; + uuid_t candidate_max = {0}; if ((*index) == 0) (*index)++; @@ -6490,13 +6628,40 @@ _select_hxlators_for_full_self_heal(xlator_t *this, glusterd_volinfo_t *volinfo, cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list) { + if (gf_uuid_compare(brickinfo->uuid, candidate_max) > 0) { + if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) { + gf_uuid_copy(candidate_max, brickinfo->uuid); + } else { + peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL); + if (peerinfo && peerinfo->connected) { + gf_uuid_copy(candidate_max, brickinfo->uuid); + } + } + } + } + + cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list) + { if (gf_uuid_is_null(brickinfo->uuid)) (void)glusterd_resolve_brick(brickinfo); - if (gf_uuid_compare(brickinfo->uuid, candidate) > 0) - gf_uuid_copy(candidate, brickinfo->uuid); + delta %= hxl_children; + if ((*index + delta) == (brick_index + hxl_children)) { + if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) { + gf_uuid_copy(candidate, brickinfo->uuid); + } else { + peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL); + if (peerinfo && peerinfo->connected) { + gf_uuid_copy(candidate, brickinfo->uuid); + } else if (peerinfo && + (!gf_uuid_compare(candidate_max, MY_UUID))) { + _add_hxlator_to_dict(dict, volinfo, + ((*index) - 1) / hxl_children, + (*hxlator_count)); + (*hxlator_count)++; + } + } - if ((*index) % hxl_children == 0) { if (!gf_uuid_compare(MY_UUID, candidate)) { _add_hxlator_to_dict(dict, volinfo, ((*index) - 1) / hxl_children, @@ -6504,6 +6669,8 @@ _select_hxlators_for_full_self_heal(xlator_t *this, glusterd_volinfo_t *volinfo, (*hxlator_count)++; } gf_uuid_clear(candidate); + brick_index += hxl_children; + delta++; } (*index)++; diff --git a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c index 82acf5bf03c..18d355cb186 100644 --- a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c @@ -367,8 +367,10 @@ glusterd_peerinfo_new(glusterd_friend_sm_state_t state, uuid_t *uuid, GF_ASSERT(conf); new_peer = GF_CALLOC(1, sizeof(*new_peer), gf_gld_mt_peerinfo_t); - if (!new_peer) + if (!new_peer) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } CDS_INIT_LIST_HEAD(&new_peer->uuid_list); @@ -564,12 +566,16 @@ glusterd_peer_hostname_new(const char *hostname, GF_ASSERT(hostname); GF_ASSERT(name); + xlator_t *this = THIS; + GF_ASSERT(this); peer_hostname = GF_CALLOC(1, sizeof(*peer_hostname), gf_gld_mt_peer_hostname_t); - if (!peer_hostname) + if (!peer_hostname) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } peer_hostname->hostname = gf_strdup(hostname); CDS_INIT_LIST_HEAD(&peer_hostname->hostname_list); @@ -900,8 +906,11 @@ gd_add_peer_hostnames_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict, { snprintf(key, sizeof(key), "%s.hostname%d", prefix, count); ret = dict_set_dynstr_with_alloc(dict, key, addr->hostname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } count++; } @@ -923,41 +932,61 @@ gd_add_peer_detail_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *friends, int keylen; char *peer_uuid_str = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); GF_ASSERT(peerinfo); GF_ASSERT(friends); peer_uuid_str = gd_peer_uuid_str(peerinfo); keylen = snprintf(key, sizeof(key), "friend%d.uuid", count); ret = dict_set_strn(friends, key, keylen, peer_uuid_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.hostname", count); ret = dict_set_strn(friends, key, keylen, peerinfo->hostname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.port", count); ret = dict_set_int32n(friends, key, keylen, peerinfo->port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.stateId", count); ret = dict_set_int32n(friends, key, keylen, peerinfo->state.state); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=%s in dict", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.state", count); ret = dict_set_strn( friends, key, keylen, glusterd_friend_sm_state_name_get(peerinfo->state.state)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "friend%d.connected", count); ret = dict_set_int32n(friends, key, keylen, (int32_t)peerinfo->connected); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } snprintf(key, sizeof(key), "friend%d", count); ret = gd_add_peer_hostnames_to_dict(peerinfo, friends, key); diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c index ec5bd1137f1..16ac628ab82 100644 --- a/xlators/mgmt/glusterd/src/glusterd-pmap.c +++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c @@ -433,17 +433,20 @@ __gluster_pmap_portbybrick(rpcsvc_request_t *req) char *brick = NULL; int port = 0; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_port_by_brick_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto fail; } brick = args.brick; - port = pmap_registry_search(THIS, brick, GF_PMAP_PORT_BRICKSERVER, + port = pmap_registry_search(this, brick, GF_PMAP_PORT_BRICKSERVER, _gf_false); if (!port) @@ -475,11 +478,14 @@ __gluster_pmap_brickbyport(rpcsvc_request_t *req) 0, }; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_brick_by_port_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto fail; } @@ -513,10 +519,13 @@ __gluster_pmap_signin(rpcsvc_request_t *req) }; int ret = -1; glusterd_brickinfo_t *brickinfo = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_signin_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto fail; } @@ -570,6 +579,7 @@ __gluster_pmap_signout(rpcsvc_request_t *req) if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto fail; } rsp.op_ret = pmap_registry_remove(THIS, args.port, args.brick, diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c index d96adcae89e..a05c90d7b10 100644 --- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c @@ -114,7 +114,7 @@ glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags) goto out; synclock_unlock(&conf->big_lock); - sleep(1); + synctask_sleep(1); synclock_lock(&conf->big_lock); if (gf_is_service_running(proc->pidfile, &pid)) { ret = kill(pid, SIGKILL); diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c index cb2d9c7c384..8370c174ce3 100644 --- a/xlators/mgmt/glusterd/src/glusterd-quota.c +++ b/xlators/mgmt/glusterd/src/glusterd-quota.c @@ -478,8 +478,9 @@ glusterd_stop_all_quota_crawl_service(glusterd_conf_t *priv, if (dir == NULL) return; - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; len = snprintf(pidfile, sizeof(pidfile), "%s/%s", pid_dir, entry->d_name); if ((len >= 0) && (len < sizeof(pidfile))) { @@ -487,8 +488,6 @@ glusterd_stop_all_quota_crawl_service(glusterd_conf_t *priv, _gf_true); sys_unlink(pidfile); } - - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } sys_closedir(dir); } @@ -1900,10 +1899,9 @@ glusterd_get_gfid_from_brick(dict_t *dict, glusterd_volinfo_t *volinfo, } ret = sys_lgetxattr(backend_path, GFID_XATTR_KEY, gfid, 16); if (ret < 0) { - gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_SETXATTR_FAIL, - "Failed to get " - "extended attribute %s for directory %s. ", - GFID_XATTR_KEY, backend_path); + gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_XATTR_FAIL, + "Attribute=%s, Directory=%s", GFID_XATTR_KEY, backend_path, + NULL); ret = 0; continue; } diff --git a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c index fc0aaddcbe3..f26d832a06d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c @@ -127,8 +127,10 @@ glusterd_quotadsvc_start(glusterd_svc_t *svc, int flags) char *options[] = {svc->name, "--process-name", NULL}; cmdline = dict_new(); - if (!cmdline) + if (!cmdline) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } for (i = 0; options[i]; i++) { ret = snprintf(key, sizeof(key), "arg%d", i); diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c index 4ce20a9e592..458bf168ede 100644 --- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c +++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c @@ -219,6 +219,9 @@ glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr, char valgrind_logfile[PATH_MAX] = { 0, }; + char msg[1024] = { + 0, + }; char *volfileserver = NULL; char *localtime_logging = NULL; @@ -270,12 +273,17 @@ glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr, "rebalance"); runinit(&runner); - if (this->ctx->cmd_args.valgrind) { + if (this->ctx->cmd_args.vgtool != _gf_none) { snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-rebalance.log", priv->logdir, volinfo->volname); - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", NULL); + if (this->ctx->cmd_args.vgtool == _gf_memcheck) + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + else + runner_add_args(&runner, "valgrind", "--tool=drd", NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } @@ -316,6 +324,10 @@ glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr, runner_add_arg(&runner, "--localtime-logging"); } + snprintf(msg, sizeof(msg), "Starting the rebalance service for volume %s", + volinfo->volname); + runner_log(&runner, this->name, GF_LOG_DEBUG, msg); + ret = runner_run_nowait(&runner); if (ret) { gf_msg_debug("glusterd", 0, "rebalance command failed"); @@ -390,8 +402,10 @@ glusterd_rebalance_rpc_create(glusterd_volinfo_t *volinfo) goto out; options = dict_new(); - if (!options) + if (!options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo); @@ -497,6 +511,7 @@ __glusterd_handle_defrag_volume(rpcsvc_request_t *req) if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c index a861240da31..43c2f4373e0 100644 --- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c +++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c @@ -227,6 +227,20 @@ glusterd_op_stage_replace_brick(dict_t *dict, char **op_errstr, is_force = _gf_true; } + if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) { + snprintf(msg, sizeof(msg), + "Volume %s has %" PRIu64 + " snapshots. " + "Changing the volume configuration will not effect snapshots." + "But the snapshot brick mount should be intact to " + "make them function.", + volname, volinfo->snap_count); + gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg); + msg[0] = '\0'; + } + + glusterd_add_peers_to_auth_list(volname); + ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr, &dst_brickinfo, &host, dict, &dup_dstbrick); diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c index a8e35f32a15..88662e3bbae 100644 --- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c @@ -183,10 +183,8 @@ glusterd_op_send_cli_response(glusterd_op_t op, int32_t op_ret, ret = dict_allocate_and_serialize(ctx, &rsp.dict.dict_val, &rsp.dict.dict_len); if (ret < 0) - gf_msg(this->name, GF_LOG_ERROR, 0, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "failed to " - "serialize buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); else free_ptr = rsp.dict.dict_val; } @@ -1464,6 +1462,7 @@ glusterd_rpc_probe(call_frame_t *frame, xlator_t *this, void *data) dict_t *dict = NULL; if (!frame || !this || !data) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); ret = -1; goto out; } @@ -1473,15 +1472,24 @@ glusterd_rpc_probe(call_frame_t *frame, xlator_t *this, void *data) GF_ASSERT(priv); ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=hostname", NULL); goto out; + } ret = dict_get_int32n(dict, "port", SLEN("port"), &port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_DEBUG, errno, GD_MSG_DICT_GET_FAILED, + "Key=port", NULL); port = GF_DEFAULT_BASE_PORT; + } ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=peerinfo", NULL); goto out; + } gf_uuid_copy(req.uuid, MY_UUID); req.hostname = gf_strdup(hostname); @@ -1510,6 +1518,7 @@ glusterd_rpc_friend_add(call_frame_t *frame, xlator_t *this, void *data) dict_t *peer_data = NULL; if (!frame || !this || !data) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); ret = -1; goto out; } @@ -1540,6 +1549,8 @@ glusterd_rpc_friend_add(call_frame_t *frame, xlator_t *this, void *data) peer_data = dict_new(); if (!peer_data) { + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL, + NULL); errno = ENOMEM; goto out; } @@ -1585,8 +1596,11 @@ glusterd_rpc_friend_add(call_frame_t *frame, xlator_t *this, void *data) if (!req.vols.vols_len) { ret = dict_allocate_and_serialize(peer_data, &req.vols.vols_val, &req.vols.vols_len); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } } ret = glusterd_submit_request( @@ -1760,8 +1774,11 @@ glusterd_mgmt_v3_lock_peers(call_frame_t *frame, xlator_t *this, void *data) GF_ASSERT(priv); ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=peerinfo", NULL); goto out; + } // peerinfo should not be in payload dict_deln(dict, "peerinfo", SLEN("peerinfo")); @@ -1771,9 +1788,8 @@ glusterd_mgmt_v3_lock_peers(call_frame_t *frame, xlator_t *this, void *data) ret = dict_allocate_and_serialize(dict, &req.dict.dict_val, &req.dict.dict_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict " - "to request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -1797,6 +1813,7 @@ glusterd_mgmt_v3_lock_peers(call_frame_t *frame, xlator_t *this, void *data) } frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); if (!frame->cookie) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -1836,8 +1853,11 @@ glusterd_mgmt_v3_unlock_peers(call_frame_t *frame, xlator_t *this, void *data) GF_ASSERT(priv); ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=peerinfo", NULL); goto out; + } // peerinfo should not be in payload dict_deln(dict, "peerinfo", SLEN("peerinfo")); @@ -1847,9 +1867,8 @@ glusterd_mgmt_v3_unlock_peers(call_frame_t *frame, xlator_t *this, void *data) ret = dict_allocate_and_serialize(dict, &req.dict.dict_val, &req.dict.dict_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict " - "to request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } @@ -1873,6 +1892,7 @@ glusterd_mgmt_v3_unlock_peers(call_frame_t *frame, xlator_t *this, void *data) } frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); if (!frame->cookie) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -1954,8 +1974,11 @@ glusterd_stage_op(call_frame_t *frame, xlator_t *this, void *data) GF_ASSERT(priv); ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=peerinfo", NULL); goto out; + } // peerinfo should not be in payload dict_deln(dict, "peerinfo", SLEN("peerinfo")); @@ -1965,9 +1988,8 @@ glusterd_stage_op(call_frame_t *frame, xlator_t *this, void *data) ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict " - "to request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } /* Sending valid transaction ID to peers */ @@ -1989,6 +2011,7 @@ glusterd_stage_op(call_frame_t *frame, xlator_t *this, void *data) } frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); if (!frame->cookie) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } @@ -2030,8 +2053,11 @@ glusterd_commit_op(call_frame_t *frame, xlator_t *this, void *data) GF_ASSERT(priv); ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=peerinfo", NULL); goto out; + } // peerinfo should not be in payload dict_deln(dict, "peerinfo", SLEN("peerinfo")); @@ -2041,9 +2067,8 @@ glusterd_commit_op(call_frame_t *frame, xlator_t *this, void *data) ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict to " - "request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; } /* Sending valid transaction ID to peers */ @@ -2065,6 +2090,7 @@ glusterd_commit_op(call_frame_t *frame, xlator_t *this, void *data) } frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); if (!frame->cookie) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -1; goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c index eab9078eb8e..c49a0eefba5 100644 --- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c @@ -117,8 +117,10 @@ glusterd_scrubsvc_start(glusterd_svc_t *svc, int flags) dict_t *cmdict = NULL; cmdict = dict_new(); - if (!cmdict) + if (!cmdict) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto error_return; + } ret = dict_set_str(cmdict, "cmdarg0", "--global-timer-wheel"); if (ret) diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c index f3781879d99..b0b8a2e4018 100644 --- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c +++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c @@ -89,12 +89,15 @@ glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict, ret = dict_get_str(dict, "volname", &volname); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); ret = 0; goto out; } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL); ret = 0; goto out; } @@ -252,8 +255,11 @@ glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo) int ret = 0; ret = dict_get_str(volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY, &quorum_type); - if (ret) + if (ret) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GLUSTERD_QUORUM_TYPE_KEY, NULL); goto out; + } if (strcmp(quorum_type, GLUSTERD_SERVER_QUORUM) == 0) res = _gf_true; @@ -287,8 +293,11 @@ does_gd_meet_server_quorum(xlator_t *this) ret = glusterd_get_quorum_cluster_counts(this, &active_count, &quorum_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, NULL); goto out; + } if (!does_quorum_meet(active_count, quorum_count)) { goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c index e106398e697..1c56384a14b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c @@ -155,6 +155,8 @@ glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo) int ret = -1; dict_t *mod_dict = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); glusterd_svc_build_shd_volfile_path(volinfo, filepath, PATH_MAX); if (!glusterd_is_shd_compatible_volume(volinfo)) { @@ -166,28 +168,42 @@ glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo) goto out; } mod_dict = dict_new(); - if (!mod_dict) + if (!mod_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.background-self-heal-count", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.data-self-heal", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.metadata-self-heal", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.entry-self-heal", NULL); goto out; + } ret = glusterd_shdsvc_generate_volfile(volinfo, filepath, mod_dict); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volfile"); goto out; } @@ -195,7 +211,7 @@ glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo) out: if (mod_dict) dict_unref(mod_dict); - gf_msg_debug(THIS->name, 0, "Returning %d", ret); + gf_msg_debug(this->name, 0, "Returning %d", ret); return ret; } @@ -270,9 +286,7 @@ glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags) } while (conf->restart_shd) { - synclock_unlock(&conf->big_lock); - sleep(2); - synclock_lock(&conf->big_lock); + synccond_wait(&conf->cond_restart_shd, &conf->big_lock); } conf->restart_shd = _gf_true; shd_restart = _gf_true; @@ -328,8 +342,10 @@ glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags) } } out: - if (shd_restart) + if (shd_restart) { conf->restart_shd = _gf_false; + synccond_broadcast(&conf->cond_restart_shd); + } if (volinfo) glusterd_volinfo_unref(volinfo); if (ret) @@ -346,6 +362,8 @@ glusterd_new_shd_svc_start(glusterd_svc_t *svc, int flags) char glusterd_uuid_option[PATH_MAX] = {0}; char client_pid[32] = {0}; dict_t *cmdline = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); cmdline = dict_new(); if (!cmdline) @@ -362,31 +380,49 @@ glusterd_new_shd_svc_start(glusterd_svc_t *svc, int flags) goto out; ret = dict_set_str(cmdline, "arg", client_pid); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=arg", NULL); goto out; + } /* Pass cmdline arguments as key-value pair. The key is merely * a carrier and is not used. Since dictionary follows LIFO the value * should be put in reverse order*/ ret = dict_set_str(cmdline, "arg4", svc->name); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=arg4", NULL); goto out; + } ret = dict_set_str(cmdline, "arg3", GD_SHD_PROCESS_NAME); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=arg3", NULL); goto out; + } ret = dict_set_str(cmdline, "arg2", glusterd_uuid_option); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=arg2", NULL); goto out; + } ret = dict_set_str(cmdline, "arg1", "--xlator-option"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=arg1", NULL); goto out; + } ret = glusterd_svc_start(svc, flags, cmdline); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_GLUSTER_SERVICE_START_FAIL, NULL); goto out; + } ret = glusterd_conn_connect(&(svc->conn)); out: @@ -539,28 +575,45 @@ glusterd_shdsvc_reconfigure(glusterd_volinfo_t *volinfo) goto out; } mod_dict = dict_new(); - if (!mod_dict) + if (!mod_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.background-self-heal-count", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.data-self-heal", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.metadata-self-heal", NULL); goto out; + } ret = dict_set_int32(mod_dict, "graph-check", 1); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=graph-check", NULL); goto out; + } ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=cluster.entry-self-heal", NULL); goto out; + } ret = glusterd_volume_svc_check_volfile_identical( "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile, diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c index 7b67592e27c..bf2d81b644a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-sm.c @@ -146,22 +146,33 @@ glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid) ctx.op = GD_FRIEND_UPDATE_DEL; friends = dict_new(); - if (!friends) + if (!friends) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "op"); ret = dict_set_int32n(friends, key, keylen, ctx.op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "hostname"); ret = dict_set_strn(friends, key, keylen, hostname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = dict_set_int32n(friends, "count", SLEN("count"), count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } RCU_READ_LOCK; cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list) @@ -370,30 +381,45 @@ glusterd_ac_friend_probe(glusterd_friend_sm_event_t *event, void *ctx) peerinfo = glusterd_peerinfo_find(NULL, probe_ctx->hostname); if (peerinfo == NULL) { // We should not reach this state ideally + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL); ret = -1; goto unlock; } - if (!peerinfo->peer) + if (!peerinfo->peer) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL, + NULL); goto unlock; + } proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY]; if (proc->fn) { frame = create_frame(this, this->ctx->pool); if (!frame) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto unlock; } frame->local = ctx; dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto unlock; + } ret = dict_set_strn(dict, "hostname", SLEN("hostname"), probe_ctx->hostname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=hostname", NULL); goto unlock; + } ret = dict_set_int32n(dict, "port", SLEN("port"), probe_ctx->port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=port", NULL); goto unlock; + } /* The peerinfo reference being set here is going to be used * only within this critical section, in glusterd_rpc_probe @@ -482,12 +508,17 @@ glusterd_ac_send_friend_remove_req(glusterd_friend_sm_event_t *event, goto unlock; } - if (!peerinfo->peer) + if (!peerinfo->peer) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL, + NULL); goto unlock; + } proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE]; if (proc->fn) { frame = create_frame(this, this->ctx->pool); if (!frame) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto unlock; } frame->local = data; @@ -556,13 +587,18 @@ glusterd_ac_send_friend_update(glusterd_friend_sm_event_t *event, void *ctx) goto out; } - if (!friends) + if (!friends) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto unlock; + } ev_ctx.op = GD_FRIEND_UPDATE_ADD; ret = dict_set_int32n(friends, key, keylen, ev_ctx.op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto unlock; + } cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list) { @@ -578,8 +614,11 @@ glusterd_ac_send_friend_update(glusterd_friend_sm_event_t *event, void *ctx) } ret = dict_set_int32n(friends, "count", SLEN("count"), count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto unlock; + } cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list) { @@ -665,13 +704,18 @@ glusterd_ac_update_friend(glusterd_friend_sm_event_t *event, void *ctx) goto unlock; } - if (!friends) + if (!friends) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ev_ctx.op = GD_FRIEND_UPDATE_ADD; ret = dict_set_int32n(friends, key, keylen, ev_ctx.op); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto unlock; + } cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list) { @@ -687,8 +731,11 @@ glusterd_ac_update_friend(glusterd_friend_sm_event_t *event, void *ctx) } ret = dict_set_int32n(friends, "count", SLEN("count"), count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto unlock; + } ret = dict_set_static_ptr(friends, "peerinfo", cur_peerinfo); if (ret) { @@ -1062,6 +1109,7 @@ glusterd_friend_sm_transition_state(uuid_t peerid, char *peername, RCU_READ_LOCK; peerinfo = glusterd_peerinfo_find(peerid, peername); if (!peerinfo) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL); goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c index 3042789916c..d75f249b29e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c @@ -87,8 +87,10 @@ glusterd_snapdsvc_init(void *data) svc = &(volinfo->snapd.svc); ret = snprintf(svc->name, sizeof(svc->name), "%s", snapd_svc_name); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } notify = glusterd_snapdsvc_rpc_notify; @@ -115,6 +117,7 @@ glusterd_snapdsvc_init(void *data) glusterd_svc_build_snapd_logfile(logfile, logdir, sizeof(logfile)); len = snprintf(volfileid, sizeof(volfileid), "snapd/%s", volinfo->volname); if ((len < 0) || (len >= sizeof(volfileid))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -301,16 +304,22 @@ glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags) } runinit(&runner); - if (this->ctx->cmd_args.valgrind) { + if (this->ctx->cmd_args.vgtool != _gf_none) { len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log", svc->proc.logdir); if ((len < 0) || (len >= PATH_MAX)) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", NULL); + if (this->ctx->cmd_args.vgtool == _gf_memcheck) + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + else + runner_add_args(&runner, "valgrind", "--tool=drd", NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c index 43735d33fee..995268b796d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c @@ -282,12 +282,10 @@ glusterd_snap_volinfo_restore(dict_t *dict, dict_t *rsp_dict, new_volinfo->volume_id, sizeof(new_volinfo->volume_id), XATTR_REPLACE); if (ret == -1) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SETXATTR_FAIL, - "Failed to " - "set extended attribute %s on %s. " - "Reason: %s, snap: %s", - GF_XATTR_VOL_ID_KEY, new_brickinfo->path, - strerror(errno), new_volinfo->volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL, + "Attribute=%s, Path=%s, Reason=%s, Snap=%s", + GF_XATTR_VOL_ID_KEY, new_brickinfo->path, + strerror(errno), new_volinfo->volname, NULL); goto out; } } @@ -1961,9 +1959,7 @@ glusterd_update_snaps_synctask(void *opaque) synclock_lock(&conf->big_lock); while (conf->restart_bricks) { - synclock_unlock(&conf->big_lock); - sleep(2); - synclock_lock(&conf->big_lock); + synccond_wait(&conf->cond_restart_bricks, &conf->big_lock); } conf->restart_bricks = _gf_true; @@ -2041,8 +2037,9 @@ glusterd_update_snaps_synctask(void *opaque) "Failed to remove snap %s", snap->snapname); goto out; } - if (dict) - dict_unref(dict); + + dict_unref(dict); + dict = NULL; } snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix); ret = dict_get_int32(peer_data, buf, &val); @@ -2070,6 +2067,7 @@ out: if (dict) dict_unref(dict); conf->restart_bricks = _gf_false; + synccond_broadcast(&conf->cond_restart_bricks); return ret; } @@ -2149,18 +2147,27 @@ glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, snprintf(base_key, sizeof(base_key), "brick%d", count); snprintf(key, sizeof(key), "%s.hostname", base_key); ret = dict_set_str(dict, key, "Snapshot Daemon"); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } snprintf(key, sizeof(key), "%s.path", base_key); ret = dict_set_dynstr(dict, key, gf_strdup(uuid_utoa(MY_UUID))); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } snprintf(key, sizeof(key), "%s.port", base_key); ret = dict_set_int32(dict, key, volinfo->snapd.port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile)); @@ -2170,8 +2177,11 @@ glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, snprintf(key, sizeof(key), "%s.pid", base_key); ret = dict_set_int32(dict, key, pid); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } snprintf(key, sizeof(key), "%s.status", base_key); ret = dict_set_int32(dict, key, brick_online); @@ -2672,8 +2682,10 @@ glusterd_missed_snapinfo_new(glusterd_missed_snap_info **missed_snapinfo) new_missed_snapinfo = GF_CALLOC(1, sizeof(*new_missed_snapinfo), gf_gld_mt_missed_snapinfo_t); - if (!new_missed_snapinfo) + if (!new_missed_snapinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } CDS_INIT_LIST_HEAD(&new_missed_snapinfo->missed_snaps); CDS_INIT_LIST_HEAD(&new_missed_snapinfo->snap_ops); @@ -2701,8 +2713,10 @@ glusterd_missed_snap_op_new(glusterd_snap_op_t **snap_op) new_snap_op = GF_CALLOC(1, sizeof(*new_snap_op), gf_gld_mt_missed_snapinfo_t); - if (!new_snap_op) + if (!new_snap_op) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } new_snap_op->brick_num = -1; new_snap_op->op = -1; @@ -3594,13 +3608,17 @@ glusterd_copy_folder(const char *source, const char *destination) continue; ret = snprintf(src_path, sizeof(src_path), "%s/%s", source, entry->d_name); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", destination, entry->d_name); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = glusterd_copy_file(src_path, dest_path); if (ret) { @@ -3756,8 +3774,10 @@ glusterd_copy_quota_files(glusterd_volinfo_t *src_vol, GLUSTERD_GET_VOLUME_DIR(dest_dir, dest_vol, priv); ret = snprintf(src_path, sizeof(src_path), "%s/quota.conf", src_dir); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } /* quota.conf is not present if quota is not enabled, Hence ignoring * the absence of this file @@ -3770,8 +3790,10 @@ glusterd_copy_quota_files(glusterd_volinfo_t *src_vol, } ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.conf", dest_dir); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = glusterd_copy_file(src_path, dest_path); if (ret) { @@ -3795,8 +3817,10 @@ glusterd_copy_quota_files(glusterd_volinfo_t *src_vol, } ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.cksum", dest_dir); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = glusterd_copy_file(src_path, dest_path); if (ret) { @@ -4066,8 +4090,10 @@ glusterd_restore_nfs_ganesha_file(glusterd_volinfo_t *src_vol, ret = snprintf(src_path, sizeof(src_path), "%s/export.%s.conf", snap_dir, snap->snapname); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = sys_lstat(src_path, &stbuf); if (ret) { @@ -4082,8 +4108,10 @@ glusterd_restore_nfs_ganesha_file(glusterd_volinfo_t *src_vol, ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf", GANESHA_EXPORT_DIRECTORY, src_vol->volname); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; + } ret = glusterd_copy_file(src_path, dest_path); if (ret) diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c index 4703a072294..aeaa8d15214 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c @@ -514,6 +514,7 @@ glusterd_copy_geo_rep_session_files(char *session, glusterd_volinfo_t *snap_vol) ret = snprintf(georep_session_dir, sizeof(georep_session_dir), "%s/%s/%s", priv->workdir, GEOREP, session); if (ret < 0) { /* Negative value is an error */ + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -521,6 +522,7 @@ glusterd_copy_geo_rep_session_files(char *session, glusterd_volinfo_t *snap_vol) priv->workdir, GLUSTERD_VOL_SNAP_DIR_PREFIX, snap_vol->snapshot->snapname, GEOREP, session); if (ret < 0) { /* Negative value is an error */ + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -568,12 +570,14 @@ glusterd_copy_geo_rep_session_files(char *session, glusterd_volinfo_t *snap_vol) ret = snprintf(src_path, sizeof(src_path), "%s/%s", georep_session_dir, files[i]->d_name); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", snap_session_dir, files[i]->d_name); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -632,12 +636,14 @@ glusterd_snapshot_backup_vol(glusterd_volinfo_t *volinfo) "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir, volinfo->volname); if ((len < 0) || (len >= sizeof(delete_path))) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH, priv->workdir); - if ((len < 0) || (len >= sizeof(delete_path))) { + if ((len < 0) || (len >= sizeof(trashdir))) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -730,6 +736,7 @@ glusterd_copy_geo_rep_files(glusterd_volinfo_t *origin_vol, * is slave volume. */ if (!origin_vol->gsync_slaves) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_SLAVE, NULL); ret = 0; goto out; } @@ -1418,6 +1425,8 @@ glusterd_handle_snapshot_config(rpcsvc_request_t *req, glusterd_op_t op, &config_command); if (ret) { snprintf(err_str, len, "Failed to get config-command type"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=config-command", NULL); goto out; } @@ -1976,6 +1985,13 @@ glusterd_snap_create_clone_common_prevalidate( "command or use [force] option in " "snapshot create to override this " "behavior."); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRICK_NOT_RUNNING, + "Please run volume status command to see brick " + "status.Please start the stopped brick and then issue " + "snapshot create command or use 'force' option in " + "snapshot create to override this behavior.", + NULL); } else { snprintf(err_str, PATH_MAX, "One or more bricks are not running. " @@ -1984,6 +2000,12 @@ glusterd_snap_create_clone_common_prevalidate( "Please start the stopped brick " "and then issue snapshot clone " "command "); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRICK_NOT_RUNNING, + "Please run snapshot status command to see brick " + "status. Please start the stopped brick and then issue " + "snapshot clone command.", + NULL); } *op_errno = EG_BRCKDWN; ret = -1; @@ -1999,6 +2021,10 @@ glusterd_snap_create_clone_common_prevalidate( if (len < 0) { strcpy(err_str, "<error>"); } + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_BRK_MNTPATH_GET_FAIL, + "Brick_hostname=%s, Brick_path=%s", brickinfo->hostname, + brickinfo->path, NULL); ret = -1; goto out; } @@ -2010,6 +2036,11 @@ glusterd_snap_create_clone_common_prevalidate( "all bricks of %s are thinly " "provisioned LV.", volinfo->volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED, + "Ensure that all bricks of volume are thinly " + "provisioned LV, Volume=%s", + volinfo->volname, NULL); ret = -1; goto out; } @@ -2022,6 +2053,9 @@ glusterd_snap_create_clone_common_prevalidate( "cannot copy the snapshot device " "name (volname: %s, snapname: %s)", volinfo->volname, snapname); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_SNAP_DEVICE_NAME_GET_FAIL, "Volname=%s, Snapname=%s", + volinfo->volname, snapname, NULL); *loglevel = GF_LOG_WARNING; ret = -1; goto out; @@ -2188,6 +2222,16 @@ glusterd_snapshot_clone_prevalidate(dict_t *dict, char **op_errstr, goto out; } + if (!glusterd_is_volume_started(snap_vol)) { + snprintf(err_str, sizeof(err_str), + "Snapshot %s is " + "not activated", + snap->snapname); + loglevel = GF_LOG_WARNING; + *op_errno = EG_VOLSTP; + goto out; + } + ret = dict_get_bin(dict, "vol1_volid", (void **)&snap_volid); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, @@ -3211,7 +3255,7 @@ glusterd_snapshot_get_snap_detail(dict_t *dict, glusterd_snap_t *snap, int volcount = 0; char key[32] = ""; /* keyprefix is quite small, up to 16 bytes */ int keylen; - char timestr[64] = ""; + char timestr[GF_TIMESTR_SIZE] = ""; char *value = NULL; glusterd_volinfo_t *snap_vol = NULL; glusterd_volinfo_t *tmp_vol = NULL; @@ -3886,7 +3930,8 @@ glusterd_handle_snapshot_create(rpcsvc_request_t *req, glusterd_op_t op, goto out; } - ret = dict_set_int64(dict, "snap-time", (int64_t)time(&snap_time)); + snap_time = gf_time(); + ret = dict_set_int64(dict, "snap-time", (int64_t)snap_time); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Unable to set snap-time"); @@ -4451,6 +4496,7 @@ glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict, snap_uuid, snap_vol->volname, brick_number, brickinfo->path, op, GD_MISSED_SNAP_PENDING); if ((len < 0) || (len >= sizeof(missed_snap_entry))) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -4458,6 +4504,8 @@ glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict, ret = dict_get_int32n(rsp_dict, "missed_snap_count", SLEN("missed_snap_count"), &missed_snap_count); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=missed_snap_count", NULL); /* Initialize the missed_snap_count for the first time */ missed_snap_count = 0; } @@ -4647,7 +4695,7 @@ glusterd_snap_brick_create(glusterd_volinfo_t *snap_volinfo, ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY, snap_volinfo->volume_id, 16, XATTR_REPLACE); if (ret == -1) { - gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SETXATTR_FAIL, + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL, "Failed to set " "extended attribute %s on %s. Reason: " "%s, snap: %s", @@ -5275,6 +5323,48 @@ glusterd_do_snap_vol(glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap, dict_deln(snap_vol->dict, "features.barrier", SLEN("features.barrier")); gd_update_volume_op_versions(snap_vol); + /* * + * Create the export file from the node where ganesha.enable "on" + * is executed + * */ + if (glusterd_is_ganesha_cluster() && + glusterd_check_ganesha_export(snap_vol)) { + if (is_origin_glusterd(dict)) { + ret = manage_export_config(clonename, "on", NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, + GD_MSG_EXPORT_FILE_CREATE_FAIL, + "Failed to create" + "export file for NFS-Ganesha\n"); + goto out; + } + } + + ret = dict_set_dynstr_with_alloc(snap_vol->dict, + "features.cache-invalidation", "on"); + ret = gd_ganesha_send_dbus(clonename, "on"); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL, + "Dynamic export addition/deletion failed." + " Please see log file for details. Clone name = %s", + clonename); + goto out; + } + } + if (!glusterd_is_ganesha_cluster() && + glusterd_check_ganesha_export(snap_vol)) { + /* This happens when a snapshot was created when Ganesha was + * enabled globally. Then Ganesha disabled from the cluster. + * In such cases, we will have the volume level option set + * on dict, So we have to disable it as it doesn't make sense + * to keep the option. + */ + + ret = dict_set_dynstr(snap_vol->dict, "ganesha.enable", "off"); + if (ret) + goto out; + } + ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL, @@ -5346,8 +5436,31 @@ out: for (i = 0; unsupported_opt[i].key; i++) GF_FREE(unsupported_opt[i].value); - if (snap_vol) + if (snap_vol) { + if (glusterd_is_ganesha_cluster() && + glusterd_check_ganesha_export(snap_vol)) { + if (is_origin_glusterd(dict)) { + ret = manage_export_config(clonename, "on", NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, + GD_MSG_EXPORT_FILE_CREATE_FAIL, + "Failed to create" + "export file for NFS-Ganesha\n"); + } + } + + ret = gd_ganesha_send_dbus(clonename, "off"); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, + GD_MSG_EXPORT_FILE_CREATE_FAIL, + "Dynamic export addition/deletion failed." + " Please see log file for details. Clone name = %s", + clonename); + } + } + glusterd_snap_volume_remove(rsp_dict, snap_vol, _gf_true, _gf_true); + } snap_vol = NULL; } @@ -5399,6 +5512,8 @@ glusterd_snapshot_activate_deactivate_prevalidate(dict_t *dict, "Snapshot (%s) does not " "exist.", snapname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND, + "Snapname=%s", snapname, NULL); *op_errno = EG_NOSNAP; ret = -1; goto out; @@ -7204,11 +7319,15 @@ glusterd_get_brick_lvm_details(dict_t *rsp_dict, if (token != NULL) { value = gf_strdup(token); if (!value) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "token=%s", token, NULL); ret = -1; goto end; } ret = snprintf(key, sizeof(key), "%s.data", key_prefix); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, + NULL); goto end; } @@ -7223,11 +7342,15 @@ glusterd_get_brick_lvm_details(dict_t *rsp_dict, if (token != NULL) { value = gf_strdup(token); if (!value) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "token=%s", token, NULL); ret = -1; goto end; } ret = snprintf(key, sizeof(key), "%s.lvsize", key_prefix); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, + NULL); goto end; } @@ -7287,6 +7410,7 @@ glusterd_get_single_brick_status(char **op_errstr, dict_t *rsp_dict, keylen = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, index); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -7294,11 +7418,14 @@ glusterd_get_single_brick_status(char **op_errstr, dict_t *rsp_dict, ret = snprintf(brick_path, sizeof(brick_path), "%s:%s", brickinfo->hostname, brickinfo->path); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; } value = gf_strdup(brick_path); if (!value) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "brick_path=%s", brick_path, NULL); ret = -1; goto out; } @@ -7374,6 +7501,8 @@ glusterd_get_single_brick_status(char **op_errstr, dict_t *rsp_dict, index); if (keylen < 0) { ret = -1; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, + NULL); goto out; } @@ -7459,6 +7588,7 @@ glusterd_get_single_snap_status(char **op_errstr, dict_t *rsp_dict, { keylen = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -7482,6 +7612,7 @@ glusterd_get_single_snap_status(char **op_errstr, dict_t *rsp_dict, } keylen = snprintf(brickkey, sizeof(brickkey), "%s.brickcount", key); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -7496,6 +7627,7 @@ glusterd_get_single_snap_status(char **op_errstr, dict_t *rsp_dict, keylen = snprintf(key, sizeof(key), "%s.volcount", keyprefix); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -7535,6 +7667,7 @@ glusterd_get_each_snap_object_status(char **op_errstr, dict_t *rsp_dict, */ keylen = snprintf(key, sizeof(key), "%s.snapname", keyprefix); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -7556,6 +7689,7 @@ glusterd_get_each_snap_object_status(char **op_errstr, dict_t *rsp_dict, keylen = snprintf(key, sizeof(key), "%s.uuid", keyprefix); if (keylen < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -7639,6 +7773,7 @@ glusterd_get_snap_status_of_volume(char **op_errstr, dict_t *rsp_dict, { ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -7690,6 +7825,7 @@ glusterd_get_all_snapshot_status(dict_t *dict, char **op_errstr, { ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i); if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -8733,6 +8869,7 @@ glusterd_snapshot_revert_partial_restored_vol(glusterd_volinfo_t *volinfo) "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir, volinfo->volname); if ((len < 0) || (len >= sizeof(trash_path))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -8793,12 +8930,10 @@ glusterd_snapshot_revert_partial_restored_vol(glusterd_volinfo_t *volinfo) snap_vol->volume_id, sizeof(snap_vol->volume_id), XATTR_REPLACE); if (ret == -1) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SETXATTR_FAIL, - "Failed to set extended " - "attribute %s on %s. " - "Reason: %s, snap: %s", - GF_XATTR_VOL_ID_KEY, brickinfo->path, - strerror(errno), snap_vol->volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL, + "Attribute=%s, Path=%s, Reason=%s, Snap=%s", + GF_XATTR_VOL_ID_KEY, brickinfo->path, + strerror(errno), snap_vol->volname, NULL); goto out; } } @@ -9178,6 +9313,7 @@ glusterd_handle_snapshot_fn(rpcsvc_request_t *req) ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 458df8dbd1d..d94dceb10b7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -74,7 +74,7 @@ glusterd_replace_slash_with_hyphen(char *str) while (ptr) { *ptr = '-'; - ptr = strchr(str, '/'); + ptr = strchr(ptr, '/'); } } @@ -660,85 +660,72 @@ out: } static int -_storeslaves(dict_t *this, char *key, data_t *value, void *data) -{ - int32_t ret = 0; - gf_store_handle_t *shandle = NULL; - xlator_t *xl = NULL; - - xl = THIS; - GF_ASSERT(xl); - - shandle = (gf_store_handle_t *)data; - - GF_ASSERT(shandle); - GF_ASSERT(shandle->fd > 0); - GF_ASSERT(shandle->path); - GF_ASSERT(key); - GF_ASSERT(value); - GF_ASSERT(value->data); - - gf_msg_debug(xl->name, 0, "Storing in volinfo:key= %s, val=%s", key, - value->data); - - ret = gf_store_save_value(shandle->fd, key, (char *)value->data); - if (ret) { - gf_msg(xl->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_WRITE_FAIL, - "Unable to write into store" - " handle for path: %s", - shandle->path); - return -1; - } - return 0; -} - -int -_storeopts(dict_t *this, char *key, data_t *value, void *data) +_storeopts(dict_t *dict_value, char *key, data_t *value, void *data) { int32_t ret = 0; int32_t exists = 0; + int32_t option_len = 0; gf_store_handle_t *shandle = NULL; - xlator_t *xl = NULL; + glusterd_volinfo_data_store_t *dict_data = NULL; + xlator_t *this = NULL; - xl = THIS; - GF_ASSERT(xl); + this = THIS; + GF_ASSERT(this); - shandle = (gf_store_handle_t *)data; + dict_data = (glusterd_volinfo_data_store_t *)data; + shandle = dict_data->shandle; GF_ASSERT(shandle); GF_ASSERT(shandle->fd > 0); - GF_ASSERT(shandle->path); GF_ASSERT(key); GF_ASSERT(value); GF_ASSERT(value->data); - if (is_key_glusterd_hooks_friendly(key)) { - exists = 1; + if (dict_data->key_check == 1) { + if (is_key_glusterd_hooks_friendly(key)) { + exists = 1; - } else { - exists = glusterd_check_option_exists(key, NULL); + } else { + exists = glusterd_check_option_exists(key, NULL); + } } - - if (1 == exists) { - gf_msg_debug(xl->name, 0, - "Storing in volinfo:key= %s, " + if (exists == 1 || dict_data->key_check == 0) { + gf_msg_debug(this->name, 0, + "Storing in buffer for volinfo:key= %s, " "val=%s", key, value->data); - } else { - gf_msg_debug(xl->name, 0, "Discarding:key= %s, val=%s", key, + gf_msg_debug(this->name, 0, "Discarding:key= %s, val=%s", key, value->data); return 0; } - ret = gf_store_save_value(shandle->fd, key, (char *)value->data); - if (ret) { - gf_msg(xl->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_WRITE_FAIL, - "Unable to write into store" - " handle for path: %s", - shandle->path); + /* + * The option_len considers the length of the key value + * pair and along with that '=' and '\n', but as value->len + * already considers a NULL at the end of the data, adding + * just 1. + */ + option_len = strlen(key) + value->len + 1; + + if ((VOLINFO_BUFFER_SIZE - dict_data->buffer_len - 1) < option_len) { + ret = gf_store_save_items(shandle->fd, dict_data->buffer); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL); + return -1; + } + dict_data->buffer_len = 0; + dict_data->buffer[0] = '\0'; + } + ret = snprintf(dict_data->buffer + dict_data->buffer_len, option_len + 1, + "%s=%s\n", key, value->data); + if (ret < 0 || ret > option_len + 1) { + gf_smsg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COPY_FAIL, NULL); return -1; } + + dict_data->buffer_len += ret; + return 0; } @@ -1013,7 +1000,7 @@ glusterd_store_create_snap_dir(glusterd_snap_t *snap) return ret; } -int32_t +static int32_t glusterd_store_volinfo_write(int fd, glusterd_volinfo_t *volinfo) { int32_t ret = -1; @@ -1021,19 +1008,47 @@ glusterd_store_volinfo_write(int fd, glusterd_volinfo_t *volinfo) GF_ASSERT(fd > 0); GF_ASSERT(volinfo); GF_ASSERT(volinfo->shandle); + xlator_t *this = NULL; + glusterd_volinfo_data_store_t *dict_data = NULL; + + this = THIS; + GF_ASSERT(this); shandle = volinfo->shandle; + + dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t), + gf_gld_mt_volinfo_dict_data_t); + if (dict_data == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL); + return -1; + } + ret = glusterd_volume_exclude_options_write(fd, volinfo); - if (ret) + if (ret) { goto out; + } + + dict_data->shandle = shandle; + dict_data->key_check = 1; shandle->fd = fd; - dict_foreach(volinfo->dict, _storeopts, shandle); + dict_foreach(volinfo->dict, _storeopts, (void *)dict_data); + + dict_data->key_check = 0; + dict_foreach(volinfo->gsync_slaves, _storeopts, (void *)dict_data); + + if (dict_data->buffer_len > 0) { + ret = gf_store_save_items(fd, dict_data->buffer); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL); + goto out; + } + } - dict_foreach(volinfo->gsync_slaves, _storeslaves, shandle); shandle->fd = 0; out: - gf_msg_debug(THIS->name, 0, "Returning %d", ret); + GF_FREE(dict_data); + gf_msg_debug(this->name, 0, "Returning %d", ret); return ret; } @@ -1274,14 +1289,6 @@ out: return ret; } -static int -_gd_store_rebalance_dict(dict_t *dict, char *key, data_t *value, void *data) -{ - int fd = *(int *)data; - - return gf_store_save_value(fd, key, value->data); -} - int32_t glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo) { @@ -1289,6 +1296,12 @@ glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo) char buf[PATH_MAX]; char uuid[UUID_SIZE + 1]; uint total_len = 0; + glusterd_volinfo_data_store_t *dict_data = NULL; + gf_store_handle_t shandle; + xlator_t *this = NULL; + + this = THIS; + GF_ASSERT(this); GF_ASSERT(fd > 0); GF_ASSERT(volinfo); @@ -1328,14 +1341,33 @@ glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo) } ret = gf_store_save_items(fd, buf); - if (ret) + if (ret) { goto out; + } if (volinfo->rebal.dict) { - dict_foreach(volinfo->rebal.dict, _gd_store_rebalance_dict, &fd); + dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t), + gf_gld_mt_volinfo_dict_data_t); + if (dict_data == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL); + return -1; + } + dict_data->shandle = &shandle; + shandle.fd = fd; + dict_foreach(volinfo->rebal.dict, _storeopts, (void *)dict_data); + if (dict_data->buffer_len > 0) { + ret = gf_store_save_items(fd, dict_data->buffer); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, + NULL); + goto out; + ; + } + } } out: - gf_msg_debug(THIS->name, 0, "Returning %d", ret); + GF_FREE(dict_data); + gf_msg_debug(this->name, 0, "Returning %d", ret); return ret; } @@ -1781,8 +1813,9 @@ glusterd_store_delete_snap(glusterd_snap_t *snap) goto out; } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; len = snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name); if ((len < 0) || (len >= PATH_MAX)) { goto stat_failed; @@ -1812,7 +1845,6 @@ glusterd_store_delete_snap(glusterd_snap_t *snap) ret ? "Failed to remove" : "Removed", entry->d_name); stat_failed: memset(path, 0, sizeof(path)); - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } ret = sys_closedir(dir); @@ -2309,7 +2341,7 @@ glusterd_store_retrieve_snapd(glusterd_volinfo_t *volinfo) ret = 0; out: - if (gf_store_iter_destroy(iter)) { + if (gf_store_iter_destroy(&iter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; @@ -2642,6 +2674,13 @@ glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo) brick_count++; } + if (gf_store_iter_destroy(&tmpiter)) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, + "Failed to destroy store iter"); + ret = -1; + goto out; + } + ret = gf_store_iter_new(volinfo->shandle, &tmpiter); if (ret) @@ -2816,13 +2855,13 @@ glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo) ret = 0; out: - if (gf_store_iter_destroy(tmpiter)) { + if (gf_store_iter_destroy(&tmpiter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; } - if (gf_store_iter_destroy(iter)) { + if (gf_store_iter_destroy(&iter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; @@ -2955,7 +2994,7 @@ glusterd_store_retrieve_node_state(glusterd_volinfo_t *volinfo) ret = 0; out: - if (gf_store_iter_destroy(iter)) { + if (gf_store_iter_destroy(&iter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; @@ -3231,7 +3270,7 @@ glusterd_store_update_volinfo(glusterd_volinfo_t *volinfo) ret = 0; out: - if (gf_store_iter_destroy(iter)) { + if (gf_store_iter_destroy(&iter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; @@ -3336,20 +3375,6 @@ glusterd_store_set_options_path(glusterd_conf_t *conf, char *path, size_t len) snprintf(path, len, "%s/options", conf->workdir); } -int -_store_global_opts(dict_t *this, char *key, data_t *value, void *data) -{ - gf_store_handle_t *shandle = data; - - if (gf_store_save_value(shandle->fd, key, (char *)value->data)) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_WRITE_FAIL, - "Unable to write into store handle for key : %s, value %s", key, - (char *)value->data); - } - - return 0; -} - int32_t glusterd_store_options(xlator_t *this, dict_t *opts) { @@ -3358,13 +3383,15 @@ glusterd_store_options(xlator_t *this, dict_t *opts) char path[PATH_MAX] = {0}; int fd = -1; int32_t ret = -1; + glusterd_volinfo_data_store_t *dict_data = NULL; conf = this->private; glusterd_store_set_options_path(conf, path, sizeof(path)); ret = gf_store_handle_new(path, &shandle); - if (ret) + if (ret) { goto out; + } fd = gf_store_mkstemp(shandle); if (fd <= 0) { @@ -3372,15 +3399,30 @@ glusterd_store_options(xlator_t *this, dict_t *opts) goto out; } + dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t), + gf_gld_mt_volinfo_dict_data_t); + if (dict_data == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL); + return -1; + } + dict_data->shandle = shandle; shandle->fd = fd; - dict_foreach(opts, _store_global_opts, shandle); - shandle->fd = 0; + dict_foreach(opts, _storeopts, (void *)dict_data); + if (dict_data->buffer_len > 0) { + ret = gf_store_save_items(fd, dict_data->buffer); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL); + goto out; + } + } + ret = gf_store_rename_tmppath(shandle); - if (ret) - goto out; out: - if ((ret < 0) && (fd > 0)) + shandle->fd = 0; + GF_FREE(dict_data); + if ((ret < 0) && (fd > 0)) { gf_store_unlink_tmppath(shandle); + } gf_store_handle_destroy(shandle); return ret; } @@ -3426,7 +3468,7 @@ glusterd_store_retrieve_options(xlator_t *this) goto out; ret = 0; out: - (void)gf_store_iter_destroy(iter); + (void)gf_store_iter_destroy(&iter); gf_store_handle_destroy(shandle); return ret; } @@ -3478,28 +3520,28 @@ glusterd_store_retrieve_volumes(xlator_t *this, glusterd_snap_t *snap) goto out; } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; if (snap && ((!strcmp(entry->d_name, "geo-replication")) || (!strcmp(entry->d_name, "info")))) - goto next; + continue; len = snprintf(entry_path, PATH_MAX, "%s/%s", path, entry->d_name); - if ((len < 0) || (len >= PATH_MAX)) { - goto next; - } + if ((len < 0) || (len >= PATH_MAX)) + continue; + ret = sys_lstat(entry_path, &st); if (ret == -1) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "Failed to stat entry %s : %s", path, strerror(errno)); - goto next; + continue; } if (!S_ISDIR(st.st_mode)) { gf_msg_debug(this->name, 0, "%s is not a valid volume", entry->d_name); - goto next; + continue; } volinfo = glusterd_store_retrieve_volume(entry->d_name, snap); @@ -3522,8 +3564,6 @@ glusterd_store_retrieve_volumes(xlator_t *this, glusterd_snap_t *snap) glusterd_store_create_nodestate_sh_on_absence(volinfo); glusterd_store_perform_node_state_store(volinfo); } - next: - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } ret = 0; @@ -3878,7 +3918,7 @@ glusterd_store_update_snap(glusterd_snap_t *snap) ret = 0; out: - if (gf_store_iter_destroy(iter)) { + if (gf_store_iter_destroy(&iter)) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL, "Failed to destroy store iter"); ret = -1; @@ -4073,9 +4113,9 @@ glusterd_store_retrieve_snaps(xlator_t *this) goto out; } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - - while (entry) { + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; if (strcmp(entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) { ret = glusterd_store_retrieve_snap(entry->d_name); if (ret) { @@ -4084,7 +4124,6 @@ glusterd_store_retrieve_snaps(xlator_t *this) goto out; } } - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); } /* Retrieve missed_snaps_list */ @@ -4399,7 +4438,7 @@ glusterd_store_create_peer_shandle(glusterd_peerinfo_t *peerinfo) static int32_t glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo) { - char buf[128]; + char buf[PATH_MAX]; uint total_len = 0; int32_t ret = 0; int32_t i = 1; @@ -4408,7 +4447,7 @@ glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo) ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n%s=%d\n", GLUSTERD_STORE_KEY_PEER_UUID, uuid_utoa(peerinfo->uuid), GLUSTERD_STORE_KEY_PEER_STATE, peerinfo->state.state); - if (ret < 0 || ret >= sizeof(buf)) { + if (ret < 0 || ret >= sizeof(buf) - total_len) { ret = -1; goto out; } @@ -4419,7 +4458,7 @@ glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo) ret = snprintf(buf + total_len, sizeof(buf) - total_len, GLUSTERD_STORE_KEY_PEER_HOSTNAME "%d=%s\n", i, hostname->hostname); - if (ret < 0 || ret >= sizeof(buf)) { + if (ret < 0 || ret >= sizeof(buf) - total_len) { ret = -1; goto out; } @@ -4531,11 +4570,9 @@ glusterd_store_retrieve_peers(xlator_t *this) goto out; } - for (;;) { - GF_SKIP_IRRELEVANT_ENTRIES(entry, dir, scratch); - if (!entry) { - break; - } + while ((entry = sys_readdir(dir, scratch))) { + if (gf_irrelevant_entry(entry)) + continue; if (gf_uuid_parse(entry->d_name, tmp_uuid) != 0) { gf_log(this->name, GF_LOG_WARNING, "skipping non-peer file %s", entry->d_name); @@ -4623,7 +4660,7 @@ glusterd_store_retrieve_peers(xlator_t *this) is_ok = _gf_true; next: - (void)gf_store_iter_destroy(iter); + (void)gf_store_iter_destroy(&iter); if (!is_ok) { gf_log(this->name, GF_LOG_WARNING, diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h index 04070549678..83f4df0783e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.h +++ b/xlators/mgmt/glusterd/src/glusterd-store.h @@ -29,7 +29,7 @@ typedef enum glusterd_store_ver_ac_ { } glusterd_volinfo_ver_ac_t; #define UUID_SIZE 36 - +#define VOLINFO_BUFFER_SIZE 4093 #define GLUSTERD_STORE_UUID_KEY "UUID" #define GLUSTERD_STORE_KEY_VOL_TYPE "type" @@ -112,6 +112,19 @@ typedef enum glusterd_store_ver_ac_ { #define GLUSTERD_STORE_KEY_GANESHA_GLOBAL "nfs-ganesha" +/* + * The structure is responsible for handling the parameter for writes into + * the buffer before it is finally written to the file. The writes will be + * of the form of key-value pairs. + */ +struct glusterd_volinfo_data_store_ { + gf_store_handle_t *shandle; /*Contains fd and path of the file */ + int16_t buffer_len; + char key_check; /* flag to check if key is to be validated before write*/ + char buffer[VOLINFO_BUFFER_SIZE]; +}; +typedef struct glusterd_volinfo_data_store_ glusterd_volinfo_data_store_t; + int32_t glusterd_store_volinfo(glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac); diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c index 18990fe365b..ca845903c4f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c +++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c @@ -239,8 +239,10 @@ glusterd_svc_check_topology_identical(char *svc_name, int tmpclean = 0; int tmpfd = -1; - if ((!identical) || (!this) || (!this->private)) + if ((!identical) || (!this) || (!this->private)) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); @@ -358,8 +360,10 @@ glusterd_volume_svc_check_topology_identical( int tmpclean = 0; int tmpfd = -1; - if ((!identical) || (!this) || (!this->private)) + if ((!identical) || (!this) || (!this->private)) { + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); @@ -634,7 +638,9 @@ my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame) conf = this->private; GF_VALIDATE_OR_GOTO(this->name, conf, out); - GF_ATOMIC_DEC(conf->blockers); + if (GF_ATOMIC_DEC(conf->blockers) == 0) { + synccond_broadcast(&conf->cond_blockers); + } STACK_DESTROY(frame->root); out: @@ -722,7 +728,9 @@ out: if (volinfo) glusterd_volinfo_unref(volinfo); - GF_ATOMIC_DEC(conf->blockers); + if (GF_ATOMIC_DEC(conf->blockers) == 0) { + synccond_broadcast(&conf->cond_blockers); + } STACK_DESTROY(frame->root); return 0; } @@ -785,12 +793,16 @@ __glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags, frame = create_frame(this, this->ctx->pool); if (!frame) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto *errlbl; } if (op == GLUSTERD_SVC_ATTACH) { dict = dict_new(); if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); ret = -ENOMEM; goto *errlbl; } @@ -808,6 +820,7 @@ __glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags, file_len = stbuf.st_size; volfile_content = GF_MALLOC(file_len + 1, gf_common_mt_char); if (!volfile_content) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); ret = -ENOMEM; goto *errlbl; } @@ -834,10 +847,8 @@ __glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags, ret = dict_allocate_and_serialize(dict, &brick_req.dict.dict_val, &brick_req.dict.dict_len); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - GD_MSG_DICT_SERL_LENGTH_GET_FAIL, - "Failed to serialize dict " - "to request buffer"); + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto *errlbl; } } @@ -969,7 +980,7 @@ glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int flags) * TBD: see if there's a better way */ synclock_unlock(&conf->big_lock); - sleep(1); + synctask_sleep(1); synclock_lock(&conf->big_lock); } ret = -1; @@ -1023,7 +1034,7 @@ glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig) * TBD: see if there's a better way */ synclock_unlock(&conf->big_lock); - sleep(1); + synctask_sleep(1); synclock_lock(&conf->big_lock); } ret = -1; diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c index 99119d69e45..18b3fb13630 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c @@ -162,6 +162,9 @@ glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline) char *localtime_logging = NULL; char *log_level = NULL; char daemon_log_level[30] = {0}; + char msg[1024] = { + 0, + }; int32_t len = 0; this = THIS; @@ -187,7 +190,7 @@ glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline) runinit(&runner); - if (this->ctx->cmd_args.valgrind) { + if (this->ctx->cmd_args.vgtool != _gf_none) { len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log", svc->proc.logdir, svc->name); if ((len < 0) || (len >= PATH_MAX)) { @@ -195,9 +198,13 @@ glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline) goto unlock; } - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", - NULL); + if (this->ctx->cmd_args.vgtool == _gf_memcheck) + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + else + runner_add_args(&runner, "valgrind", "--tool=drd", NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } @@ -226,8 +233,8 @@ glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline) if (cmdline) dict_foreach(cmdline, svc_add_args, (void *)&runner); - gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_START_SUCCESS, - "Starting %s service", svc->name); + snprintf(msg, sizeof(msg), "Starting %s service", svc->name); + runner_log(&runner, this->name, GF_LOG_DEBUG, msg); if (flags == PROC_START_NO_WAIT) { ret = runner_run_nowait(&runner); diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c index b7039f83885..b73d37ad08e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-syncop.c +++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c @@ -406,8 +406,11 @@ gd_syncop_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); gf_uuid_copy(req.txn_id, txn_id); @@ -507,8 +510,11 @@ gd_syncop_mgmt_v3_unlock(dict_t *op_ctx, glusterd_peerinfo_t *peerinfo, ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val, &req.dict.dict_len); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } gf_uuid_copy(req.uuid, my_uuid); gf_uuid_copy(req.txn_id, txn_id); @@ -842,16 +848,21 @@ gd_syncop_mgmt_stage_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args, uuid_t *peerid = NULL; req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_stage_req_t); - if (!req) + if (!req) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } gf_uuid_copy(req->uuid, my_uuid); req->op = op; ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val, &req->buf.buf_len); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); if (ret) @@ -903,6 +914,8 @@ _gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count, if (rsp.output.output_len) { args->dict = dict_new(); if (!args->dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); ret = -1; args->op_errno = ENOMEM; goto out; @@ -910,8 +923,11 @@ _gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count, ret = dict_unserialize(rsp.output.output_val, rsp.output.output_len, &args->dict); - if (ret < 0) + if (ret < 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, + GD_MSG_DICT_UNSERIALIZE_FAIL, NULL); goto out; + } } args->op_ret = rsp.op_ret; @@ -1152,16 +1168,21 @@ gd_syncop_mgmt_commit_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args, uuid_t *peerid = NULL; req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_commit_req_t); - if (!req) + if (!req) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } gf_uuid_copy(req->uuid, my_uuid); req->op = op; ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val, &req->buf.buf_len); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL); goto out; + } GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret); if (ret) @@ -1278,8 +1299,10 @@ gd_stage_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, GF_ASSERT(conf); rsp_dict = dict_new(); - if (!rsp_dict) + if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } if ((op == GD_OP_CREATE_VOLUME) || (op == GD_OP_ADD_BRICK) || (op == GD_OP_START_VOLUME)) @@ -1408,6 +1431,7 @@ gd_commit_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, rsp_dict = dict_new(); if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -1464,8 +1488,11 @@ commit_done: if (op == GD_OP_STATUS_VOLUME) { ret = dict_get_uint32(req_dict, "cmd", &cmd); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=cmd", NULL); goto out; + } if (origin_glusterd) { if ((cmd & GF_CLI_STATUS_ALL)) { @@ -1691,10 +1718,12 @@ gd_brick_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, rpc_clnt_t *rpc = NULL; dict_t *rsp_dict = NULL; int32_t cmd = GF_OP_CMD_NONE; + glusterd_volinfo_t *volinfo = NULL; this = THIS; rsp_dict = dict_new(); if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -1722,18 +1751,28 @@ gd_brick_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, cds_list_for_each_entry_safe(pending_node, tmp, &selected, list) { rpc = glusterd_pending_node_get_rpc(pending_node); + /* In the case of rebalance if the rpc object is null, we try to + * create the rpc object. if the rebalance daemon is down, it returns + * -1. otherwise, rpc object will be created and referenced. + */ if (!rpc) { - if (pending_node->type == GD_NODE_REBALANCE) { - ret = 0; - glusterd_defrag_volume_node_rsp(req_dict, NULL, op_ctx); + if (pending_node->type == GD_NODE_REBALANCE && pending_node->node) { + volinfo = pending_node->node; + ret = glusterd_rebalance_rpc_create(volinfo); + if (ret) { + ret = 0; + glusterd_defrag_volume_node_rsp(req_dict, NULL, op_ctx); + goto out; + } else { + rpc = glusterd_defrag_rpc_get(volinfo->rebal.defrag); + } + } else { + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE, + "Brick Op failed " + "due to rpc failure."); goto out; } - - ret = -1; - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE, - "Brick Op failed " - "due to rpc failure."); - goto out; } ret = gd_syncop_mgmt_brick_op(rpc, pending_node, op, req_dict, op_ctx, @@ -1759,7 +1798,7 @@ gd_brick_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, pending_node = NULL; ret = 0; out: - if (pending_node) + if (pending_node && pending_node->node) glusterd_pending_node_put_rpc(pending_node); if (rsp_dict) diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h index ce4a940c7a0..a265f2135c6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-syncop.h +++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h @@ -32,7 +32,7 @@ ret = gd_syncop_submit_request(rpc, req, stb, cookie, prog, procnum, \ cbk, (xdrproc_t)xdrproc); \ if (!ret) \ - synctask_yield(stb->task); \ + synctask_yield(stb->task, NULL); \ else \ gf_asprintf(&stb->errstr, \ "%s failed. Check log file" \ diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 57ff41d0595..90ef2cf4c9c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -79,6 +79,14 @@ #include <sys/sockio.h> #endif +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#include <sys/param.h> +#include <sys/queue.h> +#include <libprocstat.h> +#include <libutil.h> +#endif + #define NFS_PROGRAM 100003 #define NFSV3_VERSION 3 @@ -443,6 +451,8 @@ glusterd_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame, if (!iobref) { iobref = iobref_new(); if (!iobref) { + gf_smsg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, + NULL); goto out; } @@ -645,6 +655,7 @@ glusterd_volinfo_new(glusterd_volinfo_t **volinfo) new_volinfo->dict = dict_new(); if (!new_volinfo->dict) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); GF_FREE(new_volinfo); goto out; @@ -652,6 +663,7 @@ glusterd_volinfo_new(glusterd_volinfo_t **volinfo) new_volinfo->gsync_slaves = dict_new(); if (!new_volinfo->gsync_slaves) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); dict_unref(new_volinfo->dict); GF_FREE(new_volinfo); goto out; @@ -659,6 +671,7 @@ glusterd_volinfo_new(glusterd_volinfo_t **volinfo) new_volinfo->gsync_active_slaves = dict_new(); if (!new_volinfo->gsync_active_slaves) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); dict_unref(new_volinfo->dict); dict_unref(new_volinfo->gsync_slaves); GF_FREE(new_volinfo); @@ -675,7 +688,9 @@ glusterd_volinfo_new(glusterd_volinfo_t **volinfo) glusterd_gfproxydsvc_build(&new_volinfo->gfproxyd.svc); glusterd_shdsvc_build(&new_volinfo->shd.svc); + pthread_mutex_init(&new_volinfo->store_volinfo_lock, NULL); pthread_mutex_init(&new_volinfo->reflock, NULL); + *volinfo = glusterd_volinfo_ref(new_volinfo); ret = 0; @@ -956,7 +971,10 @@ glusterd_volinfo_delete(glusterd_volinfo_t *volinfo) glusterd_auth_cleanup(volinfo); glusterd_shd_svcproc_cleanup(&volinfo->shd); + pthread_mutex_destroy(&volinfo->store_volinfo_lock); pthread_mutex_destroy(&volinfo->reflock); + LOCK_DESTROY(&volinfo->lock); + GF_FREE(volinfo); ret = 0; out: @@ -1107,7 +1125,8 @@ glusterd_get_brick_mount_dir(char *brickpath, char *hostname, char *mount_dir) } brick_dir = &brickpath[strlen(mnt_pt)]; - brick_dir++; + if (brick_dir[0] == '/') + brick_dir++; snprintf(mount_dir, VALID_GLUSTERD_PATHMAX, "/%s", brick_dir); } @@ -1352,6 +1371,10 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "Reason : %s ", brickinfo->hostname, brickinfo->path, strerror(errno)); + gf_smsg( + "glusterd", GF_LOG_ERROR, errno, GD_MSG_CREATE_BRICK_DIR_FAILED, + "Brick_hostname=%s, Brick_path=%s, Reason=%s", + brickinfo->hostname, brickinfo->path, strerror(errno), NULL); goto out; } } else { @@ -1364,6 +1387,9 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "lstat failed on %s. " "Reason : %s", brickinfo->path, strerror(errno)); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL, + "Failed on Brick_path=%s, Reason=%s", brickinfo->path, + strerror(errno), NULL); goto out; } @@ -1372,6 +1398,8 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "The provided path %s " "which is already present, is not a directory", brickinfo->path); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, + "Brick_path=%s", brickinfo->path, NULL); ret = -1; goto out; } @@ -1388,6 +1416,8 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "lstat failed on /. " "Reason : %s", strerror(errno)); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL, + "Failed on /, Reason=%s", strerror(errno), NULL); goto out; } @@ -1397,6 +1427,9 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "lstat failed on %s. " "Reason : %s", parentdir, strerror(errno)); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL, + "Failed on parentdir=%s, Reason=%s", parentdir, strerror(errno), + NULL); goto out; } if (strncmp(volname, GLUSTER_SHARED_STORAGE, @@ -1407,6 +1440,8 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, len = snprintf(msg, sizeof(msg), "Brick isn't allowed to be " "created inside glusterd's working directory."); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATION_FAIL, + NULL); ret = -1; goto out; } @@ -1422,6 +1457,10 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "command if you want to override this " "behavior.", brickinfo->hostname, brickinfo->path); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATE_MNTPNT, + "Use 'force' at the end of the command if you want to " + "override this behavior, Brick_hostname=%s, Brick_path=%s", + brickinfo->hostname, brickinfo->path, NULL); ret = -1; goto out; } else if (parent_st.st_dev == root_st.st_dev) { @@ -1435,6 +1474,10 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, "command if you want to override this " "behavior.", brickinfo->hostname, brickinfo->path); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATE_ROOT, + "Use 'force' at the end of the command if you want to " + "override this behavior, Brick_hostname=%s, Brick_path=%s", + brickinfo->hostname, brickinfo->path, NULL); /* If --wignore-partition flag is used, ignore warnings * related to bricks being on root partition when 'force' @@ -1466,6 +1509,10 @@ glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo, ".glusterfs directory for brick %s:%s. " "Reason : %s ", brickinfo->hostname, brickinfo->path, strerror(errno)); + gf_smsg("glusterd", GF_LOG_ERROR, errno, + GD_MSG_CREATE_GLUSTER_DIR_FAILED, + "Brick_hostname=%s, Brick_path=%s, Reason=%s", + brickinfo->hostname, brickinfo->path, strerror(errno), NULL); goto out; } @@ -1608,8 +1655,10 @@ glusterd_volinfo_find_by_volume_id(uuid_t volume_id, glusterd_volinfo_t *voliter = NULL; glusterd_conf_t *priv = NULL; - if (!volume_id) + if (!volume_id) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); return -1; + } this = THIS; priv = this->private; @@ -1881,8 +1930,11 @@ glusterd_brick_connect(glusterd_volinfo_t *volinfo, * connections is too long for unix domain socket connections. */ options = dict_new(); - if (!options) + if (!options) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, + NULL); goto out; + } ret = rpc_transport_unix_options_build(options, socketpath, 600); if (ret) @@ -2025,8 +2077,8 @@ glusterd_volume_start_glusterfs(glusterd_volinfo_t *volinfo, retry: runinit(&runner); - if (this->ctx->cmd_args.valgrind) { - /* Run bricks with valgrind */ + if (this->ctx->cmd_args.vgtool != _gf_none) { + /* Run bricks with valgrind. */ if (volinfo->logdir) { len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-%s.log", volinfo->logdir, volinfo->volname, exp_path); @@ -2040,8 +2092,13 @@ retry: goto out; } - runner_add_args(&runner, "valgrind", "--leak-check=full", - "--trace-children=yes", "--track-origins=yes", NULL); + if (this->ctx->cmd_args.vgtool == _gf_memcheck) + runner_add_args(&runner, "valgrind", "--leak-check=full", + "--trace-children=yes", "--track-origins=yes", + NULL); + else + runner_add_args(&runner, "valgrind", "--tool=drd", NULL); + runner_argprintf(&runner, "--log-file=%s", valgrind_logfile); } @@ -2154,7 +2211,7 @@ retry: if (is_brick_mx_enabled()) runner_add_arg(&runner, "--brick-mux"); - runner_log(&runner, "", 0, "Starting GlusterFS"); + runner_log(&runner, "", GF_LOG_DEBUG, "Starting GlusterFS"); brickinfo->port = port; brickinfo->rdma_port = rdma_port; @@ -2163,7 +2220,10 @@ retry: if (wait) { synclock_unlock(&priv->big_lock); + errno = 0; ret = runner_run(&runner); + if (errno != 0) + ret = errno; synclock_lock(&priv->big_lock); if (ret == EADDRINUSE) { @@ -2745,6 +2805,15 @@ glusterd_volume_compute_cksum(glusterd_volinfo_t *volinfo, char *cksum_path, ret = -1; goto out; } + } else if (priv->op_version < GD_OP_VERSION_7_0) { + ret = get_checksum_for_path(filepath, &cksum, priv->op_version); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_GET_FAIL, + "unable to get " + "checksum for path: %s", + filepath); + goto out; + } } ret = get_checksum_for_file(fd, &cksum, priv->op_version); @@ -2864,13 +2933,19 @@ glusterd_add_bricks_hname_path_to_dict(dict_t *dict, { ret = snprintf(key, sizeof(key), "%d-hostname", index); ret = dict_set_strn(dict, key, ret, brickinfo->hostname); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = snprintf(key, sizeof(key), "%d-path", index); ret = dict_set_strn(dict, key, ret, brickinfo->path); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } index++; } @@ -2998,11 +3073,16 @@ glusterd_add_volume_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, goto out; ret = gd_add_vol_snap_details_to_dict(dict, pfx, volinfo); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "vol snap details", NULL); goto out; + } volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id)); if (!volume_id_str) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "volume id=%s", volinfo->volume_id, NULL); ret = -1; goto out; } @@ -3035,6 +3115,8 @@ glusterd_add_volume_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, rebalance_id_str = gf_strdup(uuid_utoa(volinfo->rebal.rebalance_id)); if (!rebalance_id_str) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "rebalance_id=%s", volinfo->rebal.rebalance_id, NULL); ret = -1; goto out; } @@ -3186,6 +3268,9 @@ out: GF_FREE(rebalance_id_str); GF_FREE(rb_id_str); + if (key[0] != '\0' && ret != 0) + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); gf_msg_debug(this->name, 0, "Returning with %d", ret); return ret; } @@ -3244,29 +3329,44 @@ glusterd_vol_add_quota_conf_to_dict(glusterd_volinfo_t *volinfo, dict_t *load, snprintf(key, sizeof(key) - 1, "%s.gfid%d", key_prefix, gfid_idx); ret = dict_set_dynstr_with_alloc(load, key, uuid_utoa(buf)); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } snprintf(key, sizeof(key) - 1, "%s.gfid-type%d", key_prefix, gfid_idx); ret = dict_set_int8(load, key, type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } ret = snprintf(key, sizeof(key), "%s.gfid-count", key_prefix); ret = dict_set_int32n(load, key, ret, gfid_idx); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } snprintf(key, sizeof(key), "%s.quota-cksum", key_prefix); ret = dict_set_uint32(load, key, volinfo->quota_conf_cksum); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } snprintf(key, sizeof(key), "%s.quota-version", key_prefix); ret = dict_set_uint32(load, key, volinfo->quota_conf_version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = 0; out: @@ -3599,8 +3699,11 @@ glusterd_compare_friend_volume(dict_t *peer_data, int32_t count, snprintf(key_prefix, sizeof(key_prefix), "volume%d", count); keylen = snprintf(key, sizeof(key), "%s.name", key_prefix); ret = dict_get_strn(peer_data, key, keylen, &volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { @@ -3617,8 +3720,11 @@ glusterd_compare_friend_volume(dict_t *peer_data, int32_t count, keylen = snprintf(key, sizeof(key), "%s.version", key_prefix); ret = dict_get_int32n(peer_data, key, keylen, &version); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } if (version > volinfo->version) { // Mismatch detected @@ -3626,6 +3732,7 @@ glusterd_compare_friend_volume(dict_t *peer_data, int32_t count, "Version of volume %s differ. local version = %d, " "remote version = %d on peer %s", volinfo->volname, volinfo->version, version, hostname); + GF_ATOMIC_INIT(volinfo->volpeerupdate, 1); *status = GLUSTERD_VOL_COMP_UPDATE_REQ; goto out; } else if (version < volinfo->version) { @@ -3637,8 +3744,11 @@ glusterd_compare_friend_volume(dict_t *peer_data, int32_t count, // snprintf(key, sizeof(key), "%s.ckusm", key_prefix); ret = dict_get_uint32(peer_data, key, &cksum); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } if (cksum != volinfo->cksum) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_VERS_MISMATCH, @@ -4164,8 +4274,11 @@ glusterd_import_quota_conf(dict_t *peer_data, int vol_idx, keylen = snprintf(key, sizeof(key), "%s.gfid-count", key_prefix); ret = dict_get_int32n(peer_data, key, keylen, &gfid_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = glusterd_quota_conf_write_header(fd); if (ret) @@ -4175,8 +4288,11 @@ glusterd_import_quota_conf(dict_t *peer_data, int vol_idx, keylen = snprintf(key, sizeof(key) - 1, "%s.gfid%d", key_prefix, gfid_idx); ret = dict_get_strn(peer_data, key, keylen, &gfid_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } snprintf(key, sizeof(key) - 1, "%s.gfid-type%d", key_prefix, gfid_idx); ret = dict_get_int8(peer_data, key, &gfid_type); @@ -4237,18 +4353,23 @@ gd_import_friend_volume_rebal_dict(dict_t *dict, int count, GF_ASSERT(dict); GF_ASSERT(volinfo); + xlator_t *this = THIS; + GF_ASSERT(this); snprintf(key_prefix, sizeof(key_prefix), "volume%d", count); ret = snprintf(key, sizeof(key), "%s.rebal-dict-count", key_prefix); ret = dict_get_int32n(dict, key, ret, &dict_count); if (ret) { /* Older peers will not have this dict */ + gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); ret = 0; goto out; } volinfo->rebal.dict = dict_new(); if (!volinfo->rebal.dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -4258,7 +4379,7 @@ gd_import_friend_volume_rebal_dict(dict_t *dict, int count, out: if (ret && volinfo->rebal.dict) dict_unref(volinfo->rebal.dict); - gf_msg_debug(THIS->name, 0, "Returning with %d", ret); + gf_msg_debug(this->name, 0, "Returning with %d", ret); return ret; } @@ -4746,7 +4867,7 @@ glusterd_volinfo_stop_stale_bricks(glusterd_volinfo_t *new_volinfo, * brick multiplexing enabled, then stop the brick process */ if (ret || (new_brickinfo->snap_status == -1) || - is_brick_mx_enabled()) { + GF_ATOMIC_GET(old_volinfo->volpeerupdate)) { /*TODO: may need to switch to 'atomic' flavour of * brick_stop, once we make peer rpc program also * synctask enabled*/ @@ -4915,8 +5036,15 @@ glusterd_import_friend_volume(dict_t *peer_data, int count) ret = snprintf(key, sizeof(key), "volume%d.update", count); ret = dict_get_int32n(peer_data, key, ret, &update); - if (ret || !update) { + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); + goto out; + } + + if (!update) { /* if update is 0 that means the volume is not imported */ + gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_VOLUME_NOT_IMPORTED, NULL); goto out; } @@ -4964,6 +5092,13 @@ glusterd_import_friend_volume(dict_t *peer_data, int count) goto out; } + ret = glusterd_create_volfiles(new_volinfo); + if (ret) + goto out; + + glusterd_list_add_order(&new_volinfo->vol_list, &priv->volumes, + glusterd_compare_volume_name); + if (glusterd_is_volume_started(new_volinfo)) { (void)glusterd_start_bricks(new_volinfo); if (glusterd_is_snapd_enabled(new_volinfo)) { @@ -4978,19 +5113,14 @@ glusterd_import_friend_volume(dict_t *peer_data, int count) } } - ret = glusterd_create_volfiles_and_notify_services(new_volinfo); - if (ret) - goto out; - ret = glusterd_import_quota_conf(peer_data, count, new_volinfo, "volume"); if (ret) { gf_event(EVENT_IMPORT_QUOTA_CONF_FAILED, "volume=%s", new_volinfo->volname); goto out; } - glusterd_list_add_order(&new_volinfo->vol_list, &priv->volumes, - glusterd_compare_volume_name); + ret = glusterd_fetchspec_notify(this); out: gf_msg_debug("glusterd", 0, "Returning with ret: %d", ret); return ret; @@ -5018,18 +5148,25 @@ glusterd_import_friend_volumes_synctask(void *opaque) goto out; peer_data = dict_new(); - if (!peer_data) + if (!peer_data) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_unserialize(arg->dict_buf, arg->dictlen, &peer_data); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_UNSERIALIZE_FAIL, + NULL); errno = ENOMEM; goto out; } ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); goto out; + } synclock_lock(&conf->big_lock); @@ -5038,22 +5175,22 @@ glusterd_import_friend_volumes_synctask(void *opaque) * restarted (refer glusterd_restart_bricks ()) */ while (conf->restart_bricks) { - synclock_unlock(&conf->big_lock); - sleep(2); - synclock_lock(&conf->big_lock); + synccond_wait(&conf->cond_restart_bricks, &conf->big_lock); } conf->restart_bricks = _gf_true; while (i <= count) { ret = glusterd_import_friend_volume(peer_data, i); if (ret) { - conf->restart_bricks = _gf_false; - goto out; + break; } i++; } - glusterd_svcs_manager(NULL); + if (i > count) { + glusterd_svcs_manager(NULL); + } conf->restart_bricks = _gf_false; + synccond_broadcast(&conf->cond_restart_bricks); out: if (peer_data) dict_unref(peer_data); @@ -5077,8 +5214,11 @@ glusterd_import_friend_volumes(dict_t *peer_data) GF_ASSERT(peer_data); ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); goto out; + } while (i <= count) { ret = glusterd_import_friend_volume(peer_data, i); @@ -5097,11 +5237,16 @@ glusterd_get_global_server_quorum_ratio(dict_t *opts, double *quorum) { int ret = -1; char *quorum_str = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_strn(opts, GLUSTERD_QUORUM_RATIO_KEY, SLEN(GLUSTERD_QUORUM_RATIO_KEY), &quorum_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GLUSTERD_QUORUM_RATIO_KEY, NULL); goto out; + } ret = gf_string2percent(quorum_str, quorum); if (ret) @@ -5116,11 +5261,16 @@ glusterd_get_global_opt_version(dict_t *opts, uint32_t *version) { int ret = -1; char *version_str = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_strn(opts, GLUSTERD_GLOBAL_OPT_VERSION, SLEN(GLUSTERD_GLOBAL_OPT_VERSION), &version_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL); goto out; + } ret = gf_string2uint(version_str, version); if (ret) @@ -5169,13 +5319,17 @@ glusterd_import_global_opts(dict_t *friend_data) SLEN("global-opt-count"), &count); if (ret) { // old version peer + gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED, + "Key=global-opt-count", NULL); ret = 0; goto out; } import_options = dict_new(); - if (!import_options) + if (!import_options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = import_prdict_dict(friend_data, import_options, "key", "val", count, "global"); if (ret) { @@ -5246,8 +5400,11 @@ glusterd_compare_friend_data(dict_t *peer_data, int32_t *status, char *hostname) } ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); goto out; + } while (i <= count) { ret = glusterd_compare_friend_volume(peer_data, i, status, hostname); @@ -5515,13 +5672,19 @@ glusterd_add_node_to_dict(char *server, dict_t *dict, int count, else if (!strcmp(server, priv->scrub_svc.name)) ret = dict_set_nstrn(dict, key, keylen, "Scrubber Daemon", SLEN("Scrubber Daemon")); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "brick%d.path", count); ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(uuid_utoa(MY_UUID))); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } #ifdef BUILD_GNFS /* Port is available only for the NFS server. @@ -5532,26 +5695,38 @@ glusterd_add_node_to_dict(char *server, dict_t *dict, int count, if (dict_getn(vol_opts, "nfs.port", SLEN("nfs.port"))) { ret = dict_get_int32n(vol_opts, "nfs.port", SLEN("nfs.port"), &port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=nfs.port", NULL); goto out; + } } else port = GF_NFS3_PORT; } #endif keylen = snprintf(key, sizeof(key), "brick%d.port", count); ret = dict_set_int32n(dict, key, keylen, port); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "brick%d.pid", count); ret = dict_set_int32n(dict, key, keylen, pid); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "brick%d.status", count); ret = dict_set_int32n(dict, key, keylen, running); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } out: gf_msg_debug(THIS->name, 0, "Returning %d", ret); @@ -5733,7 +5908,9 @@ my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame) call_frame_t *frame = v_frame; glusterd_conf_t *conf = frame->this->private; - GF_ATOMIC_DEC(conf->blockers); + if (GF_ATOMIC_DEC(conf->blockers) == 0) { + synccond_broadcast(&conf->cond_blockers); + } STACK_DESTROY(frame->root); return 0; @@ -5835,7 +6012,9 @@ attach_brick_callback(struct rpc_req *req, struct iovec *iov, int count, } } out: - GF_ATOMIC_DEC(conf->blockers); + if (GF_ATOMIC_DEC(conf->blockers) == 0) { + synccond_broadcast(&conf->cond_blockers); + } STACK_DESTROY(frame->root); return 0; } @@ -5891,12 +6070,15 @@ send_attach_req(xlator_t *this, struct rpc_clnt *rpc, char *path, iobref = iobref_new(); if (!iobref) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto *errlbl; } errlbl = &&free_iobref; frame = create_frame(this, this->ctx->pool); if (!frame) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL, + NULL); goto *errlbl; } @@ -5927,7 +6109,6 @@ send_attach_req(xlator_t *this, struct rpc_clnt *rpc, char *path, GF_ATOMIC_INC(conf->blockers); ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL); - return ret; free_iobref: iobref_unref(iobref); @@ -5936,7 +6117,7 @@ maybe_free_iobuf: iobuf_unref(iobuf); } err: - return -1; + return ret; } extern size_t @@ -6020,7 +6201,7 @@ attach_brick(xlator_t *this, glusterd_brickinfo_t *brickinfo, * TBD: see if there's a better way */ synclock_unlock(&conf->big_lock); - sleep(1); + synctask_sleep(1); synclock_lock(&conf->big_lock); } @@ -6160,7 +6341,7 @@ find_compat_brick_in_vol(glusterd_conf_t *conf, "brick %s is still" " starting, waiting for 2 seconds ", other_brick->path); - sleep(2); + synctask_sleep(2); synclock_lock(&conf->big_lock); retries--; } @@ -6264,7 +6445,6 @@ find_compatible_brick(glusterd_conf_t *conf, glusterd_volinfo_t *volinfo, int glusterd_get_sock_from_brick_pid(int pid, char *sockpath, size_t len) { - char fname[128] = ""; char buf[1024] = ""; char cmdline[2048] = ""; xlator_t *this = NULL; @@ -6279,6 +6459,22 @@ glusterd_get_sock_from_brick_pid(int pid, char *sockpath, size_t len) this = THIS; GF_ASSERT(this); +#ifdef __FreeBSD__ + blen = sizeof(buf); + int mib[4]; + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_ARGS; + mib[3] = pid; + + if (sys_sysctl(mib, 4, buf, &blen, NULL, blen) != 0) { + gf_log(this->name, GF_LOG_ERROR, "brick process %d is not running", + pid); + return ret; + } +#else + char fname[128] = ""; snprintf(fname, sizeof(fname), "/proc/%d/cmdline", pid); if (sys_access(fname, R_OK) != 0) { @@ -6295,6 +6491,7 @@ glusterd_get_sock_from_brick_pid(int pid, char *sockpath, size_t len) strerror(errno), fname); return ret; } +#endif /* convert cmdline to single string */ for (i = 0, j = 0; i < blen; i++) { @@ -6343,6 +6540,43 @@ glusterd_get_sock_from_brick_pid(int pid, char *sockpath, size_t len) char * search_brick_path_from_proc(pid_t brick_pid, char *brickpath) { + char *brick_path = NULL; +#ifdef __FreeBSD__ + struct filestat *fst; + struct procstat *ps; + struct kinfo_proc *kp; + struct filestat_list *head; + + ps = procstat_open_sysctl(); + if (ps == NULL) + goto out; + + kp = kinfo_getproc(brick_pid); + if (kp == NULL) + goto out; + + head = procstat_getfiles(ps, (void *)kp, 0); + if (head == NULL) + goto out; + + STAILQ_FOREACH(fst, head, next) + { + if (fst->fs_fd < 0) + continue; + + if (!strcmp(fst->fs_path, brickpath)) { + brick_path = gf_strdup(fst->fs_path); + break; + } + } + +out: + if (head != NULL) + procstat_freefiles(ps, head); + if (kp != NULL) + free(kp); + procstat_close(ps); +#else struct dirent *dp = NULL; DIR *dirp = NULL; size_t len = 0; @@ -6353,7 +6587,6 @@ search_brick_path_from_proc(pid_t brick_pid, char *brickpath) 0, }, }; - char *brick_path = NULL; if (!brickpath) goto out; @@ -6389,7 +6622,9 @@ search_brick_path_from_proc(pid_t brick_pid, char *brickpath) } } out: - sys_closedir(dirp); + if (dirp) + sys_closedir(dirp); +#endif return brick_path; } @@ -6417,8 +6652,10 @@ glusterd_brick_start(glusterd_volinfo_t *volinfo, GF_ASSERT(this); conf = this->private; - if ((!brickinfo) || (!volinfo)) + if ((!brickinfo) || (!volinfo)) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } if (gf_uuid_is_null(brickinfo->uuid)) { ret = glusterd_resolve_brick(brickinfo); @@ -6443,7 +6680,8 @@ glusterd_brick_start(glusterd_volinfo_t *volinfo, * three different triggers for an attempt to start the brick process * due to the quorum handling code in glusterd_friend_sm. */ - if (brickinfo->status == GF_BRICK_STARTING || brickinfo->start_triggered) { + if (brickinfo->status == GF_BRICK_STARTING || brickinfo->start_triggered || + GF_ATOMIC_GET(volinfo->volpeerupdate)) { gf_msg_debug(this->name, 0, "brick %s is already in starting " "phase", @@ -6649,9 +6887,7 @@ glusterd_restart_bricks(void *opaque) * glusterd_compare_friend_data ()) */ while (conf->restart_bricks) { - synclock_unlock(&conf->big_lock); - sleep(2); - synclock_lock(&conf->big_lock); + synccond_wait(&conf->cond_restart_bricks, &conf->big_lock); } conf->restart_bricks = _gf_true; @@ -6765,9 +7001,12 @@ glusterd_restart_bricks(void *opaque) ret = 0; out: - GF_ATOMIC_DEC(conf->blockers); conf->restart_done = _gf_true; conf->restart_bricks = _gf_false; + if (GF_ATOMIC_DEC(conf->blockers) == 0) { + synccond_broadcast(&conf->cond_blockers); + } + synccond_broadcast(&conf->cond_restart_bricks); return_block: return ret; @@ -7105,22 +7344,26 @@ glusterd_get_brick_root(char *path, char **mount_point) char *mnt_pt = NULL; struct stat brickstat = {0}; struct stat buf = {0}; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!path) + if (!path) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto err; + } mnt_pt = gf_strdup(path); - if (!mnt_pt) + if (!mnt_pt) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto err; + } if (sys_stat(mnt_pt, &brickstat)) goto err; while ((ptr = strrchr(mnt_pt, '/')) && ptr != mnt_pt) { *ptr = '\0'; if (sys_stat(mnt_pt, &buf)) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, - "error in " - "stat: %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, + "Error in stat=%s", strerror(errno), NULL); goto err; } @@ -7132,10 +7375,8 @@ glusterd_get_brick_root(char *path, char **mount_point) if (ptr == mnt_pt) { if (sys_stat("/", &buf)) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, - "error in " - "stat: %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, + "Error in stat=%s", strerror(errno), NULL); goto err; } if (brickstat.st_dev == buf.st_dev) @@ -7200,11 +7441,16 @@ glusterd_add_inode_size_to_dict(dict_t *dict, int count) }; struct fs_info *fs = NULL; static dict_t *cached_fs = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = snprintf(key, sizeof(key), "brick%d.device", count); ret = dict_get_strn(dict, key, ret, &device); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } if (cached_fs) { if (dict_get_str(cached_fs, device, &cur_word) == 0) { @@ -7216,8 +7462,11 @@ glusterd_add_inode_size_to_dict(dict_t *dict, int count) ret = snprintf(key, sizeof(key), "brick%d.fs_name", count); ret = dict_get_strn(dict, key, ret, &fs_name); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } runinit(&runner); runner_redir(&runner, STDOUT_FILENO, RUN_PIPE); @@ -7226,11 +7475,9 @@ glusterd_add_inode_size_to_dict(dict_t *dict, int count) if (strcmp(fs_name, fs->fs_type_name) == 0) { if (!fs->fs_tool_name) { /* dynamic inodes */ - gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_INODE_SIZE_GET_FAIL, - "the " - "brick on %s (%s) uses dynamic inode " - "sizes", - device, fs_name); + gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_INODE_SIZE_GET_FAIL, + "The brick on device uses dynamic inode sizes", + "Device=%s (%s)", device, fs_name, NULL); cur_word = "N/A"; goto cached; } @@ -7244,19 +7491,17 @@ glusterd_add_inode_size_to_dict(dict_t *dict, int count) runner_add_arg(&runner, fs->fs_tool_arg); runner_add_arg(&runner, device); } else { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, - "could not find %s to get" - "inode size for %s (%s): %s package missing?", - fs->fs_tool_name, device, fs_name, fs->fs_tool_pkg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, + "Could not find tool to get inode size for device", "Tool=%s", + fs->fs_tool_name, "Device=%s (%s)", device, fs_name, + "Missing package=%s ?", fs->fs_tool_pkg, NULL); goto out; } ret = runner_start(&runner); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL, - "failed to execute " - "\"%s\"", - fs->fs_tool_name); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL, + "Failed to execute \"%s\"", fs->fs_tool_name, NULL); /* * Runner_start might return an error after the child has * been forked, e.g. if the program isn't there. In that @@ -7284,21 +7529,22 @@ glusterd_add_inode_size_to_dict(dict_t *dict, int count) ret = runner_end(&runner); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL, - "%s exited with non-zero exit status", fs->fs_tool_name); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL, + "Tool exited with non-zero exit status", "Tool=%s", + fs->fs_tool_name, NULL); goto out; } if (!cur_word) { ret = -1; - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, - "Unable to retrieve inode size using %s", fs->fs_tool_name); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, + "Using Tool=%s", fs->fs_tool_name, NULL); goto out; } if (dict_set_dynstr_with_alloc(cached_fs, device, cur_word)) { /* not fatal if not entered into the cache */ - gf_msg_debug(THIS->name, 0, "failed to cache fs inode size for %s", + gf_msg_debug(this->name, 0, "failed to cache fs inode size for %s", device); } @@ -7309,8 +7555,7 @@ cached: out: if (ret) - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, - "failed to get inode size"); + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, NULL); return ret; } @@ -7366,16 +7611,23 @@ glusterd_add_brick_mount_details(glusterd_brickinfo_t *brickinfo, dict_t *dict, struct mntent save_entry = {0}; char *mnt_pt = NULL; struct mntent *entry = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); snprintf(base_key, sizeof(base_key), "brick%d", count); ret = glusterd_get_brick_root(brickinfo->path, &mnt_pt); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICKPATH_ROOT_GET_FAIL, + NULL); goto out; + } entry = glusterd_get_mnt_entry_info(mnt_pt, buff, sizeof(buff), &save_entry); if (!entry) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GET_MNT_ENTRY_INFO_FAIL, + NULL); ret = -1; goto out; } @@ -7384,15 +7636,21 @@ glusterd_add_brick_mount_details(glusterd_brickinfo_t *brickinfo, dict_t *dict, snprintf(key, sizeof(key), "%s.device", base_key); ret = dict_set_dynstr_with_alloc(dict, key, entry->mnt_fsname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } /* fs type */ snprintf(key, sizeof(key), "%s.fs_name", base_key); ret = dict_set_dynstr_with_alloc(dict, key, entry->mnt_type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } /* mount options */ snprintf(key, sizeof(key), "%s.mnt_options", base_key); @@ -7486,43 +7744,61 @@ glusterd_add_brick_detail_to_dict(glusterd_volinfo_t *volinfo, block_size = brickstat.f_bsize; snprintf(key, sizeof(key), "%s.block_size", base_key); ret = dict_set_uint64(dict, key, block_size); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } /* free space in brick */ memfree = brickstat.f_bfree * brickstat.f_bsize; snprintf(key, sizeof(key), "%s.free", base_key); ret = dict_set_uint64(dict, key, memfree); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } /* total space of brick */ memtotal = brickstat.f_blocks * brickstat.f_bsize; snprintf(key, sizeof(key), "%s.total", base_key); ret = dict_set_uint64(dict, key, memtotal); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } /* inodes: total and free counts only for ext2/3/4 and xfs */ inodes_total = brickstat.f_files; if (inodes_total) { snprintf(key, sizeof(key), "%s.total_inodes", base_key); ret = dict_set_uint64(dict, key, inodes_total); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } inodes_free = brickstat.f_ffree; if (inodes_free) { snprintf(key, sizeof(key), "%s.free_inodes", base_key); ret = dict_set_uint64(dict, key, inodes_free); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } ret = glusterd_add_brick_mount_details(brickinfo, dict, count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_ADD_BRICK_MNT_INFO_FAIL, + NULL); goto out; + } ret = glusterd_add_inode_size_to_dict(dict, count); out: @@ -7628,8 +7904,11 @@ glusterd_add_brick_to_dict(glusterd_volinfo_t *volinfo, ret = dict_set_int32n(dict, key, keylen, brick_online); out: - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); gf_msg_debug(this->name, 0, "Returning %d", ret); + } return ret; } @@ -7710,8 +7989,10 @@ glusterd_brick_stop(glusterd_volinfo_t *volinfo, conf = this->private; GF_ASSERT(conf); - if ((!brickinfo) || (!volinfo)) + if ((!brickinfo) || (!volinfo)) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } if (gf_uuid_is_null(brickinfo->uuid)) { ret = glusterd_resolve_brick(brickinfo); @@ -7854,8 +8135,10 @@ glusterd_rb_check_bricks(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *src, rb = &volinfo->rep_brick; - if (!rb->src_brick || !rb->dst_brick) + if (!rb->src_brick || !rb->dst_brick) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); return -1; + } if (strcmp(rb->src_brick->hostname, src->hostname) || strcmp(rb->src_brick->path, src->path)) { @@ -8001,6 +8284,8 @@ glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid, char msg[2048] = ""; gf_boolean_t in_use = _gf_false; int flags = 0; + xlator_t *this = THIS; + GF_ASSERT(this); /* Check for xattr support in backend fs */ ret = sys_lsetxattr(path, "trusted.glusterfs.test", "working", 8, 0); @@ -8011,6 +8296,8 @@ glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid, " extended attributes failed, reason:" " %s.", host, path, strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_BRICK_FAIL, + "Host=%s, Path=%s", host, path, NULL); goto out; } else { @@ -8020,6 +8307,8 @@ glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid, "Removing test extended" " attribute failed, reason: %s", strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_REMOVE_XATTR_FAIL, + NULL); goto out; } } @@ -8042,6 +8331,8 @@ glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid, "Failed to set extended " "attributes %s, reason: %s", GF_XATTR_VOL_ID_KEY, strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL, + "Attriutes=%s", GF_XATTR_VOL_ID_KEY, NULL); goto out; } @@ -8061,7 +8352,7 @@ glusterd_sm_tr_log_transition_add_to_dict(dict_t *dict, int ret = -1; char key[64] = ""; int keylen; - char timestr[64] = ""; + char timestr[GF_TIMESTR_SIZE] = ""; char *str = NULL; GF_ASSERT(dict); @@ -8093,6 +8384,9 @@ glusterd_sm_tr_log_transition_add_to_dict(dict_t *dict, goto out; out: + if (key[0] != '\0' && ret != 0) + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); gf_msg_debug("glusterd", 0, "returning %d", ret); return ret; } @@ -8201,7 +8495,8 @@ glusterd_sm_tr_log_transition_add(glusterd_sm_tr_log_t *log, int old_state, transitions[next].old_state = old_state; transitions[next].new_state = new_state; transitions[next].event = event; - time(&transitions[next].time); + transitions[next].time = gf_time(); + log->current = next; if (log->count < log->size) log->count++; @@ -8317,8 +8612,10 @@ glusterd_get_local_brickpaths(glusterd_volinfo_t *volinfo, char **pathlist) int i = 0; glusterd_brickinfo_t *brickinfo = NULL; - if ((!volinfo) || (!pathlist)) + if ((!volinfo) || (!pathlist)) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } path_tokens = GF_CALLOC(sizeof(char *), volinfo->brick_count, gf_gld_mt_charptr); @@ -8772,6 +9069,8 @@ glusterd_nfs_statedump(char *options, int option_cnt, char **op_errstr) snprintf(msg, sizeof(msg), "for nfs statedump, options should" " be after the key nfs"); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, + "Options misplaced", NULL); *op_errstr = gf_strdup(msg); ret = -1; goto out; @@ -8839,6 +9138,8 @@ glusterd_client_statedump(char *volname, char *options, int option_cnt, dup_options = gf_strdup(options); if (!dup_options) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, + "options=%s", options, NULL); goto out; } option = strtok_r(dup_options, " ", &tmpptr); @@ -8846,6 +9147,8 @@ glusterd_client_statedump(char *volname, char *options, int option_cnt, snprintf(msg, sizeof(msg), "for gluster client statedump, options " "should be after the key 'client'"); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, + "Options misplaced", NULL); *op_errstr = gf_strdup(msg); ret = -1; goto out; @@ -8853,6 +9156,8 @@ glusterd_client_statedump(char *volname, char *options, int option_cnt, target_ip = strtok_r(NULL, " ", &tmpptr); if (target_ip == NULL) { snprintf(msg, sizeof(msg), "ip address not specified"); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, msg, + NULL); *op_errstr = gf_strdup(msg); ret = -1; goto out; @@ -8861,6 +9166,8 @@ glusterd_client_statedump(char *volname, char *options, int option_cnt, pid = strtok_r(NULL, " ", &tmpptr); if (pid == NULL) { snprintf(msg, sizeof(msg), "pid not specified"); + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, msg, + NULL); *op_errstr = gf_strdup(msg); ret = -1; goto out; @@ -8901,6 +9208,8 @@ glusterd_quotad_statedump(char *options, int option_cnt, char **op_errstr) snprintf(msg, sizeof(msg), "for quotad statedump, options " "should be after the key 'quotad'"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, + "Options misplaced", NULL); *op_errstr = gf_strdup(msg); ret = -1; goto out; @@ -9763,6 +10072,8 @@ glusterd_append_gsync_status(dict_t *dst, dict_t *src) ret = dict_get_strn(src, "gsync-status", SLEN("gsync-status"), &stop_msg); if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=gsync-status", NULL); ret = 0; goto out; } @@ -10017,8 +10328,11 @@ glusterd_sync_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict) int ret = 0; GF_ASSERT(rsp_dict); + xlator_t *this = THIS; + GF_ASSERT(this); if (!rsp_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; } @@ -10066,6 +10380,8 @@ glusterd_profile_volume_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict) ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &brick_count); if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=count", NULL); ret = 0; // no bricks in the rsp goto out; } @@ -10345,6 +10661,8 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) glusterd_volinfo_t *volinfo = NULL; GF_ASSERT(rsp_dict); + xlator_t *this = THIS; + GF_ASSERT(this); if (aggr) { ctx_dict = aggr; @@ -10354,8 +10672,11 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) } ret = dict_get_int32n(ctx_dict, "cmd", SLEN("cmd"), &cmd); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=cmd", + NULL); goto out; + } if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd(ctx_dict)) { ret = dict_get_int32n(rsp_dict, "vol_count", SLEN("vol_count"), @@ -10363,18 +10684,27 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) if (ret == 0) { ret = dict_set_int32n(ctx_dict, "vol_count", SLEN("vol_count"), vol_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=vol_count", NULL); goto out; + } for (i = 0; i < vol_count; i++) { keylen = snprintf(key, sizeof(key), "vol%d", i); ret = dict_get_strn(rsp_dict, key, keylen, &volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=%s", key, NULL); goto out; + } ret = dict_set_strn(ctx_dict, key, keylen, volname); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=%s", key, NULL); goto out; + } } } else { /* Ignore the error as still the aggregation applies in @@ -10388,6 +10718,8 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &rsp_node_count); if (ret) { + gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED, "Key=count", + NULL); ret = 0; // no bricks in the rsp goto out; } @@ -10395,8 +10727,8 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) ret = dict_get_int32n(rsp_dict, "other-count", SLEN("other-count"), &rsp_other_count); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get other count from rsp_dict"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=other-count", NULL); goto out; } @@ -10406,18 +10738,27 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) if (!dict_getn(ctx_dict, "brick-index-max", SLEN("brick-index-max"))) { ret = dict_get_int32n(rsp_dict, "brick-index-max", SLEN("brick-index-max"), &brick_index_max); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=brick-index-max", NULL); goto out; + } ret = dict_set_int32n(ctx_dict, "brick-index-max", SLEN("brick-index-max"), brick_index_max); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=brick-index-max", NULL); goto out; + } } else { ret = dict_get_int32n(ctx_dict, "brick-index-max", SLEN("brick-index-max"), &brick_index_max); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=brick-index-max", NULL); goto out; + } } rsp_ctx.count = node_count; @@ -10430,45 +10771,45 @@ glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict) ret = dict_set_int32n(ctx_dict, "count", SLEN("count"), node_count + rsp_node_count); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Failed to update node count"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; } ret = dict_set_int32n(ctx_dict, "other-count", SLEN("other-count"), (other_count + rsp_other_count)); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Failed to update other-count"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=other-count", NULL); goto out; } ret = dict_get_strn(ctx_dict, "volname", SLEN("volname"), &volname); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, - "Failed to get volname"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=volname", NULL); goto out; } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, - "Failed to get volinfo for volume: %s", volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, + "Volume=%s", volname, NULL); goto out; } ret = dict_set_int32n(ctx_dict, "hot_brick_count", SLEN("hot_brick_count"), hot_brick_count); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, - "Failed to update hot_brick_count"); + gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=hot_brick_count", NULL); goto out; } ret = dict_set_int32n(ctx_dict, "type", SLEN("type"), type); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, - "Failed to update type"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=type", NULL); goto out; } @@ -12910,7 +13251,7 @@ glusterd_get_value_for_vme_entry(struct volopt_map_entry *vme, char **def_val) ret = xlator_option_info_list(&vol_opt_handle, key, &local_def_val, &descr); if (ret) { /*Swallow Error if option not found*/ - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_KEY_FAILED, + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GET_KEY_FAILED, "Failed to get option for %s " "key", key); @@ -13067,7 +13408,9 @@ glusterd_get_global_options_for_all_vols(rpcsvc_request_t *req, dict_t *ctx, gf_asprintf(&def_val, "%d", priv->op_version); need_free = _gf_true; } else { - def_val = valid_all_vol_opts[i].dflt_val; + gf_asprintf(&def_val, "%s (DEFAULT)", + valid_all_vol_opts[i].dflt_val); + need_free = _gf_true; } } @@ -13153,6 +13496,7 @@ glusterd_get_default_val_for_volopt(dict_t *ctx, gf_boolean_t all_opts, int count = 0; xlator_t *this = NULL; char *def_val = NULL; + char *def_val_str = NULL; char dict_key[50] = ""; int keylen; gf_boolean_t key_found = _gf_false; @@ -13213,7 +13557,13 @@ glusterd_get_default_val_for_volopt(dict_t *ctx, gf_boolean_t all_opts, goto out; } sprintf(dict_key, "value%d", count); - ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val); + if (get_value_vme) { // the value was never changed - DEFAULT is used + gf_asprintf(&def_val_str, "%s (DEFAULT)", def_val); + ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val_str); + GF_FREE(def_val_str); + def_val_str = NULL; + } else + ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Failed to " @@ -13738,23 +14088,24 @@ glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo, char vpath[PATH_MAX] = ""; char *volfileserver = NULL; - priv = THIS->private; - GF_VALIDATE_OR_GOTO(THIS->name, priv, out); + xlator_t *this = THIS; + GF_ASSERT(this); + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); dirty[2] = hton32(1); ret = sys_lsetxattr(brickinfo->path, GF_AFR_DIRTY, dirty, sizeof(dirty), 0); if (ret == -1) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_SETXATTR_FAIL, - "Failed to set extended" - " attribute %s : %s.", - GF_AFR_DIRTY, strerror(errno)); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL, + "Attribute=%s", GF_AFR_DIRTY, "Reason=%s", strerror(errno), + NULL); goto out; } if (mkdtemp(tmpmount) == NULL) { - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, - "failed to create a temporary mount directory."); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED, + NULL); ret = -1; goto out; } @@ -13765,7 +14116,7 @@ glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo, switch (op) { case GD_OP_REPLACE_BRICK: - if (dict_get_strn(THIS->options, "transport.socket.bind-address", + if (dict_get_strn(this->options, "transport.socket.bind-address", SLEN("transport.socket.bind-address"), &volfileserver) != 0) volfileserver = "localhost"; @@ -13808,7 +14159,7 @@ glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo, ret = runner_run(&runner); if (ret) { - gf_log(THIS->name, GF_LOG_ERROR, + gf_log(this->name, GF_LOG_ERROR, "mount command" " failed."); goto lock; @@ -13818,19 +14169,18 @@ glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo, (op == GD_OP_REPLACE_BRICK) ? GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK, brickinfo->brick_id, sizeof(brickinfo->brick_id), 0); if (ret == -1) - gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_SETXATTR_FAIL, - "Failed to set extended" - " attribute %s : %s", - (op == GD_OP_REPLACE_BRICK) ? GF_AFR_REPLACE_BRICK - : GF_AFR_ADD_BRICK, - strerror(errno)); - gf_umount_lazy(THIS->name, tmpmount, 1); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL, + "Attribute=%s, Reason=%s", + (op == GD_OP_REPLACE_BRICK) ? GF_AFR_REPLACE_BRICK + : GF_AFR_ADD_BRICK, + strerror(errno), NULL); + gf_umount_lazy(this->name, tmpmount, 1); lock: synclock_lock(&priv->big_lock); out: if (pid) GF_FREE(pid); - gf_msg_debug("glusterd", 0, "Returning with ret"); + gf_msg_debug(this->name, 0, "Returning with ret"); return ret; } @@ -14039,6 +14389,8 @@ glusterd_brick_op_prerequisites(dict_t *dict, char **op, glusterd_op_t *gd_op, "brick: %s does not exist in " "volume: %s", *src_brick, *volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND, + "Brick=%s, Volume=%s", *src_brick, *volname, NULL); *op_errstr = gf_strdup(msg); goto out; } @@ -14263,8 +14615,11 @@ glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, keylen = snprintf(key, sizeof(key), "brick%d.hostname", count); ret = dict_set_nstrn(dict, key, keylen, "Self-heal Daemon", SLEN("Self-heal Daemon")); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "brick%d.path", count); uuid_str = gf_strdup(uuid_utoa(MY_UUID)); @@ -14273,8 +14628,11 @@ glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, goto out; } ret = dict_set_dynstrn(dict, key, keylen, uuid_str); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } uuid_str = NULL; /* shd doesn't have a port. but the cli needs a port key with @@ -14283,8 +14641,11 @@ glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, keylen = snprintf(key, sizeof(key), "brick%d.port", count); ret = dict_set_int32n(dict, key, keylen, 0); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } pidfile = volinfo->shd.svc.proc.pidfile; @@ -14295,8 +14656,11 @@ glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, pid = -1; keylen = snprintf(key, sizeof(key), "brick%d.pid", count); ret = dict_set_int32n(dict, key, keylen, pid); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s", + key, NULL); goto out; + } keylen = snprintf(key, sizeof(key), "brick%d.status", count); ret = dict_set_int32n(dict, key, keylen, brick_online); @@ -14311,3 +14675,372 @@ out: return ret; } + +static gf_ai_compare_t +glusterd_compare_addrinfo(struct addrinfo *first, struct addrinfo *next) +{ + int ret = -1; + struct addrinfo *tmp1 = NULL; + struct addrinfo *tmp2 = NULL; + char firstip[NI_MAXHOST] = {0.}; + char nextip[NI_MAXHOST] = { + 0, + }; + + for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) { + ret = getnameinfo(tmp1->ai_addr, tmp1->ai_addrlen, firstip, NI_MAXHOST, + NULL, 0, NI_NUMERICHOST); + if (ret) + return GF_AI_COMPARE_ERROR; + for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) { + ret = getnameinfo(tmp2->ai_addr, tmp2->ai_addrlen, nextip, + NI_MAXHOST, NULL, 0, NI_NUMERICHOST); + if (ret) + return GF_AI_COMPARE_ERROR; + if (!strcmp(firstip, nextip)) { + return GF_AI_COMPARE_MATCH; + } + } + } + return GF_AI_COMPARE_NO_MATCH; +} + +/* Check for non optimal brick order for Replicate/Disperse : + * Checks if bricks belonging to a replicate or disperse + * volume are present on the same server + */ +int32_t +glusterd_check_brick_order(dict_t *dict, char *err_str, int32_t type, + char **volname, char **brick_list, + int32_t *brick_count, int32_t sub_count) +{ + int ret = -1; + int i = 0; + int j = 0; + int k = 0; + xlator_t *this = NULL; + addrinfo_list_t *ai_list = NULL; + addrinfo_list_t *ai_list_tmp1 = NULL; + addrinfo_list_t *ai_list_tmp2 = NULL; + char *brick = NULL; + char *brick_list_dup = NULL; + char *brick_list_ptr = NULL; + char *tmpptr = NULL; + struct addrinfo *ai_info = NULL; + char brick_addr[128] = { + 0, + }; + int addrlen = 0; + + const char failed_string[2048] = + "Failed to perform brick order " + "check. Use 'force' at the end of the command" + " if you want to override this behavior. "; + const char found_string[2048] = + "Multiple bricks of a %s " + "volume are present on the same server. This " + "setup is not optimal. Bricks should be on " + "different nodes to have best fault tolerant " + "configuration. Use 'force' at the end of the " + "command if you want to override this " + "behavior. "; + + this = THIS; + + GF_ASSERT(this); + + ai_list = MALLOC(sizeof(addrinfo_list_t)); + ai_list->info = NULL; + CDS_INIT_LIST_HEAD(&ai_list->list); + + if (!(*volname)) { + ret = dict_get_strn(dict, "volname", SLEN("volname"), &(*volname)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Unable to get volume name"); + goto out; + } + } + + if (!(*brick_list)) { + ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &(*brick_list)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Bricks check : Could not " + "retrieve bricks list"); + goto out; + } + } + + if (!(*brick_count)) { + ret = dict_get_int32n(dict, "count", SLEN("count"), &(*brick_count)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Bricks check : Could not " + "retrieve brick count"); + goto out; + } + } + + brick_list_dup = brick_list_ptr = gf_strdup(*brick_list); + /* Resolve hostnames and get addrinfo */ + while (i < *brick_count) { + ++i; + brick = strtok_r(brick_list_dup, " \n", &tmpptr); + brick_list_dup = tmpptr; + if (brick == NULL) + goto check_failed; + tmpptr = strrchr(brick, ':'); + if (tmpptr == NULL) + goto check_failed; + addrlen = strlen(brick) - strlen(tmpptr); + strncpy(brick_addr, brick, addrlen); + brick_addr[addrlen] = '\0'; + ret = getaddrinfo(brick_addr, NULL, NULL, &ai_info); + if (ret != 0) { + ret = 0; + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL, + "unable to resolve host name for addr %s", brick_addr); + goto out; + } + ai_list_tmp1 = MALLOC(sizeof(addrinfo_list_t)); + if (ai_list_tmp1 == NULL) { + ret = 0; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, + "failed to allocate " + "memory"); + freeaddrinfo(ai_info); + goto out; + } + ai_list_tmp1->info = ai_info; + cds_list_add_tail(&ai_list_tmp1->list, &ai_list->list); + ai_list_tmp1 = NULL; + } + + i = 0; + ai_list_tmp1 = cds_list_entry(ai_list->list.next, addrinfo_list_t, list); + + if (*brick_count < sub_count) { + sub_count = *brick_count; + } + + /* Check for bad brick order */ + while (i < *brick_count) { + ++i; + ai_info = ai_list_tmp1->info; + ai_list_tmp1 = cds_list_entry(ai_list_tmp1->list.next, addrinfo_list_t, + list); + if (0 == i % sub_count) { + j = 0; + continue; + } + ai_list_tmp2 = ai_list_tmp1; + k = j; + while (k < sub_count - 1) { + ++k; + ret = glusterd_compare_addrinfo(ai_info, ai_list_tmp2->info); + if (GF_AI_COMPARE_ERROR == ret) + goto check_failed; + if (GF_AI_COMPARE_MATCH == ret) + goto found_bad_brick_order; + ai_list_tmp2 = cds_list_entry(ai_list_tmp2->list.next, + addrinfo_list_t, list); + } + ++j; + } + gf_msg_debug(this->name, 0, "Brick order okay"); + ret = 0; + goto out; + +check_failed: + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER_CHECK_FAIL, + "Failed bad brick order check"); + snprintf(err_str, sizeof(failed_string), failed_string); + ret = -1; + goto out; + +found_bad_brick_order: + gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BAD_BRKORDER, + "Bad brick order found"); + if (type == GF_CLUSTER_TYPE_DISPERSE) { + snprintf(err_str, sizeof(found_string), found_string, "disperse"); + } else { + snprintf(err_str, sizeof(found_string), found_string, "replicate"); + } + + ret = -1; +out: + ai_list_tmp2 = NULL; + GF_FREE(brick_list_ptr); + cds_list_for_each_entry(ai_list_tmp1, &ai_list->list, list) + { + if (ai_list_tmp1->info) + freeaddrinfo(ai_list_tmp1->info); + free(ai_list_tmp2); + ai_list_tmp2 = ai_list_tmp1; + } + free(ai_list); + free(ai_list_tmp2); + gf_msg_debug("glusterd", 0, "Returning %d", ret); + return ret; +} + +static gf_boolean_t +search_peer_in_auth_list(char *peer_hostname, char *auth_allow_list) +{ + if (strstr(auth_allow_list, peer_hostname)) { + return _gf_true; + } + + return _gf_false; +} + +/* glusterd_add_peers_to_auth_list() adds peers into auth.allow list + * if auth.allow list is not empty. This is called for add-brick and + * replica brick operations to avoid failing the temporary mount. New + * volfiles will be generated and clients are notified reg new volfiles. + */ +void +glusterd_add_peers_to_auth_list(char *volname) +{ + int ret = 0; + glusterd_volinfo_t *volinfo = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + int32_t len = 0; + char *auth_allow_list = NULL; + char *new_auth_allow_list = NULL; + + this = THIS; + GF_ASSERT(this); + conf = this->private; + GF_ASSERT(conf); + + GF_VALIDATE_OR_GOTO(this->name, volname, out); + + ret = glusterd_volinfo_find(volname, &volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, + "Unable to find volume: %s", volname); + goto out; + } + + ret = dict_get_str_sizen(volinfo->dict, "auth.allow", &auth_allow_list); + if (ret) { + gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED, + "auth allow list is not set"); + goto out; + } + cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list) + { + len += strlen(peerinfo->hostname); + } + len += strlen(auth_allow_list) + 1; + + new_auth_allow_list = GF_CALLOC(1, len, gf_common_mt_char); + + new_auth_allow_list = strncat(new_auth_allow_list, auth_allow_list, + strlen(auth_allow_list)); + cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list) + { + ret = search_peer_in_auth_list(peerinfo->hostname, new_auth_allow_list); + if (!ret) { + gf_log(this->name, GF_LOG_DEBUG, + "peer %s not found in auth.allow list", peerinfo->hostname); + new_auth_allow_list = strcat(new_auth_allow_list, ","); + new_auth_allow_list = strncat(new_auth_allow_list, + peerinfo->hostname, + strlen(peerinfo->hostname)); + } + } + if (strcmp(new_auth_allow_list, auth_allow_list) != 0) { + /* In case, new_auth_allow_list is not same as auth_allow_list, + * we need to update the volinfo->dict with new_auth_allow_list. + * we delete the auth_allow_list and replace it with + * new_auth_allow_list. for reverting the changes in post commit, we + * keep the copy of auth_allow_list as old_auth_allow_list in + * volinfo->dict. + */ + dict_del_sizen(volinfo->dict, "auth.allow"); + ret = dict_set_strn(volinfo->dict, "auth.allow", SLEN("auth.allow"), + new_auth_allow_list); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Unable to set new auth.allow list"); + goto out; + } + ret = dict_set_strn(volinfo->dict, "old.auth.allow", + SLEN("old.auth.allow"), auth_allow_list); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Unable to set old auth.allow list"); + goto out; + } + ret = glusterd_create_volfiles_and_notify_services(volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL, + "failed to create volfiles"); + goto out; + } + } +out: + GF_FREE(new_auth_allow_list); + return; +} + +int +glusterd_replace_old_auth_allow_list(char *volname) +{ + int ret = 0; + glusterd_volinfo_t *volinfo = NULL; + xlator_t *this = NULL; + char *old_auth_allow_list = NULL; + + this = THIS; + GF_ASSERT(this); + + GF_VALIDATE_OR_GOTO(this->name, volname, out); + + ret = glusterd_volinfo_find(volname, &volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, + "Unable to find volume: %s", volname); + goto out; + } + + ret = dict_get_str_sizen(volinfo->dict, "old.auth.allow", + &old_auth_allow_list); + if (ret) { + gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED, + "old auth allow list is not set, no need to replace the list"); + ret = 0; + goto out; + } + + dict_del_sizen(volinfo->dict, "auth.allow"); + ret = dict_set_strn(volinfo->dict, "auth.allow", SLEN("auth.allow"), + old_auth_allow_list); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Unable to replace auth.allow list"); + goto out; + } + + dict_del_sizen(volinfo->dict, "old.auth.allow"); + + ret = glusterd_create_volfiles_and_notify_services(volinfo); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL, + "failed to create volfiles"); + goto out; + } + ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_STORE_FAIL, + "failed to store volinfo"); + goto out; + } +out: + return ret; +} diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index b58f158fd14..bf6ac295e26 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -857,4 +857,9 @@ search_brick_path_from_proc(pid_t brick_pid, char *brickpath); int32_t glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict, int32_t count); +int32_t +glusterd_check_brick_order(dict_t *dict, char *err_str, int32_t type, + char **volname, char **bricks, int32_t *brick_count, + int32_t sub_count); + #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index a085a0ff8d6..8d6fb5e0fac 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -90,6 +90,8 @@ xlator_instantiate_va(const char *type, const char *format, va_list arg) xlator_t *xl = NULL; char *volname = NULL; int ret = 0; + xlator_t *this = THIS; + GF_ASSERT(this); ret = gf_vasprintf(&volname, format, arg); if (ret < 0) { @@ -99,14 +101,21 @@ xlator_instantiate_va(const char *type, const char *format, va_list arg) } xl = GF_CALLOC(1, sizeof(*xl), gf_common_mt_xlator_t); - if (!xl) + if (!xl) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto error; + } ret = xlator_set_type_virtual(xl, type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_SET_OPT_FAIL, + NULL); goto error; + } xl->options = dict_new(); - if (!xl->options) + if (!xl->options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto error; + } xl->name = volname; CDS_INIT_LIST_HEAD(&xl->volume_options); @@ -115,8 +124,8 @@ xlator_instantiate_va(const char *type, const char *format, va_list arg) return xl; error: - gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XLATOR_CREATE_FAIL, - "creating xlator of type %s failed", type); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_CREATE_FAIL, "Type=%s", + type, NULL); GF_FREE(volname); if (xl) xlator_destroy(xl); @@ -865,6 +874,8 @@ _xl_link_children(xlator_t *parent, xlator_t *children, size_t child_count) xlator_t *trav = NULL; size_t seek = 0; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); if (child_count == 0) goto out; @@ -873,9 +884,12 @@ _xl_link_children(xlator_t *parent, xlator_t *children, size_t child_count) ; for (; child_count--; trav = trav->prev) { ret = volgen_xlator_link(parent, trav); - gf_msg_debug(THIS->name, 0, "%s:%s", parent->name, trav->name); - if (ret) + gf_msg_debug(this->name, 0, "%s:%s", parent->name, trav->name); + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_LINK_FAIL, + NULL); goto out; + } } ret = 0; out: @@ -933,8 +947,10 @@ volgen_apply_filters(char *orig_volfile) entry = sys_readdir(filterdir, scratch); - if (!entry || errno != 0) + if (!entry || errno != 0) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_READ_ERROR, NULL); break; + } if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; @@ -1472,14 +1488,22 @@ volgen_graph_set_xl_options(volgen_graph_t *graph, dict_t *dict) }; /* for posix* -> *posix* */ char *loglevel = NULL; xlator_t *trav = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_str_sizen(dict, "xlator", &xlator); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=xlator", NULL); goto out; + } ret = dict_get_str_sizen(dict, "loglevel", &loglevel); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=loglevel", NULL); goto out; + } snprintf(xlator_match, 1024, "*%s", xlator); @@ -1579,14 +1603,22 @@ gfproxy_server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char *password = NULL; /*int rclusters = 0;*/ + xlator_t *this = THIS; + GF_ASSERT(this); /* We are a trusted client */ ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret != 0) + if (ret != 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } ret = dict_set_int32_sizen(set_dict, "gfproxy-server", 1); - if (ret != 0) + if (ret != 0) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=gfproxy-server", NULL); goto out; + } /* Build the client section of the graph first */ build_client_graph(graph, volinfo, set_dict); @@ -1647,11 +1679,13 @@ brick_graph_add_posix(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, xlator_t *this = NULL; glusterd_conf_t *priv = NULL; - if (!graph || !volinfo || !set_dict || !brickinfo) + this = THIS; + + if (!graph || !volinfo || !set_dict || !brickinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } - this = THIS; - GF_VALIDATE_OR_GOTO("glusterd", this, out); priv = this->private; GF_VALIDATE_OR_GOTO("glusterd", priv, out); @@ -1716,9 +1750,13 @@ brick_graph_add_selinux(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo) + if (!graph || !volinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/selinux", volinfo->volname); if (!xl) @@ -1785,8 +1823,10 @@ brick_graph_add_bitrot_stub(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char *value = NULL; xlator_t *this = THIS; - if (!graph || !volinfo || !set_dict || !brickinfo) + if (!graph || !volinfo || !set_dict || !brickinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/bitrot-stub", volinfo->volname); if (!xl) @@ -1821,9 +1861,13 @@ brick_graph_add_changelog(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, }; int ret = -1; int32_t len = 0; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict || !brickinfo) + if (!graph || !volinfo || !set_dict || !brickinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/changelog", volinfo->volname); if (!xl) @@ -1836,6 +1880,7 @@ brick_graph_add_changelog(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, len = snprintf(changelog_basepath, sizeof(changelog_basepath), "%s/%s", brickinfo->path, ".glusterfs/changelogs"); if ((len < 0) || (len >= sizeof(changelog_basepath))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -1865,14 +1910,31 @@ brick_graph_add_acl(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } - xl = volgen_graph_add(graph, "features/access-control", volinfo->volname); - if (!xl) + ret = dict_get_str_boolean(set_dict, "features.acl", 1); + if (!ret) { + /* Skip creating this volume if option is disabled */ + /* By default, this is 'true' */ goto out; + } else if (ret < 0) { + /* lets not treat this as error, as this option is not critical, + and implemented for debug help */ + gf_log(THIS->name, GF_LOG_INFO, + "failed to get 'features.acl' flag from dict"); + } + xl = volgen_graph_add(graph, "features/access-control", volinfo->volname); + if (!xl) { + ret = -1; + goto out; + } ret = 0; out: return ret; @@ -1884,9 +1946,13 @@ brick_graph_add_locks(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/locks", volinfo->volname); if (!xl) @@ -1903,9 +1969,13 @@ brick_graph_add_iot(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "performance/io-threads", volinfo->volname); if (!xl) @@ -1921,9 +1991,12 @@ brick_graph_add_barrier(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; - if (!graph || !volinfo) + if (!graph || !volinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/barrier", volinfo->volname); if (!xl) @@ -1940,9 +2013,13 @@ brick_graph_add_sdfs(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo) + if (!graph || !volinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } if (!dict_get_str_boolean(set_dict, "features.sdfs", 0)) { /* update only if option is enabled */ @@ -1970,9 +2047,13 @@ brick_graph_add_namespace(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } ret = dict_get_str_boolean(set_dict, "features.tag-namespaces", 0); if (ret == -1) @@ -2025,9 +2106,13 @@ brick_graph_add_index(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char index_basepath[PATH_MAX] = {0}; int ret = -1; int32_t len = 0; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !brickinfo || !set_dict) + if (!graph || !volinfo || !brickinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/index", volinfo->volname); if (!xl) @@ -2036,6 +2121,7 @@ brick_graph_add_index(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, len = snprintf(index_basepath, sizeof(index_basepath), "%s/%s", brickinfo->path, ".glusterfs/indices"); if ((len < 0) || (len >= sizeof(index_basepath))) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); goto out; } @@ -2082,9 +2168,13 @@ brick_graph_add_marker(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char buf[32] = { 0, }; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/marker", volinfo->volname); if (!xl) @@ -2115,9 +2205,13 @@ brick_graph_add_quota(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, int ret = -1; xlator_t *xl = NULL; char *value = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/quota", volinfo->volname); if (!xl) @@ -2143,9 +2237,13 @@ brick_graph_add_ro(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { int ret = -1; xlator_t *xl = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } if (dict_get_str_boolean(set_dict, "features.read-only", 0) && (dict_get_str_boolean(set_dict, "features.worm", 0) || @@ -2175,9 +2273,13 @@ brick_graph_add_worm(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { int ret = -1; xlator_t *xl = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } if (dict_get_str_boolean(set_dict, "features.read-only", 0) && (dict_get_str_boolean(set_dict, "features.worm", 0) || @@ -2204,9 +2306,13 @@ brick_graph_add_cdc(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { int ret = -1; xlator_t *xl = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } /* Check for compress volume option, and add it to the graph on * server side */ @@ -2236,8 +2342,10 @@ brick_graph_add_io_stats(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, xlator_t *this = THIS; glusterd_conf_t *priv = this->private; - if (!graph || !set_dict || !brickinfo) + if (!graph || !set_dict || !brickinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add_as(graph, "debug/io-stats", brickinfo->path); if (!xl) @@ -2265,9 +2373,13 @@ brick_graph_add_upcall(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/upcall", volinfo->volname); if (!xl) { @@ -2287,9 +2399,13 @@ brick_graph_add_leases(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, { xlator_t *xl = NULL; int ret = -1; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } xl = volgen_graph_add(graph, "features/leases", volinfo->volname); if (!xl) { @@ -2319,9 +2435,13 @@ brick_graph_add_server(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char *volname = NULL; char *address_family_data = NULL; int32_t len = 0; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict || !brickinfo) + if (!graph || !volinfo || !set_dict || !brickinfo) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } get_vol_transport_type(volinfo, transt); @@ -2430,13 +2550,20 @@ brick_graph_add_pump(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, char *password = NULL; char *ptranst = NULL; char *address_family_data = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); - if (!graph || !volinfo || !set_dict) + if (!graph || !volinfo || !set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } ret = dict_get_int32(volinfo->dict, "enable-pump", &pump); - if (ret == -ENOENT) + if (ret == -ENOENT) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=enable-pump", NULL); ret = pump = 0; + } if (ret) return -1; @@ -3234,11 +3361,20 @@ volgen_link_bricks(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, j); j++; } + if (!xl) { ret = -1; goto out; } + if (strncmp(xl_type, "performance/readdir-ahead", + SLEN("performance/readdir-ahead")) == 0) { + ret = xlator_set_fixed_option(xl, "performance.readdir-ahead", + "on"); + if (ret) + goto out; + } + ret = volgen_xlator_link(xl, trav); if (ret) goto out; @@ -3466,13 +3602,13 @@ volgen_graph_build_readdir_ahead(volgen_graph_t *graph, int32_t clusters = 0; if (graph->type == GF_QUOTAD || graph->type == GF_SNAPD || - !glusterd_volinfo_get_boolean(volinfo, VKEY_PARALLEL_READDIR) || - !glusterd_volinfo_get_boolean(volinfo, VKEY_READDIR_AHEAD)) + !glusterd_volinfo_get_boolean(volinfo, VKEY_PARALLEL_READDIR)) goto out; clusters = volgen_link_bricks_from_list_tail( graph, volinfo, "performance/readdir-ahead", "%s-readdir-ahead-%d", child_count, 1); + out: return clusters; } @@ -3674,6 +3810,38 @@ out: } static int +set_volfile_id_option(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + int clusters) +{ + xlator_t *xlator = NULL; + int i = 0; + int ret = -1; + glusterd_conf_t *conf = NULL; + xlator_t *this = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO("glusterd", this, out); + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); + + if (conf->op_version < GD_OP_VERSION_9_0) + return 0; + xlator = first_of(graph); + + for (i = 0; i < clusters; i++) { + ret = xlator_set_fixed_option(xlator, "volume-id", + uuid_utoa(volinfo->volume_id)); + if (ret) + goto out; + + xlator = xlator->next; + } + +out: + return ret; +} + +static int volgen_graph_build_afr_clusters(volgen_graph_t *graph, glusterd_volinfo_t *volinfo) { @@ -3715,6 +3883,13 @@ volgen_graph_build_afr_clusters(volgen_graph_t *graph, clusters = -1; goto out; } + + ret = set_volfile_id_option(graph, volinfo, clusters); + if (ret) { + clusters = -1; + goto out; + } + if (!volinfo->arbiter_count && !volinfo->thin_arbiter_count) goto out; @@ -4419,11 +4594,15 @@ nfs_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme, volinfo = param; - if (!volinfo || (volinfo->volname[0] == '\0')) + if (!volinfo || (volinfo->volname[0] == '\0')) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); return 0; + } - if (!vme || !(vme->option)) + if (!vme || !(vme->option)) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); return 0; + } xl = first_of(graph); @@ -4554,8 +4733,11 @@ prepare_shd_volume_options(glusterd_volinfo_t *volinfo, dict_t *mod_dict, goto out; ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } dict_copy(volinfo->dict, set_dict); if (mod_dict) @@ -4742,6 +4924,7 @@ build_shd_graph(glusterd_volinfo_t *volinfo, volgen_graph_t *graph, set_dict = dict_new(); if (!set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -ENOMEM; goto out; } @@ -4880,25 +5063,40 @@ build_nfs_graph(volgen_graph_t *graph, dict_t *mod_dict) ret = dict_set_sizen_str_sizen(set_dict, "performance.stat-prefetch", "off"); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=performance.stat-prefetch", NULL); goto out; + } ret = dict_set_sizen_str_sizen(set_dict, "performance.client-io-threads", "off"); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=performance.client-io-threads", NULL); goto out; + } ret = dict_set_str_sizen(set_dict, "client-transport-type", nfs_xprt); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=client-transport-type", NULL); goto out; + } ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } ret = dict_set_sizen_str_sizen(set_dict, "nfs-volume-file", "yes"); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=nfs-volume-file", NULL); goto out; + } if (mod_dict && (data = dict_get_sizen(mod_dict, "volume-name"))) { volname = data->data; @@ -5140,8 +5338,11 @@ build_quotad_graph(volgen_graph_t *graph, dict_t *mod_dict) continue; ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } dict_copy(voliter->dict, set_dict); if (mod_dict) @@ -5337,14 +5538,21 @@ glusterd_generate_client_per_brick_volfile(glusterd_volinfo_t *volinfo) int ret = -1; char *ssl_str = NULL; gf_boolean_t ssl_bool = _gf_false; + xlator_t *this = THIS; + GF_ASSERT(this); dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = dict_set_uint32(dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto free_dict; + } if (dict_get_str_sizen(volinfo->dict, "client.ssl", &ssl_str) == 0) { if (gf_string2boolean(ssl_str, &ssl_bool) == 0) { @@ -5426,17 +5634,25 @@ generate_dummy_client_volfiles(glusterd_volinfo_t *volinfo) enumerate_transport_reqs(volinfo->transport_type, types); dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } for (i = 0; types[i]; i++) { ret = dict_set_str(dict, "client-transport-type", types[i]); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=client-transport-type", NULL); goto out; + } type = transport_str_to_type(types[i]); ret = dict_set_uint32(dict, "trusted-client", GF_CLIENT_OTHER); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } ret = glusterd_get_dummy_client_filepath(filepath, volinfo, type); if (ret) { @@ -5497,17 +5713,25 @@ generate_client_volfiles(glusterd_volinfo_t *volinfo, enumerate_transport_reqs(volinfo->transport_type, types); dict = dict_new(); - if (!dict) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } for (i = 0; types[i]; i++) { ret = dict_set_str(dict, "client-transport-type", types[i]); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=client-transport-type", NULL); goto out; + } type = transport_str_to_type(types[i]); ret = dict_set_uint32(dict, "trusted-client", client_type); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } if (client_type == GF_CLIENT_TRUSTED) { ret = glusterd_get_trusted_client_filepath(filepath, volinfo, type); @@ -5657,10 +5881,15 @@ prepare_bitrot_scrub_volume_options(glusterd_volinfo_t *volinfo, dict_t *mod_dict, dict_t *set_dict) { int ret = 0; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=trusted-client", NULL); goto out; + } dict_copy(volinfo->dict, set_dict); if (mod_dict) @@ -5728,6 +5957,7 @@ build_bitd_volume_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo, set_dict = dict_new(); if (!set_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } @@ -6134,8 +6364,11 @@ validate_shdopts(glusterd_volinfo_t *volinfo, dict_t *val_dict, goto out; } ret = dict_set_int32_sizen(val_dict, "graph-check", 1); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=graph-check", NULL); goto out; + } ret = build_shd_graph(volinfo, &graph, val_dict); if (!ret) ret = graph_reconf_validateopt(&graph.graph, op_errstr); @@ -6189,6 +6422,8 @@ validate_nfsopts(glusterd_volinfo_t *volinfo, dict_t *val_dict, "wrong transport " "type %s", tt); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INCOMPATIBLE_VALUE, + "Type=%s", tt, NULL); *op_errstr = gf_strdup(err_str); ret = -1; goto out; @@ -6257,6 +6492,7 @@ validate_brickopts(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *brickinfo, graph.errstr = op_errstr; full_dict = dict_new(); if (!full_dict) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); ret = -1; goto out; } diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c index 1c5a2508d9c..814ab14fb27 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c @@ -37,240 +37,6 @@ #define glusterd_op_start_volume_args_get(dict, volname, flags) \ glusterd_op_stop_volume_args_get(dict, volname, flags) -gf_ai_compare_t -glusterd_compare_addrinfo(struct addrinfo *first, struct addrinfo *next) -{ - int ret = -1; - struct addrinfo *tmp1 = NULL; - struct addrinfo *tmp2 = NULL; - char firstip[NI_MAXHOST] = {0.}; - char nextip[NI_MAXHOST] = { - 0, - }; - - for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) { - ret = getnameinfo(tmp1->ai_addr, tmp1->ai_addrlen, firstip, NI_MAXHOST, - NULL, 0, NI_NUMERICHOST); - if (ret) - return GF_AI_COMPARE_ERROR; - for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) { - ret = getnameinfo(tmp2->ai_addr, tmp2->ai_addrlen, nextip, - NI_MAXHOST, NULL, 0, NI_NUMERICHOST); - if (ret) - return GF_AI_COMPARE_ERROR; - if (!strcmp(firstip, nextip)) { - return GF_AI_COMPARE_MATCH; - } - } - } - return GF_AI_COMPARE_NO_MATCH; -} - -/* Check for non optimal brick order for replicate : - * Checks if bricks belonging to a replicate volume - * are present on the same server - */ -int32_t -glusterd_check_brick_order(dict_t *dict, char *err_str) -{ - int ret = -1; - int i = 0; - int j = 0; - int k = 0; - xlator_t *this = NULL; - addrinfo_list_t *ai_list = NULL; - addrinfo_list_t *ai_list_tmp1 = NULL; - addrinfo_list_t *ai_list_tmp2 = NULL; - char *brick = NULL; - char *brick_list = NULL; - char *brick_list_dup = NULL; - char *brick_list_ptr = NULL; - char *tmpptr = NULL; - char *volname = NULL; - int32_t brick_count = 0; - int32_t type = GF_CLUSTER_TYPE_NONE; - int32_t sub_count = 0; - struct addrinfo *ai_info = NULL; - char brick_addr[128] = { - 0, - }; - int addrlen = 0; - - const char failed_string[2048] = - "Failed to perform brick order " - "check. Use 'force' at the end of the command" - " if you want to override this behavior. "; - const char found_string[2048] = - "Multiple bricks of a %s " - "volume are present on the same server. This " - "setup is not optimal. Bricks should be on " - "different nodes to have best fault tolerant " - "configuration. Use 'force' at the end of the " - "command if you want to override this " - "behavior. "; - - this = THIS; - - GF_ASSERT(this); - - ai_list = MALLOC(sizeof(addrinfo_list_t)); - ai_list->info = NULL; - CDS_INIT_LIST_HEAD(&ai_list->list); - - ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get volume name"); - goto out; - } - - ret = dict_get_int32n(dict, "type", SLEN("type"), &type); - if (ret) { - snprintf(err_str, 512, "Unable to get type of volume %s", volname); - gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED, "%s", - err_str); - goto out; - } - - ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &brick_list); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Bricks check : Could not " - "retrieve bricks list"); - goto out; - } - - ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Bricks check : Could not " - "retrieve brick count"); - goto out; - } - - if (type != GF_CLUSTER_TYPE_DISPERSE) { - ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"), - &sub_count); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Bricks check : Could" - " not retrieve replica count"); - goto out; - } - gf_msg_debug(this->name, 0, - "Replicate cluster type " - "found. Checking brick order."); - } else { - ret = dict_get_int32n(dict, "disperse-count", SLEN("disperse-count"), - &sub_count); - if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Bricks check : Could" - " not retrieve disperse count"); - goto out; - } - gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DISPERSE_CLUSTER_FOUND, - "Disperse cluster type" - " found. Checking brick order."); - } - - brick_list_dup = brick_list_ptr = gf_strdup(brick_list); - /* Resolve hostnames and get addrinfo */ - while (i < brick_count) { - ++i; - brick = strtok_r(brick_list_dup, " \n", &tmpptr); - brick_list_dup = tmpptr; - if (brick == NULL) - goto check_failed; - tmpptr = strrchr(brick, ':'); - if (tmpptr == NULL) - goto check_failed; - addrlen = strlen(brick) - strlen(tmpptr); - strncpy(brick_addr, brick, addrlen); - brick_addr[addrlen] = '\0'; - ret = getaddrinfo(brick_addr, NULL, NULL, &ai_info); - if (ret != 0) { - ret = 0; - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL, - "unable to resolve host name for addr %s", brick_addr); - goto out; - } - ai_list_tmp1 = MALLOC(sizeof(addrinfo_list_t)); - if (ai_list_tmp1 == NULL) { - ret = 0; - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, - "failed to allocate " - "memory"); - freeaddrinfo(ai_info); - goto out; - } - ai_list_tmp1->info = ai_info; - cds_list_add_tail(&ai_list_tmp1->list, &ai_list->list); - ai_list_tmp1 = NULL; - } - - i = 0; - ai_list_tmp1 = cds_list_entry(ai_list->list.next, addrinfo_list_t, list); - - /* Check for bad brick order */ - while (i < brick_count) { - ++i; - ai_info = ai_list_tmp1->info; - ai_list_tmp1 = cds_list_entry(ai_list_tmp1->list.next, addrinfo_list_t, - list); - if (0 == i % sub_count) { - j = 0; - continue; - } - ai_list_tmp2 = ai_list_tmp1; - k = j; - while (k < sub_count - 1) { - ++k; - ret = glusterd_compare_addrinfo(ai_info, ai_list_tmp2->info); - if (GF_AI_COMPARE_ERROR == ret) - goto check_failed; - if (GF_AI_COMPARE_MATCH == ret) - goto found_bad_brick_order; - ai_list_tmp2 = cds_list_entry(ai_list_tmp2->list.next, - addrinfo_list_t, list); - } - ++j; - } - gf_msg_debug(this->name, 0, "Brick order okay"); - ret = 0; - goto out; - -check_failed: - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER_CHECK_FAIL, - "Failed bad brick order check"); - snprintf(err_str, sizeof(failed_string), failed_string); - ret = -1; - goto out; - -found_bad_brick_order: - gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BAD_BRKORDER, - "Bad brick order found"); - if (type == GF_CLUSTER_TYPE_DISPERSE) { - snprintf(err_str, sizeof(found_string), found_string, "disperse"); - } else { - snprintf(err_str, sizeof(found_string), found_string, "replicate"); - } - - ret = -1; -out: - ai_list_tmp2 = NULL; - GF_FREE(brick_list_ptr); - cds_list_for_each_entry(ai_list_tmp1, &ai_list->list, list) - { - if (ai_list_tmp1->info) - freeaddrinfo(ai_list_tmp1->info); - free(ai_list_tmp2); - ai_list_tmp2 = ai_list_tmp1; - } - free(ai_list_tmp2); - return ret; -} - int __glusterd_handle_create_volume(rpcsvc_request_t *req) { @@ -809,10 +575,14 @@ glusterd_handle_heal_options_enable_disable(rpcsvc_request_t *req, dict_t *dict, int ret = 0; char *key = NULL; char *value = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"), (int32_t *)&heal_op); if (ret || (heal_op == GF_SHD_OP_INVALID)) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=heal-op", NULL); ret = -1; goto out; } @@ -850,21 +620,33 @@ glusterd_handle_heal_options_enable_disable(rpcsvc_request_t *req, dict_t *dict, } else { key = "cluster.granular-entry-heal"; ret = dict_set_int8(dict, "is-special-key", 1); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=is-special-key", NULL); goto out; + } } ret = dict_set_strn(dict, "key1", SLEN("key1"), key); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=key1", NULL); goto out; + } ret = dict_set_strn(dict, "value1", SLEN("value1"), value); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=value1", NULL); goto out; + } ret = dict_set_int32n(dict, "count", SLEN("count"), 1); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; + } ret = glusterd_op_begin_synctask(req, GD_OP_SET_VOLUME, dict); @@ -888,18 +670,19 @@ __glusterd_handle_cli_heal_volume(rpcsvc_request_t *req) 0, }; + this = THIS; + GF_ASSERT(this); + GF_ASSERT(req); ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { // failed to decode msg; req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } - this = THIS; - GF_ASSERT(this); - if (cli_req.dict.dict_len) { /* Unserialize the dictionary */ dict = dict_new(); @@ -960,8 +743,11 @@ __glusterd_handle_cli_heal_volume(rpcsvc_request_t *req) goto out; ret = dict_set_int32n(dict, "count", SLEN("count"), volinfo->brick_count); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=count", NULL); goto out; + } ret = glusterd_op_begin_synctask(req, GD_OP_HEAL_VOLUME, dict); @@ -1013,6 +799,7 @@ __glusterd_handle_cli_statedump_volume(rpcsvc_request_t *req) ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { req->rpc_err = GARBAGE_ARGS; + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL); goto out; } if (cli_req.dict.dict_len) { @@ -1107,6 +894,8 @@ glusterd_op_stage_create_volume(dict_t *dict, char **op_errstr, int32_t local_brick_count = 0; int32_t i = 0; int32_t type = 0; + int32_t replica_count = 0; + int32_t disperse_count = 0; char *brick = NULL; char *tmpptr = NULL; xlator_t *this = NULL; @@ -1201,16 +990,44 @@ glusterd_op_stage_create_volume(dict_t *dict, char **op_errstr, } if (!is_force) { - if ((type == GF_CLUSTER_TYPE_REPLICATE) || - (type == GF_CLUSTER_TYPE_DISPERSE)) { - ret = glusterd_check_brick_order(dict, msg); + if (type == GF_CLUSTER_TYPE_REPLICATE) { + ret = dict_get_int32n(dict, "replica-count", + SLEN("replica-count"), &replica_count); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Bricks check : Could" + " not retrieve replica count"); + goto out; + } + gf_msg_debug(this->name, 0, + "Replicate cluster type " + "found. Checking brick order."); + ret = glusterd_check_brick_order(dict, msg, type, &volname, + &bricks, &brick_count, + replica_count); + } else if (type == GF_CLUSTER_TYPE_DISPERSE) { + ret = dict_get_int32n(dict, "disperse-count", + SLEN("disperse-count"), &disperse_count); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER, - "Not " - "creating volume because of " - "bad brick order"); + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Bricks check : Could" + " not retrieve disperse count"); goto out; } + gf_msg_debug(this->name, 0, + "Disperse cluster type" + " found. Checking brick order."); + ret = glusterd_check_brick_order(dict, msg, type, &volname, + &bricks, &brick_count, + disperse_count); + } + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER, + "Not creating the volume because of " + "bad brick order. %s", + msg); + *op_errstr = gf_strdup(msg); + goto out; } } } @@ -1325,20 +1142,32 @@ glusterd_op_stop_volume_args_get(dict_t *dict, char **volname, int *flags) this = THIS; GF_ASSERT(this); - if (!dict || !volname || !flags) + if (!dict) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL); + goto out; + } + + if (!volname) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } + + if (!flags) { + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL); + goto out; + } ret = dict_get_strn(dict, "volname", SLEN("volname"), volname); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get volume name"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; } ret = dict_get_int32n(dict, "flags", SLEN("flags"), flags); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get flags"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=flags", NULL); goto out; } out: @@ -1351,27 +1180,29 @@ glusterd_op_statedump_volume_args_get(dict_t *dict, char **volname, { int ret = -1; - if (!dict || !volname || !options || !option_cnt) + if (!dict || !volname || !options || !option_cnt) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); goto out; + } ret = dict_get_strn(dict, "volname", SLEN("volname"), volname); if (ret) { - gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get volname"); + gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; } ret = dict_get_strn(dict, "options", SLEN("options"), options); if (ret) { - gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get options"); + gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=options", NULL); goto out; } ret = dict_get_int32n(dict, "option_cnt", SLEN("option_cnt"), option_cnt); if (ret) { - gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Unable to get option count"); + gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=option_cnt", NULL); goto out; } @@ -1598,8 +1429,13 @@ glusterd_op_stage_stop_volume(dict_t *dict, char **op_errstr) GF_ASSERT(this); ret = glusterd_op_stop_volume_args_get(dict, &volname, &flags); - if (ret) + if (ret) { + snprintf(msg, sizeof(msg), "Failed to get details of volume %s", + volname); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_STOP_ARGS_GET_FAILED, + "Volume name=%s", volname, NULL); goto out; + } ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { @@ -1881,14 +1717,15 @@ glusterd_op_stage_heal_volume(dict_t *dict, char **op_errstr) if (!glusterd_is_volume_started(volinfo)) { ret = -1; snprintf(msg, sizeof(msg), "Volume %s is not started.", volname); - gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_STARTED, "%s", - msg); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_STARTED, + "Volume=%s", volname, NULL); *op_errstr = gf_strdup(msg); goto out; } opt_dict = volinfo->dict; if (!opt_dict) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, NULL); ret = 0; goto out; } @@ -1944,6 +1781,8 @@ glusterd_op_stage_statedump_volume(dict_t *dict, char **op_errstr) ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, + "Volume=%s", volname, NULL); goto out; } @@ -2107,8 +1946,6 @@ glusterd_op_create_volume(dict_t *dict, char **op_errstr) goto out; } - pthread_mutex_init(&volinfo->store_volinfo_lock, NULL); - ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); if (ret) { @@ -3060,33 +2897,35 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) char *mntpt = NULL; char **xl_opts = NULL; glusterd_volinfo_t *volinfo = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get volume name"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, + "Key=volname", NULL); goto out; } gf_msg_debug("glusterd", 0, "Performing clearlocks on volume %s", volname); ret = dict_get_strn(dict, "path", SLEN("path"), &path); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get path"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=path", + NULL); goto out; } ret = dict_get_strn(dict, "kind", SLEN("kind"), &kind); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get kind"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=kind", + NULL); goto out; } ret = dict_get_strn(dict, "type", SLEN("type"), &type); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, - "Failed to get type"); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=type", + NULL); goto out; } @@ -3094,10 +2933,9 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) if (ret) ret = 0; - gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_CLRCLK_VOL_REQ_RCVD, - "Received clear-locks request for " - "volume %s with kind %s type %s and options %s", - volname, kind, type, opts); + gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_CLRCLK_VOL_REQ_RCVD, + "Volume=%s, Kind=%s, Type=%s, Options=%s", volname, kind, type, + opts, NULL); if (opts) ret = gf_asprintf(&cmd_str, GF_XATTR_CLRLK_CMD ".t%s.k%s.%s", type, @@ -3110,22 +2948,25 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) ret = glusterd_volinfo_find(volname, &volinfo); if (ret) { snprintf(msg, sizeof(msg), "Volume %s doesn't exist.", volname); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "Volume=%s", + volname, NULL); goto out; } xl_opts = GF_CALLOC(volinfo->brick_count + 1, sizeof(char *), gf_gld_mt_charptr); - if (!xl_opts) + if (!xl_opts) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto out; + } ret = glusterd_clearlocks_get_local_client_ports(volinfo, xl_opts); if (ret) { snprintf(msg, sizeof(msg), "Couldn't get port numbers of " "local bricks"); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRK_PORT_NUM_GET_FAIL, "%s", - msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_PORT_NUM_GET_FAIL, + NULL); goto out; } @@ -3134,8 +2975,8 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) snprintf(msg, sizeof(msg), "Creating mount directory " "for clear-locks failed."); - gf_msg(THIS->name, GF_LOG_ERROR, 0, - GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL, "%s", msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, + GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL, NULL); goto out; } @@ -3144,16 +2985,15 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) snprintf(msg, sizeof(msg), "Failed to mount clear-locks " "maintenance client."); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL, - "%s", msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL, + NULL); goto out; } ret = glusterd_clearlocks_send_cmd(volinfo, cmd_str, path, result, msg, sizeof(msg), mntpt); if (ret) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_CLRCLK_SND_CMD_FAIL, "%s", - msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLRCLK_SND_CMD_FAIL, NULL); goto umount; } @@ -3164,16 +3004,16 @@ glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) snprintf(msg, sizeof(msg), "Failed to set clear-locks " "result"); - gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s", msg); + gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Key=lk-summary", NULL); } umount: glusterd_clearlocks_unmount(volinfo, mntpt); if (glusterd_clearlocks_rmdir_mount(volinfo, mntpt)) - gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL, - "Couldn't unmount " - "clear-locks mount point"); + gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL, + NULL); out: if (ret) diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 63de3ef685c..398b4d76f52 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -8,6 +8,7 @@ later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. */ +#include <glusterfs/syscall.h> #include "glusterd-volgen.h" #include "glusterd-utils.h" @@ -786,6 +787,32 @@ out: return ret; } +static int +is_directory(const char *path) +{ + struct stat statbuf; + if (sys_stat(path, &statbuf) != 0) + return 0; + return S_ISDIR(statbuf.st_mode); +} +static int +validate_statedump_path(glusterd_volinfo_t *volinfo, dict_t *dict, char *key, + char *value, char **op_errstr) +{ + xlator_t *this = NULL; + this = THIS; + GF_ASSERT(this); + + int ret = 0; + if (!is_directory(value)) { + gf_asprintf(op_errstr, "Failed: %s is not a directory", value); + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s", + *op_errstr); + } + + return ret; +} /* dispatch table for VOLUME SET * ----------------------------- @@ -1239,10 +1266,21 @@ struct volopt_map_entry glusterd_volopt_map[] = { .option = "priority", .op_version = 1, .flags = VOLOPT_FLAG_CLIENT_OPT}, - {.key = "performance.cache-size", + {.key = "performance.io-cache-size", .voltype = "performance/io-cache", - .op_version = 1, + .option = "cache-size", + .op_version = GD_OP_VERSION_8_0, .flags = VOLOPT_FLAG_CLIENT_OPT}, + { + .key = "performance.cache-size", + .voltype = "performance/io-cache", + .op_version = 1, + .flags = VOLOPT_FLAG_CLIENT_OPT, + .description = "Deprecated option. Use performance.io-cache-size " + "to adjust the cache size of the io-cache translator, " + "and use performance.quick-read-cache-size to adjust " + "the cache size of the quick-read translator.", + }, /* IO-threads xlator options */ {.key = "performance.io-thread-count", @@ -1282,16 +1320,29 @@ struct volopt_map_entry glusterd_volopt_map[] = { .voltype = "performance/io-cache", .option = "pass-through", .op_version = GD_OP_VERSION_4_1_0}, + {.key = "performance.quick-read-cache-size", + .voltype = "performance/quick-read", + .option = "cache-size", + .op_version = GD_OP_VERSION_8_0, + .flags = VOLOPT_FLAG_CLIENT_OPT}, {.key = "performance.cache-size", .voltype = "performance/quick-read", .type = NO_DOC, .op_version = 1, .flags = VOLOPT_FLAG_CLIENT_OPT}, + {.key = "performance.quick-read-cache-timeout", + .voltype = "performance/quick-read", + .option = "cache-timeout", + .op_version = GD_OP_VERSION_8_0, + .flags = VOLOPT_FLAG_CLIENT_OPT}, {.key = "performance.qr-cache-timeout", .voltype = "performance/quick-read", .option = "cache-timeout", .op_version = 1, - .flags = VOLOPT_FLAG_CLIENT_OPT}, + .flags = VOLOPT_FLAG_CLIENT_OPT, + .description = + "Deprecated option. Use performance.quick-read-cache-timeout " + "instead."}, {.key = "performance.quick-read-cache-invalidation", .voltype = "performance/quick-read", .option = "quick-read-cache-invalidation", @@ -1588,7 +1639,8 @@ struct volopt_map_entry glusterd_volopt_map[] = { {.key = "server.statedump-path", .voltype = "protocol/server", .option = "statedump-path", - .op_version = 1}, + .op_version = 1, + .validate_fn = validate_statedump_path}, {.key = "server.outstanding-rpc-limit", .voltype = "protocol/server", .option = "rpc.outstanding-rpc-limit", @@ -1761,7 +1813,7 @@ struct volopt_map_entry glusterd_volopt_map[] = { {.key = "performance.readdir-ahead", .voltype = "performance/readdir-ahead", .option = "!perf", - .value = "on", + .value = "off", .op_version = 3, .description = "enable/disable readdir-ahead translator in the volume.", .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT}, @@ -2434,7 +2486,6 @@ struct volopt_map_entry glusterd_volopt_map[] = { .voltype = "storage/posix", .op_version = GD_OP_VERSION_4_1_0, }, - {.key = "storage.bd-aio", .voltype = "storage/bd", .op_version = 3}, {.key = "config.memory-accounting", .voltype = "mgmt/glusterd", .option = "!config", @@ -3076,4 +3127,20 @@ struct volopt_map_entry glusterd_volopt_map[] = { .voltype = "features/cloudsync", .op_version = GD_OP_VERSION_7_0, .flags = VOLOPT_FLAG_CLIENT_OPT}, + { + .key = "features.acl", + .voltype = "features/access-control", + .value = "enable", + .option = "!features", + .op_version = GD_OP_VERSION_8_0, + .description = "(WARNING: for debug purpose only) enable/disable " + "access-control xlator in volume", + .type = NO_DOC, + }, + + {.key = "cluster.use-anonymous-inode", + .voltype = "cluster/replicate", + .op_version = GD_OP_VERSION_9_0, + .value = "yes", + .flags = VOLOPT_FLAG_CLIENT_OPT}, {.key = NULL}}; diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index cd2c5da628b..7a86c2997b1 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -67,7 +67,7 @@ extern struct rpcsvc_program gd_svc_cli_trusted_progs; extern struct rpc_clnt_program gd_brick_prog; extern struct rpcsvc_program glusterd_mgmt_hndsk_prog; -extern char snap_mount_dir[PATH_MAX]; +extern char snap_mount_dir[VALID_GLUSTERD_PATHMAX]; rpcsvc_cbk_program_t glusterd_cbk_prog = { .progname = "Gluster Callback", @@ -202,8 +202,10 @@ glusterd_options_init(xlator_t *this) priv = this->private; priv->opts = dict_new(); - if (!priv->opts) + if (!priv->opts) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } ret = glusterd_store_retrieve_options(this); if (ret == 0) { @@ -247,6 +249,7 @@ glusterd_client_statedump_submit_req(char *volname, char *target_ip, char *pid) GF_ASSERT(conf); if (target_ip == NULL || pid == NULL) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL); ret = -1; goto out; } @@ -447,14 +450,19 @@ glusterd_rpcsvc_options_build(dict_t *options) { int ret = 0; uint32_t backlog = 0; + xlator_t *this = THIS; + GF_ASSERT(this); ret = dict_get_uint32(options, "transport.listen-backlog", &backlog); if (ret) { backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG; ret = dict_set_uint32(options, "transport.listen-backlog", backlog); - if (ret) + if (ret) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Key=transport.listen-backlog", NULL); goto out; + } } gf_msg_debug("glusterd", 0, "listen-backlog value: %d", backlog); @@ -574,6 +582,7 @@ glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf) len = snprintf(georepdir, PATH_MAX, "%s/" GEOREP, conf->workdir); if ((len < 0) || (len >= PATH_MAX)) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -585,9 +594,11 @@ glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf) } ret = dict_get_str(THIS->options, GEOREP "-log-group", &greplg_s); - if (ret) + if (ret) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Key=log-group", NULL); ret = 0; - else { + } else { gr = getgrnam(greplg_s); if (!gr) { gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_LOGGROUP_INVALID, @@ -628,6 +639,7 @@ glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf) } len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-slaves", conf->logdir); if ((len < 0) || (len >= PATH_MAX)) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -654,6 +666,7 @@ glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf) len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-slaves/mbr", conf->logdir); if ((len < 0) || (len >= PATH_MAX)) { + gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL); ret = -1; goto out; } @@ -1045,6 +1058,8 @@ _install_mount_spec(dict_t *opts, char *key, data_t *value, void *data) int rv = 0; gf_mount_spec_t *mspec = NULL; char *user = NULL; + xlator_t *this = THIS; + GF_ASSERT(this); label = strtail(key, "mountbroker."); @@ -1059,8 +1074,10 @@ _install_mount_spec(dict_t *opts, char *key, data_t *value, void *data) return 0; mspec = GF_CALLOC(1, sizeof(*mspec), gf_gld_mt_mount_spec); - if (!mspec) + if (!mspec) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL); goto err; + } mspec->label = label; if (georep) { @@ -1116,8 +1133,10 @@ glusterd_init_uds_listener(xlator_t *this) GF_ASSERT(this); options = dict_new(); - if (!options) + if (!options) { + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL); goto out; + } sock_data = dict_get(this->options, "glusterd-sockfile"); (void)snprintf(sockfile, sizeof(sockfile), "%s", @@ -1404,7 +1423,7 @@ init(xlator_t *this) char *mountbroker_root = NULL; int i = 0; int total_transport = 0; - gf_boolean_t valgrind = _gf_false; + gf_valgrind_tool vgtool; char *valgrind_str = NULL; char *transport_type = NULL; char var_run_dir[PATH_MAX] = { @@ -1417,6 +1436,14 @@ init(xlator_t *this) int32_t len = 0; int op_version = 0; +#if defined(RUN_WITH_MEMCHECK) + vgtool = _gf_memcheck; +#elif defined(RUN_WITH_DRD) + vgtool = _gf_drd; +#else + vgtool = _gf_none; +#endif + #ifndef GF_DARWIN_HOST_OS { struct rlimit lim; @@ -1424,9 +1451,8 @@ init(xlator_t *this) lim.rlim_max = 65536; if (setrlimit(RLIMIT_NOFILE, &lim) == -1) { - gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SETXATTR_FAIL, - "Failed to set 'ulimit -n " - " 65536'"); + gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL, + "Failed to set 'ulimit -n 65536'", NULL); } else { gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FILE_DESC_LIMIT_SET, "Maximum allowed open file descriptors " @@ -1872,6 +1898,9 @@ init(xlator_t *this) (void)strncpy(conf->logdir, logdir, sizeof(conf->logdir)); synclock_init(&conf->big_lock, SYNC_LOCK_RECURSIVE); + synccond_init(&conf->cond_restart_bricks); + synccond_init(&conf->cond_restart_shd); + synccond_init(&conf->cond_blockers); pthread_mutex_init(&conf->xprt_lock, NULL); INIT_LIST_HEAD(&conf->xprt_list); pthread_mutex_init(&conf->import_volumes, NULL); @@ -1904,18 +1933,24 @@ init(xlator_t *this) } /* Set option to run bricks on valgrind if enabled in glusterd.vol */ - this->ctx->cmd_args.valgrind = valgrind; + this->ctx->cmd_args.vgtool = vgtool; ret = dict_get_str(this->options, "run-with-valgrind", &valgrind_str); if (ret < 0) { gf_msg_debug(this->name, 0, "cannot get run-with-valgrind value"); } if (valgrind_str) { - if (gf_string2boolean(valgrind_str, &valgrind)) { + gf_boolean_t vg = _gf_false; + + if (!strcmp(valgrind_str, "memcheck")) + this->ctx->cmd_args.vgtool = _gf_memcheck; + else if (!strcmp(valgrind_str, "drd")) + this->ctx->cmd_args.vgtool = _gf_drd; + else if (!gf_string2boolean(valgrind_str, &vg)) + this->ctx->cmd_args.vgtool = (vg ? _gf_memcheck : _gf_none); + else gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY, - "run-with-valgrind value not a boolean string"); - } else { - this->ctx->cmd_args.valgrind = valgrind; - } + "run-with-valgrind is neither boolean" + " nor one of 'memcheck' or 'drd'"); } /* Store ping-timeout in conf */ diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index d7e4da8425a..cc4f98ecf47 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -209,6 +209,9 @@ typedef struct { gf_boolean_t restart_done; dict_t *opts; synclock_t big_lock; + synccond_t cond_restart_bricks; + synccond_t cond_restart_shd; + synccond_t cond_blockers; rpcsvc_t *uds_rpc; /* RPCSVC for the unix domain socket */ uint32_t base_port; uint32_t max_port; @@ -510,6 +513,10 @@ struct glusterd_volinfo_ { * volfile generation code, we are * temporarily appending either "-hot" * or "-cold" */ + gf_atomic_t volpeerupdate; + /* Flag to check about volume has received updates + from peer + */ }; typedef enum gd_snap_status_ { @@ -1190,6 +1197,8 @@ glusterd_op_set_ganesha(dict_t *dict, char **errstr); int ganesha_manage_export(dict_t *dict, char *value, gf_boolean_t update_cache_invalidation, char **op_errstr); +int +gd_ganesha_send_dbus(char *volname, char *value); gf_boolean_t glusterd_is_ganesha_cluster(); gf_boolean_t @@ -1211,6 +1220,9 @@ int glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr); int +glusterd_set_rebalance_id_for_remove_brick(dict_t *req_dict, dict_t *rsp_dict); + +int glusterd_set_rebalance_id_in_rsp_dict(dict_t *req_dict, dict_t *rsp_dict); int @@ -1354,4 +1366,10 @@ glusterd_options_init(xlator_t *this); int32_t glusterd_recreate_volfiles(glusterd_conf_t *conf); +void +glusterd_add_peers_to_auth_list(char *volname); + +int +glusterd_replace_old_auth_allow_list(char *volname); + #endif diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c index 3df1135a2ed..0e22fe411ee 100644 --- a/xlators/mount/fuse/src/fuse-bridge.c +++ b/xlators/mount/fuse/src/fuse-bridge.c @@ -8,6 +8,8 @@ cases as published by the Free Software Foundation. */ +#include <config.h> + #include <sys/wait.h> #include "fuse-bridge.h" #include <glusterfs/glusterfs.h> @@ -177,7 +179,7 @@ fusedump_gettime(struct fusedump_timespec *fts) 0, }; - clock_gettime(CLOCK_REALTIME, &ts); + timespec_now_realtime(&ts); fts->sec = ts.tv_sec; fts->nsec = ts.tv_nsec; @@ -225,14 +227,30 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count, if (res == -1) { const char *errdesc = NULL; gf_loglevel_t loglevel = GF_LOG_ERROR; + gf_boolean_t errno_degraded = _gf_false; + gf_boolean_t errno_promoted = _gf_false; + +#define ACCOUNT_ERRNO(eno) \ + do { \ + if (errno_degraded) { \ + pthread_mutex_lock(&priv->fusedev_errno_cnt_mutex); \ + { \ + if (!++priv->fusedev_errno_cnt[FUSEDEV_##eno]) \ + errno_promoted = _gf_true; \ + } \ + pthread_mutex_unlock(&priv->fusedev_errno_cnt_mutex); \ + } \ + } while (0) /* If caller masked the errno, then it * does not indicate an error at the application * level, so we degrade the log severity to DEBUG. */ if (errnomask && errno < ERRNOMASK_MAX && - GET_ERRNO_MASK(errnomask, errno)) + GET_ERRNO_MASK(errnomask, errno)) { loglevel = GF_LOG_DEBUG; + errno_degraded = _gf_true; + } switch (errno) { /* The listed errnos are FUSE status indicators, @@ -242,33 +260,43 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count, */ case ENOENT: errdesc = "ENOENT"; + ACCOUNT_ERRNO(ENOENT); break; case ENOTDIR: errdesc = "ENOTDIR"; + ACCOUNT_ERRNO(ENOTDIR); break; case ENODEV: errdesc = "ENODEV"; + ACCOUNT_ERRNO(ENODEV); break; case EPERM: errdesc = "EPERM"; + ACCOUNT_ERRNO(EPERM); break; case ENOMEM: errdesc = "ENOMEM"; + ACCOUNT_ERRNO(ENOMEM); break; case ENOTCONN: errdesc = "ENOTCONN"; + ACCOUNT_ERRNO(ENOTCONN); break; case ECONNREFUSED: errdesc = "ECONNREFUSED"; + ACCOUNT_ERRNO(ECONNREFUSED); break; case EOVERFLOW: errdesc = "EOVERFLOW"; + ACCOUNT_ERRNO(EOVERFLOW); break; case EBUSY: errdesc = "EBUSY"; + ACCOUNT_ERRNO(EBUSY); break; case ENOTEMPTY: errdesc = "ENOTEMPTY"; + ACCOUNT_ERRNO(ENOTEMPTY); break; default: errdesc = strerror(errno); @@ -276,7 +304,13 @@ check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count, gf_log_callingfn("glusterfs-fuse", loglevel, "writing to fuse device failed: %s", errdesc); + if (errno_promoted) + gf_log("glusterfs-fuse", GF_LOG_WARNING, + "writing to fuse device yielded %s %d times", errdesc, + UINT8_MAX + 1); return errno; + +#undef ACCOUNT_ERRNO } fouh = iov_out[0].iov_base; @@ -373,7 +407,7 @@ send_fuse_data(xlator_t *this, fuse_in_header_t *finh, void *data, size_t size) static int32_t fuse_invalidate_entry(xlator_t *this, uint64_t fuse_ino) { -#if FUSE_KERNEL_MINOR_VERSION >= 11 +#if (FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS)) struct fuse_out_header *fouh = NULL; struct fuse_notify_inval_entry_out *fnieo = NULL; fuse_private_t *priv = NULL; @@ -464,7 +498,7 @@ fuse_invalidate_entry(xlator_t *this, uint64_t fuse_ino) static int32_t fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino) { -#if FUSE_KERNEL_MINOR_VERSION >= 11 +#if (FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS)) struct fuse_out_header *fouh = NULL; struct fuse_notify_inval_inode_out *fniio = NULL; fuse_private_t *priv = NULL; @@ -799,18 +833,22 @@ fuse_interrupt_finish_fop(call_frame_t *frame, xlator_t *this, { intstat_orig = fir->interrupt_state; if (fir->interrupt_state == INTERRUPT_NONE) { - fir->interrupt_state = INTERRUPT_SQUELCHED; if (sync) { - while (fir->interrupt_state == INTERRUPT_NONE) { + fir->interrupt_state = INTERRUPT_WAITING_HANDLER; + while (fir->interrupt_state != INTERRUPT_SQUELCHED) { pthread_cond_wait(&fir->handler_cond, &fir->handler_mutex); } - } + } else + fir->interrupt_state = INTERRUPT_SQUELCHED; } } pthread_mutex_unlock(&fir->handler_mutex); } + GF_ASSERT(intstat_orig == INTERRUPT_NONE || + intstat_orig == INTERRUPT_HANDLED || + intstat_orig == INTERRUPT_SQUELCHED); gf_log("glusterfs-fuse", GF_LOG_DEBUG, "intstat_orig=%d", intstat_orig); /* @@ -860,19 +898,29 @@ fuse_interrupt_finish_interrupt(xlator_t *this, fuse_interrupt_record_t *fir, }; fuse_interrupt_state_t intstat_orig = INTERRUPT_NONE; + GF_ASSERT(intstat == INTERRUPT_HANDLED || intstat == INTERRUPT_SQUELCHED); + pthread_mutex_lock(&fir->handler_mutex); { intstat_orig = fir->interrupt_state; - if (fir->interrupt_state == INTERRUPT_NONE) { - fir->interrupt_state = intstat; - if (sync) { + switch (intstat_orig) { + case INTERRUPT_NONE: + fir->interrupt_state = intstat; + break; + case INTERRUPT_WAITING_HANDLER: + fir->interrupt_state = INTERRUPT_SQUELCHED; pthread_cond_signal(&fir->handler_cond); - } + break; + default: + break; } finh = fir->fuse_in_header; } pthread_mutex_unlock(&fir->handler_mutex); + GF_ASSERT(intstat_orig == INTERRUPT_NONE || + (sync && intstat_orig == INTERRUPT_WAITING_HANDLER) || + (!sync && intstat_orig == INTERRUPT_SQUELCHED)); gf_log("glusterfs-fuse", GF_LOG_DEBUG, "intstat_orig=%d", intstat_orig); /* @@ -2185,7 +2233,6 @@ fuse_mknod(xlator_t *this, fuse_in_header_t *finh, void *msg, fuse_state_t *state = NULL; #if FUSE_KERNEL_MINOR_VERSION >= 12 fuse_private_t *priv = NULL; - int32_t ret = -1; priv = this->private; if (priv->proto_minor < 12) @@ -2257,7 +2304,6 @@ fuse_mkdir(xlator_t *this, fuse_in_header_t *finh, void *msg, char *name = (char *)(fmi + 1); #if FUSE_KERNEL_MINOR_VERSION >= 12 fuse_private_t *priv = NULL; - int32_t ret = -1; #endif fuse_state_t *state; @@ -2783,7 +2829,6 @@ fuse_create(xlator_t *this, fuse_in_header_t *finh, void *msg, #if FUSE_KERNEL_MINOR_VERSION >= 12 struct fuse_create_in *fci = msg; fuse_private_t *priv = NULL; - int32_t ret = -1; #else struct fuse_open_in *fci = msg; #endif @@ -3381,6 +3426,8 @@ fuse_release(xlator_t *this, fuse_in_header_t *finh, void *msg, gf_log("glusterfs-fuse", GF_LOG_TRACE, "finh->unique: %" PRIu64 ": RELEASE %p", finh->unique, state->fd); + fd_close(state->fd); + fuse_fd_ctx_destroy(this, state->fd); fd_unref(fd); @@ -4690,12 +4737,10 @@ fuse_setlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, fuse_state_t *state = NULL; int ret = 0; - ret = fuse_interrupt_finish_fop(frame, this, _gf_false, (void **)&state); - if (state) { - GF_FREE(state->name); - dict_unref(state->xdata); - GF_FREE(state); - } + ret = fuse_interrupt_finish_fop(frame, this, _gf_true, (void **)&state); + GF_FREE(state->name); + dict_unref(state->xdata); + GF_FREE(state); if (ret) { return 0; } @@ -4751,28 +4796,12 @@ fuse_setlk_interrupt_handler_cbk(call_frame_t *frame, void *cookie, int32_t op_errno, dict_t *dict, dict_t *xdata) { fuse_interrupt_state_t intstat = INTERRUPT_NONE; - fuse_interrupt_record_t *fir; - fuse_state_t *state = NULL; - int ret = 0; - - ret = dict_get_bin(xdata, "fuse-interrupt-record", (void **)&fir); - if (ret < 0) { - gf_log("glusterfs-fuse", GF_LOG_ERROR, "interrupt record not found"); - - goto out; - } + fuse_interrupt_record_t *fir = cookie; intstat = op_ret >= 0 ? INTERRUPT_HANDLED : INTERRUPT_SQUELCHED; - fuse_interrupt_finish_interrupt(this, fir, intstat, _gf_false, - (void **)&state); - if (state) { - GF_FREE(state->name); - dict_unref(state->xdata); - GF_FREE(state); - } + fuse_interrupt_finish_interrupt(this, fir, intstat, _gf_true, NULL); -out: STACK_DESTROY(frame->root); return 0; @@ -4810,9 +4839,10 @@ fuse_setlk_interrupt_handler(xlator_t *this, fuse_interrupt_record_t *fir) frame->op = GF_FOP_GETXATTR; state->name = xattr_name; - STACK_WIND(frame, fuse_setlk_interrupt_handler_cbk, state->active_subvol, - state->active_subvol->fops->fgetxattr, state->fd, xattr_name, - state->xdata); + STACK_WIND_COOKIE(frame, fuse_setlk_interrupt_handler_cbk, fir, + state->active_subvol, + state->active_subvol->fops->fgetxattr, state->fd, + xattr_name, state->xdata); return; @@ -4835,15 +4865,9 @@ fuse_setlk_resume(fuse_state_t *state) fir = fuse_interrupt_record_new(state->finh, fuse_setlk_interrupt_handler); state_clone = gf_memdup(state, sizeof(*state)); if (state_clone) { - /* - * Calling this allocator with fir casted to (char *) seems like - * an abuse of this API, but in fact the API is stupid to assume - * a (char *) argument (in the funcion it's casted to (void *) - * anyway). - */ - state_clone->xdata = dict_for_key_value( - "fuse-interrupt-record", (char *)fir, sizeof(*fir), _gf_true); + state_clone->xdata = dict_new(); } + if (!fir || !state_clone || !state_clone->xdata) { if (fir) { GF_FREE(fir); @@ -4901,7 +4925,7 @@ fuse_setlk(xlator_t *this, fuse_in_header_t *finh, void *msg, return; } -#if FUSE_KERNEL_MINOR_VERSION >= 11 +#if FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS) static void * notify_kernel_loop(void *data) { @@ -5148,6 +5172,7 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg, priv->timed_response_fuse_thread_started = _gf_true; /* Used for 'reverse invalidation of inode' */ +#ifdef HAVE_FUSE_NOTIFICATIONS if (fini->minor >= 12) { ret = gf_thread_create(&messenger, NULL, notify_kernel_loop, this, "fusenoti"); @@ -5159,7 +5184,9 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg, goto out; } priv->reverse_fuse_thread_started = _gf_true; - } else { + } else +#endif + { /* * FUSE minor < 12 does not implement invalidate notifications. * This mechanism is required for fopen-keep-cache to operate @@ -5586,6 +5613,7 @@ fuse_migrate_fd(xlator_t *this, fd_t *basefd, xlator_t *old_subvol, char create_in_progress = 0; fuse_fd_ctx_t *basefd_ctx = NULL; fd_t *oldfd = NULL; + dict_t *xdata = NULL; basefd_ctx = fuse_fd_ctx_get(this, basefd); GF_VALIDATE_OR_GOTO("glusterfs-fuse", basefd_ctx, out); @@ -5622,10 +5650,23 @@ fuse_migrate_fd(xlator_t *this, fd_t *basefd, xlator_t *old_subvol, } if (oldfd->inode->table->xl == old_subvol) { - if (IA_ISDIR(oldfd->inode->ia_type)) + if (IA_ISDIR(oldfd->inode->ia_type)) { ret = syncop_fsyncdir(old_subvol, oldfd, 0, NULL, NULL); - else - ret = syncop_fsync(old_subvol, oldfd, 0, NULL, NULL, NULL, NULL); + } else { + xdata = dict_new(); + if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) { + gf_log("glusterfs-fuse", GF_LOG_WARNING, + "last-fsync set failed (%s) on fd (%p)" + "(basefd:%p basefd-inode.gfid:%s) " + "(old-subvolume:%s-%d new-subvolume:%s-%d)", + strerror(ENOMEM), oldfd, basefd, + uuid_utoa(basefd->inode->gfid), old_subvol->name, + old_subvol->graph->id, new_subvol->name, + new_subvol->graph->id); + } + + ret = syncop_fsync(old_subvol, oldfd, 0, NULL, NULL, xdata, NULL); + } if (ret < 0) { gf_log("glusterfs-fuse", GF_LOG_WARNING, @@ -5680,6 +5721,9 @@ out: fd_unref(oldfd); + if (xdata) + dict_unref(xdata); + return ret; } @@ -5855,7 +5899,9 @@ fuse_graph_sync(xlator_t *this) new_graph_id = priv->next_graph->id; priv->next_graph = NULL; need_first_lookup = 1; - priv->handle_graph_switch = _gf_true; + if (old_subvol) { + priv->handle_graph_switch = _gf_true; + } while (!priv->event_recvd) { ret = pthread_cond_wait(&priv->sync_cond, &priv->sync_mutex); @@ -5891,13 +5937,6 @@ unlock: if (winds_on_old_subvol == 0) { xlator_notify(old_subvol, GF_EVENT_PARENT_DOWN, old_subvol, NULL); } - } else { - pthread_mutex_lock(&priv->sync_mutex); - { - priv->handle_graph_switch = _gf_false; - pthread_cond_broadcast(&priv->migrate_cond); - } - pthread_mutex_unlock(&priv->sync_mutex); } return 0; @@ -6284,14 +6323,12 @@ out: int dump_history_fuse(circular_buffer_t *cb, void *data) { - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; - gf_time_fmt(timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_F_HMS); + gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_F_HMS); - snprintf(timestr + strlen(timestr), 256 - strlen(timestr), - ".%" GF_PRI_SUSECONDS, cb->tv.tv_usec); gf_proc_dump_write("TIME", "%s", timestr); gf_proc_dump_write("message", "%s\n", (char *)cb->data); @@ -6374,6 +6411,7 @@ notify(xlator_t *this, int32_t event, void *data, ...) fuse_private_t *private = NULL; gf_boolean_t start_thread = _gf_false; glusterfs_graph_t *graph = NULL; + struct pollfd pfd = {0}; private = this->private; @@ -6441,6 +6479,32 @@ notify(xlator_t *this, int32_t event, void *data, ...) /* Authentication failure is an error and glusterfs should stop */ gf_log(this->name, GF_LOG_ERROR, "Server authenication failed. Shutting down."); + pthread_mutex_lock(&private->sync_mutex); + { + /*Wait for mount to finish*/ + if (!private->mount_finished) { + pfd.fd = private->status_pipe[0]; + pfd.events = POLLIN | POLLHUP | POLLERR; + if (poll(&pfd, 1, -1) < 0) { + gf_log(this->name, GF_LOG_ERROR, "poll error %s", + strerror(errno)); + goto auth_fail_unlock; + } + if (pfd.revents & POLLIN) { + if (fuse_get_mount_status(this) != 0) { + goto auth_fail_unlock; + } + private + ->mount_finished = _gf_true; + } else if (pfd.revents) { + gf_log(this->name, GF_LOG_ERROR, + "mount pipe closed without status"); + goto auth_fail_unlock; + } + } + } + auth_fail_unlock: + pthread_mutex_unlock(&private->sync_mutex); fini(this); break; } @@ -6644,6 +6708,8 @@ init(xlator_t *this_xl) INIT_LIST_HEAD(&priv->interrupt_list); pthread_mutex_init(&priv->interrupt_mutex, NULL); + pthread_mutex_init(&priv->fusedev_errno_cnt_mutex, NULL); + /* get options from option dictionary */ ret = dict_get_str(options, ZR_MOUNTPOINT_OPT, &value_string); if (ret == -1 || value_string == NULL) { diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h index f4c6ad8f2fb..4cb94c23cad 100644 --- a/xlators/mount/fuse/src/fuse-bridge.h +++ b/xlators/mount/fuse/src/fuse-bridge.h @@ -78,6 +78,20 @@ typedef struct fuse_in_header fuse_in_header_t; typedef void(fuse_handler_t)(xlator_t *this, fuse_in_header_t *finh, void *msg, struct iobuf *iobuf); +enum fusedev_errno { + FUSEDEV_ENOENT, + FUSEDEV_ENOTDIR, + FUSEDEV_ENODEV, + FUSEDEV_EPERM, + FUSEDEV_ENOMEM, + FUSEDEV_ENOTCONN, + FUSEDEV_ECONNREFUSED, + FUSEDEV_EOVERFLOW, + FUSEDEV_EBUSY, + FUSEDEV_ENOTEMPTY, + FUSEDEV_EMAXPLUS +}; + struct fuse_private { int fd; uint32_t proto_minor; @@ -193,6 +207,10 @@ struct fuse_private { uint32_t lru_limit; uint32_t invalidate_limit; uint32_t fuse_dev_eperm_ratelimit_ns; + + /* counters for fusdev errnos */ + uint8_t fusedev_errno_cnt[FUSEDEV_EMAXPLUS]; + pthread_mutex_t fusedev_errno_cnt_mutex; }; typedef struct fuse_private fuse_private_t; @@ -226,6 +244,7 @@ enum fuse_interrupt_state { INTERRUPT_NONE, INTERRUPT_SQUELCHED, INTERRUPT_HANDLED, + INTERRUPT_WAITING_HANDLER, }; typedef enum fuse_interrupt_state fuse_interrupt_state_t; struct fuse_interrupt_record; @@ -341,30 +360,6 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t; return; \ } \ state->umask = fci->umask; \ - \ - /* TODO: remove this after 3.4.0 release. keeping it for the \ - sake of backward compatibility with old (3.3.[01]) \ - releases till then. */ \ - ret = dict_set_int16(state->xdata, "umask", fci->umask); \ - if (ret < 0) { \ - gf_log("glusterfs-fuse", GF_LOG_WARNING, \ - "%s Failed adding umask" \ - " to request", \ - op); \ - send_fuse_err(this, finh, ENOMEM); \ - free_fuse_state(state); \ - return; \ - } \ - ret = dict_set_int16(state->xdata, "mode", fci->mode); \ - if (ret < 0) { \ - gf_log("glusterfs-fuse", GF_LOG_WARNING, \ - "%s Failed adding mode " \ - "to request", \ - op); \ - send_fuse_err(this, finh, ENOMEM); \ - free_fuse_state(state); \ - return; \ - } \ } \ } while (0) diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c index fd11f2ba652..a2b0ad11fe4 100644 --- a/xlators/mount/fuse/src/fuse-helpers.c +++ b/xlators/mount/fuse/src/fuse-helpers.c @@ -139,8 +139,6 @@ get_fuse_state(xlator_t *this, fuse_in_header_t *finh) return state; } -#define FUSE_MAX_AUX_GROUPS \ - 32 /* We can get only up to 32 aux groups from /proc */ void frame_fill_groups(call_frame_t *frame) { @@ -150,8 +148,6 @@ frame_fill_groups(call_frame_t *frame) char filename[32]; char line[4096]; char *ptr = NULL; - FILE *fp = NULL; - int idx = 0; long int id = 0; char *saveptr = NULL; char *endptr = NULL; @@ -191,45 +187,72 @@ frame_fill_groups(call_frame_t *frame) call_stack_set_groups(frame->root, ngroups, &mygroups); } else { + FILE *fp = NULL; + ret = snprintf(filename, sizeof filename, "/proc/%d/status", frame->root->pid); - if (ret >= sizeof filename) + if (ret >= sizeof filename) { + gf_log(this->name, GF_LOG_ERROR, "procfs path exceeds buffer size"); goto out; + } fp = fopen(filename, "r"); - if (!fp) + if (!fp) { + gf_log(this->name, GF_LOG_ERROR, "failed to open %s: %s", filename, + strerror(errno)); goto out; + } - if (call_stack_alloc_groups(frame->root, ngroups) != 0) - goto out; + for (;;) { + gf_boolean_t found_groups = _gf_false; + int idx = 0; - while ((ptr = fgets(line, sizeof line, fp))) { - if (strncmp(ptr, "Groups:", 7) != 0) - continue; + if (call_stack_alloc_groups(frame->root, ngroups) != 0) { + gf_log(this->name, GF_LOG_ERROR, + "failed to allocate gid buffer"); + goto out; + } + while ((ptr = fgets(line, sizeof line, fp))) { + if (strncmp(ptr, "Groups:", 7) == 0) { + found_groups = _gf_true; + break; + } + } + if (!found_groups) { + gf_log(this->name, GF_LOG_ERROR, "cannot find gid list in %s", + filename); + break; + } ptr = line + 8; for (ptr = strtok_r(ptr, " \t\r\n", &saveptr); ptr; ptr = strtok_r(NULL, " \t\r\n", &saveptr)) { errno = 0; id = strtol(ptr, &endptr, 0); - if (errno == ERANGE) - break; - if (!endptr || *endptr) + if (errno == ERANGE || !endptr || *endptr) { + gf_log(this->name, GF_LOG_ERROR, "failed to parse %s", + filename); break; - frame->root->groups[idx++] = id; - if (idx == FUSE_MAX_AUX_GROUPS) + } + if (idx < call_stack_groups_capacity(frame->root)) + frame->root->groups[idx] = id; + idx++; + if (idx == GF_MAX_AUX_GROUPS) break; } - - frame->root->ngrps = idx; - break; + if (idx > call_stack_groups_capacity(frame->root)) { + ngroups = idx; + rewind(fp); + } else { + frame->root->ngrps = idx; + break; + } } + out: + if (fp) + fclose(fp); } - -out: - if (fp) - fclose(fp); #elif defined(GF_SOLARIS_HOST_OS) char filename[32]; char scratch[128]; @@ -245,7 +268,7 @@ out: fp = fopen(filename, "r"); if (fp != NULL) { if (fgets(scratch, sizeof scratch, fp) != NULL) { - ngrps = MIN(prcred->pr_ngroups, FUSE_MAX_AUX_GROUPS); + ngrps = MIN(prcred->pr_ngroups, GF_MAX_AUX_GROUPS); if (call_stack_alloc_groups(frame->root, ngrps) != 0) { fclose(fp); return; diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in index 99513b4e8c3..ac4d94cb743 100755 --- a/xlators/mount/fuse/utils/mount.glusterfs.in +++ b/xlators/mount/fuse/utils/mount.glusterfs.in @@ -776,6 +776,7 @@ main () [ $(echo $volume_str_temp | grep -c "/") -eq 1 ] && [ "$volume_id_temp" != "snaps" ] && { volume_id=$volume_id_temp; + [ ${first_char} = '/' ] && volume_id=/$volume_id; subdir_mount=$(echo "$volume_str_temp" | cut -f2- -d '/'); } } diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in index d43fc97d084..3a5feb606d7 100755 --- a/xlators/mount/fuse/utils/mount_glusterfs.in +++ b/xlators/mount/fuse/utils/mount_glusterfs.in @@ -469,6 +469,7 @@ parse_options() main () { +#if !defined(__FreeBSD__) ## `mount` on OSX specifies options as first argument echo $1|grep -q -- "-o" if [ $? -eq 0 ]; then @@ -478,7 +479,7 @@ main () volfile_loc=$1 mount_point=$2 fi - +#endif /* __FreeBSD__ */ while getopts "Vo:h" opt; do case "${opt}" in o) @@ -499,6 +500,12 @@ main () esac done +#ifdef __FreeBSD__ + shift $((OPTIND - 1)) + volfile_loc="$1" + mount_point="$2" +#endif /* __FreeBSD__ */ + [ -r "$volfile_loc" ] || { # '%' included to support ipv6 link local addresses server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p'); diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c index 50cd82dd422..7e3bbf16086 100644 --- a/xlators/nfs/server/src/acl3.c +++ b/xlators/nfs/server/src/acl3.c @@ -571,6 +571,12 @@ acl3_setacl_resume(void *carg) acl3_check_fh_resolve_status(cs, stat, acl3err); nfs_request_user_init(&nfu, cs->req); xattr = dict_new(); + if (xattr == NULL) { + gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); + goto acl3err; + } + if (cs->aclcount) ret = dict_set_static_bin(xattr, POSIX_ACL_ACCESS_XATTR, cs->aclxattr, posix_acl_xattr_size(cs->aclcount)); @@ -724,7 +730,6 @@ acl3svc_init(xlator_t *nfsx) struct nfs_state *nfs = NULL; dict_t *options = NULL; int ret = -1; - char *portstr = NULL; static gf_boolean_t acl3_inited = _gf_false; /* Already inited */ @@ -742,12 +747,13 @@ acl3svc_init(xlator_t *nfsx) acl3prog.private = ns; options = dict_new(); - - ret = gf_asprintf(&portstr, "%d", GF_ACL3_PORT); - if (ret == -1) + if (options == NULL) { + gf_msg(GF_ACL, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); goto err; + } - ret = dict_set_dynstr(options, "transport.socket.listen-port", portstr); + ret = dict_set_str(options, "transport.socket.listen-port", GF_ACL3_PORT); if (ret == -1) goto err; ret = dict_set_str(options, "transport-type", "socket"); @@ -783,7 +789,6 @@ acl3svc_init(xlator_t *nfsx) if (ret == -1) { gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL, "Unable to create listeners"); - dict_unref(options); goto err; } diff --git a/xlators/nfs/server/src/acl3.h b/xlators/nfs/server/src/acl3.h index dead04b4273..762fbb04a0f 100644 --- a/xlators/nfs/server/src/acl3.h +++ b/xlators/nfs/server/src/acl3.h @@ -18,7 +18,7 @@ #define ACL3_SETACL 2 #define ACL3_PROC_COUNT 3 -#define GF_ACL3_PORT 38469 +#define GF_ACL3_PORT "38469" #define GF_ACL GF_NFS "-ACL" /* Flags for the getacl/setacl mode */ diff --git a/xlators/nfs/server/src/auth-cache.c b/xlators/nfs/server/src/auth-cache.c index 64768646074..ffbf5b6cad6 100644 --- a/xlators/nfs/server/src/auth-cache.c +++ b/xlators/nfs/server/src/auth-cache.c @@ -189,7 +189,7 @@ out: static int _auth_cache_expired(struct auth_cache *cache, struct auth_cache_entry *entry) { - return ((time(NULL) - entry->timestamp) > cache->ttl_sec); + return ((gf_time() - entry->timestamp) > cache->ttl_sec); } /** @@ -474,7 +474,7 @@ cache_nfs_fh(struct auth_cache *cache, struct nfs3_fh *fh, goto out; } - entry->timestamp = time(NULL); + entry->timestamp = gf_time(); /* Update entry->item if it is pointing to a different export_item */ if (entry->item && entry->item != export_item) { GF_REF_PUT(entry->item); diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c index 88d3002a752..a34d9104c17 100644 --- a/xlators/nfs/server/src/mount3.c +++ b/xlators/nfs/server/src/mount3.c @@ -4062,6 +4062,11 @@ mnt3svc_init(xlator_t *nfsx) mnt3prog.private = mstate; options = dict_new(); + if (options == NULL) { + gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); + goto err; + } ret = gf_asprintf(&portstr, "%d", GF_MOUNTV3_PORT); if (ret == -1) @@ -4097,7 +4102,6 @@ mnt3svc_init(xlator_t *nfsx) if (ret == -1) { gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL, "Unable to create listeners"); - dict_unref(options); goto err; } @@ -4162,6 +4166,11 @@ mnt1svc_init(xlator_t *nfsx) mnt1prog.private = mstate; options = dict_new(); + if (options == NULL) { + gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); + goto err; + } ret = gf_asprintf(&portstr, "%d", GF_MOUNTV1_PORT); if (ret == -1) diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c index 0688779eb65..1a2b0f85453 100644 --- a/xlators/nfs/server/src/mount3udp_svc.c +++ b/xlators/nfs/server/src/mount3udp_svc.c @@ -216,7 +216,7 @@ mount3udp_thread(void *argv) GF_ASSERT(nfsx); - glusterfs_this_set(nfsx); + THIS = nfsx; transp = svcudp_create(RPC_ANYSOCK); if (transp == NULL) { diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c index cab57bbf3c9..39b73f88ac3 100644 --- a/xlators/nfs/server/src/nfs.c +++ b/xlators/nfs/server/src/nfs.c @@ -1157,7 +1157,7 @@ out: return ret; } -int +static int nfs_reconfigure_state(xlator_t *this, dict_t *options) { int ret = 0; @@ -1167,8 +1167,8 @@ nfs_reconfigure_state(xlator_t *this, dict_t *options) gf_boolean_t optbool; uint32_t optuint32; struct nfs_state *nfs = NULL; - char *blacklist_keys[] = {"nfs.port", "nfs.transport-type", - "nfs.mem-factor", NULL}; + static char *options_require_restart[] = {"nfs.port", "nfs.transport-type", + "nfs.mem-factor", NULL}; GF_VALIDATE_OR_GOTO(GF_NFS, this, out); GF_VALIDATE_OR_GOTO(GF_NFS, this->private, out); @@ -1176,14 +1176,14 @@ nfs_reconfigure_state(xlator_t *this, dict_t *options) nfs = (struct nfs_state *)this->private; - /* Black listed options can't be reconfigured, they need + /* Some listed options can't be reconfigured, they need * NFS to be restarted. There are two cases 1. SET 2. UNSET. * 1. SET */ - while (blacklist_keys[keyindx]) { - if (dict_get(options, blacklist_keys[keyindx])) { + while (options_require_restart[keyindx]) { + if (dict_get(options, options_require_restart[keyindx])) { gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL, "Reconfiguring %s needs NFS restart", - blacklist_keys[keyindx]); + options_require_restart[keyindx]); goto out; } keyindx++; diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c index 8a58977b53c..897fb42b071 100644 --- a/xlators/nfs/server/src/nfs3-helpers.c +++ b/xlators/nfs/server/src/nfs3-helpers.c @@ -1072,7 +1072,7 @@ nfs3_sattr3_to_setattr_valid(sattr3 *sattr, struct iatt *buf, mode_t *omode) if (sattr->atime.set_it == SET_TO_SERVER_TIME) { valid |= GF_SET_ATTR_ATIME; if (buf) - buf->ia_atime = time(NULL); + buf->ia_atime = gf_time(); } if (sattr->mtime.set_it == SET_TO_CLIENT_TIME) { @@ -1084,7 +1084,7 @@ nfs3_sattr3_to_setattr_valid(sattr3 *sattr, struct iatt *buf, mode_t *omode) if (sattr->mtime.set_it == SET_TO_SERVER_TIME) { valid |= GF_SET_ATTR_MTIME; if (buf) - buf->ia_mtime = time(NULL); + buf->ia_mtime = gf_time(); } return valid; diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c index 7cfd75f9ed1..f9042bc3b3f 100644 --- a/xlators/nfs/server/src/nfs3.c +++ b/xlators/nfs/server/src/nfs3.c @@ -5651,7 +5651,7 @@ nfs3_init_state(xlator_t *nfsx) goto free_localpool; } - nfs3->serverstart = (uint64_t)time(NULL); + nfs3->serverstart = (uint64_t)gf_time(); INIT_LIST_HEAD(&nfs3->fdlru); LOCK_INIT(&nfs3->fdlrulock); nfs3->fdcount = 0; diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c index ef65d7f55f8..577e8543966 100644 --- a/xlators/nfs/server/src/nlm4.c +++ b/xlators/nfs/server/src/nlm4.c @@ -1011,7 +1011,8 @@ nlm4_establish_callback(nfs3_call_state_t *cs, call_frame_t *cbk_frame) int port = -1; struct nlm4_notify_args *ncf = NULL; - glusterfs_this_set(cs->nfsx); + GF_ASSERT(cs->nfsx); + THIS = cs->nfsx; rpc_transport_get_peeraddr(cs->trans, NULL, 0, &sock_union.storage, sizeof(sock_union.storage)); @@ -1054,6 +1055,12 @@ nlm4_establish_callback(nfs3_call_state_t *cs, call_frame_t *cbk_frame) } options = dict_new(); + if (options == NULL) { + gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); + goto err; + } + ret = dict_set_str(options, "transport-type", "socket"); if (ret == -1) { gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED, @@ -2592,6 +2599,11 @@ nlm4svc_init(xlator_t *nfsx) nlm4prog.private = ns; options = dict_new(); + if (options == NULL) { + gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL, + "dict allocation failed"); + goto err; + } ret = gf_asprintf(&portstr, "%d", GF_NLM4_PORT); if (ret == -1) @@ -2633,7 +2645,6 @@ nlm4svc_init(xlator_t *nfsx) if (ret == -1) { gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL, "Unable to create listeners"); - dict_unref(options); goto err; } INIT_LIST_HEAD(&nlm_client_list); @@ -2704,7 +2715,7 @@ nlm4svc_init(xlator_t *nfsx) goto err; } - (void)gf_thread_create(&thr, NULL, nsm_thread, (void *)NULL, "nfsnsm"); + (void)gf_thread_create(&thr, NULL, nsm_thread, nfsx, "nfsnsm"); timeout.tv_sec = nlm_grace_period; timeout.tv_nsec = 0; diff --git a/xlators/nfs/server/src/nlmcbk_svc.c b/xlators/nfs/server/src/nlmcbk_svc.c index d18b86ce8db..eaa7b916190 100644 --- a/xlators/nfs/server/src/nlmcbk_svc.c +++ b/xlators/nfs/server/src/nlmcbk_svc.c @@ -84,9 +84,14 @@ nlmcbk_program_0(struct svc_req *rqstp, register SVCXPRT *transp) void * nsm_thread(void *argv) { + xlator_t *nfsx = argv; register SVCXPRT *transp; int ret = 0; + GF_ASSERT(nfsx); + + THIS = nfsx; + ret = pmap_unset(NLMCBK_PROGRAM, NLMCBK_V1); if (ret == 0) { gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_PMAP_UNSET_FAIL, diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c index c007e0a355d..9375d29c17f 100644 --- a/xlators/performance/io-cache/src/io-cache.c +++ b/xlators/performance/io-cache/src/io-cache.c @@ -133,23 +133,17 @@ ioc_update_pages(call_frame_t *frame, ioc_inode_t *ioc_inode, return 0; } -int32_t +static gf_boolean_t ioc_inode_need_revalidate(ioc_inode_t *ioc_inode) { - int8_t need_revalidate = 0; - struct timeval tv = { - 0, - }; ioc_table_t *table = NULL; + GF_ASSERT(ioc_inode); table = ioc_inode->table; + GF_ASSERT(table); - gettimeofday(&tv, NULL); - - if (time_elapsed(&tv, &ioc_inode->cache.tv) >= table->cache_timeout) - need_revalidate = 1; - - return need_revalidate; + return (gf_time() - ioc_inode->cache.last_revalidate >= + table->cache_timeout); } /* @@ -411,9 +405,6 @@ ioc_cache_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, ioc_inode_t *ioc_inode = NULL; size_t destroy_size = 0; struct iatt *local_stbuf = NULL; - struct timeval tv = { - 0, - }; local = frame->local; ioc_inode = local->inode; @@ -451,10 +442,9 @@ ioc_cache_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) local_stbuf = NULL; - gettimeofday(&tv, NULL); ioc_inode_lock(ioc_inode); { - memcpy(&ioc_inode->cache.tv, &tv, sizeof(struct timeval)); + ioc_inode->cache.last_revalidate = gf_time(); } ioc_inode_unlock(ioc_inode); @@ -1405,9 +1395,6 @@ ioc_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd, { ioc_inode_t *ioc_inode = NULL; uint64_t tmp_inode = 0; - struct timeval tv = { - 0, - }; inode_ctx_get(fd->inode, this, &tmp_inode); ioc_inode = (ioc_inode_t *)(long)tmp_inode; @@ -1418,10 +1405,9 @@ ioc_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd, return 0; } - gettimeofday(&tv, NULL); ioc_inode_lock(ioc_inode); { - memcpy(&ioc_inode->cache.tv, &tv, sizeof(struct timeval)); + ioc_inode->cache.last_revalidate = gf_time(); } ioc_inode_unlock(ioc_inode); @@ -1945,7 +1931,7 @@ __ioc_cache_dump(ioc_inode_t *ioc_inode, char *prefix) char key[GF_DUMP_MAX_BUF_LEN] = { 0, }; - char timestr[256] = { + char timestr[GF_TIMESTR_SIZE] = { 0, }; @@ -1955,11 +1941,9 @@ __ioc_cache_dump(ioc_inode_t *ioc_inode, char *prefix) table = ioc_inode->table; - if (ioc_inode->cache.tv.tv_sec) { - gf_time_fmt(timestr, sizeof timestr, ioc_inode->cache.tv.tv_sec, + if (ioc_inode->cache.last_revalidate) { + gf_time_fmt(timestr, sizeof timestr, ioc_inode->cache.last_revalidate, gf_timefmt_FT); - snprintf(timestr + strlen(timestr), sizeof timestr - strlen(timestr), - ".%" GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec); gf_proc_dump_write("last-cache-validation-time", "%s", timestr); } diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h index 4303c2fae13..14923c75edc 100644 --- a/xlators/performance/io-cache/src/io-cache.h +++ b/xlators/performance/io-cache/src/io-cache.h @@ -117,15 +117,13 @@ struct ioc_page { struct ioc_cache { rbthash_table_t *page_table; struct list_head page_lru; - time_t mtime; /* - * seconds component of file mtime - */ - time_t mtime_nsec; /* - * nanosecond component of file mtime - */ - struct timeval tv; /* - * time-stamp at last re-validate - */ + time_t mtime; /* + * seconds component of file mtime + */ + time_t mtime_nsec; /* + * nanosecond component of file mtime + */ + time_t last_revalidate; /* timestamp at last re-validate */ }; struct ioc_inode { @@ -270,17 +268,6 @@ ioc_frame_fill(ioc_page_t *page, call_frame_t *frame, off_t offset, size_t size, pthread_mutex_unlock(&page->page_lock); \ } while (0) -static inline uint64_t -time_elapsed(struct timeval *now, struct timeval *then) -{ - uint64_t sec = now->tv_sec - then->tv_sec; - - if (sec) - return sec; - - return 0; -} - ioc_inode_t * ioc_inode_search(ioc_table_t *table, inode_t *inode); diff --git a/xlators/performance/io-cache/src/page.c b/xlators/performance/io-cache/src/page.c index a8edbde23f2..84b1ae6cb20 100644 --- a/xlators/performance/io-cache/src/page.c +++ b/xlators/performance/io-cache/src/page.c @@ -413,9 +413,6 @@ ioc_fault_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, ioc_waitq_t *waitq = NULL; size_t iobref_page_size = 0; char zero_filled = 0; - struct timeval tv = { - 0, - }; GF_ASSERT(frame); @@ -431,7 +428,6 @@ ioc_fault_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, zero_filled = ((op_ret >= 0) && (stbuf->ia_mtime == 0)); - gettimeofday(&tv, NULL); ioc_inode_lock(ioc_inode); { if (op_ret == -1 || @@ -448,7 +444,7 @@ ioc_fault_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, ioc_inode->cache.mtime_nsec = stbuf->ia_mtime_nsec; } - memcpy(&ioc_inode->cache.tv, &tv, sizeof(struct timeval)); + ioc_inode->cache.last_revalidate = gf_time(); if (op_ret < 0) { /* error, readv returned -1 */ diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c index 6fa4d88389c..3d24cc97f4b 100644 --- a/xlators/performance/io-threads/src/io-threads.c +++ b/xlators/performance/io-threads/src/io-threads.c @@ -1016,16 +1016,13 @@ static uint32_t THRESH_LIMIT = 1209600; /* SECONDS * (EVENTS-1) */ static void iot_apply_event(xlator_t *this, threshold_t *thresh) { - struct timespec now; - time_t delta; + time_t delta, now = gf_time(); /* Refresh for manual testing/debugging. It's cheap. */ THRESH_LIMIT = THRESH_SECONDS * (THRESH_EVENTS - 1); - timespec_now(&now); - if (thresh->value && thresh->update_time) { - delta = now.tv_sec - thresh->update_time; + delta = now - thresh->update_time; /* Be careful about underflow. */ if (thresh->value <= delta) { thresh->value = 0; @@ -1046,7 +1043,7 @@ iot_apply_event(xlator_t *this, threshold_t *thresh) kill(getpid(), SIGTRAP); } - thresh->update_time = now.tv_sec; + thresh->update_time = now; } static void * @@ -1311,7 +1308,7 @@ notify(xlator_t *this, int32_t event, void *data, ...) /* Wait for draining stub from queue before notify PARENT_DOWN */ stub_cnt = GF_ATOMIC_GET(conf->stub_cnt); if (stub_cnt) { - clock_gettime(CLOCK_REALTIME, &sleep_till); + timespec_now_realtime(&sleep_till); sleep_till.tv_sec += 1; pthread_mutex_lock(&conf->mutex); { diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c index 4c76f3089d5..a405be51f02 100644 --- a/xlators/performance/md-cache/src/md-cache.c +++ b/xlators/performance/md-cache/src/md-cache.c @@ -8,7 +8,6 @@ cases as published by the Free Software Foundation. */ -#include <glusterfs/timespec.h> #include <glusterfs/glusterfs.h> #include <glusterfs/defaults.h> #include <glusterfs/logging.h> @@ -33,8 +32,7 @@ struct mdc_statfs_cache { pthread_mutex_t lock; - gf_boolean_t initialized; - struct timespec last_refreshed; + time_t last_refreshed; /* (time_t)-1 if not yet initialized. */ struct statvfs buf; }; @@ -61,7 +59,7 @@ struct mdc_statistics { }; struct mdc_conf { - int timeout; + uint32_t timeout; gf_boolean_t cache_posix_acl; gf_boolean_t cache_glusterfs_acl; gf_boolean_t cache_selinux; @@ -132,6 +130,7 @@ struct mdc_local { char *key; dict_t *xattr; uint64_t incident_time; + bool update_cache; }; int @@ -375,10 +374,9 @@ unlock: static gf_boolean_t __is_cache_valid(xlator_t *this, time_t mdc_time) { - time_t now = 0; gf_boolean_t ret = _gf_true; struct mdc_conf *conf = NULL; - int timeout = 0; + uint32_t timeout = 0; time_t last_child_down = 0; conf = this->private; @@ -392,15 +390,13 @@ __is_cache_valid(xlator_t *this, time_t mdc_time) last_child_down = conf->last_child_down; timeout = conf->timeout; - time(&now); - if ((mdc_time == 0) || ((last_child_down != 0) && (mdc_time < last_child_down))) { ret = _gf_false; goto out; } - if (now >= (mdc_time + timeout)) { + if (gf_time() >= (mdc_time + timeout)) { ret = _gf_false; } @@ -580,10 +576,9 @@ mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf, mdc_from_iatt(mdc, iatt); mdc->valid = _gf_true; if (update_time) { - time(&mdc->ia_time); - + mdc->ia_time = gf_time(); if (mdc->xa_time && update_xa_time) - time(&mdc->xa_time); + mdc->xa_time = mdc->ia_time; } gf_msg_callingfn( @@ -784,7 +779,7 @@ mdc_inode_xatt_set(xlator_t *this, inode_t *inode, dict_t *dict) if (newdict) mdc->xattr = newdict; - time(&mdc->xa_time); + mdc->xa_time = gf_time(); gf_msg_trace("md-cache", 0, "xatt cache set for (%s) time:%lld", uuid_utoa(inode->gfid), (long long)mdc->xa_time); } @@ -985,7 +980,7 @@ out: return ret; } -void +static bool mdc_load_reqs(xlator_t *this, dict_t *dict) { struct mdc_conf *conf = this->private; @@ -994,6 +989,7 @@ mdc_load_reqs(xlator_t *this, dict_t *dict) char *tmp = NULL; char *tmp1 = NULL; int ret = 0; + bool loaded = false; tmp1 = conf->mdc_xattr_str; if (!tmp1) @@ -1011,13 +1007,17 @@ mdc_load_reqs(xlator_t *this, dict_t *dict) conf->mdc_xattr_str = NULL; gf_msg("md-cache", GF_LOG_ERROR, 0, MD_CACHE_MSG_NO_XATTR_CACHE, "Disabled cache for xattrs, dict_set failed"); + goto out; } pattern = strtok_r(NULL, ",", &tmp); } - GF_FREE(mdc_xattr_str); + loaded = true; + out: - return; + GF_FREE(mdc_xattr_str); + + return loaded; } struct checkpair { @@ -1057,8 +1057,7 @@ mdc_cache_statfs(xlator_t *this, struct statvfs *buf) pthread_mutex_lock(&conf->statfs_cache.lock); { memcpy(&conf->statfs_cache.buf, buf, sizeof(struct statvfs)); - clock_gettime(CLOCK_MONOTONIC, &conf->statfs_cache.last_refreshed); - conf->statfs_cache.initialized = _gf_true; + conf->statfs_cache.last_refreshed = gf_time(); } pthread_mutex_unlock(&conf->statfs_cache.lock); } @@ -1067,8 +1066,7 @@ int mdc_load_statfs_info_from_cache(xlator_t *this, struct statvfs **buf) { struct mdc_conf *conf = this->private; - struct timespec now; - double cache_age = 0.0; + uint32_t cache_age = 0; int ret = 0; if (!buf || !conf) { @@ -1077,23 +1075,23 @@ mdc_load_statfs_info_from_cache(xlator_t *this, struct statvfs **buf) } *buf = NULL; - timespec_now(&now); pthread_mutex_lock(&conf->statfs_cache.lock); { - /* Skip if the cache is not initialized */ - if (!conf->statfs_cache.initialized) { + /* Skip if the cache is not initialized. */ + if (conf->statfs_cache.last_refreshed == (time_t)-1) { ret = -1; goto unlock; } - cache_age = (now.tv_sec - conf->statfs_cache.last_refreshed.tv_sec); + cache_age = (gf_time() - conf->statfs_cache.last_refreshed); - gf_log(this->name, GF_LOG_DEBUG, "STATFS cache age = %lf", cache_age); + gf_log(this->name, GF_LOG_DEBUG, "STATFS cache age = %u secs", + cache_age); if (cache_age > conf->timeout) { - /* Expire the cache */ + /* Expire the cache. */ gf_log(this->name, GF_LOG_DEBUG, - "Cache age %lf exceeded timeout %d", cache_age, + "Cache age %u secs exceeded timeout %u secs", cache_age, conf->timeout); ret = -1; goto unlock; @@ -1107,6 +1105,31 @@ err: return ret; } +static dict_t * +mdc_prepare_request(xlator_t *this, mdc_local_t *local, dict_t *xdata) +{ + if (xdata != NULL) { + dict_ref(xdata); + } + + if (local == NULL) { + return xdata; + } + + if (xdata == NULL) { + xdata = dict_new(); + if (xdata == NULL) { + local->update_cache = false; + + return NULL; + } + } + + local->update_cache = mdc_load_reqs(this, xdata); + + return xdata; +} + int mdc_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct statvfs *buf, @@ -1189,6 +1212,9 @@ mdc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, local = frame->local; + if (!local) + goto out; + if (op_ret != 0) { if (op_errno == ENOENT) GF_ATOMIC_INC(conf->mdc_counter.negative_lookup); @@ -1206,9 +1232,6 @@ mdc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, goto out; } - if (!local) - goto out; - if (local->loc.parent) { mdc_inode_iatt_set(this, local->loc.parent, postparent, local->incident_time); @@ -1216,7 +1239,9 @@ mdc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (local->loc.inode) { mdc_inode_iatt_set(this, local->loc.inode, stbuf, local->incident_time); - mdc_inode_xatt_set(this, local->loc.inode, dict); + if (local->update_cache) { + mdc_inode_xatt_set(this, local->loc.inode, dict); + } } out: MDC_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, dict, @@ -1235,7 +1260,6 @@ mdc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) 0, }; dict_t *xattr_rsp = NULL; - dict_t *xattr_alloc = NULL; mdc_local_t *local = NULL; struct mdc_conf *conf = this->private; @@ -1286,18 +1310,18 @@ mdc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) return 0; uncached: - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); STACK_WIND(frame, mdc_lookup_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, loc, xdata); if (xattr_rsp) dict_unref(xattr_rsp); - if (xattr_alloc) - dict_unref(xattr_alloc); + + if (xdata != NULL) { + dict_unref(xdata); + } + return 0; } @@ -1320,7 +1344,9 @@ mdc_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, } mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time); - mdc_inode_xatt_set(this, local->loc.inode, xdata); + if (local->update_cache) { + mdc_inode_xatt_set(this, local->loc.inode, xdata); + } out: MDC_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata); @@ -1334,7 +1360,6 @@ mdc_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) int ret; struct iatt stbuf; mdc_local_t *local = NULL; - dict_t *xattr_alloc = NULL; struct mdc_conf *conf = this->private; local = mdc_local_get(frame, loc->inode); @@ -1358,17 +1383,16 @@ mdc_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) return 0; uncached: - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); GF_ATOMIC_INC(conf->mdc_counter.stat_miss); STACK_WIND(frame, mdc_stat_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat, loc, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + if (xdata != NULL) { + dict_unref(xdata); + } + return 0; } @@ -1391,7 +1415,9 @@ mdc_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, } mdc_inode_iatt_set(this, local->fd->inode, buf, local->incident_time); - mdc_inode_xatt_set(this, local->fd->inode, xdata); + if (local->update_cache) { + mdc_inode_xatt_set(this, local->fd->inode, xdata); + } out: MDC_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata); @@ -1405,14 +1431,13 @@ mdc_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) int ret; struct iatt stbuf; mdc_local_t *local = NULL; - dict_t *xattr_alloc = NULL; struct mdc_conf *conf = this->private; local = mdc_local_get(frame, fd->inode); if (!local) goto uncached; - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); ret = mdc_inode_iatt_get(this, fd->inode, &stbuf); if (ret != 0) @@ -1424,17 +1449,16 @@ mdc_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) return 0; uncached: - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); GF_ATOMIC_INC(conf->mdc_counter.stat_miss); STACK_WIND(frame, mdc_fstat_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat, fd, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + if (xdata != NULL) { + dict_unref(xdata); + } + return 0; } @@ -1473,8 +1497,9 @@ mdc_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - local->loc.inode = inode_ref(loc->inode); + if (local != NULL) { + local->loc.inode = inode_ref(loc->inode); + } STACK_WIND(frame, mdc_truncate_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate, loc, offset, xdata); @@ -1517,8 +1542,9 @@ mdc_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); - - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_ftruncate_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata); @@ -1566,9 +1592,10 @@ mdc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); - local->xattr = dict_ref(xdata); + if (local != NULL) { + loc_copy(&local->loc, loc); + local->xattr = dict_ref(xdata); + } STACK_WIND(frame, mdc_mknod_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata); @@ -1616,9 +1643,10 @@ mdc_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); - local->xattr = dict_ref(xdata); + if (local != NULL) { + loc_copy(&local->loc, loc); + local->xattr = dict_ref(xdata); + } STACK_WIND(frame, mdc_mkdir_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata); @@ -1675,8 +1703,9 @@ mdc_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); + if (local != NULL) { + loc_copy(&local->loc, loc); + } STACK_WIND(frame, mdc_unlink_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata); @@ -1729,8 +1758,9 @@ mdc_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); + if (local != NULL) { + loc_copy(&local->loc, loc); + } STACK_WIND(frame, mdc_rmdir_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir, loc, flag, xdata); @@ -1777,13 +1807,22 @@ mdc_symlink(call_frame_t *frame, xlator_t *this, const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata) { mdc_local_t *local = NULL; + char *name; + name = gf_strdup(linkname); + if (name == NULL) { + goto wind; + } local = mdc_local_get(frame, loc->inode); + if (local == NULL) { + GF_FREE(name); + goto wind; + } loc_copy(&local->loc, loc); + local->linkname = name; - local->linkname = gf_strdup(linkname); - +wind: STACK_WIND(frame, mdc_symlink_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata); return 0; @@ -1841,9 +1880,10 @@ mdc_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, mdc_local_t *local = NULL; local = mdc_local_get(frame, oldloc->inode); - - loc_copy(&local->loc, oldloc); - loc_copy(&local->loc2, newloc); + if (local != NULL) { + loc_copy(&local->loc, oldloc); + loc_copy(&local->loc2, newloc); + } STACK_WIND(frame, mdc_rename_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata); @@ -1892,9 +1932,10 @@ mdc_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, mdc_local_t *local = NULL; local = mdc_local_get(frame, oldloc->inode); - - loc_copy(&local->loc, oldloc); - loc_copy(&local->loc2, newloc); + if (local != NULL) { + loc_copy(&local->loc, oldloc); + loc_copy(&local->loc2, newloc); + } STACK_WIND(frame, mdc_link_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata); @@ -1943,9 +1984,10 @@ mdc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); - local->xattr = dict_ref(xdata); + if (local != NULL) { + loc_copy(&local->loc, loc); + local->xattr = dict_ref(xdata); + } STACK_WIND(frame, mdc_create_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd, @@ -1992,8 +2034,9 @@ mdc_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd, } local = mdc_local_get(frame, loc->inode); - - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } out: STACK_WIND(frame, mdc_open_cbk, FIRST_CHILD(this), @@ -2034,8 +2077,9 @@ mdc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); - - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_readv_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata); @@ -2076,8 +2120,9 @@ mdc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector, mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); - - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_writev_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd, vector, count, offset, @@ -2093,15 +2138,14 @@ mdc_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, mdc_local_t *local = NULL; local = frame->local; + if (!local) + goto out; if (op_ret != 0) { mdc_inode_iatt_set(this, local->loc.inode, NULL, local->incident_time); goto out; } - if (!local) - goto out; - mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf, _gf_true, local->incident_time); mdc_inode_xatt_update(this, local->loc.inode, xdata); @@ -2122,6 +2166,9 @@ mdc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf, struct mdc_conf *conf = this->private; local = mdc_local_get(frame, loc->inode); + if (local == NULL) { + goto wind; + } loc_copy(&local->loc, loc); @@ -2149,6 +2196,7 @@ mdc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf, } } +wind: STACK_WIND(frame, mdc_setattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata); @@ -2194,8 +2242,11 @@ mdc_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf, struct mdc_conf *conf = this->private; local = mdc_local_get(frame, fd->inode); + if (local == NULL) { + goto wind; + } - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); if ((valid & GF_SET_ATTR_MODE) && conf->cache_glusterfs_acl) { if (!xdata) @@ -2221,6 +2272,7 @@ mdc_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf, } } +wind: STACK_WIND(frame, mdc_fsetattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata); @@ -2262,8 +2314,9 @@ mdc_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync, mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); - - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_fsync_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata); @@ -2318,9 +2371,10 @@ mdc_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr, mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); - local->xattr = dict_ref(xattr); + if (local != NULL) { + loc_copy(&local->loc, loc); + local->xattr = dict_ref(xattr); + } STACK_WIND(frame, mdc_setxattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, loc, xattr, flags, xdata); @@ -2376,9 +2430,10 @@ mdc_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr, mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); - - local->fd = fd_ref(fd); - local->xattr = dict_ref(xattr); + if (local != NULL) { + local->fd = __fd_ref(fd); + local->xattr = dict_ref(xattr); + } STACK_WIND(frame, mdc_fsetxattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr, fd, xattr, flags, xdata); @@ -2408,7 +2463,9 @@ mdc_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, goto out; } - mdc_inode_xatt_set(this, local->loc.inode, xdata); + if (local->update_cache) { + mdc_inode_xatt_set(this, local->loc.inode, xdata); + } out: MDC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata); @@ -2425,19 +2482,19 @@ mdc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key, mdc_local_t *local = NULL; dict_t *xattr = NULL; struct mdc_conf *conf = this->private; - dict_t *xattr_alloc = NULL; - gf_boolean_t key_satisfied = _gf_true; + gf_boolean_t key_satisfied = _gf_false; local = mdc_local_get(frame, loc->inode); - if (!local) + if (!local) { goto uncached; + } loc_copy(&local->loc, loc); if (!is_mdc_key_satisfied(this, key)) { - key_satisfied = _gf_false; goto uncached; } + key_satisfied = _gf_true; ret = mdc_inode_xatt_get(this, loc->inode, &xattr); if (ret != 0) @@ -2458,18 +2515,17 @@ mdc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key, uncached: if (key_satisfied) { - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); } GF_ATOMIC_INC(conf->mdc_counter.xattr_miss); STACK_WIND(frame, mdc_getxattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr, loc, key, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + if (key_satisfied && (xdata != NULL)) { + dict_unref(xdata); + } + return 0; } @@ -2496,7 +2552,9 @@ mdc_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, goto out; } - mdc_inode_xatt_set(this, local->fd->inode, xdata); + if (local->update_cache) { + mdc_inode_xatt_set(this, local->fd->inode, xdata); + } out: MDC_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata); @@ -2513,14 +2571,13 @@ mdc_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key, dict_t *xattr = NULL; int op_errno = ENODATA; struct mdc_conf *conf = this->private; - dict_t *xattr_alloc = NULL; gf_boolean_t key_satisfied = _gf_true; local = mdc_local_get(frame, fd->inode); if (!local) goto uncached; - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); if (!is_mdc_key_satisfied(this, key)) { key_satisfied = _gf_false; @@ -2546,18 +2603,17 @@ mdc_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key, uncached: if (key_satisfied) { - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); } GF_ATOMIC_INC(conf->mdc_counter.xattr_miss); STACK_WIND(frame, mdc_fgetxattr_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + if (key_satisfied && (xdata != NULL)) { + dict_unref(xdata); + } + return 0; } @@ -2613,12 +2669,21 @@ mdc_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, int ret = 0; dict_t *xattr = NULL; struct mdc_conf *conf = this->private; + char *name2; + + name2 = gf_strdup(name); + if (name2 == NULL) { + goto uncached; + } local = mdc_local_get(frame, loc->inode); + if (local == NULL) { + GF_FREE(name2); + goto uncached; + } loc_copy(&local->loc, loc); - - local->key = gf_strdup(name); + local->key = name2; if (!is_mdc_key_satisfied(this, name)) goto uncached; @@ -2704,12 +2769,21 @@ mdc_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, int ret = 0; dict_t *xattr = NULL; struct mdc_conf *conf = this->private; + char *name2; - local = mdc_local_get(frame, fd->inode); + name2 = gf_strdup(name); + if (name2 == NULL) { + goto uncached; + } - local->fd = fd_ref(fd); + local = mdc_local_get(frame, fd->inode); + if (local == NULL) { + GF_FREE(name2); + goto uncached; + } - local->key = gf_strdup(name); + local->fd = __fd_ref(fd); + local->key = name2; if (!is_mdc_key_satisfied(this, name)) goto uncached; @@ -2767,27 +2841,23 @@ int mdc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata) { - dict_t *xattr_alloc = NULL; mdc_local_t *local = NULL; local = mdc_local_get(frame, loc->inode); - - loc_copy(&local->loc, loc); - - if (!xdata) - xdata = xattr_alloc = dict_new(); - - if (xdata) { - /* Tell readdir-ahead to include these keys in xdata when it - * internally issues readdirp() in it's opendir_cbk */ - mdc_load_reqs(this, xdata); + if (local != NULL) { + loc_copy(&local->loc, loc); } + /* Tell readdir-ahead to include these keys in xdata when it + * internally issues readdirp() in it's opendir_cbk */ + xdata = mdc_prepare_request(this, local, xdata); + STACK_WIND(frame, mdc_opendir_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir, loc, fd, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + if (xdata != NULL) { + dict_unref(xdata); + } return 0; } @@ -2815,7 +2885,9 @@ mdc_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, continue; mdc_inode_iatt_set(this, entry->inode, &entry->d_stat, local->incident_time); - mdc_inode_xatt_set(this, entry->inode, entry->dict); + if (local->update_cache) { + mdc_inode_xatt_set(this, entry->inode, entry->dict); + } } unwind: @@ -2827,24 +2899,23 @@ int mdc_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, dict_t *xdata) { - dict_t *xattr_alloc = NULL; mdc_local_t *local = NULL; local = mdc_local_get(frame, fd->inode); if (!local) goto out; - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); - if (!xdata) - xdata = xattr_alloc = dict_new(); - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata); - if (xattr_alloc) - dict_unref(xattr_alloc); + + if (xdata != NULL) { + dict_unref(xdata); + } + return 0; out: MDC_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL); @@ -2875,7 +2946,6 @@ int mdc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, dict_t *xdata) { - int need_unref = 0; mdc_local_t *local = NULL; struct mdc_conf *conf = this->private; @@ -2883,7 +2953,7 @@ mdc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, if (!local) goto unwind; - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); if (!conf->force_readdirp) { STACK_WIND(frame, mdc_readdir_cbk, FIRST_CHILD(this), @@ -2891,19 +2961,14 @@ mdc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, return 0; } - if (!xdata) { - xdata = dict_new(); - need_unref = 1; - } - - if (xdata) - mdc_load_reqs(this, xdata); + xdata = mdc_prepare_request(this, local, xdata); STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata); - if (need_unref && xdata) + if (xdata != NULL) { dict_unref(xdata); + } return 0; unwind: @@ -2945,7 +3010,9 @@ mdc_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, mdc_local_t *local; local = mdc_local_get(frame, fd->inode); - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_fallocate_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len, @@ -2987,7 +3054,9 @@ mdc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, mdc_local_t *local; local = mdc_local_get(frame, fd->inode); - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_discard_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata); @@ -3028,7 +3097,9 @@ mdc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, mdc_local_t *local; local = mdc_local_get(frame, fd->inode); - local->fd = fd_ref(fd); + if (local != NULL) { + local->fd = __fd_ref(fd); + } STACK_WIND(frame, mdc_zerofill_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata); @@ -3110,7 +3181,7 @@ mdc_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, if (!local) goto unwind; - local->fd = fd_ref(fd); + local->fd = __fd_ref(fd); STACK_WIND(frame, mdc_fsyncdir_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata); @@ -3483,7 +3554,12 @@ mdc_register_xattr_inval(xlator_t *this) goto out; } - mdc_load_reqs(this, xattr); + if (!mdc_load_reqs(this, xattr)) { + gf_msg(this->name, GF_LOG_WARNING, ENOMEM, MD_CACHE_MSG_NO_MEMORY, + "failed to populate cache entries"); + ret = -1; + goto out; + } frame = create_frame(this, this->ctx->pool); if (!frame) { @@ -3532,7 +3608,7 @@ int mdc_reconfigure(xlator_t *this, dict_t *options) { struct mdc_conf *conf = NULL; - int timeout = 0; + int timeout = 0, ret = 0; char *tmp_str = NULL; conf = this->private; @@ -3572,7 +3648,10 @@ mdc_reconfigure(xlator_t *this, dict_t *options) GF_OPTION_RECONF("md-cache-statfs", conf->cache_statfs, options, bool, out); GF_OPTION_RECONF("xattr-cache-list", tmp_str, options, str, out); - mdc_xattr_list_populate(conf, tmp_str); + + ret = mdc_xattr_list_populate(conf, tmp_str); + if (ret < 0) + goto out; /* If timeout is greater than 60s (default before the patch that added * cache invalidation support was added) then, cache invalidation @@ -3585,25 +3664,22 @@ mdc_reconfigure(xlator_t *this, dict_t *options) } conf->timeout = timeout; - (void)mdc_register_xattr_inval(this); + ret = mdc_register_xattr_inval(this); out: - return 0; + return ret; } int32_t mdc_mem_acct_init(xlator_t *this) { - int ret = -1; - - ret = xlator_mem_acct_init(this, gf_mdc_mt_end + 1); - return ret; + return xlator_mem_acct_init(this, gf_mdc_mt_end + 1); } int mdc_init(xlator_t *this) { struct mdc_conf *conf = NULL; - int timeout = 0; + uint32_t timeout = 0; char *tmp_str = NULL; conf = GF_CALLOC(sizeof(*conf), 1, gf_mdc_mt_mdc_conf_t); @@ -3615,7 +3691,7 @@ mdc_init(xlator_t *this) LOCK_INIT(&conf->lock); - GF_OPTION_INIT("md-cache-timeout", timeout, int32, out); + GF_OPTION_INIT("md-cache-timeout", timeout, uint32, out); GF_OPTION_INIT("cache-selinux", conf->cache_selinux, bool, out); @@ -3649,7 +3725,9 @@ mdc_init(xlator_t *this) GF_OPTION_INIT("xattr-cache-list", tmp_str, str, out); mdc_xattr_list_populate(conf, tmp_str); - time(&conf->last_child_down); + conf->last_child_down = gf_time(); + conf->statfs_cache.last_refreshed = (time_t)-1; + /* initialize gf_atomic_t counters */ GF_ATOMIC_INIT(conf->mdc_counter.stat_hit, 0); GF_ATOMIC_INIT(conf->mdc_counter.stat_miss, 0); @@ -3680,7 +3758,7 @@ out: } void -mdc_update_child_down_time(xlator_t *this, time_t *now) +mdc_update_child_down_time(xlator_t *this, time_t now) { struct mdc_conf *conf = NULL; @@ -3688,7 +3766,7 @@ mdc_update_child_down_time(xlator_t *this, time_t *now) LOCK(&conf->lock); { - conf->last_child_down = *now; + conf->last_child_down = now; } UNLOCK(&conf->lock); } @@ -3698,14 +3776,12 @@ mdc_notify(xlator_t *this, int event, void *data, ...) { int ret = 0; struct mdc_conf *conf = NULL; - time_t now = 0; conf = this->private; switch (event) { case GF_EVENT_CHILD_DOWN: case GF_EVENT_SOME_DESCENDENT_DOWN: - time(&now); - mdc_update_child_down_time(this, &now); + mdc_update_child_down_time(this, gf_time()); break; case GF_EVENT_UPCALL: if (conf->mdc_invalidation) diff --git a/xlators/performance/nl-cache/src/nl-cache-helper.c b/xlators/performance/nl-cache/src/nl-cache-helper.c index 03dedf8ea08..29b99b5b8ea 100644 --- a/xlators/performance/nl-cache/src/nl-cache-helper.c +++ b/xlators/performance/nl-cache/src/nl-cache-helper.c @@ -113,7 +113,7 @@ out: } void -nlc_update_child_down_time(xlator_t *this, time_t *now) +nlc_update_child_down_time(xlator_t *this, time_t now) { nlc_conf_t *conf = NULL; @@ -121,7 +121,7 @@ nlc_update_child_down_time(xlator_t *this, time_t *now) LOCK(&conf->lock); { - conf->last_child_down = *now; + conf->last_child_down = now; } UNLOCK(&conf->lock); @@ -262,7 +262,7 @@ nlc_init_invalid_ctx(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx) if (nlc_ctx->timer) { gf_tw_mod_timer_pending(conf->timer_wheel, nlc_ctx->timer, conf->cache_timeout); - time(&nlc_ctx->cache_time); + nlc_ctx->cache_time = gf_time(); goto unlock; } @@ -496,7 +496,7 @@ __nlc_inode_ctx_timer_start(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx) nlc_ctx->timer_data = tmp; gf_tw_add_timer(conf->timer_wheel, timer); - time(&nlc_ctx->cache_time); + nlc_ctx->cache_time = gf_time(); gf_msg_trace(this->name, 0, "Registering timer:%p, inode:%p, " "gfid:%s", diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c index cd0e1d195fd..33a7c471663 100644 --- a/xlators/performance/nl-cache/src/nl-cache.c +++ b/xlators/performance/nl-cache/src/nl-cache.c @@ -520,15 +520,13 @@ int nlc_notify(xlator_t *this, int event, void *data, ...) { int ret = 0; - time_t now = 0; switch (event) { case GF_EVENT_CHILD_DOWN: case GF_EVENT_SOME_DESCENDENT_DOWN: case GF_EVENT_CHILD_UP: case GF_EVENT_SOME_DESCENDENT_UP: - time(&now); - nlc_update_child_down_time(this, &now); + nlc_update_child_down_time(this, gf_time()); /* TODO: nlc_clear_all_cache (this); else lru prune will lazily clear it*/ break; @@ -731,7 +729,7 @@ nlc_init(xlator_t *this) GF_ATOMIC_INIT(conf->nlc_counter.nlc_invals, 0); INIT_LIST_HEAD(&conf->lru); - time(&conf->last_child_down); + conf->last_child_down = gf_time(); conf->timer_wheel = glusterfs_ctx_tw_get(this->ctx); if (!conf->timer_wheel) { diff --git a/xlators/performance/nl-cache/src/nl-cache.h b/xlators/performance/nl-cache/src/nl-cache.h index 8b09972bb09..85fcc176342 100644 --- a/xlators/performance/nl-cache/src/nl-cache.h +++ b/xlators/performance/nl-cache/src/nl-cache.h @@ -155,7 +155,7 @@ nlc_local_init(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop, loc_t *loc, loc_t *loc2); void -nlc_update_child_down_time(xlator_t *this, time_t *now); +nlc_update_child_down_time(xlator_t *this, time_t now); void nlc_inode_clear_cache(xlator_t *this, inode_t *inode, int reason); diff --git a/xlators/performance/open-behind/src/open-behind-messages.h b/xlators/performance/open-behind/src/open-behind-messages.h index f25082433f8..0e789177684 100644 --- a/xlators/performance/open-behind/src/open-behind-messages.h +++ b/xlators/performance/open-behind/src/open-behind-messages.h @@ -23,6 +23,10 @@ */ GLFS_MSGID(OPEN_BEHIND, OPEN_BEHIND_MSG_XLATOR_CHILD_MISCONFIGURED, - OPEN_BEHIND_MSG_VOL_MISCONFIGURED, OPEN_BEHIND_MSG_NO_MEMORY); + OPEN_BEHIND_MSG_VOL_MISCONFIGURED, OPEN_BEHIND_MSG_NO_MEMORY, + OPEN_BEHIND_MSG_FAILED, OPEN_BEHIND_MSG_BAD_STATE); + +#define OPEN_BEHIND_MSG_FAILED_STR "Failed to submit fop" +#define OPEN_BEHIND_MSG_BAD_STATE_STR "Unexpected state" #endif /* _OPEN_BEHIND_MESSAGES_H_ */ diff --git a/xlators/performance/open-behind/src/open-behind.c b/xlators/performance/open-behind/src/open-behind.c index 95e00dbd790..600c3b62ffe 100644 --- a/xlators/performance/open-behind/src/open-behind.c +++ b/xlators/performance/open-behind/src/open-behind.c @@ -16,6 +16,18 @@ #include "open-behind-messages.h" #include <glusterfs/glusterfs-acl.h> +/* Note: The initial design of open-behind was made to cover the simple case + * of open, read, close for small files. This pattern combined with + * quick-read can do the whole operation without a single request to the + * bricks (except the initial lookup). + * + * The way to do this has been improved, but the logic remains the same. + * Basically, this means that any operation sent to the fd or the inode + * that it's not a read, causes the open request to be sent to the + * bricks, and all future operations will be executed synchronously, + * including opens (it's reset once all fd's are closed). + */ + typedef struct ob_conf { gf_boolean_t use_anonymous_fd; /* use anonymous FDs wherever safe e.g - fstat() readv() @@ -32,1091 +44,811 @@ typedef struct ob_conf { */ } ob_conf_t; -typedef struct ob_inode { - inode_t *inode; - struct list_head resume_fops; - struct list_head ob_fds; - int count; - int op_ret; - int op_errno; - gf_boolean_t open_in_progress; - int unlinked; -} ob_inode_t; +/* A negative state represents an errno value negated. In this case the + * current operation cannot be processed. */ +typedef enum _ob_state { + /* There are no opens on the inode or the first open is already + * completed. The current operation can be sent directly. */ + OB_STATE_READY = 0, -typedef struct ob_fd { - call_frame_t *open_frame; - loc_t loc; - dict_t *xdata; - int flags; - int op_errno; - ob_inode_t *ob_inode; - fd_t *fd; - gf_boolean_t opened; - gf_boolean_t ob_inode_fops_waiting; - struct list_head list; - struct list_head ob_fds_on_inode; -} ob_fd_t; + /* There's an open pending and it has been triggered. The current + * operation should be "stubbified" and processed with + * ob_stub_dispatch(). */ + OB_STATE_OPEN_TRIGGERED, -ob_inode_t * -ob_inode_alloc(inode_t *inode) -{ - ob_inode_t *ob_inode = NULL; + /* There's an open pending but it has not been triggered. The current + * operation can be processed directly but using an anonymous fd. */ + OB_STATE_OPEN_PENDING, - ob_inode = GF_CALLOC(1, sizeof(*ob_inode), gf_ob_mt_inode_t); - if (ob_inode == NULL) - goto out; + /* The current operation is the first open on the inode. */ + OB_STATE_FIRST_OPEN +} ob_state_t; - ob_inode->inode = inode; - INIT_LIST_HEAD(&ob_inode->resume_fops); - INIT_LIST_HEAD(&ob_inode->ob_fds); -out: - return ob_inode; -} - -void -ob_inode_free(ob_inode_t *ob_inode) -{ - if (ob_inode == NULL) - goto out; +typedef struct ob_inode { + /* List of stubs pending on the first open. Once the first open is + * complete, all these stubs will be resubmitted, and dependencies + * will be checked again. */ + struct list_head resume_fops; - list_del_init(&ob_inode->resume_fops); - list_del_init(&ob_inode->ob_fds); + /* The inode this object references. */ + inode_t *inode; - GF_FREE(ob_inode); -out: - return; -} + /* The fd from the first open sent to this inode. It will be set + * from the moment the open is processed until the open if fully + * executed or closed before actually opened. It's NULL in all + * other cases. */ + fd_t *first_fd; + + /* The stub from the first open operation. When open fop starts + * being processed, it's assigned the OB_OPEN_PREPARING value + * until the actual stub is created. This is necessary to avoid + * creating the stub inside a locked region. Once the stub is + * successfully created, it's assigned here. This value is set + * to NULL once the stub is resumed. */ + call_stub_t *first_open; + + /* The total number of currently open fd's on this inode. */ + int32_t open_count; + + /* This flag is set as soon as we know that the open will be + * sent to the bricks, even before the stub is ready. */ + bool triggered; +} ob_inode_t; -ob_inode_t * -ob_inode_get(xlator_t *this, inode_t *inode) +/* Dummy pointer used temporarily while the actual open stub is being created */ +#define OB_OPEN_PREPARING ((call_stub_t *)-1) + +#define OB_POST_COMMON(_fop, _xl, _frame, _fd, _args...) \ + case OB_STATE_FIRST_OPEN: \ + gf_smsg((_xl)->name, GF_LOG_ERROR, EINVAL, OPEN_BEHIND_MSG_BAD_STATE, \ + "fop=%s", #_fop, "state=%d", __ob_state, NULL); \ + default_##_fop##_failure_cbk(_frame, EINVAL); \ + break; \ + case OB_STATE_READY: \ + default_##_fop(_frame, _xl, ##_args); \ + break; \ + case OB_STATE_OPEN_TRIGGERED: { \ + call_stub_t *__ob_stub = fop_##_fop##_stub(_frame, ob_##_fop, \ + ##_args); \ + if (__ob_stub != NULL) { \ + ob_stub_dispatch(_xl, __ob_inode, _fd, __ob_stub); \ + break; \ + } \ + __ob_state = -ENOMEM; \ + } \ + default: \ + gf_smsg((_xl)->name, GF_LOG_ERROR, -__ob_state, \ + OPEN_BEHIND_MSG_FAILED, "fop=%s", #_fop, NULL); \ + default_##_fop##_failure_cbk(_frame, -__ob_state) + +#define OB_POST_FD(_fop, _xl, _frame, _fd, _trigger, _args...) \ + do { \ + ob_inode_t *__ob_inode; \ + fd_t *__first_fd; \ + ob_state_t __ob_state = ob_open_and_resume_fd( \ + _xl, _fd, 0, true, _trigger, &__ob_inode, &__first_fd); \ + switch (__ob_state) { \ + case OB_STATE_OPEN_PENDING: \ + if (!(_trigger)) { \ + fd_t *__ob_fd = fd_anonymous_with_flags((_fd)->inode, \ + (_fd)->flags); \ + if (__ob_fd != NULL) { \ + default_##_fop(_frame, _xl, ##_args); \ + fd_unref(__ob_fd); \ + break; \ + } \ + __ob_state = -ENOMEM; \ + } \ + OB_POST_COMMON(_fop, _xl, _frame, __first_fd, ##_args); \ + } \ + } while (0) + +#define OB_POST_FLUSH(_xl, _frame, _fd, _args...) \ + do { \ + ob_inode_t *__ob_inode; \ + fd_t *__first_fd; \ + ob_state_t __ob_state = ob_open_and_resume_fd( \ + _xl, _fd, 0, true, false, &__ob_inode, &__first_fd); \ + switch (__ob_state) { \ + case OB_STATE_OPEN_PENDING: \ + default_flush_cbk(_frame, NULL, _xl, 0, 0, NULL); \ + break; \ + OB_POST_COMMON(flush, _xl, _frame, __first_fd, ##_args); \ + } \ + } while (0) + +#define OB_POST_INODE(_fop, _xl, _frame, _inode, _trigger, _args...) \ + do { \ + ob_inode_t *__ob_inode; \ + fd_t *__first_fd; \ + ob_state_t __ob_state = ob_open_and_resume_inode( \ + _xl, _inode, NULL, 0, true, _trigger, &__ob_inode, &__first_fd); \ + switch (__ob_state) { \ + case OB_STATE_OPEN_PENDING: \ + OB_POST_COMMON(_fop, _xl, _frame, __first_fd, ##_args); \ + } \ + } while (0) + +static ob_inode_t * +ob_inode_get_locked(xlator_t *this, inode_t *inode) { ob_inode_t *ob_inode = NULL; uint64_t value = 0; - int ret = 0; - if (!inode) - goto out; + if ((__inode_ctx_get(inode, this, &value) == 0) && (value != 0)) { + return (ob_inode_t *)(uintptr_t)value; + } - LOCK(&inode->lock); - { - __inode_ctx_get(inode, this, &value); - if (value == 0) { - ob_inode = ob_inode_alloc(inode); - if (ob_inode == NULL) - goto unlock; - - value = (uint64_t)(uintptr_t)ob_inode; - ret = __inode_ctx_set(inode, this, &value); - if (ret < 0) { - ob_inode_free(ob_inode); - ob_inode = NULL; - } - } else { - ob_inode = (ob_inode_t *)(uintptr_t)value; + ob_inode = GF_CALLOC(1, sizeof(*ob_inode), gf_ob_mt_inode_t); + if (ob_inode != NULL) { + ob_inode->inode = inode; + INIT_LIST_HEAD(&ob_inode->resume_fops); + + value = (uint64_t)(uintptr_t)ob_inode; + if (__inode_ctx_set(inode, this, &value) < 0) { + GF_FREE(ob_inode); + ob_inode = NULL; } } -unlock: - UNLOCK(&inode->lock); -out: return ob_inode; } -ob_fd_t * -__ob_fd_ctx_get(xlator_t *this, fd_t *fd) +static ob_state_t +ob_open_and_resume_inode(xlator_t *xl, inode_t *inode, fd_t *fd, + int32_t open_count, bool synchronous, bool trigger, + ob_inode_t **pob_inode, fd_t **pfd) { - uint64_t value = 0; - int ret = -1; - ob_fd_t *ob_fd = NULL; + ob_conf_t *conf; + ob_inode_t *ob_inode; + call_stub_t *open_stub; - ret = __fd_ctx_get(fd, this, &value); - if (ret) - return NULL; + if (inode == NULL) { + return OB_STATE_READY; + } - ob_fd = (void *)((long)value); + conf = xl->private; - return ob_fd; -} + *pfd = NULL; -ob_fd_t * -ob_fd_ctx_get(xlator_t *this, fd_t *fd) -{ - ob_fd_t *ob_fd = NULL; - - LOCK(&fd->lock); + LOCK(&inode->lock); { - ob_fd = __ob_fd_ctx_get(this, fd); - } - UNLOCK(&fd->lock); - - return ob_fd; -} + ob_inode = ob_inode_get_locked(xl, inode); + if (ob_inode == NULL) { + UNLOCK(&inode->lock); -int -__ob_fd_ctx_set(xlator_t *this, fd_t *fd, ob_fd_t *ob_fd) -{ - uint64_t value = 0; - int ret = -1; + return -ENOMEM; + } + *pob_inode = ob_inode; + + ob_inode->open_count += open_count; + + /* If first_fd is not NULL, it means that there's a previous open not + * yet completed. */ + if (ob_inode->first_fd != NULL) { + *pfd = ob_inode->first_fd; + /* If the current request doesn't trigger the open and it hasn't + * been triggered yet, we can continue without issuing the open + * only if the current request belongs to the same fd as the + * first one. */ + if (!trigger && !ob_inode->triggered && + (ob_inode->first_fd == fd)) { + UNLOCK(&inode->lock); + + return OB_STATE_OPEN_PENDING; + } - value = (long)((void *)ob_fd); + /* We need to issue the open. It could have already been triggered + * before. In this case open_stub will be NULL. Or the initial open + * may not be completely ready yet. In this case open_stub will be + * OB_OPEN_PREPARING. */ + open_stub = ob_inode->first_open; + ob_inode->first_open = NULL; + ob_inode->triggered = true; - ret = __fd_ctx_set(fd, this, value); + UNLOCK(&inode->lock); - return ret; -} + if ((open_stub != NULL) && (open_stub != OB_OPEN_PREPARING)) { + call_resume(open_stub); + } -int -ob_fd_ctx_set(xlator_t *this, fd_t *fd, ob_fd_t *ob_fd) -{ - int ret = -1; + return OB_STATE_OPEN_TRIGGERED; + } - LOCK(&fd->lock); - { - ret = __ob_fd_ctx_set(this, fd, ob_fd); - } - UNLOCK(&fd->lock); + /* There's no pending open. Only opens can be non synchronous, so all + * regular fops will be processed directly. For non synchronous opens, + * we'll still process them normally (i.e. synchornous) if there are + * more file descriptors open. */ + if (synchronous || (ob_inode->open_count > open_count)) { + UNLOCK(&inode->lock); - return ret; -} + return OB_STATE_READY; + } -ob_fd_t * -ob_fd_new(void) -{ - ob_fd_t *ob_fd = NULL; + *pfd = fd; - ob_fd = GF_CALLOC(1, sizeof(*ob_fd), gf_ob_mt_fd_t); + /* This is the first open. We keep a reference on the fd and set + * first_open stub to OB_OPEN_PREPARING until the actual stub can + * be assigned (we don't create the stub here to avoid doing memory + * allocations inside the mutex). */ + ob_inode->first_fd = __fd_ref(fd); + ob_inode->first_open = OB_OPEN_PREPARING; - INIT_LIST_HEAD(&ob_fd->list); - INIT_LIST_HEAD(&ob_fd->ob_fds_on_inode); + /* If lazy_open is not set, we'll need to immediately send the open, + * so we set triggered right now. */ + ob_inode->triggered = !conf->lazy_open; + } + UNLOCK(&inode->lock); - return ob_fd; + return OB_STATE_FIRST_OPEN; } -void -ob_fd_free(ob_fd_t *ob_fd) +static ob_state_t +ob_open_and_resume_fd(xlator_t *xl, fd_t *fd, int32_t open_count, + bool synchronous, bool trigger, ob_inode_t **pob_inode, + fd_t **pfd) { - LOCK(&ob_fd->fd->inode->lock); - { - list_del_init(&ob_fd->ob_fds_on_inode); - } - UNLOCK(&ob_fd->fd->inode->lock); - - loc_wipe(&ob_fd->loc); - - if (ob_fd->xdata) - dict_unref(ob_fd->xdata); + uint64_t err; - if (ob_fd->open_frame) - STACK_DESTROY(ob_fd->open_frame->root); + if ((fd_ctx_get(fd, xl, &err) == 0) && (err != 0)) { + return (ob_state_t)-err; + } - GF_FREE(ob_fd); + return ob_open_and_resume_inode(xl, fd->inode, fd, open_count, synchronous, + trigger, pob_inode, pfd); } -int -ob_wake_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, - int op_errno, fd_t *fd_ret, dict_t *xdata) +static ob_state_t +ob_open_behind(xlator_t *xl, fd_t *fd, int32_t flags, ob_inode_t **pob_inode, + fd_t **pfd) { - fd_t *fd = NULL; - int count = 0; - int ob_inode_op_ret = 0; - int ob_inode_op_errno = 0; - ob_fd_t *ob_fd = NULL; - call_stub_t *stub = NULL, *tmp = NULL; - ob_inode_t *ob_inode = NULL; - gf_boolean_t ob_inode_fops_waiting = _gf_false; - struct list_head fops_waiting_on_fd, fops_waiting_on_inode; + bool synchronous; - fd = frame->local; - frame->local = NULL; - - INIT_LIST_HEAD(&fops_waiting_on_fd); - INIT_LIST_HEAD(&fops_waiting_on_inode); + /* TODO: If O_CREAT, O_APPEND, O_WRONLY or O_DIRECT are specified, shouldn't + * we also execute this open synchronously ? */ + synchronous = (flags & O_TRUNC) != 0; - ob_inode = ob_inode_get(this, fd->inode); + return ob_open_and_resume_fd(xl, fd, 1, synchronous, true, pob_inode, pfd); +} - LOCK(&fd->lock); +static int32_t +ob_stub_dispatch(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd, + call_stub_t *stub) +{ + LOCK(&ob_inode->inode->lock); { - ob_fd = __ob_fd_ctx_get(this, fd); - ob_fd->opened = _gf_true; - - ob_inode_fops_waiting = ob_fd->ob_inode_fops_waiting; - - list_splice_init(&ob_fd->list, &fops_waiting_on_fd); - - if (op_ret < 0) { - /* mark fd BAD for ever */ - ob_fd->op_errno = op_errno; - ob_fd = NULL; /*shouldn't be freed*/ - } else { - __fd_ctx_del(fd, this, NULL); - } - } - UNLOCK(&fd->lock); - - if (ob_inode_fops_waiting) { - LOCK(&fd->inode->lock); - { - count = --ob_inode->count; - if (op_ret < 0) { - /* TODO: when to reset the error? */ - ob_inode->op_ret = -1; - ob_inode->op_errno = op_errno; - } - - if (count == 0) { - ob_inode->open_in_progress = _gf_false; - ob_inode_op_ret = ob_inode->op_ret; - ob_inode_op_errno = ob_inode->op_errno; - list_splice_init(&ob_inode->resume_fops, - &fops_waiting_on_inode); - } + /* We only queue a stub if the open has not been completed or + * cancelled. */ + if (ob_inode->first_fd == fd) { + list_add_tail(&stub->list, &ob_inode->resume_fops); + stub = NULL; } - UNLOCK(&fd->inode->lock); - } - - if (ob_fd) - ob_fd_free(ob_fd); - - list_for_each_entry_safe(stub, tmp, &fops_waiting_on_fd, list) - { - list_del_init(&stub->list); - - if (op_ret < 0) - call_unwind_error(stub, -1, op_errno); - else - call_resume(stub); } + UNLOCK(&ob_inode->inode->lock); - list_for_each_entry_safe(stub, tmp, &fops_waiting_on_inode, list) - { - list_del_init(&stub->list); - - if (ob_inode_op_ret < 0) - call_unwind_error(stub, -1, ob_inode_op_errno); - else - call_resume(stub); + if (stub != NULL) { + call_resume(stub); } - fd_unref(fd); - - STACK_DESTROY(frame->root); - return 0; } -int -ob_fd_wake(xlator_t *this, fd_t *fd, ob_fd_t *ob_fd) +static void +ob_open_destroy(call_stub_t *stub, fd_t *fd) { - call_frame_t *frame = NULL; + stub->frame->local = NULL; + STACK_DESTROY(stub->frame->root); + call_stub_destroy(stub); + fd_unref(fd); +} - if (ob_fd == NULL) { - LOCK(&fd->lock); - { - ob_fd = __ob_fd_ctx_get(this, fd); - if (!ob_fd) - goto unlock; +static int32_t +ob_open_dispatch(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd, + call_stub_t *stub) +{ + bool closed; - frame = ob_fd->open_frame; - ob_fd->open_frame = NULL; - } - unlock: - UNLOCK(&fd->lock); - } else { - LOCK(&fd->lock); - { - frame = ob_fd->open_frame; - ob_fd->open_frame = NULL; + LOCK(&ob_inode->inode->lock); + { + closed = ob_inode->first_fd != fd; + if (!closed) { + if (ob_inode->triggered) { + ob_inode->first_open = NULL; + } else { + ob_inode->first_open = stub; + stub = NULL; + } } - UNLOCK(&fd->lock); } + UNLOCK(&ob_inode->inode->lock); - if (frame) { - frame->local = fd_ref(fd); - - STACK_WIND(frame, ob_wake_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->open, &ob_fd->loc, ob_fd->flags, fd, - ob_fd->xdata); + if (stub != NULL) { + if (closed) { + ob_open_destroy(stub, fd); + } else { + call_resume(stub); + } } return 0; } -void -ob_inode_wake(xlator_t *this, struct list_head *ob_fds) +static void +ob_resume_pending(struct list_head *list) { - ob_fd_t *ob_fd = NULL, *tmp = NULL; - fd_t *fd = NULL; - - if (!list_empty(ob_fds)) { - list_for_each_entry_safe(ob_fd, tmp, ob_fds, ob_fds_on_inode) - { - ob_fd_wake(this, ob_fd->fd, ob_fd); - fd = ob_fd->fd; - ob_fd_free(ob_fd); - fd_unref(fd); - } - } -} + call_stub_t *stub; -/* called holding inode->lock and fd->lock */ -void -ob_fd_copy(ob_fd_t *src, ob_fd_t *dst) -{ - if (!src || !dst) - goto out; - - dst->fd = __fd_ref(src->fd); - dst->loc.inode = inode_ref(src->loc.inode); - gf_uuid_copy(dst->loc.gfid, src->loc.gfid); - dst->flags = src->flags; - dst->xdata = dict_ref(src->xdata); - dst->ob_inode = src->ob_inode; -out: - return; + while (!list_empty(list)) { + stub = list_first_entry(list, call_stub_t, list); + list_del_init(&stub->list); + + call_resume(stub); + } } -int -open_all_pending_fds_and_resume(xlator_t *this, inode_t *inode, - call_stub_t *stub) +static void +ob_open_completed(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd, int32_t op_ret, + int32_t op_errno) { - ob_inode_t *ob_inode = NULL; - ob_fd_t *ob_fd = NULL, *tmp = NULL; - gf_boolean_t was_open_in_progress = _gf_false; - gf_boolean_t wait_for_open = _gf_false; - struct list_head ob_fds; + struct list_head list; - ob_inode = ob_inode_get(this, inode); - if (ob_inode == NULL) - goto out; + INIT_LIST_HEAD(&list); - INIT_LIST_HEAD(&ob_fds); + if (op_ret < 0) { + fd_ctx_set(fd, xl, op_errno <= 0 ? EIO : op_errno); + } - LOCK(&inode->lock); + LOCK(&ob_inode->inode->lock); { - was_open_in_progress = ob_inode->open_in_progress; - ob_inode->unlinked = 1; - - if (was_open_in_progress) { - list_add_tail(&stub->list, &ob_inode->resume_fops); - goto inode_unlock; - } - - list_for_each_entry(ob_fd, &ob_inode->ob_fds, ob_fds_on_inode) - { - LOCK(&ob_fd->fd->lock); - { - if (ob_fd->opened) - goto fd_unlock; - - ob_inode->count++; - ob_fd->ob_inode_fops_waiting = _gf_true; - - if (ob_fd->open_frame == NULL) { - /* open in progress no need of wake */ - } else { - tmp = ob_fd_new(); - tmp->open_frame = ob_fd->open_frame; - ob_fd->open_frame = NULL; - - ob_fd_copy(ob_fd, tmp); - list_add_tail(&tmp->ob_fds_on_inode, &ob_fds); - } - } - fd_unlock: - UNLOCK(&ob_fd->fd->lock); - } - - if (ob_inode->count) { - wait_for_open = ob_inode->open_in_progress = _gf_true; - list_add_tail(&stub->list, &ob_inode->resume_fops); + /* Only update the fields if the file has not been closed before + * getting here. */ + if (ob_inode->first_fd == fd) { + list_splice_init(&ob_inode->resume_fops, &list); + ob_inode->first_fd = NULL; + ob_inode->first_open = NULL; + ob_inode->triggered = false; } } -inode_unlock: - UNLOCK(&inode->lock); + UNLOCK(&ob_inode->inode->lock); -out: - if (!was_open_in_progress) { - if (!wait_for_open) { - call_resume(stub); - } else { - ob_inode_wake(this, &ob_fds); - } - } + ob_resume_pending(&list); - return 0; + fd_unref(fd); } -int -open_and_resume(xlator_t *this, fd_t *fd, call_stub_t *stub) +static int32_t +ob_open_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, int32_t op_ret, + int32_t op_errno, fd_t *fd, dict_t *xdata) { - ob_fd_t *ob_fd = NULL; - int op_errno = 0; - - if (!fd) - goto nofd; - - LOCK(&fd->lock); - { - ob_fd = __ob_fd_ctx_get(this, fd); - if (!ob_fd) - goto unlock; + ob_inode_t *ob_inode; - if (ob_fd->op_errno) { - op_errno = ob_fd->op_errno; - goto unlock; - } + ob_inode = frame->local; + frame->local = NULL; - list_add_tail(&stub->list, &ob_fd->list); - } -unlock: - UNLOCK(&fd->lock); + ob_open_completed(xl, ob_inode, cookie, op_ret, op_errno); -nofd: - if (op_errno) - call_unwind_error(stub, -1, op_errno); - else if (ob_fd) - ob_fd_wake(this, fd, NULL); - else - call_resume(stub); + STACK_DESTROY(frame->root); return 0; } -int -ob_open_behind(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, +static int32_t +ob_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd, dict_t *xdata) { - ob_fd_t *ob_fd = NULL; - int ret = -1; - ob_conf_t *conf = NULL; - ob_inode_t *ob_inode = NULL; - gf_boolean_t open_in_progress = _gf_false; - int unlinked = 0; + STACK_WIND_COOKIE(frame, ob_open_cbk, fd, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata); - conf = this->private; + return 0; +} - if (flags & O_TRUNC) { - STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata); - return 0; +static int32_t +ob_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd, + dict_t *xdata) +{ + ob_inode_t *ob_inode; + call_frame_t *open_frame; + call_stub_t *stub; + fd_t *first_fd; + ob_state_t state; + + state = ob_open_behind(this, fd, flags, &ob_inode, &first_fd); + if (state == OB_STATE_READY) { + /* There's no pending open, but there are other file descriptors opened + * or the current flags require a synchronous open. */ + return default_open(frame, this, loc, flags, fd, xdata); } - ob_inode = ob_inode_get(this, fd->inode); - - ob_fd = ob_fd_new(); - if (!ob_fd) - goto enomem; - - ob_fd->ob_inode = ob_inode; + if (state == OB_STATE_OPEN_TRIGGERED) { + /* The first open is in progress (either because it was already issued + * or because this request triggered it). We try to create a new stub + * to retry the operation once the initial open completes. */ + stub = fop_open_stub(frame, ob_open, loc, flags, fd, xdata); + if (stub != NULL) { + return ob_stub_dispatch(this, ob_inode, first_fd, stub); + } - /* don't do fd_ref, it'll cause leaks */ - ob_fd->fd = fd; + state = -ENOMEM; + } - ob_fd->open_frame = copy_frame(frame); - if (!ob_fd->open_frame) - goto enomem; - ret = loc_copy(&ob_fd->loc, loc); - if (ret) - goto enomem; + if (state == OB_STATE_FIRST_OPEN) { + /* We try to create a stub for the new open. A new frame needs to be + * used because the current one may be destroyed soon after sending + * the open's reply. */ + open_frame = copy_frame(frame); + if (open_frame != NULL) { + stub = fop_open_stub(open_frame, ob_open_resume, loc, flags, fd, + xdata); + if (stub != NULL) { + open_frame->local = ob_inode; - ob_fd->flags = flags; - if (xdata) - ob_fd->xdata = dict_ref(xdata); + /* TODO: Previous version passed xdata back to the caller, but + * probably this doesn't make sense since it won't contain + * any requested data. I think it would be better to pass + * NULL for xdata. */ + default_open_cbk(frame, NULL, this, 0, 0, fd, xdata); - LOCK(&fd->inode->lock); - { - open_in_progress = ob_inode->open_in_progress; - unlinked = ob_inode->unlinked; - if (!open_in_progress && !unlinked) { - ret = ob_fd_ctx_set(this, fd, ob_fd); - if (ret) { - UNLOCK(&fd->inode->lock); - goto enomem; + return ob_open_dispatch(this, ob_inode, first_fd, stub); } - list_add(&ob_fd->ob_fds_on_inode, &ob_inode->ob_fds); + STACK_DESTROY(open_frame->root); } - } - UNLOCK(&fd->inode->lock); - if (!open_in_progress && !unlinked) { - fd_ref(fd); + /* In case of error, simulate a regular completion but with an error + * code. */ + ob_open_completed(this, ob_inode, first_fd, -1, ENOMEM); - STACK_UNWIND_STRICT(open, frame, 0, 0, fd, xdata); - - if (!conf->lazy_open) - ob_fd_wake(this, fd, NULL); - - fd_unref(fd); - } else { - ob_fd_free(ob_fd); - STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata); + state = -ENOMEM; } - return 0; -enomem: - if (ob_fd) { - if (ob_fd->open_frame) - STACK_DESTROY(ob_fd->open_frame->root); - - loc_wipe(&ob_fd->loc); - if (ob_fd->xdata) - dict_unref(ob_fd->xdata); + /* In case of failure we need to decrement the number of open files because + * ob_fdclose() won't be called. */ - GF_FREE(ob_fd); + LOCK(&fd->inode->lock); + { + ob_inode->open_count--; } + UNLOCK(&fd->inode->lock); - return -1; + gf_smsg(this->name, GF_LOG_ERROR, -state, OPEN_BEHIND_MSG_FAILED, "fop=%s", + "open", "path=%s", loc->path, NULL); + + return default_open_failure_cbk(frame, -state); } -int -ob_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd, - dict_t *xdata) +static int32_t +ob_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, + mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) { - fd_t *old_fd = NULL; - int ret = -1; - int op_errno = ENOMEM; - call_stub_t *stub = NULL; - - old_fd = fd_lookup(fd->inode, 0); - if (old_fd) { - /* open-behind only when this is the first FD */ - stub = fop_open_stub(frame, default_open_resume, loc, flags, fd, xdata); - if (!stub) { - fd_unref(old_fd); - goto err; - } - - open_and_resume(this, old_fd, stub); - - fd_unref(old_fd); - - return 0; - } - - ret = ob_open_behind(frame, this, loc, flags, fd, xdata); - if (ret) { - goto err; + ob_inode_t *ob_inode; + call_stub_t *stub; + fd_t *first_fd; + ob_state_t state; + + /* Create requests are never delayed. We always send them synchronously. */ + state = ob_open_and_resume_fd(this, fd, 1, true, true, &ob_inode, + &first_fd); + if (state == OB_STATE_READY) { + /* There's no pending open, but there are other file descriptors opened + * so we simply forward the request synchronously. */ + return default_create(frame, this, loc, flags, mode, umask, fd, xdata); } - return 0; -err: - gf_msg(this->name, GF_LOG_ERROR, op_errno, OPEN_BEHIND_MSG_NO_MEMORY, "%s", - loc->path); - - STACK_UNWIND_STRICT(open, frame, -1, op_errno, 0, 0); - - return 0; -} + if (state == OB_STATE_OPEN_TRIGGERED) { + /* The first open is in progress (either because it was already issued + * or because this request triggered it). We try to create a new stub + * to retry the operation once the initial open completes. */ + stub = fop_create_stub(frame, ob_create, loc, flags, mode, umask, fd, + xdata); + if (stub != NULL) { + return ob_stub_dispatch(this, ob_inode, first_fd, stub); + } -fd_t * -ob_get_wind_fd(xlator_t *this, fd_t *fd, uint32_t *flag) -{ - fd_t *wind_fd = NULL; - ob_fd_t *ob_fd = NULL; - ob_conf_t *conf = NULL; + state = -ENOMEM; + } - conf = this->private; + /* Since we forced a synchronous request, OB_STATE_FIRST_OPEN will never + * be returned by ob_open_and_resume_fd(). If we are here it can only be + * because there has been a problem. */ - ob_fd = ob_fd_ctx_get(this, fd); + /* In case of failure we need to decrement the number of open files because + * ob_fdclose() won't be called. */ - if (ob_fd && ob_fd->open_frame && conf->use_anonymous_fd) { - wind_fd = fd_anonymous(fd->inode); - if ((ob_fd->flags & O_DIRECT) && (flag)) - *flag = *flag | O_DIRECT; - } else { - wind_fd = fd_ref(fd); + LOCK(&fd->inode->lock); + { + ob_inode->open_count--; } + UNLOCK(&fd->inode->lock); - return wind_fd; + gf_smsg(this->name, GF_LOG_ERROR, -state, OPEN_BEHIND_MSG_FAILED, "fop=%s", + "create", "path=%s", loc->path, NULL); + + return default_create_failure_cbk(frame, -state); } -int +static int32_t ob_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata) { - call_stub_t *stub = NULL; - fd_t *wind_fd = NULL; - ob_conf_t *conf = NULL; + ob_conf_t *conf = this->private; + bool trigger = conf->read_after_open || !conf->use_anonymous_fd; - conf = this->private; - - if (!conf->read_after_open) - wind_fd = ob_get_wind_fd(this, fd, &flags); - else - wind_fd = fd_ref(fd); - - stub = fop_readv_stub(frame, default_readv_resume, wind_fd, size, offset, - flags, xdata); - fd_unref(wind_fd); - - if (!stub) - goto err; - - open_and_resume(this, wind_fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0); + OB_POST_FD(readv, this, frame, fd, trigger, fd, size, offset, flags, xdata); return 0; } -int +static int32_t ob_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov, int count, off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_writev_stub(frame, default_writev_resume, fd, iov, count, offset, - flags, iobref, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(writev, frame, -1, ENOMEM, 0, 0, 0); + OB_POST_FD(writev, this, frame, fd, true, fd, iov, count, offset, flags, + iobref, xdata); return 0; } -int +static int32_t ob_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) { - call_stub_t *stub = NULL; - fd_t *wind_fd = NULL; - - wind_fd = ob_get_wind_fd(this, fd, NULL); - - stub = fop_fstat_stub(frame, default_fstat_resume, wind_fd, xdata); - - fd_unref(wind_fd); - - if (!stub) - goto err; + ob_conf_t *conf = this->private; + bool trigger = !conf->use_anonymous_fd; - open_and_resume(this, wind_fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, 0, 0); + OB_POST_FD(fstat, this, frame, fd, trigger, fd, xdata); return 0; } -int +static int32_t ob_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, gf_seek_what_t what, dict_t *xdata) { - call_stub_t *stub = NULL; - fd_t *wind_fd = NULL; + ob_conf_t *conf = this->private; + bool trigger = !conf->use_anonymous_fd; - wind_fd = ob_get_wind_fd(this, fd, NULL); - - stub = fop_seek_stub(frame, default_seek_resume, wind_fd, offset, what, - xdata); - - fd_unref(wind_fd); - - if (!stub) - goto err; - - open_and_resume(this, wind_fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, 0, 0); + OB_POST_FD(seek, this, frame, fd, trigger, fd, offset, what, xdata); return 0; } -int +static int32_t ob_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) { - call_stub_t *stub = NULL; - ob_fd_t *ob_fd = NULL; - gf_boolean_t unwind = _gf_false; - - LOCK(&fd->lock); - { - ob_fd = __ob_fd_ctx_get(this, fd); - if (ob_fd && ob_fd->open_frame) - /* if open() was never wound to backend, - no need to wind flush() either. - */ - unwind = _gf_true; - } - UNLOCK(&fd->lock); - - if (unwind) - goto unwind; - - stub = fop_flush_stub(frame, default_flush_resume, fd, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, 0); - - return 0; - -unwind: - STACK_UNWIND_STRICT(flush, frame, 0, 0, 0); + OB_POST_FLUSH(this, frame, fd, fd, xdata); return 0; } -int +static int32_t ob_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int flag, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_fsync_stub(frame, default_fsync_resume, fd, flag, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM, 0, 0, 0); + OB_POST_FD(fsync, this, frame, fd, true, fd, flag, xdata); return 0; } -int +static int32_t ob_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_lk_stub(frame, default_lk_resume, fd, cmd, flock, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(lk, frame, -1, ENOMEM, 0, 0); + OB_POST_FD(lk, this, frame, fd, true, fd, cmd, flock, xdata); return 0; } -int +static int32_t ob_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_ftruncate_stub(frame, default_ftruncate_resume, fd, offset, - xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOMEM, 0, 0, 0); + OB_POST_FD(ftruncate, this, frame, fd, true, fd, offset, xdata); return 0; } -int +static int32_t ob_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr, int flags, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_fsetxattr_stub(frame, default_fsetxattr_resume, fd, xattr, flags, - xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fsetxattr, frame, -1, ENOMEM, 0); + OB_POST_FD(fsetxattr, this, frame, fd, true, fd, xattr, flags, xdata); return 0; } -int +static int32_t ob_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_fgetxattr_stub(frame, default_fgetxattr_resume, fd, name, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOMEM, 0, 0); + OB_POST_FD(fgetxattr, this, frame, fd, true, fd, name, xdata); return 0; } -int +static int32_t ob_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_fremovexattr_stub(frame, default_fremovexattr_resume, fd, name, - xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fremovexattr, frame, -1, ENOMEM, 0); + OB_POST_FD(fremovexattr, this, frame, fd, true, fd, name, xdata); return 0; } -int +static int32_t ob_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata) { - call_stub_t *stub = fop_finodelk_stub(frame, default_finodelk_resume, - volume, fd, cmd, flock, xdata); - if (stub) - open_and_resume(this, fd, stub); - else - STACK_UNWIND_STRICT(finodelk, frame, -1, ENOMEM, 0); + OB_POST_FD(finodelk, this, frame, fd, true, volume, fd, cmd, flock, xdata); return 0; } -int +static int32_t ob_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type, dict_t *xdata) { - call_stub_t *stub = fop_fentrylk_stub( - frame, default_fentrylk_resume, volume, fd, basename, cmd, type, xdata); - if (stub) - open_and_resume(this, fd, stub); - else - STACK_UNWIND_STRICT(fentrylk, frame, -1, ENOMEM, 0); + OB_POST_FD(fentrylk, this, frame, fd, true, volume, fd, basename, cmd, type, + xdata); return 0; } -int +static int32_t ob_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) { - call_stub_t *stub = fop_fxattrop_stub(frame, default_fxattrop_resume, fd, - optype, xattr, xdata); - if (stub) - open_and_resume(this, fd, stub); - else - STACK_UNWIND_STRICT(fxattrop, frame, -1, ENOMEM, 0, 0); + OB_POST_FD(fxattrop, this, frame, fd, true, fd, optype, xattr, xdata); return 0; } -int +static int32_t ob_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *iatt, int valid, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_fsetattr_stub(frame, default_fsetattr_resume, fd, iatt, valid, - xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(fsetattr, frame, -1, ENOMEM, 0, 0, 0); + OB_POST_FD(fsetattr, this, frame, fd, true, fd, iatt, valid, xdata); return 0; } -int +static int32_t ob_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, off_t offset, size_t len, dict_t *xdata) { - call_stub_t *stub; - - stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode, offset, - len, xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); + OB_POST_FD(fallocate, this, frame, fd, true, fd, mode, offset, len, xdata); return 0; -err: - STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL); - return 0; } -int +static int32_t ob_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, size_t len, dict_t *xdata) { - call_stub_t *stub; - - stub = fop_discard_stub(frame, default_discard_resume, fd, offset, len, - xdata); - if (!stub) - goto err; + OB_POST_FD(discard, this, frame, fd, true, fd, offset, len, xdata); - open_and_resume(this, fd, stub); - - return 0; -err: - STACK_UNWIND_STRICT(discard, frame, -1, ENOMEM, NULL, NULL, NULL); return 0; } -int +static int32_t ob_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, off_t len, dict_t *xdata) { - call_stub_t *stub; - - stub = fop_zerofill_stub(frame, default_zerofill_resume, fd, offset, len, - xdata); - if (!stub) - goto err; - - open_and_resume(this, fd, stub); + OB_POST_FD(zerofill, this, frame, fd, true, fd, offset, len, xdata); return 0; -err: - STACK_UNWIND_STRICT(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL); - return 0; } -int +static int32_t ob_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_unlink_stub(frame, default_unlink_resume, loc, xflags, xdata); - if (!stub) - goto err; - - open_all_pending_fds_and_resume(this, loc->inode, stub); - - return 0; -err: - STACK_UNWIND_STRICT(unlink, frame, -1, ENOMEM, 0, 0, 0); + OB_POST_INODE(unlink, this, frame, loc->inode, true, loc, xflags, xdata); return 0; } -int +static int32_t ob_rename(call_frame_t *frame, xlator_t *this, loc_t *src, loc_t *dst, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_rename_stub(frame, default_rename_resume, src, dst, xdata); - if (!stub) - goto err; - - open_all_pending_fds_and_resume(this, dst->inode, stub); - - return 0; -err: - STACK_UNWIND_STRICT(rename, frame, -1, ENOMEM, 0, 0, 0, 0, 0, 0); + OB_POST_INODE(rename, this, frame, dst->inode, true, src, dst, xdata); return 0; } -int32_t +static int32_t ob_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata) { - call_stub_t *stub = NULL; - - stub = fop_setattr_stub(frame, default_setattr_resume, loc, stbuf, valid, - xdata); - if (!stub) - goto err; - - open_all_pending_fds_and_resume(this, loc->inode, stub); + OB_POST_INODE(setattr, this, frame, loc->inode, true, loc, stbuf, valid, + xdata); return 0; -err: - STACK_UNWIND_STRICT(setattr, frame, -1, ENOMEM, NULL, NULL, NULL); - return 0; } -int32_t +static int32_t ob_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata) { - call_stub_t *stub = NULL; - gf_boolean_t access_xattr = _gf_false; - if (dict_get(dict, POSIX_ACL_DEFAULT_XATTR) || dict_get(dict, POSIX_ACL_ACCESS_XATTR) || - dict_get(dict, GF_SELINUX_XATTR_KEY)) - access_xattr = _gf_true; - - if (!access_xattr) + dict_get(dict, GF_SELINUX_XATTR_KEY)) { return default_setxattr(frame, this, loc, dict, flags, xdata); + } - stub = fop_setxattr_stub(frame, default_setxattr_resume, loc, dict, flags, - xdata); - if (!stub) - goto err; - - open_all_pending_fds_and_resume(this, loc->inode, stub); + OB_POST_INODE(setxattr, this, frame, loc->inode, true, loc, dict, flags, + xdata); return 0; -err: - STACK_UNWIND_STRICT(setxattr, frame, -1, ENOMEM, NULL); - return 0; } -int -ob_release(xlator_t *this, fd_t *fd) +static void +ob_fdclose(xlator_t *this, fd_t *fd) { - ob_fd_t *ob_fd = NULL; + struct list_head list; + ob_inode_t *ob_inode; + call_stub_t *stub; - ob_fd = ob_fd_ctx_get(this, fd); + INIT_LIST_HEAD(&list); + stub = NULL; - ob_fd_free(ob_fd); + LOCK(&fd->inode->lock); + { + ob_inode = ob_inode_get_locked(this, fd->inode); + if (ob_inode != NULL) { + ob_inode->open_count--; + + /* If this fd is the same as ob_inode->first_fd, it means that + * the initial open has not fully completed. We'll try to cancel + * it. */ + if (ob_inode->first_fd == fd) { + if (ob_inode->first_open == OB_OPEN_PREPARING) { + /* In this case ob_open_dispatch() has not been called yet. + * We clear first_fd and first_open to allow that function + * to know that the open is not really needed. This also + * allows other requests to work as expected if they + * arrive before the dispatch function is called. If there + * are pending fops, we can directly process them here. + * (note that there shouldn't be any fd related fops, but + * if there are, it's fine if they fail). */ + ob_inode->first_fd = NULL; + ob_inode->first_open = NULL; + ob_inode->triggered = false; + list_splice_init(&ob_inode->resume_fops, &list); + } else if (!ob_inode->triggered) { + /* If the open has already been dispatched, we can only + * cancel it if it has not been triggered. Otherwise we + * simply wait until it completes. While it's not triggered, + * first_open must be a valid stub and there can't be any + * pending fops. */ + GF_ASSERT((ob_inode->first_open != NULL) && + list_empty(&ob_inode->resume_fops)); + + ob_inode->first_fd = NULL; + stub = ob_inode->first_open; + ob_inode->first_open = NULL; + } + } + } + } + UNLOCK(&fd->inode->lock); - return 0; + if (stub != NULL) { + ob_open_destroy(stub, fd); + } + + ob_resume_pending(&list); } int ob_forget(xlator_t *this, inode_t *inode) { - ob_inode_t *ob_inode = NULL; + ob_inode_t *ob_inode; uint64_t value = 0; - inode_ctx_del(inode, this, &value); - - if (value) { + if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) { ob_inode = (ob_inode_t *)(uintptr_t)value; - ob_inode_free(ob_inode); + GF_FREE(ob_inode); } return 0; @@ -1148,20 +880,18 @@ ob_priv_dump(xlator_t *this) int ob_fdctx_dump(xlator_t *this, fd_t *fd) { - ob_fd_t *ob_fd = NULL; char key_prefix[GF_DUMP_MAX_BUF_LEN] = { 0, }; - int ret = 0; + uint64_t value = 0; + int ret = 0, error = 0; ret = TRY_LOCK(&fd->lock); if (ret) return 0; - ob_fd = __ob_fd_ctx_get(this, fd); - if (!ob_fd) { - UNLOCK(&fd->lock); - return 0; + if ((__fd_ctx_get(fd, this, &value) == 0) && (value != 0)) { + error = (int32_t)value; } gf_proc_dump_build_key(key_prefix, "xlator.performance.open-behind", @@ -1170,17 +900,7 @@ ob_fdctx_dump(xlator_t *this, fd_t *fd) gf_proc_dump_write("fd", "%p", fd); - gf_proc_dump_write("open_frame", "%p", ob_fd->open_frame); - - if (ob_fd->open_frame) - gf_proc_dump_write("open_frame.root.unique", "%" PRIu64, - ob_fd->open_frame->root->unique); - - gf_proc_dump_write("loc.path", "%s", ob_fd->loc.path); - - gf_proc_dump_write("loc.ino", "%s", uuid_utoa(ob_fd->loc.gfid)); - - gf_proc_dump_write("flags", "%d", ob_fd->flags); + gf_proc_dump_write("error", "%d", error); UNLOCK(&fd->lock); @@ -1277,6 +997,7 @@ fini(xlator_t *this) struct xlator_fops fops = { .open = ob_open, + .create = ob_create, .readv = ob_readv, .writev = ob_writev, .flush = ob_flush, @@ -1302,7 +1023,7 @@ struct xlator_fops fops = { }; struct xlator_cbks cbks = { - .release = ob_release, + .fdclose = ob_fdclose, .forget = ob_forget, }; diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c index 4f16d148262..7fe4b3c3a4b 100644 --- a/xlators/performance/quick-read/src/quick-read.c +++ b/xlators/performance/quick-read/src/quick-read.c @@ -421,9 +421,6 @@ qr_content_update(xlator_t *this, qr_inode_t *qr_inode, void *data, qr_private_t *priv = NULL; qr_inode_table_t *table = NULL; uint32_t rollover = 0; - struct timeval tv = { - 0, - }; rollover = gen >> 32; gen = gen & 0xffffffff; @@ -431,7 +428,6 @@ qr_content_update(xlator_t *this, qr_inode_t *qr_inode, void *data, priv = this->private; table = &priv->table; - gettimeofday(&tv, NULL); LOCK(&table->lock); { if ((rollover != qr_inode->gen_rollover) || @@ -453,8 +449,7 @@ qr_content_update(xlator_t *this, qr_inode_t *qr_inode, void *data, qr_inode->ia_ctime_nsec = buf->ia_ctime_nsec; qr_inode->buf = *buf; - - memcpy(&qr_inode->last_refresh, &tv, sizeof(struct timeval)); + qr_inode->last_refresh = gf_time(); __qr_inode_register(this, table, qr_inode); } @@ -524,9 +519,7 @@ __qr_content_refresh(xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf, if (qr_size_fits(conf, buf) && qr_time_equal(conf, qr_inode, buf)) { qr_inode->buf = *buf; - - gettimeofday(&qr_inode->last_refresh, NULL); - + qr_inode->last_refresh = gf_time(); __qr_inode_register(this, table, qr_inode); } else { __qr_inode_prune(this, table, qr_inode, gen); @@ -558,20 +551,14 @@ __qr_cache_is_fresh(xlator_t *this, qr_inode_t *qr_inode) { qr_conf_t *conf = NULL; qr_private_t *priv = NULL; - struct timeval now; - struct timeval diff; priv = this->private; conf = &priv->conf; - gettimeofday(&now, NULL); - - timersub(&now, &qr_inode->last_refresh, &diff); - - if (qr_inode->last_refresh.tv_sec < priv->last_child_down) + if (qr_inode->last_refresh < priv->last_child_down) return _gf_false; - if (diff.tv_sec >= conf->cache_timeout) + if (gf_time() - qr_inode->last_refresh >= conf->cache_timeout) return _gf_false; return _gf_true; @@ -1034,7 +1021,7 @@ qr_inodectx_dump(xlator_t *this, inode_t *inode) char key_prefix[GF_DUMP_MAX_BUF_LEN] = { 0, }; - char buf[256] = { + char buf[GF_TIMESTR_SIZE] = { 0, }; @@ -1049,12 +1036,8 @@ qr_inodectx_dump(xlator_t *this, inode_t *inode) gf_proc_dump_write("entire-file-cached", "%s", qr_inode->data ? "yes" : "no"); - if (qr_inode->last_refresh.tv_sec) { - gf_time_fmt(buf, sizeof buf, qr_inode->last_refresh.tv_sec, - gf_timefmt_FT); - snprintf(buf + strlen(buf), sizeof buf - strlen(buf), - ".%" GF_PRI_SUSECONDS, qr_inode->last_refresh.tv_usec); - + if (qr_inode->last_refresh) { + gf_time_fmt(buf, sizeof buf, qr_inode->last_refresh, gf_timefmt_FT); gf_proc_dump_write("last-cache-validation-time", "%s", buf); } @@ -1407,7 +1390,7 @@ qr_init(xlator_t *this) ret = 0; - time(&priv->last_child_down); + priv->last_child_down = gf_time(); GF_ATOMIC_INIT(priv->generation, 0); this->private = priv; out: @@ -1457,7 +1440,7 @@ qr_conf_destroy(qr_conf_t *conf) } void -qr_update_child_down_time(xlator_t *this, time_t *now) +qr_update_child_down_time(xlator_t *this, time_t now) { qr_private_t *priv = NULL; @@ -1465,7 +1448,7 @@ qr_update_child_down_time(xlator_t *this, time_t *now) LOCK(&priv->lock); { - priv->last_child_down = *now; + priv->last_child_down = now; } UNLOCK(&priv->lock); } @@ -1511,7 +1494,6 @@ qr_notify(xlator_t *this, int event, void *data, ...) { int ret = 0; qr_private_t *priv = NULL; - time_t now = 0; qr_conf_t *conf = NULL; priv = this->private; @@ -1520,8 +1502,7 @@ qr_notify(xlator_t *this, int event, void *data, ...) switch (event) { case GF_EVENT_CHILD_DOWN: case GF_EVENT_SOME_DESCENDENT_DOWN: - time(&now); - qr_update_child_down_time(this, &now); + qr_update_child_down_time(this, gf_time()); break; case GF_EVENT_UPCALL: if (conf->qr_invalidation) diff --git a/xlators/performance/quick-read/src/quick-read.h b/xlators/performance/quick-read/src/quick-read.h index 67850821b8e..20fcc70b3a7 100644 --- a/xlators/performance/quick-read/src/quick-read.h +++ b/xlators/performance/quick-read/src/quick-read.h @@ -39,7 +39,7 @@ struct qr_inode { uint32_t ia_ctime_nsec; uint32_t gen_rollover; struct iatt buf; - struct timeval last_refresh; + time_t last_refresh; struct list_head lru; uint64_t gen; uint64_t invalidation_time; diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c index ab6b76cace8..00cfca016e6 100644 --- a/xlators/performance/write-behind/src/write-behind.c +++ b/xlators/performance/write-behind/src/write-behind.c @@ -1283,14 +1283,14 @@ __wb_pick_unwinds(wb_inode_t *wb_inode, list_head_t *lies) wb_inode->window_current += req->orig_size; + wb_inode->gen++; + if (!req->ordering.fulfilled) { /* burden increased */ list_add_tail(&req->lie, &wb_inode->liability); req->ordering.lied = 1; - wb_inode->gen++; - uuid_utoa_r(req->gfid, gfid); gf_msg_debug(wb_inode->this->name, 0, "(unique=%" PRIu64 @@ -2489,7 +2489,7 @@ wb_mark_readdirp_start(xlator_t *this, inode_t *directory) wb_directory_inode = wb_inode_create(this, directory); - if (!wb_directory_inode || !wb_directory_inode->lock.spinlock) + if (!wb_directory_inode) return; LOCK(&wb_directory_inode->lock); @@ -2509,7 +2509,7 @@ wb_mark_readdirp_end(xlator_t *this, inode_t *directory) wb_directory_inode = wb_inode_ctx_get(this, directory); - if (!wb_directory_inode || !wb_directory_inode->lock.spinlock) + if (!wb_directory_inode) return; LOCK(&wb_directory_inode->lock); diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c index a5cc3d5cd35..d83d9c14899 100644 --- a/xlators/protocol/client/src/client-callback.c +++ b/xlators/protocol/client/src/client-callback.c @@ -16,24 +16,21 @@ static int client_cbk_null(struct rpc_clnt *rpc, void *mydata, void *data) { - gf_msg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, - "this function should not be called"); + gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL); return 0; } static int client_cbk_fetchspec(struct rpc_clnt *rpc, void *mydata, void *data) { - gf_msg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, - "this function should not be called"); + gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL); return 0; } static int client_cbk_ino_flush(struct rpc_clnt *rpc, void *mydata, void *data) { - gf_msg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, - "this function should not be called"); + gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL); return 0; } @@ -61,8 +58,8 @@ client_cbk_recall_lease(struct rpc_clnt *rpc, void *mydata, void *data) (xdrproc_t)xdr_gfs3_recall_lease_req); if (ret < 0) { - gf_msg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_RECALL_LEASE_FAIL, - "XDR decode of recall lease failed."); + gf_smsg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_RECALL_LEASE_FAIL, + NULL); goto out; } @@ -113,8 +110,8 @@ client_cbk_cache_invalidation(struct rpc_clnt *rpc, void *mydata, void *data) (xdrproc_t)xdr_gfs3_cbk_cache_invalidation_req); if (ret < 0) { - gf_msg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_CACHE_INVALIDATION_FAIL, - "XDR decode of cache_invalidation failed."); + gf_smsg(THIS->name, GF_LOG_WARNING, -ret, + PC_MSG_CACHE_INVALIDATION_FAIL, NULL); goto out; } @@ -205,8 +202,8 @@ client_cbk_inodelk_contention(struct rpc_clnt *rpc, void *mydata, void *data) (xdrproc_t)xdr_gfs4_inodelk_contention_req); if (ret < 0) { - gf_msg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_INODELK_CONTENTION_FAIL, - "XDR decode of inodelk contention failed."); + gf_smsg(THIS->name, GF_LOG_WARNING, -ret, + PC_MSG_INODELK_CONTENTION_FAIL, NULL); goto out; } @@ -256,8 +253,8 @@ client_cbk_entrylk_contention(struct rpc_clnt *rpc, void *mydata, void *data) (xdrproc_t)xdr_gfs4_entrylk_contention_req); if (ret < 0) { - gf_msg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_ENTRYLK_CONTENTION_FAIL, - "XDR decode of entrylk contention failed."); + gf_smsg(THIS->name, GF_LOG_WARNING, -ret, + PC_MSG_ENTRYLK_CONTENTION_FAIL, NULL); goto out; } diff --git a/xlators/protocol/client/src/client-common.c b/xlators/protocol/client/src/client-common.c index 64db98d661b..c112820e407 100644 --- a/xlators/protocol/client/src/client-common.c +++ b/xlators/protocol/client/src/client-common.c @@ -449,7 +449,8 @@ client_pre_fsync(xlator_t *this, gfs3_fsync_req *req, fd_t *fd, int32_t flags, int64_t remote_fd = -1; int op_errno = 0; - CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out); + CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno, + out); req->fd = remote_fd; req->data = flags; @@ -714,8 +715,8 @@ client_pre_lk(xlator_t *this, gfs3_lk_req *req, int32_t cmd, ret = client_cmd_to_gf_cmd(cmd, &gf_cmd); if (ret) { op_errno = EINVAL; - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); goto out; } @@ -827,8 +828,8 @@ client_pre_inodelk(xlator_t *this, gfs3_inodelk_req *req, loc_t *loc, int cmd, else if (cmd == F_SETLKW || cmd == F_SETLKW64) gf_cmd = GF_LK_SETLKW; else { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); op_errno = EINVAL; goto out; } @@ -877,8 +878,8 @@ client_pre_finodelk(xlator_t *this, gfs3_finodelk_req *req, fd_t *fd, int cmd, else if (cmd == F_SETLKW || cmd == F_SETLKW64) gf_cmd = GF_LK_SETLKW; else { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); goto out; } @@ -2641,7 +2642,8 @@ client_pre_fsync_v2(xlator_t *this, gfx_fsync_req *req, fd_t *fd, int32_t flags, int64_t remote_fd = -1; int op_errno = 0; - CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out); + CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno, + out); req->fd = remote_fd; req->data = flags; @@ -2895,8 +2897,8 @@ client_pre_lk_v2(xlator_t *this, gfx_lk_req *req, int32_t cmd, ret = client_cmd_to_gf_cmd(cmd, &gf_cmd); if (ret) { op_errno = EINVAL; - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); goto out; } @@ -3005,8 +3007,8 @@ client_pre_inodelk_v2(xlator_t *this, gfx_inodelk_req *req, loc_t *loc, int cmd, else if (cmd == F_SETLKW || cmd == F_SETLKW64) gf_cmd = GF_LK_SETLKW; else { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); op_errno = EINVAL; goto out; } @@ -3055,8 +3057,8 @@ client_pre_finodelk_v2(xlator_t *this, gfx_finodelk_req *req, fd_t *fd, int cmd, else if (cmd == F_SETLKW || cmd == F_SETLKW64) gf_cmd = GF_LK_SETLKW; else { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_INVALID_ENTRY, - "Unknown cmd (%d)!", gf_cmd); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD, + "gf_cmd=%d", gf_cmd, NULL); goto out; } diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c index e49647faa6f..189dfddd021 100644 --- a/xlators/protocol/client/src/client-helpers.c +++ b/xlators/protocol/client/src/client-helpers.c @@ -91,23 +91,23 @@ this_fd_set_ctx(fd_t *file, xlator_t *this, loc_t *loc, clnt_fd_ctx_t *ctx) ret = fd_ctx_get(file, this, &oldaddr); if (ret >= 0) { if (loc) - gf_msg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY, - "%s (%s): trying duplicate remote fd set. ", loc->path, - uuid_utoa(loc->inode->gfid)); + gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY, + "path=%s", loc->path, "gfid=%s", + uuid_utoa(loc->inode->gfid), NULL); else - gf_msg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY, - "%p: trying duplicate remote fd set. ", file); + gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY, + "file=%p", file, NULL); } ret = fd_ctx_set(file, this, (uint64_t)(unsigned long)ctx); if (ret < 0) { if (loc) - gf_msg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL, - "%s (%s): failed to set remote fd", loc->path, - uuid_utoa(loc->inode->gfid)); + gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL, + "path=%s", loc->path, "gfid=%s", + uuid_utoa(loc->inode->gfid), NULL); else - gf_msg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL, - "%p: failed to set remote fd", file); + gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL, + "file=%p", file, NULL); } out: return; @@ -213,9 +213,8 @@ unserialize_rsp_direntp(xlator_t *this, fd_t *fd, struct gfs3_readdirp_rsp *rsp, ret = dict_unserialize(trav->dict.dict_val, trav->dict.dict_len, &entry->dict); if (ret < 0) { - gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, - PC_MSG_DICT_UNSERIALIZE_FAIL, - "failed to unserialize xattr dict"); + gf_smsg(THIS->name, GF_LOG_WARNING, EINVAL, + PC_MSG_DICT_UNSERIALIZE_FAIL, "xattr", NULL); goto out; } } @@ -538,7 +537,7 @@ clnt_unserialize_rsp_locklist(xlator_t *this, struct gfs3_getactivelk_rsp *rsp, while (trav) { temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig); if (temp == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, 0, "No memory"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_NO_MEM, NULL); goto out; } @@ -578,7 +577,7 @@ clnt_unserialize_rsp_locklist_v2(xlator_t *this, while (trav) { temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig); if (temp == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, 0, "No memory"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_NO_MEM, NULL); goto out; } @@ -662,8 +661,8 @@ serialize_req_locklist(lock_migration_info_t *locklist, break; default: - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, - "Unknown lock type: %" PRId32 "!", tmp->flock.l_type); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_UNKNOWN_LOCK_TYPE, + "type=%" PRId32, tmp->flock.l_type, NULL); break; } @@ -673,8 +672,8 @@ serialize_req_locklist(lock_migration_info_t *locklist, trav->client_uid = gf_strdup(tmp->client_uid); if (!trav->client_uid) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, - "client_uid could not be allocated"); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_CLIENT_UID_ALLOC_FAILED, + NULL); ret = -1; goto out; } @@ -725,8 +724,8 @@ serialize_req_locklist_v2(lock_migration_info_t *locklist, break; default: - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, - "Unknown lock type: %" PRId32 "!", tmp->flock.l_type); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_UNKNOWN_LOCK_TYPE, + "type=%" PRId32, tmp->flock.l_type, NULL); break; } @@ -736,8 +735,8 @@ serialize_req_locklist_v2(lock_migration_info_t *locklist, trav->client_uid = gf_strdup(tmp->client_uid); if (!trav->client_uid) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, - "client_uid could not be allocated"); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_CLIENT_UID_ALLOC_FAILED, + NULL); ret = -1; goto out; } diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c index c1fb055fc0e..795839734c5 100644 --- a/xlators/protocol/client/src/client-lk.c +++ b/xlators/protocol/client/src/client-lk.c @@ -24,14 +24,12 @@ __dump_client_lock(client_posix_lock_t *lock) this = THIS; - gf_msg(this->name, GF_LOG_INFO, 0, PC_MSG_CLIENT_LOCK_INFO, - "{fd=%p}" - "{%s lk-owner:%s %" PRId64 " - %" PRId64 - "}" - "{start=%" PRId64 " end=%" PRId64 "}", - lock->fd, lock->fl_type == F_WRLCK ? "Write-Lock" : "Read-Lock", - lkowner_utoa(&lock->owner), lock->user_flock.l_start, - lock->user_flock.l_len, lock->fl_start, lock->fl_end); + gf_smsg( + this->name, GF_LOG_INFO, 0, PC_MSG_CLIENT_LOCK_INFO, "fd=%p", lock->fd, + "fl_type=%s", lock->fl_type == F_WRLCK ? "Write-Lock" : "Read-Lock", + "lk-owner=%s", lkowner_utoa(&lock->owner), "l_start=%" PRId64, + lock->user_flock.l_start, "l_len=%" PRId64, lock->user_flock.l_len, + "start=%" PRId64, lock->fl_start, "end=%" PRId64, lock->fl_end, NULL); } static int @@ -203,9 +201,7 @@ subtract_locks(client_posix_lock_t *big, client_posix_lock_t *small) memcpy(v.locks[1], small, sizeof(client_posix_lock_t)); } else { /* LOG-TODO : decide what more info is required here*/ - gf_msg("client-protocol", GF_LOG_CRITICAL, 0, PC_MSG_LOCK_ERROR, - "Unexpected case in subtract_locks. Please send " - "a bug report to gluster-devel@gluster.org"); + gf_smsg("client-protocol", GF_LOG_CRITICAL, 0, PC_MSG_LOCK_ERROR, NULL); } return v; @@ -343,8 +339,8 @@ delete_granted_locks_owner(fd_t *fd, gf_lkowner_t *owner) if (!fdctx) { pthread_spin_unlock(&conf->fd_lock); - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_FD_CTX_INVALID, - "fdctx not valid"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_FD_CTX_INVALID, + NULL); ret = -1; goto out; } @@ -461,8 +457,7 @@ client_add_lock_for_recovery(fd_t *fd, struct gf_flock *flock, if (!fdctx) { pthread_spin_unlock(&conf->fd_lock); - gf_msg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_GET_FAIL, - "failed to get fd context. sending EBADFD"); + gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_GET_FAIL, NULL); ret = -EBADFD; goto out; } @@ -498,8 +493,8 @@ client_dump_locks(char *name, inode_t *inode, dict_t *dict) ret = dict_set_dynstr(new_dict, CLIENT_DUMP_LOCKS, dict_string); if (ret) { - gf_msg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAILED, - "could not set dict with %s", CLIENT_DUMP_LOCKS); + gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAIL, "lock=%s", + CLIENT_DUMP_LOCKS, NULL); goto out; } diff --git a/xlators/protocol/client/src/client-messages.h b/xlators/protocol/client/src/client-messages.h index 2c8049f93d9..25a851d80b9 100644 --- a/xlators/protocol/client/src/client-messages.h +++ b/xlators/protocol/client/src/client-messages.h @@ -60,7 +60,8 @@ GLFS_MSGID( PC_MSG_NOREMOTE_HOST, PC_MSG_REMOTE_SUBVOL_NOT_GIVEN, PC_MSG_FATAL_CLIENT_PROTOCOL, PC_MSG_VOL_DANGLING, PC_MSG_CREATE_MEM_POOL_FAILED, PC_MSG_PVT_XLATOR_NULL, PC_MSG_XLATOR_NULL, - PC_MSG_LEASE_FOP_FAILED); + PC_MSG_LEASE_FOP_FAILED, PC_MSG_DICT_SET_FAIL, PC_MSG_NO_MEM, + PC_MSG_UNKNOWN_LOCK_TYPE, PC_MSG_CLIENT_UID_ALLOC_FAILED); #define PC_MSG_REMOTE_OP_FAILED_STR "remote operation failed." #define PC_MSG_XDR_DECODING_FAILED_STR "XDR decoding failed" @@ -148,4 +149,26 @@ GLFS_MSGID( #define PC_MSG_XLATOR_NULL_STR "xlator is NULL" #define PC_MSG_PVT_XLATOR_NULL_STR "private structure of the xlator is NULL" #define PC_MSG_LEASE_FOP_FAILED_STR "Lease fop failed" +#define PC_MSG_LOCK_ERROR_STR \ + "Unexpected case in subtract_locks. Please send a bug report to " \ + "gluster-devel@gluster.org" +#define PC_MSG_FD_CTX_INVALID_STR "fdctx not valid" +#define PC_MSG_FD_GET_FAIL_STR "failed to get fd context. sending EBADFD" +#define PC_MSG_DICT_SET_FAIL_STR "could not set dict" +#define PC_MSG_CLIENT_LOCK_INFO_STR "client lock info" +#define PC_MSG_BAD_FD_STR "remote_fd is -1. EBADFD" +#define PC_MSG_FUNCTION_CALL_ERROR_STR "this function should not be called" +#define PC_MSG_RECALL_LEASE_FAIL_STR "XDR decode of recall lease failed" +#define PC_MSG_CACHE_INVALIDATION_FAIL_STR \ + "XDR decode of cache_invalidation failed" +#define PC_MSG_INODELK_CONTENTION_FAIL_STR \ + "XDR decode of inodelk contention failed" +#define PC_MSG_ENTRYLK_CONTENTION_FAIL_STR \ + "XDR decode of entrylk contention failed" +#define PC_MSG_FD_DUPLICATE_TRY_STR "trying duplicate remote fd set" +#define PC_MSG_FD_SET_FAIL_STR "failed to set remote-fd" +#define PC_MSG_NO_MEM_STR "No memory" +#define PC_MSG_UNKNOWN_LOCK_TYPE_STR "Unknown lock type" +#define PC_MSG_CLIENT_UID_ALLOC_FAILED_STR "client-uid could not be allocated" + #endif /* !_PC_MESSAGES_H__ */ diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h index 3b47d1207bc..ab799c41755 100644 --- a/xlators/protocol/client/src/client.h +++ b/xlators/protocol/client/src/client.h @@ -63,10 +63,8 @@ typedef enum { goto label; \ } \ if (remote_fd == -1) { \ - gf_msg(xl->name, GF_LOG_WARNING, EBADFD, PC_MSG_BAD_FD, \ - " (%s) " \ - "remote_fd is -1. EBADFD", \ - uuid_utoa(fd->inode->gfid)); \ + gf_smsg(xl->name, GF_LOG_WARNING, EBADFD, PC_MSG_BAD_FD, \ + "gfid=%s", uuid_utoa(fd->inode->gfid), NULL); \ op_errno = EBADFD; \ goto label; \ } \ diff --git a/xlators/protocol/server/src/Makefile.am b/xlators/protocol/server/src/Makefile.am index 01edbd35d9c..5e875c8df0b 100644 --- a/xlators/protocol/server/src/Makefile.am +++ b/xlators/protocol/server/src/Makefile.am @@ -4,11 +4,11 @@ endif xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol -server_la_LDFLAGS = $(LIB_DL) -module $(GF_XLATOR_DEFAULT_LDFLAGS) +server_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ - $(top_builddir)/rpc/xdr/src/libgfxdr.la + $(top_builddir)/rpc/xdr/src/libgfxdr.la $(LIB_DL) server_la_SOURCES = server.c server-resolve.c server-helpers.c \ server-rpc-fops.c server-handshake.c authenticate.c \ diff --git a/xlators/protocol/server/src/server-common.c b/xlators/protocol/server/src/server-common.c index 4bb84042a9e..cd79cf4d930 100644 --- a/xlators/protocol/server/src/server-common.c +++ b/xlators/protocol/server/src/server-common.c @@ -828,7 +828,7 @@ server4_post_lease(gfx_lease_rsp *rsp, struct gf_lease *lease) void server4_post_link(server_state_t *state, gfx_common_3iatt_rsp *rsp, inode_t *inode, struct iatt *stbuf, struct iatt *preparent, - struct iatt *postparent, dict_t *xdata) + struct iatt *postparent) { inode_t *link_inode = NULL; diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c index 510dd09c268..6e644912a46 100644 --- a/xlators/protocol/server/src/server-helpers.c +++ b/xlators/protocol/server/src/server-helpers.c @@ -52,14 +52,14 @@ gid_resolve(server_conf_t *conf, call_stack_t *root) ret = getpwuid_r(root->uid, &mypw, mystrs, sizeof(mystrs), &result); if (ret != 0) { - gf_msg("gid-cache", GF_LOG_ERROR, errno, PS_MSG_GET_UID_FAILED, - "getpwuid_r(%u) failed", root->uid); + gf_smsg("gid-cache", GF_LOG_ERROR, errno, PS_MSG_GET_UID_FAILED, + "uid=%u", root->uid, NULL); return -1; } if (!result) { - gf_msg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_UID_NOT_FOUND, - "getpwuid_r(%u) found nothing", root->uid); + gf_smsg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_UID_NOT_FOUND, "uid=%u", + root->uid, NULL); return -1; } @@ -67,9 +67,9 @@ gid_resolve(server_conf_t *conf, call_stack_t *root) ngroups = gf_getgrouplist(result->pw_name, root->gid, &mygroups); if (ngroups == -1) { - gf_msg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_MAPPING_ERROR, - "could not map %s to group list (%d gids)", result->pw_name, - root->ngrps); + gf_smsg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_MAPPING_ERROR, + "pw_name=%s", result->pw_name, "root->ngtps=%d", root->ngrps, + NULL); return -1; } root->ngrps = (uint16_t)ngroups; @@ -313,13 +313,12 @@ do_fd_cleanup(xlator_t *this, client_t *client, fdentry_t *fdentries, ret = inode_path(fd->inode, NULL, &path); if (ret > 0) { - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP, - "fd cleanup on %s", path); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP, + "path=%s", path, NULL); GF_FREE(path); } else { - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP, - "fd cleanup on inode with gfid %s", - uuid_utoa(fd->inode->gfid)); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP, + "inode-gfid=%s", uuid_utoa(fd->inode->gfid), NULL); } tmp_frame->local = fd; @@ -363,9 +362,7 @@ server_connection_cleanup(xlator_t *this, client_t *client, int32_t flags, serv_ctx = server_ctx_get(client, client->this); if (serv_ctx == NULL) { - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, - "server_ctx_get() " - "failed"); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, NULL); goto out; } @@ -413,8 +410,7 @@ server_connection_cleanup(xlator_t *this, client_t *client, int32_t flags, fd_count); ret = do_fd_cleanup(this, client, fdentries, fd_count, detach); } else - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_FDENTRY_NULL, - "no fdentries to clean"); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FDENTRY_NULL, NULL); if (cd_ret || ret) ret = -1; @@ -598,9 +594,8 @@ server_build_config(xlator_t *this, server_conf_t *conf) if (data) { ret = gf_string2boolean(data->data, &conf->verify_volfile); if (ret != 0) { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "wrong value for '" - "verify-volfile-checksum', Neglecting option"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_WRONG_VALUE, + NULL); } } @@ -608,9 +603,8 @@ server_build_config(xlator_t *this, server_conf_t *conf) if (data) { ret = gf_string2boolean(data->data, &conf->trace); if (ret != 0) { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "'trace' takes on only " - "boolean values. Neglecting option"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, + NULL); } } @@ -631,18 +625,16 @@ server_build_config(xlator_t *this, server_conf_t *conf) or directory specified is non standard */ ret = sys_stat(data->data, &buf); if ((ret != 0) || !S_ISDIR(buf.st_mode)) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_DIR_NOT_FOUND, - "Directory '%s' doesn't " - "exist, exiting.", - data->data); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_DIR_NOT_FOUND, + "data=%s", data->data, NULL); ret = -1; goto out; } /* Make sure that conf-dir doesn't contain ".." in path */ if ((gf_strstr(data->data, "/", "..")) == -1) { ret = -1; - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_CONF_DIR_INVALID, - "%s: invalid conf_dir", data->data); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_CONF_DIR_INVALID, + "data=%s", data->data, NULL); goto out; } @@ -824,8 +816,9 @@ server_print_reply(call_frame_t *frame, int op_ret, int op_errno) if (state->fd) snprintf(fdstr, 32, " fd=%p", state->fd); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SERVER_MSG, - "%s%s => (%d, %d)%s", op, caller, op_ret, op_errno, fdstr); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SERVER_MSG, "op=%s", op, + "caller=%s", caller, "op_ret=%d", op_ret, "op_errno=%d", op_errno, + "fdstr=%s", fdstr, NULL); out: return; } @@ -885,9 +878,10 @@ server_print_request(call_frame_t *frame) break; } - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_MSG, "%s%s%s%s%s%s%s", op, - caller, resolve_vars, loc_vars, resolve2_vars, loc2_vars, - other_vars); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_MSG, "op=%s", op, + "caller=%s", caller, "resolve_vars=%s", resolve_vars, "loc_vars=%s", + loc_vars, "resolve2_vars=%s", resolve2_vars, "loc2_vars=%s", + loc2_vars, "other_vars=%s", other_vars, NULL); out: return; } @@ -923,8 +917,8 @@ serialize_rsp_direntp(gf_dirent_t *entries, gfs3_readdirp_rsp *rsp) (char **)&trav->dict.dict_val, &trav->dict.dict_len); if (ret != 0) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_DICT_SERIALIZE_FAIL, - "failed to serialize reply dict"); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_DICT_SERIALIZE_FAIL, + NULL); errno = -ret; trav->dict.dict_len = 0; goto out; @@ -1155,8 +1149,8 @@ common_rsp_locklist(lock_migration_info_t *locklist, gfs3_locklist **reply) break; default: - gf_msg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_LOCK_ERROR, - "Unknown lock type: %" PRId32 "!", tmp->flock.l_type); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_LOCK_ERROR, + "lock_type=%" PRId32, tmp->flock.l_type, NULL); break; } @@ -1255,8 +1249,9 @@ gf_server_check_getxattr_cmd(call_frame_t *frame, const char *key) { list_for_each_entry(xprt, &conf->xprt_list, list) { - gf_msg("mount-point-list", GF_LOG_INFO, 0, PS_MSG_MOUNT_PT_FAIL, - "%s", xprt->peerinfo.identifier); + gf_smsg("mount-point-list", GF_LOG_INFO, 0, + PS_MSG_MOUNT_PT_FAIL, "identifier=%s", + xprt->peerinfo.identifier, NULL); } } pthread_mutex_unlock(&conf->mutex); @@ -1286,9 +1281,8 @@ gf_server_check_setxattr_cmd(call_frame_t *frame, dict_t *dict) total_read += xprt->total_bytes_read; total_write += xprt->total_bytes_write; } - gf_msg("stats", GF_LOG_INFO, 0, PS_MSG_RW_STAT, - "total-read %" PRIu64 ", total-write %" PRIu64, total_read, - total_write); + gf_smsg("stats", GF_LOG_INFO, 0, PS_MSG_RW_STAT, "total-read=%" PRIu64, + total_read, "total-write=%" PRIu64, total_write, NULL); } return 0; @@ -1364,15 +1358,15 @@ auth_set_username_passwd(dict_t *input_params, dict_t *config_params, ret = dict_get_str(input_params, "password", &password); if (ret) { - gf_msg("auth/login", GF_LOG_WARNING, 0, PS_MSG_DICT_GET_FAILED, - "password not found, returning DONT-CARE"); + gf_smsg("auth/login", GF_LOG_WARNING, 0, PS_MSG_PASSWORD_NOT_FOUND, + NULL); goto out; } ret = dict_get_str(input_params, "remote-subvolume", &brick_name); if (ret) { - gf_msg("auth/login", GF_LOG_ERROR, 0, PS_MSG_DICT_GET_FAILED, - "remote-subvolume not specified"); + gf_smsg("auth/login", GF_LOG_ERROR, 0, + PS_MSG_REMOTE_SUBVOL_NOT_SPECIFIED, NULL); ret = -1; goto out; } @@ -1404,10 +1398,8 @@ auth_set_username_passwd(dict_t *input_params, dict_t *config_params, GF_FREE(searchstr); if (!passwd_data) { - gf_msg("auth/login", GF_LOG_ERROR, 0, PS_MSG_LOGIN_ERROR, - "wrong " - "username/password " - "combination"); + gf_smsg("auth/login", GF_LOG_ERROR, 0, PS_MSG_LOGIN_ERROR, + NULL); ret = -1; goto out; } @@ -1417,10 +1409,8 @@ auth_set_username_passwd(dict_t *input_params, dict_t *config_params, client->auth.username = gf_strdup(username); client->auth.passwd = gf_strdup(password); } else { - gf_msg("auth/login", GF_LOG_ERROR, 0, PS_MSG_LOGIN_ERROR, - "wrong " - "password for user %s", - username); + gf_smsg("auth/login", GF_LOG_ERROR, 0, PS_MSG_LOGIN_ERROR, + "username=%s", username, NULL); } break; } @@ -1457,7 +1447,7 @@ unserialize_req_locklist(gfs3_setactivelk_req *req, lock_migration_info_t *lmi) while (trav) { temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig); if (temp == NULL) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "No memory"); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_NO_MEM, NULL); goto out; } @@ -1494,7 +1484,7 @@ unserialize_req_locklist_v2(gfx_setactivelk_req *req, while (trav) { temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig); if (temp == NULL) { - gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "No memory"); + gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_NO_MEM, NULL); goto out; } diff --git a/xlators/protocol/server/src/server-messages.h b/xlators/protocol/server/src/server-messages.h index 9f00421fe92..1b2e149cb7d 100644 --- a/xlators/protocol/server/src/server-messages.h +++ b/xlators/protocol/server/src/server-messages.h @@ -23,43 +23,146 @@ * glfs-message-id.h. */ -GLFS_MSGID(PS, PS_MSG_AUTHENTICATE_ERROR, PS_MSG_VOL_VALIDATE_FAILED, - PS_MSG_AUTH_INIT_FAILED, PS_MSG_REMOTE_CLIENT_REFUSED, - PS_MSG_GFID_RESOLVE_FAILED, PS_MSG_ANONYMOUS_FD_CREATE_FAILED, - PS_MSG_NO_MEMORY, PS_MSG_FD_NOT_FOUND, PS_MSG_INVALID_ENTRY, - PS_MSG_GET_UID_FAILED, PS_MSG_UID_NOT_FOUND, PS_MSG_MAPPING_ERROR, - PS_MSG_FD_CLEANUP, PS_MSG_SERVER_CTX_GET_FAILED, PS_MSG_FDENTRY_NULL, - PS_MSG_DIR_NOT_FOUND, PS_MSG_SERVER_MSG, PS_MSG_DICT_SERIALIZE_FAIL, - PS_MSG_RW_STAT, PS_MSG_DICT_GET_FAILED, PS_MSG_LOGIN_ERROR, - PS_MSG_REMOUNT_CLIENT_REQD, PS_MSG_DEFAULTING_FILE, - PS_MSG_VOL_FILE_OPEN_FAILED, PS_MSG_STAT_ERROR, - PS_MSG_SSL_NAME_SET_FAILED, PS_MSG_ASPRINTF_FAILED, - PS_MSG_CLIENT_VERSION_NOT_SET, PS_MSG_CLIENT_ACCEPTED, - PS_MSG_CLIENT_LK_VERSION_ERROR, PS_MSG_GRACE_TIMER_EXPD, - PS_MSG_SERIALIZE_REPLY_FAILED, PS_MSG_AUTH_IP_ERROR, - PS_MSG_SKIP_FORMAT_CHK, PS_MSG_INTERNET_ADDR_ERROR, - PS_MSG_CLIENT_DISCONNECTING, PS_MSG_GRACE_TIMER_START, - PS_MSG_STATEDUMP_PATH_ERROR, PS_MSG_GRP_CACHE_ERROR, - PS_MSG_RPC_CONF_ERROR, PS_MSG_TRANSPORT_ERROR, PS_MSG_SUBVOL_NULL, - PS_MSG_PARENT_VOL_ERROR, PS_MSG_RPCSVC_CREATE_FAILED, - PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, PS_MSG_RPCSVC_NOTIFY, - PS_MSG_PGM_REG_FAILED, PS_MSG_ULIMIT_SET_FAILED, PS_MSG_STATFS, - PS_MSG_LOOKUP_INFO, PS_MSG_LK_INFO, PS_MSG_LOCK_ERROR, - PS_MSG_INODELK_INFO, PS_MSG_ENTRYLK_INFO, PS_MSG_ACCESS_INFO, - PS_MSG_DIR_INFO, PS_MSG_MKNOD_INFO, PS_MSG_REMOVEXATTR_INFO, - PS_MSG_GETXATTR_INFO, PS_MSG_SETXATTR_INFO, PS_MSG_RENAME_INFO, - PS_MSG_LINK_INFO, PS_MSG_TRUNCATE_INFO, PS_MSG_FSTAT_INFO, - PS_MSG_FLUSH_INFO, PS_MSG_SYNC_INFO, PS_MSG_WRITE_INFO, - PS_MSG_READ_INFO, PS_MSG_CHKSUM_INFO, PS_MSG_OPEN_INFO, - PS_MSG_CREATE_INFO, PS_MSG_SETATTR_INFO, PS_MSG_XATTROP_INFO, - PS_MSG_ALLOC_INFO, PS_MSG_DISCARD_INFO, PS_MSG_ZEROFILL_INFO, - PS_MSG_FD_CREATE_FAILED, PS_MSG_WRONG_STATE, PS_MSG_CONF_DIR_INVALID, - PS_MSG_MOUNT_PT_FAIL, PS_MSG_STAT_INFO, PS_MSG_FILE_OP_FAILED, - PS_MSG_GRACE_TIMER_CANCELLED, PS_MSG_ENCODE_MSG_FAILED, - PS_MSG_REPLY_SUBMIT_FAILED, PS_MSG_RPC_NOTIFY_ERROR, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, PS_MSG_SERVER_IPC_INFO, - PS_MSG_SEEK_INFO, PS_MSG_COMPOUND_INFO, - PS_MSG_CLIENT_OPVERSION_GET_FAILED, PS_MSG_CHILD_STATUS_FAILED, - PS_MSG_PUT_INFO); +GLFS_MSGID( + PS, PS_MSG_AUTHENTICATE_ERROR, PS_MSG_VOL_VALIDATE_FAILED, + PS_MSG_AUTH_INIT_FAILED, PS_MSG_REMOTE_CLIENT_REFUSED, + PS_MSG_GFID_RESOLVE_FAILED, PS_MSG_ANONYMOUS_FD_CREATE_FAILED, + PS_MSG_NO_MEMORY, PS_MSG_FD_NOT_FOUND, PS_MSG_INVALID_ENTRY, + PS_MSG_GET_UID_FAILED, PS_MSG_UID_NOT_FOUND, PS_MSG_MAPPING_ERROR, + PS_MSG_FD_CLEANUP, PS_MSG_SERVER_CTX_GET_FAILED, PS_MSG_FDENTRY_NULL, + PS_MSG_DIR_NOT_FOUND, PS_MSG_SERVER_MSG, PS_MSG_DICT_SERIALIZE_FAIL, + PS_MSG_RW_STAT, PS_MSG_DICT_GET_FAILED, PS_MSG_LOGIN_ERROR, + PS_MSG_REMOUNT_CLIENT_REQD, PS_MSG_DEFAULTING_FILE, + PS_MSG_VOL_FILE_OPEN_FAILED, PS_MSG_STAT_ERROR, PS_MSG_SSL_NAME_SET_FAILED, + PS_MSG_ASPRINTF_FAILED, PS_MSG_CLIENT_VERSION_NOT_SET, + PS_MSG_CLIENT_ACCEPTED, PS_MSG_CLIENT_LK_VERSION_ERROR, + PS_MSG_GRACE_TIMER_EXPD, PS_MSG_SERIALIZE_REPLY_FAILED, + PS_MSG_AUTH_IP_ERROR, PS_MSG_SKIP_FORMAT_CHK, PS_MSG_INTERNET_ADDR_ERROR, + PS_MSG_CLIENT_DISCONNECTING, PS_MSG_GRACE_TIMER_START, + PS_MSG_STATEDUMP_PATH_ERROR, PS_MSG_GRP_CACHE_ERROR, PS_MSG_RPC_CONF_ERROR, + PS_MSG_TRANSPORT_ERROR, PS_MSG_SUBVOL_NULL, PS_MSG_PARENT_VOL_ERROR, + PS_MSG_RPCSVC_CREATE_FAILED, PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, + PS_MSG_RPCSVC_NOTIFY, PS_MSG_PGM_REG_FAILED, PS_MSG_ULIMIT_SET_FAILED, + PS_MSG_STATFS, PS_MSG_LOOKUP_INFO, PS_MSG_LK_INFO, PS_MSG_LOCK_ERROR, + PS_MSG_INODELK_INFO, PS_MSG_ENTRYLK_INFO, PS_MSG_ACCESS_INFO, + PS_MSG_DIR_INFO, PS_MSG_MKNOD_INFO, PS_MSG_REMOVEXATTR_INFO, + PS_MSG_GETXATTR_INFO, PS_MSG_SETXATTR_INFO, PS_MSG_RENAME_INFO, + PS_MSG_LINK_INFO, PS_MSG_TRUNCATE_INFO, PS_MSG_FSTAT_INFO, + PS_MSG_FLUSH_INFO, PS_MSG_SYNC_INFO, PS_MSG_WRITE_INFO, PS_MSG_READ_INFO, + PS_MSG_CHKSUM_INFO, PS_MSG_OPEN_INFO, PS_MSG_CREATE_INFO, + PS_MSG_SETATTR_INFO, PS_MSG_XATTROP_INFO, PS_MSG_ALLOC_INFO, + PS_MSG_DISCARD_INFO, PS_MSG_ZEROFILL_INFO, PS_MSG_FD_CREATE_FAILED, + PS_MSG_WRONG_STATE, PS_MSG_CONF_DIR_INVALID, PS_MSG_MOUNT_PT_FAIL, + PS_MSG_STAT_INFO, PS_MSG_FILE_OP_FAILED, PS_MSG_GRACE_TIMER_CANCELLED, + PS_MSG_ENCODE_MSG_FAILED, PS_MSG_REPLY_SUBMIT_FAILED, + PS_MSG_RPC_NOTIFY_ERROR, PS_MSG_SERVER_EVENT_UPCALL_FAILED, + PS_MSG_SERVER_IPC_INFO, PS_MSG_SEEK_INFO, PS_MSG_COMPOUND_INFO, + PS_MSG_CLIENT_OPVERSION_GET_FAILED, PS_MSG_CHILD_STATUS_FAILED, + PS_MSG_PUT_INFO, PS_MSG_UNAUTHORIZED_CLIENT, PS_MSG_RECONFIGURE_FAILED, + PS_MSG_SET_STATEDUMP_PATH_ERROR, PS_MSG_INIT_GRP_CACHE_ERROR, + PS_MSG_RPC_CONFIGURE_FAILED, PS_MSG_TRANSPORT_TYPE_NOT_SET, + PS_MSG_GET_TOTAL_AVAIL_TRANSPORT_FAILED, PS_MSG_INVLAID_UPCALL_EVENT, + PS_MSG_SERVER_CHILD_EVENT_FAILED, PS_MSG_SETACTIVELK_INFO, + PS_MSG_GETACTIVELK_INFO, PS_MSG_WRONG_VALUE, PS_MSG_PASSWORD_NOT_FOUND, + PS_MSG_REMOTE_SUBVOL_NOT_SPECIFIED, PS_MSG_NO_MEM); +#define PS_MSG_SERIALIZE_REPLY_FAILED_STR "Failed to serialize reply" +#define PS_MSG_AUTH_IP_ERROR_STR "assuming 'auth.ip' to be 'auth.addr'" +#define PS_MSG_SKIP_FORMAT_CHK_STR "skip format check for non-addr auth option" +#define PS_MSG_INTERNET_ADDR_ERROR_STR \ + "internet address does not confirm to standards" +#define PS_MSG_AUTHENTICATE_ERROR_STR \ + "volume defined as subvolume, but no authentication defined for the same" +#define PS_MSG_CLIENT_DISCONNECTING_STR "disconnecting connection" +#define PS_MSG_DICT_GET_FAILED_STR "failed to get" +#define PS_MSG_NO_MEMORY_STR "Memory accounting init failed" +#define PS_MSG_INVALID_ENTRY_STR \ + "'trace' takes on only boolean values. Neglecting option" +#define PS_MSG_STATEDUMP_PATH_ERROR_STR \ + "Error while reconfiguring statedump path" +#define PS_MSG_GRP_CACHE_ERROR_STR "Failed to reconfigure group cache." +#define PS_MSG_RPC_CONF_ERROR_STR "No rpc_conf !!!!" +#define PS_MSG_CLIENT_ACCEPTED_STR \ + "authorized client, hence we continue with this connection" +#define PS_MSG_UNAUTHORIZED_CLIENT_STR \ + "unauthorized client, hence terminating the connection" +#define PS_MSG_RECONFIGURE_FAILED_STR \ + "Failed to reconfigure outstanding-rpc-limit" +#define PS_MSG_TRANSPORT_ERROR_STR "Reconfigure not found for transport" +#define PS_MSG_SUBVOL_NULL_STR "protocol/server should have subvolume" +#define PS_MSG_PARENT_VOL_ERROR_STR \ + "protocol/server should not have parent volumes" +#define PS_MSG_SET_STATEDUMP_PATH_ERROR_STR "Error setting statedump path" +#define PS_MSG_INIT_GRP_CACHE_ERROR_STR "Failed to initialize group cache." +#define PS_MSG_RPCSVC_CREATE_FAILED_STR "creation of rpcsvc failed" +#define PS_MSG_RPC_CONFIGURE_FAILED_STR \ + "Failed to configure outstanding-rpc-limit" +#define PS_MSG_TRANSPORT_TYPE_NOT_SET_STR "option transport-type not set" +#define PS_MSG_GET_TOTAL_AVAIL_TRANSPORT_FAILED_STR \ + "failed to get total number of available tranpsorts" +#define PS_MSG_RPCSVC_LISTENER_CREATE_FAILED_STR "creation of listener failed" +#define PS_MSG_RPCSVC_NOTIFY_STR "registration of notify with rpcsvc failed" +#define PS_MSG_PGM_REG_FAILED_STR "registration of program failed" +#define PS_MSG_ULIMIT_SET_FAILED_STR "WARNING: Failed to set 'ulimit -n 1M'" +#define PS_MSG_FD_NOT_FOUND_STR "Failed to set max open fd to 64k" +#define PS_MSG_VOL_FILE_OPEN_FAILED_STR \ + "volfile-id argument not given. This is mandatory argument, defaulting " \ + "to 'gluster'" +#define PS_MSG_INVLAID_UPCALL_EVENT_STR "Received invalid upcall event" +#define PS_MSG_CHILD_STATUS_FAILED_STR "No xlator is found in child status list" +#define PS_MSG_SERVER_EVENT_UPCALL_FAILED_STR \ + "server_process_event_upcall failed" +#define PS_MSG_SERVER_CHILD_EVENT_FAILED_STR "server_process_child_event failed" +#define PS_MSG_STATFS_STR "STATFS" +#define PS_MSG_LOOKUP_INFO_STR "LOOKUP info" +#define PS_MSG_LK_INFO_STR "LEASE info" +#define PS_MSG_INODELK_INFO_STR "INODELK info" +#define PS_MSG_DIR_INFO_STR "MKDIR info" +#define PS_MSG_MKNOD_INFO_STR "MKNOD info" +#define PS_MSG_REMOVEXATTR_INFO_STR "REMOVEXATTR info" +#define PS_MSG_GETXATTR_INFO_STR "GETXATTR info" +#define PS_MSG_SETXATTR_INFO_STR "SETXATTR info" +#define PS_MSG_RENAME_INFO_STR "RENAME inf" +#define PS_MSG_LINK_INFO_STR "LINK info" +#define PS_MSG_TRUNCATE_INFO_STR "TRUNCATE info" +#define PS_MSG_STAT_INFO_STR "STAT info" +#define PS_MSG_FLUSH_INFO_STR "FLUSH info" +#define PS_MSG_SYNC_INFO_STR "SYNC info" +#define PS_MSG_WRITE_INFO_STR "WRITE info" +#define PS_MSG_READ_INFO_STR "READ info" +#define PS_MSG_CHKSUM_INFO_STR "CHKSUM info" +#define PS_MSG_OPEN_INFO_STR "OPEN info" +#define PS_MSG_XATTROP_INFO_STR "XATTROP info" +#define PS_MSG_ALLOC_INFO_STR "ALLOC info" +#define PS_MSG_DISCARD_INFO_STR "DISCARD info" +#define PS_MSG_ZEROFILL_INFO_STR "ZEROFILL info" +#define PS_MSG_SERVER_IPC_INFO_STR "IPC info" +#define PS_MSG_SEEK_INFO_STR "SEEK info" +#define PS_MSG_SETACTIVELK_INFO_STR "SETACTIVELK info" +#define PS_MSG_CREATE_INFO_STR "CREATE info" +#define PS_MSG_PUT_INFO_STR "PUT info" +#define PS_MSG_FD_CREATE_FAILED_STR "could not create the fd" +#define PS_MSG_GETACTIVELK_INFO_STR "GETACTIVELK info" +#define PS_MSG_ENTRYLK_INFO_STR "ENTRYLK info" +#define PS_MSG_ACCESS_INFO_STR "ACCESS info" +#define PS_MSG_SETATTR_INFO_STR "SETATTR info" +#define PS_MSG_SERVER_CTX_GET_FAILED_STR "server_ctx_get() failed" +#define PS_MSG_LOCK_ERROR_STR "Unknown lock type" +#define PS_MSG_GET_UID_FAILED_STR "getpwuid_r failed" +#define PS_MSG_UID_NOT_FOUND_STR "getpwuid_r found nothing" +#define PS_MSG_MAPPING_ERROR_STR "could not map to group list" +#define PS_MSG_FD_CLEANUP_STR "fd cleanup" +#define PS_MSG_FDENTRY_NULL_STR "no fdentry to clean" +#define PS_MSG_WRONG_VALUE_STR \ + "wrong value for 'verify-volfile-checksum', Neglecting option" +#define PS_MSG_DIR_NOT_FOUND_STR "Directory doesnot exist" +#define PS_MSG_CONF_DIR_INVALID_STR "invalid conf_dir" +#define PS_MSG_SERVER_MSG_STR "server msg" +#define PS_MSG_DICT_SERIALIZE_FAIL_STR "failed to serialize reply dict" +#define PS_MSG_MOUNT_PT_FAIL_STR "mount point fail" +#define PS_MSG_RW_STAT_STR "stat" +#define PS_MSG_PASSWORD_NOT_FOUND_STR "password not found, returning DONT-CARE" +#define PS_MSG_REMOTE_SUBVOL_NOT_SPECIFIED_STR "remote-subvolume not specified" +#define PS_MSG_LOGIN_ERROR_STR "wrong password for user" +#define PS_MSG_NO_MEM_STR "No memory" #endif /* !_PS_MESSAGES_H__ */ diff --git a/xlators/protocol/server/src/server-rpc-fops_v2.c b/xlators/protocol/server/src/server-rpc-fops_v2.c index ec0769873a9..c267f8f7832 100644 --- a/xlators/protocol/server/src/server-rpc-fops_v2.c +++ b/xlators/protocol/server/src/server-rpc-fops_v2.c @@ -56,10 +56,10 @@ server4_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, dict_to_xdr(xdata, &rsp.xdata); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_WARNING, op_errno, PS_MSG_STATFS, - "%" PRId64 ": STATFS, client: %s, error-xlator: %s", - frame->root->unique, STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_WARNING, op_errno, PS_MSG_STATFS, + "frame=%" PRId64, frame->root->unique, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -146,25 +146,20 @@ out: if (op_ret) { if (state->resolve.bname) { - gf_msg(this->name, fop_log_level(GF_FOP_LOOKUP, op_errno), op_errno, - PS_MSG_LOOKUP_INFO, - "%" PRId64 - ": LOOKUP %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_LOOKUP, op_errno), + op_errno, PS_MSG_LOOKUP_INFO, "frame=%" PRId64, + frame->root->unique, "path=%s", state->loc.path, + "uuid_utoa=%s", uuid_utoa(state->resolve.pargfid), + "bname=%s", state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); } else { - gf_msg(this->name, fop_log_level(GF_FOP_LOOKUP, op_errno), op_errno, - PS_MSG_LOOKUP_INFO, - "%" PRId64 - ": LOOKUP %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), - STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_LOOKUP, op_errno), + op_errno, PS_MSG_LOOKUP_INFO, "frame=%" PRId64, + frame->root->unique, "path=%s", state->loc.path, + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); } } @@ -192,12 +187,12 @@ server4_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_LEASE, op_errno), op_errno, - PS_MSG_LK_INFO, - "%" PRId64 ": LEASE %s (%s), client: %s, error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_LEASE, op_errno), op_errno, + PS_MSG_LK_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); } server4_post_lease(&rsp, lease); @@ -228,14 +223,12 @@ server4_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_LK, op_errno), op_errno, - PS_MSG_LK_INFO, - "%" PRId64 ": LK %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_LK, op_errno), op_errno, + PS_MSG_LK_INFO, "frame=%" PRId64, frame->root->unique, + "fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -268,14 +261,12 @@ server4_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_INODELK, op_errno), op_errno, - PS_MSG_INODELK_INFO, - "%" PRId64 - ": INODELK %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_INODELK, op_errno), op_errno, + PS_MSG_INODELK_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -307,14 +298,12 @@ server4_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_FINODELK, op_errno), op_errno, - PS_MSG_INODELK_INFO, - "%" PRId64 ": FINODELK %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FINODELK, op_errno), op_errno, + PS_MSG_INODELK_INFO, "frame=%" PRId64, frame->root->unique, + "FINODELK_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -346,14 +335,12 @@ server4_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_ENTRYLK, op_errno), op_errno, - PS_MSG_ENTRYLK_INFO, - "%" PRId64 - ": ENTRYLK %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_ENTRYLK, op_errno), op_errno, + PS_MSG_ENTRYLK_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -385,14 +372,12 @@ server4_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_FENTRYLK, op_errno), op_errno, - PS_MSG_ENTRYLK_INFO, - "%" PRId64 ": FENTRYLK %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FENTRYLK, op_errno), op_errno, + PS_MSG_ENTRYLK_INFO, "frame=%" PRId64, frame->root->unique, + "FENTRYLK_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -423,13 +408,12 @@ server4_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_ACCESS_INFO, - "%" PRId64 - ": ACCESS %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_ACCESS_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", + (state->loc.path) ? state->loc.path : "", "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -462,13 +446,13 @@ server4_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_DIR_INFO, - "%" PRId64 - ": RMDIR %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_DIR_INFO, + "frame=%" PRId64, frame->root->unique, "RMDIR_pat=%s", + (state->loc.path) ? state->loc.path : "", "uuid_utoa=%s", + uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -504,14 +488,13 @@ server4_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_MKDIR, op_errno), op_errno, - PS_MSG_DIR_INFO, - "%" PRId64 - ": MKDIR %s (%s/%s) client: %s, " - "error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_MKDIR, op_errno), op_errno, + PS_MSG_DIR_INFO, "frame=%" PRId64, frame->root->unique, + "MKDIR_path=%s", (state->loc.path) ? state->loc.path : "", + "uuid_utoa=%s", uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -546,14 +529,13 @@ server4_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, fop_log_level(GF_FOP_MKNOD, op_errno), op_errno, - PS_MSG_MKNOD_INFO, - "%" PRId64 - ": MKNOD %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_MKNOD, op_errno), op_errno, + PS_MSG_MKNOD_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -585,14 +567,12 @@ server4_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FSYNCDIR, op_errno), op_errno, - PS_MSG_DIR_INFO, - "%" PRId64 ": FSYNCDIR %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FSYNCDIR, op_errno), op_errno, + PS_MSG_DIR_INFO, "frame=%" PRId64, frame->root->unique, + "FSYNCDIR_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -625,14 +605,12 @@ server4_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_READDIR, op_errno), op_errno, - PS_MSG_DIR_INFO, - "%" PRId64 ": READDIR %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_READDIR, op_errno), op_errno, + PS_MSG_DIR_INFO, "frame=%" PRId64, frame->root->unique, + "READDIR_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -676,14 +654,12 @@ server4_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_OPENDIR, op_errno), op_errno, - PS_MSG_DIR_INFO, - "%" PRId64 - ": OPENDIR %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_OPENDIR, op_errno), op_errno, + PS_MSG_DIR_INFO, "frame=%" PRId64, frame->root->unique, + "OPENDIR_path=%s", (state->loc.path) ? state->loc.path : "", + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -725,13 +701,12 @@ server4_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, else loglevel = GF_LOG_INFO; - gf_msg(this->name, loglevel, op_errno, PS_MSG_REMOVEXATTR_INFO, - "%" PRId64 - ": REMOVEXATTR %s (%s) of key %s, client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), state->name, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, loglevel, op_errno, PS_MSG_REMOVEXATTR_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", + state->loc.path, "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), + "name=%s", state->name, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -762,14 +737,13 @@ server4_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret == -1) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FREMOVEXATTR, op_errno), - op_errno, PS_MSG_REMOVEXATTR_INFO, - "%" PRId64 ": FREMOVEXATTR %" PRId64 - " (%s) (%s), " - "client: %s, error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), state->name, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FREMOVEXATTR, op_errno), + op_errno, PS_MSG_REMOVEXATTR_INFO, "frame=%" PRId64, + frame->root->unique, "FREMOVEXATTR_fd_no%" PRId64, + state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "name=%s", state->name, + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -801,14 +775,12 @@ server4_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret == -1) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_GETXATTR, op_errno), op_errno, - PS_MSG_GETXATTR_INFO, - "%" PRId64 - ": GETXATTR %s (%s) (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), state->name, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_GETXATTR, op_errno), op_errno, + PS_MSG_GETXATTR_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "name=%s", state->name, + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -843,14 +815,12 @@ server4_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret == -1) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FGETXATTR, op_errno), op_errno, - PS_MSG_GETXATTR_INFO, - "%" PRId64 ": FGETXATTR %" PRId64 - " (%s) (%s), " - "client: %s, error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), state->name, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FGETXATTR, op_errno), op_errno, + PS_MSG_GETXATTR_INFO, "frame=%" PRId64, frame->root->unique, + "FGETXATTR_fd_no=%" PRId64, state->resolve.fd_no, + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "name=%s", + state->name, "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -890,11 +860,9 @@ server4_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_errno == ENOTSUP) { gf_msg_debug(THIS->name, 0, "%s", strerror(op_errno)); } else { - gf_msg(THIS->name, GF_LOG_INFO, op_errno, PS_MSG_SETXATTR_INFO, - "client: %s, " - "error-xlator: %s", - STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(THIS->name, GF_LOG_INFO, op_errno, PS_MSG_SETXATTR_INFO, + "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); } goto out; } @@ -932,11 +900,9 @@ server4_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_errno == ENOTSUP) { gf_msg_debug(THIS->name, 0, "%s", strerror(op_errno)); } else { - gf_msg(THIS->name, GF_LOG_INFO, op_errno, PS_MSG_SETXATTR_INFO, - "client: %s, " - "error-xlator: %s", - STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(THIS->name, GF_LOG_INFO, op_errno, PS_MSG_SETXATTR_INFO, + "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); } goto out; } @@ -980,14 +946,14 @@ server4_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret == -1) { uuid_utoa_r(state->resolve.pargfid, oldpar_str); uuid_utoa_r(state->resolve2.pargfid, newpar_str); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_RENAME_INFO, - "%" PRId64 - ": RENAME %s (%s/%s) -> %s (%s/%s), " - "client: %s, error-xlator: %s", - frame->root->unique, state->loc.path, oldpar_str, - state->resolve.bname, state->loc2.path, newpar_str, - state->resolve2.bname, STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_RENAME_INFO, + "frame=%" PRId64, frame->root->unique, "loc.path=%s", + state->loc.path, "oldpar_str=%s", oldpar_str, "resolve-name=%s", + state->resolve.bname, "loc2.path=%s", state->loc2.path, + "newpar_str=%s", newpar_str, "resolve2=%s", + state->resolve2.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1022,14 +988,13 @@ server4_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret) { - gf_msg(this->name, fop_log_level(GF_FOP_UNLINK, op_errno), op_errno, - PS_MSG_LINK_INFO, - "%" PRId64 - ": UNLINK %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_UNLINK, op_errno), op_errno, + PS_MSG_LINK_INFO, "frame=%" PRId64, frame->root->unique, + "UNLINK_path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1072,13 +1037,13 @@ server4_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, - "%" PRId64 - ": SYMLINK %s (%s/%s), client: %s, " - "error-xlator:%s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, + "frame=%" PRId64, frame->root->unique, "SYMLINK_path= %s", + (state->loc.path) ? state->loc.path : "", "uuid_utoa=%s", + uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1123,13 +1088,12 @@ server4_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, uuid_utoa_r(state->resolve.gfid, gfid_str); uuid_utoa_r(state->resolve2.pargfid, newpar_str); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, - "%" PRId64 - ": LINK %s (%s) -> %s/%s, client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, gfid_str, newpar_str, - state->resolve2.bname, STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, + "frame=%" PRId64, frame->root->unique, "LINK_path=%s", + state->loc.path, "gfid_str=%s", gfid_str, "newpar_str=%s", + newpar_str, "resolve2.bname=%s", state->resolve2.bname, + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1163,13 +1127,11 @@ server4_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_TRUNCATE_INFO, - "%" PRId64 - ": TRUNCATE %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_TRUNCATE_INFO, + "frame=%" PRId64, frame->root->unique, "TRUNCATE_path=%s", + state->loc.path, "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1203,14 +1165,12 @@ server4_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret) { - gf_msg(this->name, fop_log_level(GF_FOP_FSTAT, op_errno), op_errno, - PS_MSG_STAT_INFO, - "%" PRId64 ": FSTAT %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FSTAT, op_errno), op_errno, + PS_MSG_STAT_INFO, "frame=%" PRId64, frame->root->unique, + "FSTAT_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1242,14 +1202,12 @@ server4_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FTRUNCATE, op_errno), op_errno, - PS_MSG_TRUNCATE_INFO, - "%" PRId64 ": FTRUNCATE %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FTRUNCATE, op_errno), op_errno, + PS_MSG_TRUNCATE_INFO, "frame=%" PRId64, frame->root->unique, + "FTRUNCATE_fd_no=%" PRId64, state->resolve.fd_no, + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1282,14 +1240,12 @@ server4_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FLUSH, op_errno), op_errno, - PS_MSG_FLUSH_INFO, - "%" PRId64 ": FLUSH %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FLUSH, op_errno), op_errno, + PS_MSG_FLUSH_INFO, "frame=%" PRId64, frame->root->unique, + "FLUSH_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1321,14 +1277,12 @@ server4_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FSYNC, op_errno), op_errno, - PS_MSG_SYNC_INFO, - "%" PRId64 ": FSYNC %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FSYNC, op_errno), op_errno, + PS_MSG_SYNC_INFO, "frame=%" PRId64, frame->root->unique, + "FSYNC_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1362,14 +1316,12 @@ server4_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_WRITE, op_errno), op_errno, - PS_MSG_WRITE_INFO, - "%" PRId64 ": WRITEV %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_WRITE, op_errno), op_errno, + PS_MSG_WRITE_INFO, "frame=%" PRId64, frame->root->unique, + "WRITEV_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1414,14 +1366,12 @@ server4_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_READ, op_errno), op_errno, - PS_MSG_READ_INFO, - "%" PRId64 ": READV %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_READ, op_errno), op_errno, + PS_MSG_READ_INFO, "frame=%" PRId64, frame->root->unique, + "READV_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1455,14 +1405,12 @@ server4_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_RCHECKSUM, op_errno), op_errno, - PS_MSG_CHKSUM_INFO, - "%" PRId64 ": RCHECKSUM %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_RCHECKSUM, op_errno), op_errno, + PS_MSG_CHKSUM_INFO, "frame=%" PRId64, frame->root->unique, + "RCHECKSUM_fd_no=%" PRId64, state->resolve.fd_no, + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1494,12 +1442,12 @@ server4_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno, - PS_MSG_OPEN_INFO, - "%" PRId64 ": OPEN %s (%s), client: %s, error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno, + PS_MSG_OPEN_INFO, "frame=%" PRId64, frame->root->unique, + "OPEN_path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1536,13 +1484,12 @@ server4_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_CREATE_INFO, - "%" PRId64 - ": CREATE %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg( + this->name, GF_LOG_INFO, op_errno, PS_MSG_CREATE_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", state->loc.path, + "uuid_utoa=%s", uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1592,13 +1539,11 @@ server4_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, - "%" PRId64 - ": READLINK %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_LINK_INFO, + "frame=%" PRId64, frame->root->unique, "READLINK_path=%s", + state->loc.path, "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1633,12 +1578,12 @@ server4_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret) { - gf_msg(this->name, fop_log_level(GF_FOP_STAT, op_errno), op_errno, - PS_MSG_STAT_INFO, - "%" PRId64 ": STAT %s (%s), client: %s, error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_STAT, op_errno), op_errno, + PS_MSG_STAT_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", (state->loc.path) ? state->loc.path : "", + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1671,13 +1616,12 @@ server4_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SETATTR_INFO, - "%" PRId64 - ": SETATTR %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, (state->loc.path) ? state->loc.path : "", - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SETATTR_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", + (state->loc.path) ? state->loc.path : "", "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1711,14 +1655,12 @@ server4_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FSETATTR, op_errno), op_errno, - PS_MSG_SETATTR_INFO, - "%" PRId64 ": FSETATTR %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FSETATTR, op_errno), op_errno, + PS_MSG_SETATTR_INFO, "frame=%" PRId64, + "FSETATTR_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1752,14 +1694,12 @@ server4_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_XATTROP, op_errno), op_errno, - PS_MSG_XATTROP_INFO, - "%" PRId64 - ": XATTROP %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_XATTROP, op_errno), op_errno, + PS_MSG_XATTROP_INFO, "frame=%" PRId64, frame->root->unique, + "path=%s", state->loc.path, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1794,14 +1734,12 @@ server4_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FXATTROP, op_errno), op_errno, - PS_MSG_XATTROP_INFO, - "%" PRId64 ": FXATTROP %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FXATTROP, op_errno), op_errno, + PS_MSG_XATTROP_INFO, "frame=%" PRId64, frame->root->unique, + "FXATTROP_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1840,14 +1778,12 @@ server4_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_READDIRP, op_errno), op_errno, - PS_MSG_DIR_INFO, - "%" PRId64 ": READDIRP %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_READDIRP, op_errno), op_errno, + PS_MSG_DIR_INFO, "frame=%" PRId64, frame->root->unique, + "READDIRP_fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1893,14 +1829,12 @@ server4_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_FALLOCATE, op_errno), op_errno, - PS_MSG_ALLOC_INFO, - "%" PRId64 ": FALLOCATE %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_FALLOCATE, op_errno), op_errno, + PS_MSG_ALLOC_INFO, "frame=%" PRId64, frame->root->unique, + "FALLOCATE_fd_no=%" PRId64, state->resolve.fd_no, + "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1934,14 +1868,12 @@ server4_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret) { state = CALL_STATE(frame); - gf_msg(this->name, fop_log_level(GF_FOP_DISCARD, op_errno), op_errno, - PS_MSG_DISCARD_INFO, - "%" PRId64 ": DISCARD %" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_DISCARD, op_errno), op_errno, + PS_MSG_DISCARD_INFO, "frame=%" PRId64, frame->root->unique, + "fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator: %s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -1977,14 +1909,12 @@ server4_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, dict_to_xdr(xdata, &rsp.xdata); if (op_ret) { - gf_msg(this->name, fop_log_level(GF_FOP_ZEROFILL, op_errno), op_errno, - PS_MSG_ZEROFILL_INFO, - "%" PRId64 ": ZEROFILL%" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_ZEROFILL, op_errno), op_errno, + PS_MSG_ZEROFILL_INFO, "frame=%" PRId64, frame->root->unique, + "fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2018,13 +1948,12 @@ server4_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, dict_to_xdr(xdata, &rsp.xdata); if (op_ret) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SERVER_IPC_INFO, - "%" PRId64 ": IPC%" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SERVER_IPC_INFO, + "frame=%" PRId64, frame->root->unique, "IPC=%" PRId64, + state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2056,14 +1985,12 @@ server4_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, dict_to_xdr(xdata, &rsp.xdata); if (op_ret) { - gf_msg(this->name, fop_log_level(GF_FOP_SEEK, op_errno), op_errno, - PS_MSG_SEEK_INFO, - "%" PRId64 ": SEEK%" PRId64 - " (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_SEEK, op_errno), op_errno, + PS_MSG_SEEK_INFO, "frame=%" PRId64, frame->root->unique, + "fd_no=%" PRId64, state->resolve.fd_no, "uuid_utoa=%s", + uuid_utoa(state->resolve.gfid), "client=%s", + STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2096,13 +2023,11 @@ server4_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, 0, - "%" PRId64 - ": SETACTIVELK %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SETACTIVELK_INFO, + "frame=%" PRId64, frame->root->unique, "path==%s", + state->loc.path, "uuid_utoa=%s", uuid_utoa(state->resolve.gfid), + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2171,9 +2096,10 @@ server4_icreate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_CREATE_INFO, - "%" PRId64 ": ICREATE [%s] ==> (%s)", frame->root->unique, - uuid_utoa(state->resolve.gfid), strerror(op_errno)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_CREATE_INFO, + "frame=%" PRId64, uuid_utoa(state->resolve.gfid), + "ICREATE_gfid=%s", uuid_utoa(state->resolve.gfid), + "op_errno=%s", strerror(op_errno), NULL); goto out; } @@ -2226,13 +2152,12 @@ server4_put_cbk(call_frame_t *frame, void *cookie, xlator_t *this, state = CALL_STATE(frame); if (op_ret < 0) { - gf_msg(this->name, GF_LOG_INFO, op_errno, PS_MSG_PUT_INFO, - "%" PRId64 - ": PUT %s (%s/%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.pargfid), state->resolve.bname, - STACK_CLIENT_NAME(frame->root), STACK_ERR_XL_NAME(frame->root)); + gf_smsg( + this->name, GF_LOG_INFO, op_errno, PS_MSG_PUT_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", state->loc.path, + "uuid_utoa=%s", uuid_utoa(state->resolve.pargfid), "bname=%s", + state->resolve.bname, "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2280,14 +2205,13 @@ server4_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this, uuid_utoa_r(state->resolve.gfid, in_gfid); uuid_utoa_r(state->resolve2.gfid, out_gfid); - gf_msg(this->name, fop_log_level(GF_FOP_COPY_FILE_RANGE, op_errno), - op_errno, PS_MSG_WRITE_INFO, - "%" PRId64 ": COPY_FILE_RANGE %" PRId64 " (%s), %" PRId64 - " (%s) client: %s, " - "error-xlator: %s", - frame->root->unique, state->resolve.fd_no, in_gfid, - state->resolve2.fd_no, out_gfid, STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, fop_log_level(GF_FOP_COPY_FILE_RANGE, op_errno), + op_errno, PS_MSG_WRITE_INFO, "frame=%" PRId64, + frame->root->unique, "COPY_FILE_RANGE_fd_no=%" PRId64, + state->resolve.fd_no, "in_gfid=%s", in_gfid, + "resolve2_fd_no=%" PRId64, state->resolve2.fd_no, "out_gfid=%s", + out_gfid, "client=%s", STACK_CLIENT_NAME(frame->root), + "error-xlator=%s", STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -2773,8 +2697,7 @@ server4_opendir_resume(call_frame_t *frame, xlator_t *bound_xl) state->fd = fd_create(state->loc.inode, frame->root->pid); if (!state->fd) { - gf_msg("server", GF_LOG_ERROR, 0, PS_MSG_FD_CREATE_FAILED, - "could not create the fd"); + gf_smsg("server", GF_LOG_ERROR, 0, PS_MSG_FD_CREATE_FAILED, NULL); goto err; } @@ -3137,9 +3060,9 @@ server4_create_resume(call_frame_t *frame, xlator_t *bound_xl) state->fd = fd_create(state->loc.inode, frame->root->pid); if (!state->fd) { - gf_msg("server", GF_LOG_ERROR, 0, PS_MSG_FD_CREATE_FAILED, - "fd creation for the inode %s failed", - state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : NULL); + gf_smsg("server", GF_LOG_ERROR, 0, PS_MSG_FD_CREATE_FAILED, "inode=%s", + state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : NULL, + NULL); state->resolve.op_ret = -1; state->resolve.op_errno = ENOMEM; goto err; @@ -3385,13 +3308,11 @@ server4_getactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, if (op_ret < 0) { state = CALL_STATE(frame); - gf_msg(this->name, GF_LOG_INFO, op_errno, 0, - "%" PRId64 - ": GETACTIVELK %s (%s), client: %s, " - "error-xlator: %s", - frame->root->unique, state->loc.path, - uuid_utoa(state->resolve.gfid), STACK_CLIENT_NAME(frame->root), - STACK_ERR_XL_NAME(frame->root)); + gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_GETACTIVELK_INFO, + "frame=%" PRId64, frame->root->unique, "path=%s", + state->loc.path, "gfid=%s", uuid_utoa(state->resolve.gfid), + "client=%s", STACK_CLIENT_NAME(frame->root), "error-xlator=%s", + STACK_ERR_XL_NAME(frame->root), NULL); goto out; } @@ -4112,9 +4033,8 @@ server4_0_release(rpcsvc_request_t *req) serv_ctx = server_ctx_get(client, client->this); if (serv_ctx == NULL) { - gf_msg(req->trans->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, - "server_ctx_get() " - "failed"); + gf_smsg(req->trans->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, + NULL); req->rpc_err = SYSTEM_ERR; goto out; } @@ -4158,9 +4078,8 @@ server4_0_releasedir(rpcsvc_request_t *req) serv_ctx = server_ctx_get(client, client->this); if (serv_ctx == NULL) { - gf_msg(req->trans->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, - "server_ctx_get() " - "failed"); + gf_smsg(req->trans->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, + NULL); req->rpc_err = SYSTEM_ERR; goto out; } @@ -5531,13 +5450,10 @@ server4_0_lk(rpcsvc_request_t *req) state->flock.l_type = F_UNLCK; break; default: - gf_msg(frame->root->client->bound_xl->name, GF_LOG_ERROR, 0, - PS_MSG_LOCK_ERROR, - "fd - %" PRId64 - " (%s): Unknown " - "lock type: %" PRId32 "!", - state->resolve.fd_no, uuid_utoa(state->fd->inode->gfid), - state->type); + gf_smsg(frame->root->client->bound_xl->name, GF_LOG_ERROR, 0, + PS_MSG_LOCK_ERROR, "fd=%" PRId64, state->resolve.fd_no, + "uuid_utoa=%s", uuid_utoa(state->fd->inode->gfid), + "lock type=" PRId32, state->type, NULL); break; } diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 222dc9a18f1..721968004a0 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -108,8 +108,7 @@ server_submit_reply(call_frame_t *frame, rpcsvc_request_t *req, void *arg, iob = gfs_serialize_reply(req, arg, &rsp, xdrproc); if (!iob) { - gf_msg("", GF_LOG_ERROR, 0, PS_MSG_SERIALIZE_REPLY_FAILED, - "Failed to serialize reply"); + gf_smsg("", GF_LOG_ERROR, 0, PS_MSG_SERIALIZE_REPLY_FAILED, NULL); goto ret; } @@ -268,6 +267,8 @@ server_priv(xlator_t *this) gf_proc_dump_build_key(key, "server", "total-bytes-write"); gf_proc_dump_write(key, "%" PRIu64, total_write); + rpcsvc_statedump(conf->rpc); + ret = 0; out: if (ret) @@ -303,8 +304,7 @@ get_auth_types(dict_t *this, char *key, data_t *value, void *data) /* TODO: backward compatibility, remove when newer versions are available */ tmp = "addr"; - gf_msg("server", GF_LOG_WARNING, 0, PS_MSG_AUTH_IP_ERROR, - "assuming 'auth.ip' to be 'auth.addr'"); + gf_smsg("server", GF_LOG_WARNING, 0, PS_MSG_AUTH_IP_ERROR, NULL); } ret = dict_set_dynptr(auth_dict, tmp, NULL, 0); if (ret < 0) { @@ -333,8 +333,8 @@ _check_for_auth_option(dict_t *d, char *k, data_t *v, void *tmp) goto out; if (strncmp(tail, "addr.", 5) != 0) { - gf_msg(xl->name, GF_LOG_TRACE, 0, PS_MSG_SKIP_FORMAT_CHK, - "skip format check for non-addr auth option %s", k); + gf_smsg(xl->name, GF_LOG_TRACE, 0, PS_MSG_SKIP_FORMAT_CHK, "option=%s", + k, NULL); goto out; } @@ -356,10 +356,8 @@ _check_for_auth_option(dict_t *d, char *k, data_t *v, void *tmp) ret = xlator_option_validate_addr_list(xl, "auth-*", v->data, NULL, NULL); if (ret) - gf_msg(xl->name, GF_LOG_ERROR, 0, PS_MSG_INTERNET_ADDR_ERROR, - "internet address '%s' does not conform " - "to standards.", - v->data); + gf_smsg(xl->name, GF_LOG_ERROR, 0, PS_MSG_INTERNET_ADDR_ERROR, + "data=%s", v->data, NULL); } out: return ret; @@ -379,11 +377,8 @@ validate_auth_options(xlator_t *this, dict_t *dict) error = dict_foreach(dict, _check_for_auth_option, trav->xlator); if (-1 == error) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR, - "volume '%s' " - "defined as subvolume, but no authentication " - "defined for the same", - trav->xlator->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR, + "name=%s", trav->xlator->name, NULL); break; } trav = trav->next; @@ -410,7 +405,14 @@ server_call_xlator_mem_cleanup(xlator_t *this, char *victim_name) arg = calloc(1, sizeof(*arg)); arg->this = this; - arg->victim_name = gf_strdup(victim_name); + arg->victim_name = strdup(victim_name); + if (!arg->victim_name) { + gf_smsg(this->name, GF_LOG_CRITICAL, ENOMEM, LG_MSG_NO_MEMORY, + "Memory allocation is failed"); + free(arg); + return; + } + th_ret = gf_thread_create_detached(&th_id, server_graph_janitor_threads, arg, "graphjanitor"); if (th_ret) { @@ -418,7 +420,7 @@ server_call_xlator_mem_cleanup(xlator_t *this, char *victim_name) "graph janitor Thread" " creation is failed for brick %s", victim_name); - GF_FREE(arg->victim_name); + free(arg->victim_name); free(arg); } } @@ -494,15 +496,13 @@ server_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, void *data) if (!client) goto unref_transport; - gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_DISCONNECTING, - "disconnecting connection" - " from %s", - client->client_uid); + gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_DISCONNECTING, + "client-uid=%s", client->client_uid, NULL); ret = dict_get_str_sizen(this->options, "auth-path", &auth_path); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_DICT_GET_FAILED, - "failed to get auth-path"); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_DICT_GET_FAILED, + "type=auth-path", NULL); auth_path = NULL; } @@ -630,7 +630,7 @@ server_graph_janitor_threads(void *data) } out: - GF_FREE(arg->victim_name); + free(arg->victim_name); free(arg); return NULL; } @@ -645,8 +645,7 @@ server_mem_acct_init(xlator_t *this) ret = xlator_mem_acct_init(this, gf_server_mt_end + 1); if (ret != 0) { - gf_msg(this->name, GF_LOG_ERROR, ENOMEM, PS_MSG_NO_MEMORY, - "Memory accounting init failed"); + gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, PS_MSG_NO_MEMORY, NULL); return ret; } out: @@ -775,9 +774,8 @@ server_reconfigure(xlator_t *this, dict_t *options) if (data) { ret = gf_string2boolean(data->data, &trace); if (ret != 0) { - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "'trace' takes on only " - "boolean values. Neglecting option"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, + NULL); ret = -1; goto out; } @@ -787,8 +785,7 @@ server_reconfigure(xlator_t *this, dict_t *options) GF_OPTION_RECONF("statedump-path", statedump_path, options, path, do_auth); if (!statedump_path) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, - "Error while reconfiguring statedump path"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, NULL); goto do_auth; } gf_path_strip_trailing_slashes(statedump_path); @@ -821,16 +818,14 @@ do_auth: GF_OPTION_RECONF("gid-timeout", conf->gid_cache_timeout, options, int32, do_rpc); if (gid_cache_reconf(&conf->gid_cache, conf->gid_cache_timeout) < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, - "Failed to reconfigure group cache."); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, NULL); goto do_rpc; } do_rpc: rpc_conf = conf->rpc; if (!rpc_conf) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "No rpc_conf !!!!"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, NULL); goto out; } @@ -884,9 +879,8 @@ do_rpc: ret = gf_authenticate(xprt->clnt_options, options, conf->auth_modules); if (ret == AUTH_ACCEPT) { - gf_msg(kid->name, GF_LOG_TRACE, 0, PS_MSG_CLIENT_ACCEPTED, - "authorized client, hence we " - "continue with this connection"); + gf_smsg(kid->name, GF_LOG_TRACE, 0, PS_MSG_CLIENT_ACCEPTED, + NULL); } else { gf_event(EVENT_CLIENT_AUTH_REJECT, "client_uid=%s;" @@ -896,11 +890,10 @@ do_rpc: xprt->xl_private->client_uid, xprt->peerinfo.identifier, xprt->myinfo.identifier, auth_path); - gf_msg(this->name, GF_LOG_INFO, EACCES, - PS_MSG_AUTHENTICATE_ERROR, - "unauthorized client, hence " - "terminating the connection %s", - xprt->peerinfo.identifier); + gf_smsg(this->name, GF_LOG_INFO, EACCES, + PS_MSG_UNAUTHORIZED_CLIENT, + "peerinfo-identifier=%s", xprt->peerinfo.identifier, + NULL); rpc_transport_disconnect(xprt, _gf_false); } } @@ -911,8 +904,7 @@ do_rpc: ret = rpcsvc_set_outstanding_rpc_limit( rpc_conf, options, RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "Failed to reconfigure outstanding-rpc-limit"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RECONFIGURE_FAILED, NULL); goto out; } @@ -922,9 +914,8 @@ do_rpc: if (listeners->trans->reconfigure) listeners->trans->reconfigure(listeners->trans, options); else - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "Reconfigure " - "not found for transport"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, + NULL); } } @@ -1052,14 +1043,12 @@ server_init(xlator_t *this) GF_VALIDATE_OR_GOTO("init", this, out); if (this->children == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_SUBVOL_NULL, - "protocol/server should have subvolume"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_SUBVOL_NULL, NULL); goto out; } if (this->parents != NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_PARENT_VOL_ERROR, - "protocol/server should not have parent volumes"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_PARENT_VOL_ERROR, NULL); goto out; } @@ -1096,8 +1085,8 @@ server_init(xlator_t *this) gf_path_strip_trailing_slashes(statedump_path); this->ctx->statedump_path = gf_strdup(statedump_path); } else { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_STATEDUMP_PATH_ERROR, - "Error setting statedump path"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_SET_STATEDUMP_PATH_ERROR, + NULL); ret = -1; goto out; } @@ -1128,8 +1117,7 @@ server_init(xlator_t *this) GF_OPTION_INIT("gid-timeout", conf->gid_cache_timeout, int32, out); if (gid_cache_init(&conf->gid_cache, conf->gid_cache_timeout) < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_GRP_CACHE_ERROR, - "Failed to initialize group cache."); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_INIT_GRP_CACHE_ERROR, NULL); goto out; } @@ -1148,9 +1136,7 @@ server_init(xlator_t *this) /* RPC related */ conf->rpc = rpcsvc_init(this, this->ctx, this->options, 0); if (conf->rpc == NULL) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPCSVC_CREATE_FAILED, - "creation of rpcsvc " - "failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPCSVC_CREATE_FAILED, NULL); ret = -1; goto out; } @@ -1158,8 +1144,7 @@ server_init(xlator_t *this) ret = rpcsvc_set_outstanding_rpc_limit( conf->rpc, this->options, RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); if (ret < 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONF_ERROR, - "Failed to configure outstanding-rpc-limit"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_RPC_CONFIGURE_FAILED, NULL); goto out; } @@ -1171,15 +1156,15 @@ server_init(xlator_t *this) ret = dict_get_str_sizen(this->options, "transport-type", &transport_type); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "option transport-type not set"); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_TYPE_NOT_SET, + NULL); ret = -1; goto out; } total_transport = rpc_transport_count(transport_type); if (total_transport <= 0) { - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_TRANSPORT_ERROR, - "failed to get total number of available tranpsorts"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_GET_TOTAL_AVAIL_TRANSPORT_FAILED, NULL); ret = -1; goto out; } @@ -1188,24 +1173,21 @@ server_init(xlator_t *this) ret = rpcsvc_create_listeners(conf->rpc, this->options, this->name); if (ret < 1) { - gf_msg(this->name, GF_LOG_WARNING, 0, - PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, - "creation of listener failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, + PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, NULL); if (ret != -EADDRINUSE) ret = -1; goto out; } else if (ret < total_transport) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, - "creation of %d listeners failed, continuing with " - "succeeded transport", - (total_transport - ret)); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_RPCSVC_LISTENER_CREATE_FAILED, "number=%d", + "continuing with succeeded transport", (total_transport - ret), + NULL); } ret = rpcsvc_register_notify(conf->rpc, server_rpc_notify, this); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_RPCSVC_NOTIFY, - "registration of notify with rpcsvc failed"); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_RPCSVC_NOTIFY, NULL); goto out; } @@ -1215,11 +1197,10 @@ server_init(xlator_t *this) */ ret = rpcsvc_program_register(conf->rpc, &glusterfs3_3_fop_prog, _gf_true); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, - "registration of program (name:%s, prognum:%d, " - "progver:%d) failed", - glusterfs3_3_fop_prog.progname, glusterfs3_3_fop_prog.prognum, - glusterfs3_3_fop_prog.progver); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, "name=%s", + glusterfs3_3_fop_prog.progname, "prognum=%d", + glusterfs3_3_fop_prog.prognum, "progver=%d", + glusterfs3_3_fop_prog.progver, NULL); goto out; } @@ -1239,11 +1220,10 @@ server_init(xlator_t *this) ret = rpcsvc_program_register(conf->rpc, &gluster_handshake_prog, _gf_false); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, - "registration of program (name:%s, prognum:%d, " - "progver:%d) failed", - gluster_handshake_prog.progname, gluster_handshake_prog.prognum, - gluster_handshake_prog.progver); + gf_smsg(this->name, GF_LOG_WARNING, 0, PS_MSG_PGM_REG_FAILED, "name=%s", + gluster_handshake_prog.progname, "prognum=%d", + gluster_handshake_prog.prognum, "progver=%d", + gluster_handshake_prog.progver, NULL); rpcsvc_program_unregister(conf->rpc, &glusterfs3_3_fop_prog); rpcsvc_program_unregister(conf->rpc, &glusterfs4_0_fop_prog); goto out; @@ -1257,18 +1237,14 @@ server_init(xlator_t *this) lim.rlim_max = 1048576; if (setrlimit(RLIMIT_NOFILE, &lim) == -1) { - gf_msg(this->name, GF_LOG_WARNING, errno, PS_MSG_ULIMIT_SET_FAILED, - "WARNING: Failed to " - "set 'ulimit -n 1M': %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_WARNING, errno, PS_MSG_ULIMIT_SET_FAILED, + "errno=%s", strerror(errno), NULL); lim.rlim_cur = 65536; lim.rlim_max = 65536; if (setrlimit(RLIMIT_NOFILE, &lim) == -1) { - gf_msg(this->name, GF_LOG_WARNING, errno, PS_MSG_FD_NOT_FOUND, - "Failed to set " - "max open fd to 64k: %s", - strerror(errno)); + gf_smsg(this->name, GF_LOG_WARNING, errno, PS_MSG_FD_NOT_FOUND, + "errno=%s", strerror(errno), NULL); } else { gf_msg_trace(this->name, 0, "max open fd set " @@ -1280,9 +1256,8 @@ server_init(xlator_t *this) if (!this->ctx->cmd_args.volfile_id) { /* In some use cases this is a valid case, but document this to be annoying log in that case */ - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_VOL_FILE_OPEN_FAILED, - "volfile-id argument not given. " - "This is mandatory argument, defaulting to 'gluster'"); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_VOL_FILE_OPEN_FAILED, + NULL); this->ctx->cmd_args.volfile_id = gf_strdup("gluster"); } FIRST_CHILD(this)->volfile_id = gf_strdup(this->ctx->cmd_args.volfile_id); @@ -1418,9 +1393,9 @@ server_process_event_upcall(xlator_t *this, void *data) xdrproc = (xdrproc_t)xdr_gfs4_entrylk_contention_req; break; default: - gf_msg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY, - "Received invalid upcall event(%d)", - upcall_data->event_type); + gf_smsg(this->name, GF_LOG_WARNING, EINVAL, + PS_MSG_INVLAID_UPCALL_EVENT, "event-type=%d", + upcall_data->event_type, NULL); goto out; } @@ -1512,10 +1487,8 @@ server_process_child_event(xlator_t *this, int32_t event, void *data, } if (!tmp->name) - gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED, - "No xlator %s is found in " - "child status list", - victim->name); + gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED, + "name=%s", victim->name, NULL); } list_for_each_entry(xprt, &conf->xprt_list, list) { @@ -1564,9 +1537,8 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) ret = server_process_event_upcall(this, data); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_event_upcall failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_EVENT_UPCALL_FAILED, NULL); goto out; } break; @@ -1585,9 +1557,8 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) ret = server_process_child_event(this, event, data, GF_CBK_CHILD_UP); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_child_event failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_CHILD_EVENT_FAILED, NULL); goto out; } default_notify(this, event, data); @@ -1604,9 +1575,8 @@ server_notify(xlator_t *this, int32_t event, void *data, ...) ret = server_process_child_event(this, event, data, GF_CBK_CHILD_DOWN); if (ret) { - gf_msg(this->name, GF_LOG_ERROR, 0, - PS_MSG_SERVER_EVENT_UPCALL_FAILED, - "server_process_child_event failed"); + gf_smsg(this->name, GF_LOG_ERROR, 0, + PS_MSG_SERVER_CHILD_EVENT_FAILED, NULL); goto out; } default_notify(this, event, data); diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c index f4003942f4e..f10722ec3fb 100644 --- a/xlators/storage/posix/src/posix-common.c +++ b/xlators/storage/posix/src/posix-common.c @@ -140,6 +140,7 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...) struct timespec sleep_till = { 0, }; + glusterfs_ctx_t *ctx = this->ctx; switch (event) { case GF_EVENT_PARENT_UP: { @@ -150,8 +151,6 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...) case GF_EVENT_PARENT_DOWN: { if (!victim->cleanup_starting) break; - gf_log(this->name, GF_LOG_INFO, "Sending CHILD_DOWN for brick %s", - victim->name); if (priv->janitor) { pthread_mutex_lock(&priv->janitor_mutex); @@ -160,7 +159,7 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...) ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor); if (!ret) { - clock_gettime(CLOCK_REALTIME, &sleep_till); + timespec_now_realtime(&sleep_till); sleep_till.tv_sec += 1; /* Wait to set janitor_task flag to _gf_false by * janitor_task_done */ @@ -168,7 +167,7 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...) (void)pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_mutex, &sleep_till); - clock_gettime(CLOCK_REALTIME, &sleep_till); + timespec_now_realtime(&sleep_till); sleep_till.tv_sec += 1; } } @@ -177,6 +176,16 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...) GF_FREE(priv->janitor); } priv->janitor = NULL; + pthread_mutex_lock(&ctx->fd_lock); + { + while (priv->rel_fdcount > 0) { + pthread_cond_wait(&priv->fd_cond, &ctx->fd_lock); + } + } + pthread_mutex_unlock(&ctx->fd_lock); + + gf_log(this->name, GF_LOG_INFO, "Sending CHILD_DOWN for brick %s", + victim->name); default_notify(this->parents->xlator, GF_EVENT_CHILD_DOWN, data); } break; default: @@ -543,6 +552,30 @@ posix_create_unlink_dir(xlator_t *this) return 0; } +int +posix_create_open_directory_based_fd(xlator_t *this, int pdirfd, char *dir_name) +{ + int ret = -1; + + ret = sys_openat(pdirfd, dir_name, (O_DIRECTORY | O_RDONLY), 0); + if (ret < 0 && errno == ENOENT) { + ret = sys_mkdirat(pdirfd, dir_name, 0700); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, + "Creating directory %s failed", dir_name); + goto out; + } + ret = sys_openat(pdirfd, dir_name, (O_DIRECTORY | O_RDONLY), 0); + if (ret < 0 && errno != EEXIST) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, + "error mkdir hash-1 %s ", dir_name); + goto out; + } + } +out: + return ret; +} + /** * init - */ @@ -579,6 +612,15 @@ posix_init(xlator_t *this) int force_directory = -1; int create_mask = -1; int create_directory_mask = -1; + char dir_handle[PATH_MAX] = { + 0, + }; + int i; + char fhash[4] = { + 0, + }; + int hdirfd = -1; + char value; dir_data = dict_get(this->options, "directory"); @@ -621,6 +663,11 @@ posix_init(xlator_t *this) _private->base_path = gf_strdup(dir_data->data); _private->base_path_length = dir_data->len - 1; + _private->dirfd = -1; + _private->mount_lock = -1; + for (i = 0; i < 256; i++) + _private->arrdfd[i] = -1; + ret = dict_get_str(this->options, "hostname", &_private->hostname); if (ret) { _private->hostname = GF_CALLOC(256, sizeof(char), gf_common_mt_char); @@ -635,16 +682,11 @@ posix_init(xlator_t *this) } /* Check for Extended attribute support, if not present, log it */ - op_ret = sys_lsetxattr(dir_data->data, "trusted.glusterfs.test", "working", - 8, 0); - if (op_ret != -1) { - ret = sys_lremovexattr(dir_data->data, "trusted.glusterfs.test"); - if (ret) { - gf_msg(this->name, GF_LOG_DEBUG, errno, P_MSG_INVALID_OPTION, - "failed to remove xattr: " - "trusted.glusterfs.test"); - } - } else { + size = sys_lgetxattr(dir_data->data, "user.x", &value, sizeof(value)); + + if ((size == -1) && (errno == EOPNOTSUPP)) { + gf_msg(this->name, GF_LOG_DEBUG, 0, P_MSG_XDATA_GETXATTR, + "getxattr returned %zd", size); tmp_data = dict_get(this->options, "mandate-attribute"); if (tmp_data) { if (gf_string2boolean(tmp_data->data, &tmp_bool) == -1) { @@ -893,8 +935,9 @@ posix_init(xlator_t *this) /* performing open dir on brick dir locks the brick dir * and prevents it from being unmounted */ - _private->mount_lock = sys_opendir(dir_data->data); - if (!_private->mount_lock) { + _private->mount_lock = sys_open(dir_data->data, (O_DIRECTORY | O_RDONLY), + 0); + if (_private->mount_lock < 0) { ret = -1; op_errno = errno; gf_msg(this->name, GF_LOG_ERROR, 0, P_MSG_DIR_OPERATION_FAILED, @@ -938,6 +981,28 @@ posix_init(xlator_t *this) } this->private = (void *)_private; + snprintf(dir_handle, sizeof(dir_handle), "%s/%s", _private->base_path, + GF_HIDDEN_PATH); + hdirfd = posix_create_open_directory_based_fd(this, _private->mount_lock, + dir_handle); + if (hdirfd < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, + "error open directory failed for dir %s", dir_handle); + ret = -1; + goto out; + } + _private->dirfd = hdirfd; + for (i = 0; i < 256; i++) { + snprintf(fhash, sizeof(fhash), "%02x", i); + _private->arrdfd[i] = posix_create_open_directory_based_fd(this, hdirfd, + fhash); + if (_private->arrdfd[i] < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, + "error openat failed for file %s", fhash); + ret = -1; + goto out; + } + } op_ret = posix_handle_init(this); if (op_ret == -1) { @@ -1028,7 +1093,9 @@ posix_init(xlator_t *this) pthread_cond_init(&_private->fsync_cond, NULL); pthread_mutex_init(&_private->janitor_mutex, NULL); pthread_cond_init(&_private->janitor_cond, NULL); + pthread_cond_init(&_private->fd_cond, NULL); INIT_LIST_HEAD(&_private->fsyncs); + _private->rel_fdcount = 0; ret = posix_spawn_ctx_janitor_thread(this); if (ret) goto out; @@ -1105,9 +1172,27 @@ posix_init(xlator_t *this) out); GF_OPTION_INIT("ctime", _private->ctime, bool, out); + out: if (ret) { if (_private) { + if (_private->dirfd >= 0) { + sys_close(_private->dirfd); + _private->dirfd = -1; + } + + for (i = 0; i < 256; i++) { + if (_private->arrdfd[i] >= 0) { + sys_close(_private->arrdfd[i]); + _private->arrdfd[i] = -1; + } + } + /*unlock brick dir*/ + if (_private->mount_lock >= 0) { + (void)sys_close(_private->mount_lock); + _private->mount_lock = -1; + } + GF_FREE(_private->base_path); GF_FREE(_private->hostname); @@ -1127,7 +1212,10 @@ posix_fini(xlator_t *this) { struct posix_private *priv = this->private; gf_boolean_t health_check = _gf_false; + glusterfs_ctx_t *ctx = this->ctx; + uint32_t count; int ret = 0; + int i = 0; if (!priv) return; @@ -1138,6 +1226,18 @@ posix_fini(xlator_t *this) } UNLOCK(&priv->lock); + if (priv->dirfd >= 0) { + sys_close(priv->dirfd); + priv->dirfd = -1; + } + + for (i = 0; i < 256; i++) { + if (priv->arrdfd[i] >= 0) { + sys_close(priv->arrdfd[i]); + priv->arrdfd[i] = -1; + } + } + if (health_check) { (void)gf_thread_cleanup_xint(priv->health_check); priv->health_check = 0; @@ -1160,13 +1260,28 @@ posix_fini(xlator_t *this) priv->janitor = NULL; } + pthread_mutex_lock(&ctx->fd_lock); + { + count = --ctx->pxl_count; + if (count == 0) { + pthread_cond_signal(&ctx->fd_cond); + } + } + pthread_mutex_unlock(&ctx->fd_lock); + + if (count == 0) { + pthread_join(ctx->janitor, NULL); + } + if (priv->fsyncer) { (void)gf_thread_cleanup_xint(priv->fsyncer); priv->fsyncer = 0; } /*unlock brick dir*/ - if (priv->mount_lock) - (void)sys_closedir(priv->mount_lock); + if (priv->mount_lock >= 0) { + (void)sys_close(priv->mount_lock); + priv->mount_lock = -1; + } GF_FREE(priv->base_path); LOCK_DESTROY(&priv->lock); @@ -1351,24 +1466,21 @@ struct volume_options posix_options[] = { .min = 0000, .max = 0777, .default_value = "0000", - .validate = GF_OPT_VALIDATE_MIN, - .validate = GF_OPT_VALIDATE_MAX, + .validate = GF_OPT_VALIDATE_BOTH, .description = "Mode bit permission that will always be set on a file."}, {.key = {"force-directory-mode"}, .type = GF_OPTION_TYPE_INT, .min = 0000, .max = 0777, .default_value = "0000", - .validate = GF_OPT_VALIDATE_MIN, - .validate = GF_OPT_VALIDATE_MAX, + .validate = GF_OPT_VALIDATE_BOTH, .description = "Mode bit permission that will be always set on directory"}, {.key = {"create-mask"}, .type = GF_OPTION_TYPE_INT, .min = 0000, .max = 0777, .default_value = "0777", - .validate = GF_OPT_VALIDATE_MIN, - .validate = GF_OPT_VALIDATE_MAX, + .validate = GF_OPT_VALIDATE_BOTH, .description = "Any bit not set here will be removed from the" "modes set on a file when it is created"}, {.key = {"create-directory-mask"}, @@ -1376,8 +1488,7 @@ struct volume_options posix_options[] = { .min = 0000, .max = 0777, .default_value = "0777", - .validate = GF_OPT_VALIDATE_MIN, - .validate = GF_OPT_VALIDATE_MAX, + .validate = GF_OPT_VALIDATE_BOTH, .description = "Any bit not set here will be removed from the" "modes set on a directory when it is created"}, {.key = {"max-hardlinks"}, diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c index 667fec7d62f..8cc3ccf8c00 100644 --- a/xlators/storage/posix/src/posix-entry-ops.c +++ b/xlators/storage/posix/src/posix-entry-ops.c @@ -176,6 +176,7 @@ posix_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) struct posix_private *priv = NULL; posix_inode_ctx_t *ctx = NULL; int ret = 0; + int dfd = -1; VALIDATE_OR_GOTO(frame, out); VALIDATE_OR_GOTO(this, out); @@ -197,6 +198,19 @@ posix_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) goto out; } +#ifdef __NetBSD__ + /* Same for NetBSD's .attribute directory */ + if (__is_root_gfid(loc->pargfid) && loc->name && + (strcmp(loc->name, ".attribute") == 0)) { + gf_msg(this->name, GF_LOG_WARNING, EPERM, P_MSG_LOOKUP_NOT_PERMITTED, + "Lookup issued on .attribute," + " which is not permitted"); + op_errno = EPERM; + op_ret = -1; + goto out; + } +#endif /* __NetBSD__ */ + op_ret = dict_get_int32_sizen(xdata, GF_GFIDLESS_LOOKUP, &gfidless); op_ret = -1; if (gf_uuid_is_null(loc->pargfid) || (loc->name == NULL)) { @@ -232,12 +246,12 @@ posix_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) if (!op_errno) op_errno = ESTALE; loc_gfid(loc, gfid); - MAKE_HANDLE_ABSPATH(gfid_path, this, gfid); - ret = sys_stat(gfid_path, &statbuf); + MAKE_HANDLE_ABSPATH_FD(gfid_path, this, gfid, dfd); + ret = sys_fstatat(dfd, gfid_path, &statbuf, 0); if (ret == 0 && ((statbuf.st_mode & S_IFMT) == S_IFDIR)) /*Don't unset if it was a symlink to a dir.*/ goto parent; - ret = sys_lstat(gfid_path, &statbuf); + ret = sys_fstatat(dfd, gfid_path, &statbuf, AT_SYMLINK_NOFOLLOW); if (ret == 0 && statbuf.st_nlink == 1) { gf_msg(this->name, GF_LOG_WARNING, op_errno, P_MSG_HANDLE_DELETE, @@ -649,6 +663,19 @@ posix_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, goto out; } +#ifdef __NetBSD__ + /* Same for NetBSD's .attribute directory */ + if (__is_root_gfid(loc->pargfid) && + (strcmp(loc->name, ".attribute") == 0)) { + gf_msg(this->name, GF_LOG_WARNING, EPERM, P_MSG_MKDIR_NOT_PERMITTED, + "mkdir issued on .attribute, which" + "is not permitted"); + op_errno = EPERM; + op_ret = -1; + goto out; + } +#endif + priv = this->private; VALIDATE_OR_GOTO(priv, out); GFID_NULL_CHECK_AND_GOTO(frame, this, loc, xdata, op_ret, op_errno, @@ -1415,6 +1442,19 @@ posix_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, goto out; } +#ifdef __NetBSD__ + /* Same for NetBSD's .attribute directory */ + if (__is_root_gfid(loc->pargfid) && + (strcmp(loc->name, ".attribute") == 0)) { + gf_msg(this->name, GF_LOG_WARNING, EPERM, P_MSG_RMDIR_NOT_PERMITTED, + "rmdir issued on .attribute, which" + "is not permitted"); + op_errno = EPERM; + op_ret = -1; + goto out; + } +#endif + priv = this->private; MAKE_ENTRY_HANDLE(real_path, par_path, this, loc, &stbuf); diff --git a/xlators/storage/posix/src/posix-handle.c b/xlators/storage/posix/src/posix-handle.c index f58f5416ff0..410b38da8cb 100644 --- a/xlators/storage/posix/src/posix-handle.c +++ b/xlators/storage/posix/src/posix-handle.c @@ -25,7 +25,7 @@ #include <glusterfs/compat-errno.h> int -posix_handle_mkdir_hashes(xlator_t *this, const char *newpath); +posix_handle_mkdir_hashes(xlator_t *this, int dfd, uuid_t gfid); inode_t * posix_resolve(xlator_t *this, inode_table_t *itable, inode_t *parent, @@ -331,9 +331,23 @@ posix_handle_pump(xlator_t *this, char *buf, int len, int maxlen, int ret = 0; int blen = 0; int link_len = 0; + char tmpstr[POSIX_GFID_HASH2_LEN] = { + 0, + }; + char d2[3] = { + 0, + }; + int index = 0; + int dirfd = 0; + struct posix_private *priv = this->private; + + strncpy(tmpstr, (base_str + pfx_len + 3), 40); + strncpy(d2, (base_str + pfx_len), 2); + index = strtoul(d2, NULL, 16); + dirfd = priv->arrdfd[index]; /* is a directory's symlink-handle */ - ret = sys_readlink(base_str, linkname, 512); + ret = readlinkat(dirfd, tmpstr, linkname, 512); if (ret == -1) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_READLINK_FAILED, "internal readlink failed on %s ", base_str); @@ -398,6 +412,11 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf, int pfx_len; int maxlen; char *buf; + int index = 0; + int dfd = 0; + char newstr[POSIX_GFID_HASH2_LEN] = { + 0, + }; priv = this->private; @@ -411,12 +430,14 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf, buf = alloca(maxlen); } + index = gfid[0]; + dfd = priv->arrdfd[index]; + base_len = (priv->base_path_length + SLEN(GF_HIDDEN_PATH) + 45); base_str = alloca(base_len + 1); base_len = snprintf(base_str, base_len + 1, "%s/%s/%02x/%02x/%s", priv->base_path, GF_HIDDEN_PATH, gfid[0], gfid[1], uuid_str); - pfx_len = priv->base_path_length + 1 + SLEN(GF_HIDDEN_PATH) + 1; if (basename) { @@ -425,7 +446,8 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf, len = snprintf(buf, maxlen, "%s", base_str); } - ret = sys_lstat(base_str, &stat); + snprintf(newstr, sizeof(newstr), "%02x/%s", gfid[1], uuid_str); + ret = sys_fstatat(dfd, newstr, &stat, AT_SYMLINK_NOFOLLOW); if (!(ret == 0 && S_ISLNK(stat.st_mode) && stat.st_nlink == 1)) goto out; @@ -438,7 +460,6 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf, if (ret == -1) break; - ret = sys_lstat(buf, &stat); } while ((ret == -1) && errno == ELOOP); @@ -485,6 +506,7 @@ posix_handle_init(xlator_t *this) struct stat exportbuf; char *rootstr = NULL; static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + int dfd = 0; priv = this->private; @@ -534,9 +556,8 @@ posix_handle_init(xlator_t *this) return -1; } - MAKE_HANDLE_ABSPATH(rootstr, this, gfid); - - ret = sys_stat(rootstr, &rootbuf); + MAKE_HANDLE_ABSPATH_FD(rootstr, this, gfid, dfd); + ret = sys_fstatat(dfd, rootstr, &rootbuf, 0); switch (ret) { case -1: if (errno != ENOENT) { @@ -544,15 +565,14 @@ posix_handle_init(xlator_t *this) "%s", priv->base_path); return -1; } - - ret = posix_handle_mkdir_hashes(this, rootstr); + ret = posix_handle_mkdir_hashes(this, dfd, gfid); if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "mkdir %s failed", rootstr); return -1; } - ret = sys_symlink("../../..", rootstr); + ret = sys_symlinkat("../../..", dfd, rootstr); if (ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, "symlink %s creation failed", rootstr); @@ -681,30 +701,18 @@ out: } int -posix_handle_mkdir_hashes(xlator_t *this, const char *newpath) +posix_handle_mkdir_hashes(xlator_t *this, int dirfd, uuid_t gfid) { - char *duppath = NULL; - char *parpath = NULL; - int ret = 0; - - duppath = strdupa(newpath); - parpath = dirname(duppath); - parpath = dirname(duppath); - - ret = sys_mkdir(parpath, 0700); - if (ret == -1 && errno != EEXIST) { - gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, - "error mkdir hash-1 %s ", parpath); - return -1; - } - - strcpy(duppath, newpath); - parpath = dirname(duppath); + int ret = -1; + char d2[3] = { + 0, + }; - ret = sys_mkdir(parpath, 0700); + snprintf(d2, sizeof(d2), "%02x", gfid[1]); + ret = sys_mkdirat(dirfd, d2, 0700); if (ret == -1 && errno != EEXIST) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE, - "error mkdir hash-2 %s ", parpath); + "error mkdir hash-2 %s ", uuid_utoa(gfid)); return -1; } @@ -715,51 +723,59 @@ int posix_handle_hard(xlator_t *this, const char *oldpath, uuid_t gfid, struct stat *oldbuf) { - char *newpath = NULL; struct stat newbuf; + struct stat hashbuf; int ret = -1; gf_boolean_t link_exists = _gf_false; + char d2[3] = { + 0, + }; + int dfd = -1; + char *newstr = NULL; - MAKE_HANDLE_ABSPATH(newpath, this, gfid); + MAKE_HANDLE_ABSPATH_FD(newstr, this, gfid, dfd); + ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW); - ret = sys_lstat(newpath, &newbuf); if (ret == -1 && errno != ENOENT) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "%s", - newpath); + uuid_utoa(gfid)); return -1; } if (ret == -1 && errno == ENOENT) { - ret = posix_handle_mkdir_hashes(this, newpath); + snprintf(d2, sizeof(d2), "%02x", gfid[1]); + ret = sys_fstatat(dfd, d2, &hashbuf, 0); if (ret) { - gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "mkdir %s failed ", newpath); - return -1; + ret = posix_handle_mkdir_hashes(this, dfd, gfid); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, + "mkdir %s failed ", uuid_utoa(gfid)); + return -1; + } } - - ret = sys_link(oldpath, newpath); + ret = sys_linkat(AT_FDCWD, oldpath, dfd, newstr); if (ret) { if (errno != EEXIST) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "link %s -> %s" "failed ", - oldpath, newpath); + oldpath, newstr); return -1; } else { link_exists = _gf_true; } } + ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW); - ret = sys_lstat(newpath, &newbuf); if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "lstat on %s failed", newpath); + "lstat on %s failed", uuid_utoa(gfid)); return -1; } if ((link_exists) && (!S_ISREG(newbuf.st_mode))) { gf_msg(this->name, GF_LOG_ERROR, EINVAL, P_MSG_HANDLE_CREATE, - "%s - Expected regular file", newpath); + "%s - Expected regular file", uuid_utoa(gfid)); return -1; } } @@ -769,7 +785,8 @@ posix_handle_hard(xlator_t *this, const char *oldpath, uuid_t gfid, "mismatching ino/dev between file %s (%lld/%lld) " "and handle %s (%lld/%lld)", oldpath, (long long)oldbuf->st_ino, (long long)oldbuf->st_dev, - newpath, (long long)newbuf.st_ino, (long long)newbuf.st_dev); + uuid_utoa(gfid), (long long)newbuf.st_ino, + (long long)newbuf.st_dev); ret = -1; } @@ -783,15 +800,23 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc, char *oldpath = NULL; char *newpath = NULL; struct stat newbuf; + struct stat hashbuf; int ret = -1; + char d2[3] = { + 0, + }; + int dfd = -1; + char *newstr = NULL; MAKE_HANDLE_ABSPATH(newpath, this, gfid); + MAKE_HANDLE_ABSPATH_FD(newstr, this, gfid, dfd); MAKE_HANDLE_RELPATH(oldpath, this, loc->pargfid, loc->name); - ret = sys_lstat(newpath, &newbuf); + ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW); + if (ret == -1 && errno != ENOENT) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "%s", - newpath); + newstr); return -1; } @@ -801,24 +826,30 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc, errno = EINVAL; return -1; } - ret = posix_handle_mkdir_hashes(this, newpath); + + snprintf(d2, sizeof(d2), "%02x", gfid[1]); + ret = sys_fstatat(dfd, d2, &hashbuf, 0); + if (ret) { - gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "mkdir %s failed ", newpath); - return -1; + ret = posix_handle_mkdir_hashes(this, dfd, gfid); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, + "mkdir %s failed ", newstr); + return -1; + } } - - ret = sys_symlink(oldpath, newpath); + ret = sys_symlinkat(oldpath, dfd, newstr); if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "symlink %s -> %s failed", oldpath, newpath); + "symlink %s -> %s failed", oldpath, newstr); return -1; } - ret = sys_lstat(newpath, &newbuf); + ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW); + if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "stat on %s failed ", newpath); + "stat on %s failed ", newstr); return -1; } } @@ -826,7 +857,7 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc, ret = sys_stat(real_path, &newbuf); if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, - "stat on %s failed ", newpath); + "stat on %s failed ", real_path); return -1; } @@ -848,26 +879,33 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc, int posix_handle_unset_gfid(xlator_t *this, uuid_t gfid) { - char *path = NULL; - int ret = -1; + int ret = 0; struct stat stat; + int index = 0; + int dfd = 0; + char newstr[POSIX_GFID_HASH2_LEN] = { + 0, + }; + struct posix_private *priv = this->private; - MAKE_HANDLE_GFID_PATH(path, this, gfid); + index = gfid[0]; + dfd = priv->arrdfd[index]; - ret = sys_lstat(path, &stat); + snprintf(newstr, sizeof(newstr), "%02x/%s", gfid[1], uuid_utoa(gfid)); + ret = sys_fstatat(dfd, newstr, &stat, AT_SYMLINK_NOFOLLOW); if (ret == -1) { if (errno != ENOENT) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_DELETE, "%s", - path); + newstr); } goto out; } - ret = sys_unlink(path); - if (ret == -1) { + ret = sys_unlinkat(dfd, newstr); + if (ret) { gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_DELETE, - "unlink %s failed ", path); + "unlink %s is failed", newstr); } out: diff --git a/xlators/storage/posix/src/posix-handle.h b/xlators/storage/posix/src/posix-handle.h index 70c68c3d89c..f33ed92620d 100644 --- a/xlators/storage/posix/src/posix-handle.h +++ b/xlators/storage/posix/src/posix-handle.h @@ -141,6 +141,16 @@ __priv->base_path, gfid[0], gfid[1], uuid_utoa(gfid)); \ } while (0) +#define MAKE_HANDLE_ABSPATH_FD(var, this, gfid, dfd) \ + do { \ + struct posix_private *__priv = this->private; \ + int findex = gfid[0]; \ + int __len = POSIX_GFID_HASH2_LEN; \ + var = alloca(__len); \ + snprintf(var, __len, "%02x/%s", gfid[1], uuid_utoa(gfid)); \ + dfd = __priv->arrdfd[findex]; \ + } while (0) + #define MAKE_ENTRY_HANDLE(entp, parp, this, loc, ent_p) \ do { \ char *__parp; \ @@ -184,6 +194,7 @@ /* expand ELOOP */ \ } while (0) +#define POSIX_GFID_HASH2_LEN 45 int posix_handle_gfid_path(xlator_t *this, uuid_t gfid, char *buf, size_t len); diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 4919cbdf2e9..67db3324083 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -824,6 +824,11 @@ posix_pstat(xlator_t *this, inode_t *inode, uuid_t gfid, const char *path, gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_LSTAT_FAILED, "lstat failed on %s", path); errno = op_errno; /*gf_msg could have changed errno*/ + } else { + op_errno = errno; + gf_msg_debug(this->name, 0, "lstat failed on %s (%s)", path, + strerror(errno)); + errno = op_errno; /*gf_msg could have changed errno*/ } goto out; } @@ -1065,7 +1070,7 @@ verify_handle: ret = posix_handle_soft(this, path, loc, uuid_curr, &stat); out: - if (!(*op_errno)) + if (ret && !(*op_errno)) *op_errno = errno; return ret; } @@ -1500,7 +1505,7 @@ posix_janitor_task(void *data) if (!priv) goto out; - time(&now); + now = gf_time(); if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { if (priv->disable_landfill_purge) { gf_msg_debug(this->name, 0, @@ -1588,113 +1593,107 @@ unlock: } static struct posix_fd * -janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep) +janitor_get_next_fd(glusterfs_ctx_t *ctx) { struct posix_fd *pfd = NULL; - struct timespec timeout; + while (list_empty(&ctx->janitor_fds)) { + if (ctx->pxl_count == 0) { + return NULL; + } - pthread_mutex_lock(&ctx->janitor_lock); - { - if (list_empty(&ctx->janitor_fds)) { - time(&timeout.tv_sec); - timeout.tv_sec += janitor_sleep; - timeout.tv_nsec = 0; + pthread_cond_wait(&ctx->fd_cond, &ctx->fd_lock); + } - pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock, - &timeout); - goto unlock; - } + pfd = list_first_entry(&ctx->janitor_fds, struct posix_fd, list); + list_del_init(&pfd->list); - pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list); + return pfd; +} + +static void +posix_close_pfd(xlator_t *xl, struct posix_fd *pfd) +{ + THIS = xl; - list_del(ctx->janitor_fds.next); + if (pfd->dir == NULL) { + gf_msg_trace(xl->name, 0, "janitor: closing file fd=%d", pfd->fd); + sys_close(pfd->fd); + } else { + gf_msg_debug(xl->name, 0, "janitor: closing dir fd=%p", pfd->dir); + sys_closedir(pfd->dir); } -unlock: - pthread_mutex_unlock(&ctx->janitor_lock); - return pfd; + GF_FREE(pfd); } static void * posix_ctx_janitor_thread_proc(void *data) { - xlator_t *this = NULL; + xlator_t *xl; struct posix_fd *pfd; glusterfs_ctx_t *ctx = NULL; - struct posix_private *priv = NULL; - int32_t sleep_duration = 0; + struct posix_private *priv_fd; - this = data; - ctx = THIS->ctx; - THIS = this; + ctx = data; - priv = this->private; - sleep_duration = priv->janitor_sleep_duration; - while (1) { - pfd = janitor_get_next_fd(ctx, sleep_duration); - if (pfd) { - if (pfd->dir == NULL) { - gf_msg_trace(this->name, 0, "janitor: closing file fd=%d", - pfd->fd); - sys_close(pfd->fd); - } else { - gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", - pfd->dir); - sys_closedir(pfd->dir); - } + pthread_mutex_lock(&ctx->fd_lock); - GF_FREE(pfd); - } + while ((pfd = janitor_get_next_fd(ctx)) != NULL) { + pthread_mutex_unlock(&ctx->fd_lock); + + xl = pfd->xl; + posix_close_pfd(xl, pfd); + + pthread_mutex_lock(&ctx->fd_lock); + + priv_fd = xl->private; + priv_fd->rel_fdcount--; + if (!priv_fd->rel_fdcount) + pthread_cond_signal(&priv_fd->fd_cond); } + pthread_mutex_unlock(&ctx->fd_lock); + return NULL; } int posix_spawn_ctx_janitor_thread(xlator_t *this) { - struct posix_private *priv = NULL; int ret = 0; glusterfs_ctx_t *ctx = NULL; - priv = this->private; - ctx = THIS->ctx; + ctx = this->ctx; - LOCK(&priv->lock); + pthread_mutex_lock(&ctx->fd_lock); { - if (!ctx->janitor) { - pthread_mutex_init(&ctx->janitor_lock, NULL); - pthread_cond_init(&ctx->janitor_cond, NULL); - INIT_LIST_HEAD(&ctx->janitor_fds); - + if (ctx->pxl_count++ == 0) { ret = gf_thread_create(&ctx->janitor, NULL, - posix_ctx_janitor_thread_proc, this, + posix_ctx_janitor_thread_proc, ctx, "posixctxjan"); if (ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, - "spawning janitor " - "thread failed"); - goto unlock; + "spawning janitor thread failed"); + ctx->pxl_count--; } } } -unlock: - UNLOCK(&priv->lock); + pthread_mutex_unlock(&ctx->fd_lock); + return ret; } static int -is_fresh_file(int64_t sec, int64_t ns) +is_fresh_file(struct timespec *ts) { - struct timeval tv; + struct timespec now; int64_t elapsed; - gettimeofday(&tv, NULL); + timespec_now_realtime(&now); + elapsed = (int64_t)gf_tsdiff(ts, &now); - elapsed = (tv.tv_sec - sec) * 1000000L; - elapsed += tv.tv_usec - (ns / 1000L); if (elapsed < 0) { /* The file has been modified in the future !!! * Is it fresh ? previous implementation considered this as a @@ -1703,11 +1702,7 @@ is_fresh_file(int64_t sec, int64_t ns) } /* If the file is newer than a second, we consider it fresh. */ - if (elapsed < 1000000) { - return 1; - } - - return 0; + return elapsed < 1000000; } int @@ -1770,7 +1765,9 @@ posix_gfid_heal(xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req) if (ret != 16) { /* TODO: This is a very hacky way of doing this, and very prone to * errors and unexpected behavior. This should be changed. */ - if (is_fresh_file(stbuf.ia_ctime, stbuf.ia_ctime_nsec)) { + struct timespec ts = {.tv_sec = stbuf.ia_ctime, + .tv_nsec = stbuf.ia_ctime_nsec}; + if (is_fresh_file(&ts)) { gf_msg(this->name, GF_LOG_ERROR, ENOENT, P_MSG_FRESHFILE, "Fresh file: %s", path); return -ENOENT; @@ -1784,7 +1781,7 @@ posix_gfid_heal(xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req) if (ret != 16) { /* TODO: This is a very hacky way of doing this, and very prone to * errors and unexpected behavior. This should be changed. */ - if (is_fresh_file(stat.st_ctim.tv_sec, stat.st_ctim.tv_nsec)) { + if (is_fresh_file(&stat.st_ctim)) { gf_msg(this->name, GF_LOG_ERROR, ENOENT, P_MSG_FRESHFILE, "Fresh file: %s", path); return -ENOENT; @@ -2017,7 +2014,7 @@ posix_fs_health_check(xlator_t *this, char *file_path) { struct posix_private *priv = NULL; int ret = -1; - char timestamp[256] = { + char timestamp[GF_TIMESTR_SIZE] = { 0, }; int fd = -1; @@ -2032,9 +2029,7 @@ posix_fs_health_check(xlator_t *this, char *file_path) int timeout = 0; struct aiocb aiocb; - GF_VALIDATE_OR_GOTO(this->name, this, out); priv = this->private; - GF_VALIDATE_OR_GOTO("posix-helpers", priv, out); timeout = priv->health_check_timeout; @@ -2045,7 +2040,7 @@ posix_fs_health_check(xlator_t *this, char *file_path) goto out; } - time_sec = time(NULL); + time_sec = gf_time(); gf_time_fmt(timestamp, sizeof timestamp, time_sec, gf_timefmt_FT); timelen = strlen(timestamp); @@ -2317,7 +2312,7 @@ posix_disk_space_check(xlator_t *this) double totsz = 0; double freesz = 0; - GF_VALIDATE_OR_GOTO(this->name, this, out); + GF_VALIDATE_OR_GOTO("posix-helpers", this, out); priv = this->private; GF_VALIDATE_OR_GOTO(this->name, priv, out); @@ -2410,7 +2405,7 @@ posix_spawn_disk_space_check_thread(xlator_t *xl) ret = gf_thread_create(&priv->disk_space_check, NULL, posix_disk_space_check_thread_proc, xl, - "posix_reserve"); + "posixrsv"); if (ret) { priv->disk_space_check_active = _gf_false; gf_msg(xl->name, GF_LOG_ERROR, errno, P_MSG_DISK_SPACE_CHECK_FAILED, @@ -2490,23 +2485,8 @@ posix_fsyncer_syncfs(xlator_t *this, struct list_head *head) stub = list_entry(head->prev, call_stub_t, list); ret = posix_fd_ctx_get(stub->args.fd, this, &pfd, NULL); - if (ret) - return; - -#ifdef GF_LINUX_HOST_OS - /* syncfs() is not "declared" in RHEL's glibc even though - the kernel has support. - */ -#include <sys/syscall.h> -#include <unistd.h> -#ifdef SYS_syncfs - syscall(SYS_syncfs, pfd->fd); -#else - sync(); -#endif -#else - sync(); -#endif + if (!ret) + (void)gf_syncfs(pfd->fd); } void * @@ -3598,18 +3578,21 @@ posix_is_layout_stale(dict_t *xdata, char *par_path, xlator_t *this) gf_boolean_t have_val = _gf_false; data_t *arg_data = NULL; char *xattr_name = NULL; + size_t xattr_len = 0; gf_boolean_t is_stale = _gf_false; op_ret = dict_get_str_sizen(xdata, GF_PREOP_PARENT_KEY, &xattr_name); if (xattr_name == NULL) { op_ret = 0; - goto out; + return is_stale; } - arg_data = dict_get(xdata, xattr_name); + xattr_len = strlen(xattr_name); + arg_data = dict_getn(xdata, xattr_name, xattr_len); if (!arg_data) { op_ret = 0; - goto out; + dict_del_sizen(xdata, GF_PREOP_PARENT_KEY); + return is_stale; } size = sys_lgetxattr(par_path, xattr_name, value_buf, @@ -3653,7 +3636,7 @@ posix_is_layout_stale(dict_t *xdata, char *par_path, xlator_t *this) } out: - dict_del_sizen(xdata, xattr_name); + dict_deln(xdata, xattr_name, xattr_len); dict_del_sizen(xdata, GF_PREOP_PARENT_KEY); if (op_ret == -1) { @@ -3662,3 +3645,22 @@ out: return is_stale; } + +/* Delete user xattr from the file at the file-path specified by data and from + * dict */ +int +posix_delete_user_xattr(dict_t *dict, char *k, data_t *v, void *data) +{ + int ret; + char *real_path = data; + + ret = sys_lremovexattr(real_path, k); + if (ret) { + gf_msg("posix-helpers", GF_LOG_ERROR, P_MSG_XATTR_NOT_REMOVED, errno, + "removexattr failed. key %s path %s", k, real_path); + } + + dict_del(dict, k); + + return ret; +} diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c index a6c2b512ef1..6d54d37e5aa 100644 --- a/xlators/storage/posix/src/posix-inode-fd-ops.c +++ b/xlators/storage/posix/src/posix-inode-fd-ops.c @@ -54,6 +54,7 @@ #include <glusterfs/events.h> #include "posix-gfid-path.h" #include <glusterfs/compat-uuid.h> +#include <glusterfs/common-utils.h> extern char *marker_xattrs[]; #define ALIGN_SIZE 4096 @@ -1360,13 +1361,28 @@ out: return 0; } +static void +posix_add_fd_to_cleanup(xlator_t *this, struct posix_fd *pfd) +{ + glusterfs_ctx_t *ctx = this->ctx; + struct posix_private *priv = this->private; + + pfd->xl = this; + pthread_mutex_lock(&ctx->fd_lock); + { + list_add_tail(&pfd->list, &ctx->janitor_fds); + priv->rel_fdcount++; + pthread_cond_signal(&ctx->fd_cond); + } + pthread_mutex_unlock(&ctx->fd_lock); +} + int32_t posix_releasedir(xlator_t *this, fd_t *fd) { struct posix_fd *pfd = NULL; uint64_t tmp_pfd = 0; int ret = 0; - glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); @@ -1383,22 +1399,8 @@ posix_releasedir(xlator_t *this, fd_t *fd) "pfd->dir is NULL for fd=%p", fd); goto out; } + posix_add_fd_to_cleanup(this, pfd); - ctx = THIS->ctx; - - pthread_mutex_lock(&ctx->janitor_lock); - { - INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &ctx->janitor_fds); - pthread_cond_signal(&ctx->janitor_cond); - } - pthread_mutex_unlock(&ctx->janitor_lock); - - /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); - - sys_closedir(pfd->dir); - GF_FREE(pfd); - */ out: return 0; } @@ -2305,8 +2307,7 @@ posix_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in, flags); if (op_ret < 0) { - op_errno = -op_ret; - op_ret = -1; + op_errno = errno; gf_msg(this->name, GF_LOG_ERROR, op_errno, P_MSG_COPY_FILE_RANGE_FAILED, "copy_file_range failed: fd_in: %p (gfid: %s) ," " fd_out %p (gfid:%s)", @@ -2521,18 +2522,13 @@ out: int32_t posix_release(xlator_t *this, fd_t *fd) { - struct posix_private *priv = NULL; struct posix_fd *pfd = NULL; int ret = -1; uint64_t tmp_pfd = 0; - glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); - priv = this->private; - ctx = THIS->ctx; - ret = fd_ctx_del(fd, this, &tmp_pfd); if (ret < 0) { gf_msg(this->name, GF_LOG_WARNING, 0, P_MSG_PFD_NULL, @@ -2546,16 +2542,7 @@ posix_release(xlator_t *this, fd_t *fd) "pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd); } - pthread_mutex_lock(&ctx->janitor_lock); - { - INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &ctx->janitor_fds); - pthread_cond_signal(&ctx->janitor_cond); - } - pthread_mutex_unlock(&ctx->janitor_lock); - - if (!priv) - goto out; + posix_add_fd_to_cleanup(this, pfd); out: return 0; @@ -2725,6 +2712,7 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, int32_t ret = 0; ssize_t acl_size = 0; dict_t *xattr = NULL; + dict_t *subvol_xattrs = NULL; posix_xattr_filler_t filler = { 0, }; @@ -2740,6 +2728,10 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, struct mdata_iatt mdata_iatt = { 0, }; + int8_t sync_backend_xattrs = _gf_false; + data_pair_t *custom_xattrs; + data_t *keyval = NULL; + char **xattrs_to_heal = get_xattrs_to_heal(); DECLARE_OLD_FS_ID_VAR; SET_FS_ID(frame->root->uid, frame->root->gid); @@ -2922,6 +2914,66 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, goto out; } + ret = dict_get_int8(xdata, "sync_backend_xattrs", &sync_backend_xattrs); + if (ret) { + gf_msg_debug(this->name, -ret, "Unable to get sync_backend_xattrs"); + } + + if (sync_backend_xattrs) { + /* List all custom xattrs */ + subvol_xattrs = dict_new(); + if (!subvol_xattrs) + goto out; + + ret = dict_set_int32_sizen(xdata, "list-xattr", 1); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, + "Unable to set list-xattr in dict "); + goto out; + } + + subvol_xattrs = posix_xattr_fill(this, real_path, loc, NULL, -1, xdata, + NULL); + + /* Remove all user xattrs from the file */ + dict_foreach_fnmatch(subvol_xattrs, "user.*", posix_delete_user_xattr, + real_path); + + /* Remove all custom xattrs from the file */ + for (i = 1; xattrs_to_heal[i]; i++) { + keyval = dict_get(subvol_xattrs, xattrs_to_heal[i]); + if (keyval) { + ret = sys_lremovexattr(real_path, xattrs_to_heal[i]); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, P_MSG_XATTR_NOT_REMOVED, + errno, "removexattr failed. key %s path %s", + xattrs_to_heal[i], loc->path); + goto out; + } + + dict_del(subvol_xattrs, xattrs_to_heal[i]); + keyval = NULL; + } + } + + /* Set custom xattrs based on info provided by DHT */ + custom_xattrs = dict->members_list; + + while (custom_xattrs != NULL) { + ret = sys_lsetxattr(real_path, custom_xattrs->key, + custom_xattrs->value->data, + custom_xattrs->value->len, flags); + if (ret) { + op_errno = errno; + gf_log(this->name, GF_LOG_ERROR, "setxattr failed - %s %d", + custom_xattrs->key, ret); + goto out; + } + + custom_xattrs = custom_xattrs->next; + } + } + xattr = dict_new(); if (!xattr) goto out; @@ -3029,6 +3081,9 @@ out: if (xattr) dict_unref(xattr); + if (subvol_xattrs) + dict_unref(subvol_xattrs); + return 0; } diff --git a/xlators/storage/posix/src/posix-metadata.h b/xlators/storage/posix/src/posix-metadata.h index 63e8771d3b1..d37014af93e 100644 --- a/xlators/storage/posix/src/posix-metadata.h +++ b/xlators/storage/posix/src/posix-metadata.h @@ -15,13 +15,15 @@ /* In memory representation posix metadata xattr */ typedef struct { - /* version of structure, bumped up if any new member is added */ - uint8_t version; /* flags indicates valid fields in the structure */ uint64_t flags; struct timespec ctime; struct timespec mtime; struct timespec atime; + /* version of structure, bumped up if any new member is added */ + uint8_t version; + + char _pad[7]; /* manual padding */ } posix_mdata_t; typedef struct { diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h index 7893a9b3e35..b8db146eef2 100644 --- a/xlators/storage/posix/src/posix.h +++ b/xlators/storage/posix/src/posix.h @@ -119,12 +119,14 @@ */ struct posix_fd { - int fd; /* fd returned by the kernel */ - int32_t flags; /* flags for open/creat */ - DIR *dir; /* handle returned by the kernel */ - off_t dir_eof; /* offset at dir EOF */ - int odirect; + int fd; /* fd returned by the kernel */ + int32_t flags; /* flags for open/creat */ + DIR *dir; /* handle returned by the kernel */ + off_t dir_eof; /* offset at dir EOF */ struct list_head list; /* to add to the janitor list */ + int odirect; + xlator_t *xl; + char _pad[4]; /* manual padding */ }; struct posix_private { @@ -135,67 +137,38 @@ struct posix_private { gf_lock_t lock; char *hostname; - /* Statistics, provides activity of the server */ - - struct timeval prev_fetch_time; - struct timeval init_time; time_t last_landfill_check; - int32_t janitor_sleep_duration; gf_atomic_t read_value; /* Total read, from init */ gf_atomic_t write_value; /* Total write, from init */ - /* - In some cases, two exported volumes may reside on the same - partition on the server. Sending statvfs info for both - the volumes will lead to erroneous df output at the client, - since free space on the partition will be counted twice. - - In such cases, user can disable exporting statvfs info - on one of the volumes by setting this option. - */ - gf_boolean_t export_statfs; - - gf_boolean_t o_direct; /* always open files in O_DIRECT mode */ - - /* - decide whether posix_unlink does open (file), unlink (file), close (fd) - instead of just unlink (file). with the former approach there is no - lockout of access to parent directory during removal of very large files - for the entire duration of freeing of data blocks. - */ - gf_boolean_t background_unlink; /* janitor task which cleans up /.trash (created by replicate) */ struct gf_tw_timer_list *janitor; char *trash_path; /* lock for brick dir */ - DIR *mount_lock; + int mount_lock; struct stat handledir; /* uuid of glusterd that swapned the brick process */ uuid_t glusterd_uuid; - gf_boolean_t aio_configured; - gf_boolean_t aio_init_done; - gf_boolean_t aio_capable; #ifdef HAVE_LIBAIO io_context_t ctxp; pthread_t aiothread; #endif - /* node-uuid in pathinfo xattr */ - gf_boolean_t node_uuid_pathinfo; - pthread_t fsyncer; struct list_head fsyncs; pthread_mutex_t fsync_mutex; pthread_cond_t fsync_cond; pthread_mutex_t janitor_mutex; pthread_cond_t janitor_cond; + pthread_cond_t fd_cond; int fsync_queue_count; + int32_t janitor_sleep_duration; enum { BATCH_NONE = 0, @@ -206,8 +179,6 @@ struct posix_private { } batch_fsync_mode; uint32_t batch_fsync_delay_usec; - gf_boolean_t update_pgfid_nlinks; - gf_boolean_t gfid2path; char gfid2path_sep[8]; /* seconds to sleep between health checks */ @@ -215,13 +186,10 @@ struct posix_private { /* seconds to sleep to wait for aio write finish for health checks */ uint32_t health_check_timeout; pthread_t health_check; - gf_boolean_t health_check_active; double disk_reserve; - char disk_unit; - uint32_t disk_space_full; pthread_t disk_space_check; - gf_boolean_t disk_space_check_active; + uint32_t disk_space_full; #ifdef GF_DARWIN_HOST_OS enum { @@ -236,9 +204,6 @@ struct posix_private { same backend. Very much usable in brick-splitting feature. */ int32_t shared_brick_count; - /* This option is used for either to call a landfill_purge or not */ - gf_boolean_t disable_landfill_purge; - /*Option to set mode bit permission that will always be set on file/directory. */ mode_t force_create_mode; @@ -246,10 +211,47 @@ struct posix_private { mode_t create_mask; mode_t create_directory_mask; uint32_t max_hardlinks; + int32_t arrdfd[256]; + int dirfd; + + /* This option is used for either to call a landfill_purge or not */ + gf_boolean_t disable_landfill_purge; gf_boolean_t fips_mode_rchecksum; gf_boolean_t ctime; gf_boolean_t janitor_task_stop; + + gf_boolean_t disk_space_check_active; + char disk_unit; + gf_boolean_t health_check_active; + gf_boolean_t update_pgfid_nlinks; + gf_boolean_t gfid2path; + /* node-uuid in pathinfo xattr */ + gf_boolean_t node_uuid_pathinfo; + /* + In some cases, two exported volumes may reside on the same + partition on the server. Sending statvfs info for both + the volumes will lead to erroneous df output at the client, + since free space on the partition will be counted twice. + + In such cases, user can disable exporting statvfs info + on one of the volumes by setting this option. + */ + gf_boolean_t export_statfs; + + gf_boolean_t o_direct; /* always open files in O_DIRECT mode */ + + /* + decide whether posix_unlink does open (file), unlink (file), close (fd) + instead of just unlink (file). with the former approach there is no + lockout of access to parent directory during removal of very large files + for the entire duration of freeing of data blocks. + */ + gf_boolean_t background_unlink; + gf_boolean_t aio_configured; + gf_boolean_t aio_init_done; + gf_boolean_t aio_capable; + uint32_t rel_fdcount; }; typedef struct { @@ -263,9 +265,11 @@ typedef struct { fd_t *fd; int fdnum; int flags; - int32_t op_errno; char *list; size_t list_size; + int32_t op_errno; + + char _pad[4]; /* manual padding */ } posix_xattr_filler_t; typedef struct { @@ -325,6 +329,7 @@ posix_istat(xlator_t *this, inode_t *inode, uuid_t gfid, const char *basename, int posix_pstat(xlator_t *this, inode_t *inode, uuid_t gfid, const char *real_path, struct iatt *iatt, gf_boolean_t inode_locked); + dict_t * posix_xattr_fill(xlator_t *this, const char *path, loc_t *loc, fd_t *fd, int fdnum, dict_t *xattr, struct iatt *buf); @@ -662,4 +667,7 @@ posix_update_iatt_buf(struct iatt *buf, int fd, char *loc, dict_t *xdata); gf_boolean_t posix_is_layout_stale(dict_t *xdata, char *par_path, xlator_t *this); +int +posix_delete_user_xattr(dict_t *dict, char *k, data_t *v, void *data); + #endif /* _POSIX_H */ diff --git a/xlators/system/posix-acl/src/posix-acl.c b/xlators/system/posix-acl/src/posix-acl.c index 77c8df5a54f..fc227364b31 100644 --- a/xlators/system/posix-acl/src/posix-acl.c +++ b/xlators/system/posix-acl/src/posix-acl.c @@ -50,8 +50,8 @@ r00t() return conf->super_uid; } -int -whitelisted_xattr(const char *key) +static int +allowed_xattr(const char *key) { if (!key) return 0; @@ -2016,7 +2016,7 @@ int posix_acl_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name, dict_t *xdata) { - if (whitelisted_xattr(name)) + if (allowed_xattr(name)) goto green; if (acl_permits(frame, loc->inode, POSIX_ACL_READ)) @@ -2039,7 +2039,7 @@ int posix_acl_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name, dict_t *xdata) { - if (whitelisted_xattr(name)) + if (allowed_xattr(name)) goto green; if (acl_permits(frame, fd->inode, POSIX_ACL_READ)) @@ -2072,7 +2072,7 @@ posix_acl_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, goto red; } - if (whitelisted_xattr(name)) { + if (allowed_xattr(name)) { if (!frame_is_user(frame, ctx->uid)) { op_errno = EPERM; goto red; |