summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVarun Shastry <vshastry@redhat.com>2013-10-22 16:12:58 +0530
committerVijay Bellur <vbellur@redhat.com>2014-01-28 22:39:43 -0800
commit6cb2106c22e1ea3758ac63dc725b888470f5dd5a (patch)
tree9eba13747544c26407469be27ce68207991fbad4
parent9f8ca919eb8472fec033a70f06fd18ab9fad1fd5 (diff)
features/quota: Metadata cleanup
Quota and marker uses 'trusted.glusterfs.quota*' and 'trusted.pgfid*' xattrs to store its configurations and accounting information and also to build the parent inode chain in case of absense of path. Problem: After disabling and then enabling quota back, the xattrs may contain stale data leading to impaired accounting and thus improper enforcement. Solution: Clean up all the quota related xattrs after quota disable. Marker xlator implements a virtual xattr to cleanup quota and pgfid xattrs. In this approach glusterd mounts an auxiliary mount and sends the below command to all the files by crawling the mountpoint. #setfattr -n "glusterfs.quota-xattr-cleanup" -v 1 <path/to/file> Credit: Krishnan Parthasarathi <kparthas@redhat.com> Varun Shastry <vshastry@redhat.com> Change-Id: I9380eca58a285dc27dd572de1767aac8f2cd8049 BUG: 969461 Signed-off-by: Varun Shastry <vshastry@redhat.com> Reviewed-on: http://review.gluster.org/6369 Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com> Reviewed-on: http://review.gluster.org/6838
-rw-r--r--extras/Makefile.am10
-rwxr-xr-xextras/quota-metadata-cleanup.sh24
-rwxr-xr-xextras/quota-remove-xattr.sh24
-rw-r--r--libglusterfs/src/glusterfs.h1
-rw-r--r--xlators/features/marker/src/marker.c143
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-quota.c30
6 files changed, 169 insertions, 63 deletions
diff --git a/extras/Makefile.am b/extras/Makefile.am
index cf61932..2633e20 100644
--- a/extras/Makefile.am
+++ b/extras/Makefile.am
@@ -1,4 +1,3 @@
-
EditorModedir = $(docdir)
EditorMode_DATA = glusterfs-mode.el glusterfs.vim
@@ -13,8 +12,7 @@ vol_DATA = glusterd.vol
EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim \
migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \
- backend-cleanup.sh disk_usage_sync.sh quota-remove-xattr.sh \
- quota-metadata-cleanup.sh glusterfs-logrotate clear_xattrs.sh \
- group-virt.example glusterd-sysconfig gluster-rsyslog-7.2.conf \
- gluster-rsyslog-5.8.conf logger.conf.example glusterd.vol \
- glusterfs-georep-logrotate
+ backend-cleanup.sh disk_usage_sync.sh glusterfs-logrotate \
+ clear_xattrs.sh group-virt.example glusterd-sysconfig \
+ gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
+ logger.conf.example glusterd.vol glusterfs-georep-logrotate
diff --git a/extras/quota-metadata-cleanup.sh b/extras/quota-metadata-cleanup.sh
deleted file mode 100755
index 37ad8bc..0000000
--- a/extras/quota-metadata-cleanup.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# This script is used to cleanup xattrs setup by quota-translator in (a)
-# backend directory(ies). It takes a single or list of backend directories
-# as argument(s).
-
-usage ()
-{
- echo >&2 "usage: $0 <list-of-backend-directories>"
-}
-
-main ()
-{
- [ $# -lt 1 ] && usage
-
- INSTALL_DIR=`dirname $0`
-
- for i in $@; do
- find $i -exec $INSTALL_DIR/quota-remove-xattr.sh '{}' \;
- done
-
-}
-
-main $@
diff --git a/extras/quota-remove-xattr.sh b/extras/quota-remove-xattr.sh
deleted file mode 100755
index 7191f9b..0000000
--- a/extras/quota-remove-xattr.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# This script is used to remove xattrs set by quota translator on a path.
-# It is generally invoked from quota-metadata-cleanup.sh, but can
-# also be used stand-alone.
-
-usage ()
-{
- echo >&2 "usage: $0 <path>"
-}
-
-main ()
-{
- [ $# -ne 1 ] && usage $0
-
- XATTR_KEY_VALUE_PAIRS=`getfattr -h -d -m 'trusted.glusterfs.quota' $1 2>/dev/null | sed -e '/^# file/d'`
-
- for i in $XATTR_KEY_VALUE_PAIRS; do
- XATTR_KEY=`echo $i | sed -e 's/\([^=]*\).*/\1/g'`
- setfattr -h -x $XATTR_KEY $1
- done
-}
-
-main $@
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 4f7a523..2b26763 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -86,6 +86,7 @@
#define GF_XATTR_LOCKINFO_KEY "trusted.glusterfs.lockinfo"
#define GF_XATTR_GET_REAL_FILENAME_KEY "user.glusterfs.get_real_filename:"
#define QUOTA_LIMIT_KEY "trusted.glusterfs.quota.limit-set"
+#define VIRTUAL_QUOTA_XATTR_CLEANUP_KEY "glusterfs.quota-xattr-cleanup"
#define GF_READDIR_SKIP_DIRS "readdir-filter-directories"
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index dbe9d53..596a651 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -21,6 +21,7 @@
#include "marker-quota-helper.h"
#include "marker-common.h"
#include "byte-order.h"
+#include "syncop.h"
#define _GF_UID_GID_CHANGED 1
@@ -2086,16 +2087,150 @@ out:
return 0;
}
+int
+remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
+{
+ call_frame_t *frame = data;
+ marker_local_t *local = frame->local;
+ xlator_t *this = frame->this;
+ int ret = -1;
+
+ ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
+ "extended attribute: %s", local->loc.path, k);
+ return -1;
+ }
+ return 0;
+}
+
+int
+quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+{
+ dict_t *xdata = args;
+ int op_ret = -1;
+ int op_errno = 0;
+ marker_local_t *local = NULL;
+
+ local = frame->local;
+ frame->local = NULL;
+
+ op_ret = (ret < 0)? -1: 0;
+ op_errno = -ret;
+
+ STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+ marker_local_unref (local);
+ return ret;
+}
+
+int
+quota_xattr_cleaner (void *args)
+{
+ struct synctask *task = NULL;
+ call_frame_t *frame = NULL;
+ xlator_t *this = NULL;
+ marker_local_t *local = NULL;
+ dict_t *xdata = NULL;
+ int ret = -1;
+
+ task = synctask_get ();
+ if (!task)
+ goto out;
+
+ frame = task->frame;
+ this = frame->this;
+ local = frame->local;
+
+ ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata);
+ if (ret == -1) {
+ ret = -errno;
+ goto out;
+ }
+
+ ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
+ remove_quota_keys, frame);
+ if (ret == -1) {
+ ret = -errno;
+ goto out;
+ }
+ ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
+ remove_quota_keys, frame);
+ if (ret == -1) {
+ ret = -errno;
+ goto out;
+ }
+
+ ret = 0;
+out:
+ if (xdata)
+ dict_unref (xdata);
+
+ return ret;
+}
+
+int
+marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+ loc_t *loc)
+{
+ int ret = -1;
+ marker_local_t *local = NULL;
+
+ local = mem_get0 (this->local_pool);
+ if (!local)
+ goto out;
+
+ MARKER_INIT_LOCAL (frame, local);
+
+ loc_copy (&local->loc, loc);
+ ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
+ quota_xattr_cleaner_cbk, frame, xdata);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
+ "for cleaning up quota extended attributes");
+ goto out;
+ }
+
+ ret = 0;
+out:
+ if (ret) {
+ frame->local = NULL;
+ STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, xdata);
+ marker_local_unref (local);
+ }
+ return ret;
+}
+
+static inline gf_boolean_t
+marker_xattr_cleanup_cmd (dict_t *dict)
+{
+ return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+}
+
int32_t
marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
int32_t flags, dict_t *xdata)
{
- int32_t ret = 0;
- marker_local_t *local = NULL;
- marker_conf_t *priv = NULL;
+ int32_t ret = 0;
+ marker_local_t *local = NULL;
+ marker_conf_t *priv = NULL;
+ int op_errno = ENOMEM;
priv = this->private;
+ if (marker_xattr_cleanup_cmd (dict)) {
+ if (frame->root->uid != 0 || frame->root->gid != 0) {
+ op_errno = EPERM;
+ ret = -1;
+ goto err;
+ }
+
+ /* The following function does the cleanup and then unwinds the
+ * corresponding call*/
+ loc_path (loc, NULL);
+ marker_do_xattr_cleanup (frame, this, xdata, loc);
+ return 0;
+ }
+
if (priv->feature_enabled == 0)
goto wind;
@@ -2116,7 +2251,7 @@ wind:
FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
return 0;
err:
- STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
+ STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, NULL);
return 0;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index f46f087..979dac6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -28,6 +28,8 @@
#include <sys/wait.h>
#include <dlfcn.h>
+/* Any negative pid to make it special client */
+#define QUOTA_CRAWL_PID "-100"
const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT+1] = {
[GF_QUOTA_OPTION_TYPE_NONE] = "none",
@@ -158,7 +160,8 @@ out:
}
int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
+ int type)
{
pid_t pid;
int32_t ret = 0;
@@ -178,6 +181,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
"-s", "localhost",
"--volfile-id", volname,
"--use-readdirp=no",
+ "--client-pid", QUOTA_CRAWL_PID,
"-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
mountdir, NULL);
@@ -210,7 +214,19 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
exit (EXIT_FAILURE);
}
runinit (&runner);
- runner_add_args (&runner, "/usr/bin/find", "find", ".", NULL);
+
+ if (type == GF_QUOTA_OPTION_TYPE_ENABLE)
+
+ runner_add_args (&runner, "/usr/bin/find", "find", ".",
+ NULL);
+
+ else if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+
+ runner_add_args (&runner, "/usr/bin/find", ".",
+ "-exec", "/usr/bin/setfattr", "-n",
+ VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
+ "1", "{}", "\\", ";", NULL);
+
if (runner_start (&runner) == -1)
_exit (EXIT_FAILURE);
@@ -325,7 +341,8 @@ out:
}
int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
+ gf_boolean_t *crawl)
{
int32_t ret = -1;
int i = 0;
@@ -381,6 +398,8 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
if (ret)
goto out;
+ *crawl = _gf_true;
+
(void) glusterd_clean_up_quota_store (volinfo);
ret = 0;
@@ -1045,7 +1064,8 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
break;
case GF_QUOTA_OPTION_TYPE_DISABLE:
- ret = glusterd_quota_disable (volinfo, op_errstr);
+ ret = glusterd_quota_disable (volinfo, op_errstr,
+ &start_crawl);
if (ret < 0)
goto out;
@@ -1129,7 +1149,7 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
}
if (rsp_dict && start_crawl == _gf_true)
- glusterd_quota_initiate_fs_crawl (priv, volname);
+ glusterd_quota_initiate_fs_crawl (priv, volname, type);
if (priv->op_version > GD_OP_VERSION_MIN) {
ret = glusterd_quotad_op (type);