From 7e8eefca2caaaa61e1b31b747384f660c595d9c9 Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Sun, 15 Dec 2013 08:05:04 +0530 Subject: heal: glfs-heal implementation Thanks a lot to Niels for helping me to get build stuff right. Change-Id: I634f24d90cd856ceab3cc0c6e9a91003f443403e BUG: 1147462 Signed-off-by: Pranith Kumar K Reviewed-on: http://review.gluster.org/6529 Reviewed-by: Krutika Dhananjay Tested-by: Gluster Build System --- Makefile.am | 3 +- cli/src/cli-cmd-volume.c | 33 +- configure.ac | 2 + glusterfs.spec.in | 1 + heal/Makefile.am | 3 + heal/src/Makefile.am | 35 ++ heal/src/glfs-heal.c | 610 +++++++++++++++++++++++++ tests/basic/afr/self-heald.t | 161 +++++++ tests/basic/self-heald.t | 44 -- tests/bugs/bug-880898.t | 4 +- xlators/cluster/afr/src/afr-self-heal-common.c | 21 +- xlators/cluster/afr/src/afr-self-heal.h | 7 + xlators/cluster/afr/src/afr-self-heald.c | 2 - xlators/cluster/afr/src/afr-self-heald.h | 6 + 14 files changed, 870 insertions(+), 62 deletions(-) create mode 100644 heal/Makefile.am create mode 100644 heal/src/Makefile.am create mode 100644 heal/src/glfs-heal.c create mode 100644 tests/basic/afr/self-heald.t delete mode 100644 tests/basic/self-heald.t diff --git a/Makefile.am b/Makefile.am index 7bb4b96da2f..187bfd5ed16 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,9 +10,8 @@ EXTRA_DIST = autogen.sh \ $(shell find $(top_srcdir)/tests -type f -print) SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc api xlators glusterfsd \ - $(FUSERMOUNT_SUBDIR) doc extras cli @SYNCDAEMON_SUBDIR@ \ + $(FUSERMOUNT_SUBDIR) doc extras cli heal @SYNCDAEMON_SUBDIR@ \ @UMOUNTD_SUBDIR@ - pkgconfigdir = @pkgconfigdir@ pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c index 15f448b5ba2..a31b82011e0 100644 --- a/cli/src/cli-cmd-volume.c +++ b/cli/src/cli-cmd-volume.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #ifndef _CONFIG_H @@ -1903,6 +1904,10 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word, dict_t *options = NULL; xlator_t *this = NULL; cli_local_t *local = NULL; + int heal_op = 0; + runner_t runner = {0}; + char buff[PATH_MAX] = {0}; + char *out = NULL; this = THIS; frame = create_frame (this, this->ctx->pool); @@ -1921,13 +1926,33 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word, parse_error = 1; goto out; } + ret = dict_get_int32 (options, "heal-op", &heal_op); + if (ret < 0) + goto out; - proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME]; + if (heal_op == GF_AFR_OP_INDEX_SUMMARY) { + runinit (&runner); + runner_add_args (&runner, SBIN_DIR"/glfsheal", words[2], NULL); + runner_redir (&runner, STDOUT_FILENO, RUN_PIPE); + ret = runner_start (&runner); + if (ret == -1) + goto out; + while ((out = fgets(buff, sizeof(buff), + runner_chio (&runner, STDOUT_FILENO)))) { + printf ("%s", out); + } - CLI_LOCAL_INIT (local, words, frame, options); + ret = runner_end (&runner); + ret = WEXITSTATUS (ret); + } + else { + proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME]; - if (proc->fn) { - ret = proc->fn (frame, THIS, options); + CLI_LOCAL_INIT (local, words, frame, options); + + if (proc->fn) { + ret = proc->fn (frame, THIS, options); + } } out: diff --git a/configure.ac b/configure.ac index cc03b571fb1..3757c33df72 100644 --- a/configure.ac +++ b/configure.ac @@ -210,6 +210,8 @@ AC_CONFIG_FILES([Makefile geo-replication/Makefile geo-replication/src/Makefile geo-replication/syncdaemon/Makefile + heal/Makefile + heal/src/Makefile glusterfs.spec]) AC_CANONICAL_HOST diff --git a/glusterfs.spec.in b/glusterfs.spec.in index 1ba37c1eb16..2caecddce97 100644 --- a/glusterfs.spec.in +++ b/glusterfs.spec.in @@ -995,6 +995,7 @@ fi %endif # binaries %{_sbindir}/glusterd +%{_sbindir}/glfsheal %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage* %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix* %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server* diff --git a/heal/Makefile.am b/heal/Makefile.am new file mode 100644 index 00000000000..a985f42a877 --- /dev/null +++ b/heal/Makefile.am @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = diff --git a/heal/src/Makefile.am b/heal/src/Makefile.am new file mode 100644 index 00000000000..65093df35cc --- /dev/null +++ b/heal/src/Makefile.am @@ -0,0 +1,35 @@ +sbin_PROGRAMS = glfsheal + +glfsheal_SOURCES = glfs-heal.c + +glfsheal_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\ + $(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \ + $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \ + $(top_builddir)/api/src/libgfapi.la \ + $(top_builddir)/xlators/cluster/afr/src/afr.la \ + $(GF_GLUSTERFS_LIBS) $(XML_LIBS) + +glfsheal_LDFLAGS = $(GF_LDFLAGS) + +AM_CPPFLAGS = $(GF_CPPFLAGS) \ + -I$(top_srcdir)/xlators/lib/src\ + -I$(top_srcdir)/xlators/cluster/afr/src\ + -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\ + -I$(top_srcdir)/rpc/xdr/src\ + -I$(top_srcdir)/api/src\ + -I$(top_srcdir)/contrib/argp-standalone\ + -DDATADIR=\"$(localstatedir)\" \ + -DCONFDIR=\"$(sysconfdir)/glusterfs\" \ + -DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\ + -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\ + $(XML_CPPFLAGS) + +AM_CFLAGS = -Wall $(GF_GLUSTERFS_CFLAGS) + +CLEANFILES = + +$(top_builddir)/libglusterfs/src/libglusterfs.la: + $(MAKE) -C $(top_builddir)/libglusterfs/src/ all + +$(top_builddir)/xlators/cluster/afr/src/afr.la: + $(MAKE) -C $(top_builddir)/xlators/cluster/afr/src/ all diff --git a/heal/src/glfs-heal.c b/heal/src/glfs-heal.c new file mode 100644 index 00000000000..d02858642ab --- /dev/null +++ b/heal/src/glfs-heal.c @@ -0,0 +1,610 @@ +/* + Copyright (c) 2014 Red Hat, Inc. + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include "glfs.h" +#include "glfs-handles.h" +#include "glfs-internal.h" +#include "syncop.h" +#include +#include +#include "afr.h" +#include "afr-self-heal.h" +#include "afr-self-heald.h" + +#define DEFAULT_HEAL_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs" + +int +glfsh_link_inode_update_loc (loc_t *loc, struct iatt *iattr) +{ + inode_t *link_inode = NULL; + int ret = -1; + + link_inode = inode_link (loc->inode, NULL, NULL, iattr); + if (link_inode == NULL) + goto out; + + inode_unref (loc->inode); + loc->inode = link_inode; + ret = 0; +out: + return ret; +} + +int +glfsh_get_index_dir_loc (loc_t *rootloc, xlator_t *xl, loc_t *dirloc, + int32_t *op_errno) +{ + void *index_gfid = NULL; + int ret = 0; + dict_t *xattr = NULL; + struct iatt iattr = {0}; + struct iatt parent = {0}; + + ret = syncop_getxattr (xl, rootloc, &xattr, GF_XATTROP_INDEX_GFID); + if (ret < 0) { + *op_errno = -ret; + goto out; + } + + ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid); + if (ret < 0) { + *op_errno = EINVAL; + goto out; + } + + uuid_copy (dirloc->gfid, index_gfid); + dirloc->path = ""; + dirloc->inode = inode_new (rootloc->inode->table); + ret = syncop_lookup (xl, dirloc, NULL, + &iattr, NULL, &parent); + dirloc->path = NULL; + if (ret < 0) { + *op_errno = -ret; + goto out; + } + ret = glfsh_link_inode_update_loc (dirloc, &iattr); + if (ret) + goto out; + glfs_loc_touchup (dirloc); + + ret = 0; +out: + if (xattr) + dict_unref (xattr); + return ret; +} + +int +afr_selfheal_locked_metadata_inspect (call_frame_t *frame, xlator_t *this, + inode_t *inode, + gf_boolean_t *metadata_selfheal) +{ + int ret = -1; + unsigned char *locked_on = NULL; + afr_private_t *priv = this->private; + + locked_on = alloca0 (priv->child_count); + + ret = afr_selfheal_inodelk (frame, this, inode, this->name, + LLONG_MAX - 1, 0, locked_on); + { + if (ret == 0) { + /* Not a single lock */ + ret = -afr_final_errno (frame->local, priv); + if (ret == 0) + ret = -ENOTCONN;/* all invalid responses */ + goto out; + } + ret = afr_selfheal_unlocked_inspect (frame, this, inode->gfid, + NULL, NULL, + metadata_selfheal, NULL); + } + afr_selfheal_uninodelk (frame, this, inode, this->name, + LLONG_MAX - 1, 0, locked_on); +out: + return ret; +} + +int +afr_selfheal_locked_data_inspect (call_frame_t *frame, xlator_t *this, + inode_t *inode, gf_boolean_t *data_selfheal) +{ + int ret = -1; + afr_private_t *priv = NULL; + unsigned char *locked_on = NULL; + unsigned char *data_lock = NULL; + + priv = this->private; + locked_on = alloca0 (priv->child_count); + data_lock = alloca0 (priv->child_count); + + ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, + 0, 0, locked_on); + { + if (ret == 0) { + ret = -afr_final_errno (frame->local, priv); + if (ret == 0) + ret = -ENOTCONN;/* all invalid responses */ + goto out; + } + ret = afr_selfheal_inodelk (frame, this, inode, this->name, + 0, 0, data_lock); + { + if (ret == 0) { + ret = -afr_final_errno (frame->local, priv); + if (ret == 0) + ret = -ENOTCONN; + /* all invalid responses */ + goto unlock; + } + ret = afr_selfheal_unlocked_inspect (frame, this, + inode->gfid, NULL, + data_selfheal, + NULL, NULL); + } + afr_selfheal_uninodelk (frame, this, inode, this->name, 0, 0, + data_lock); + } +unlock: + afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, + locked_on); +out: + return ret; +} + +int +afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this, + inode_t *inode, gf_boolean_t *entry_selfheal) +{ + int ret = -1; + afr_private_t *priv = NULL; + unsigned char *locked_on = NULL; + unsigned char *data_lock = NULL; + + priv = this->private; + locked_on = alloca0 (priv->child_count); + data_lock = alloca0 (priv->child_count); + + ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain, + NULL, locked_on); + { + if (ret == 0) { + ret = -afr_final_errno (frame->local, priv); + if (ret == 0) + ret = -ENOTCONN;/* all invalid responses */ + goto out; + } + + ret = afr_selfheal_entrylk (frame, this, inode, this->name, + NULL, data_lock); + { + if (ret == 0) { + ret = -afr_final_errno (frame->local, priv); + if (ret == 0) + ret = -ENOTCONN; + /* all invalid responses */ + goto unlock; + } + ret = afr_selfheal_unlocked_inspect (frame, this, + inode->gfid, + NULL, NULL, NULL, + entry_selfheal); + } + afr_selfheal_unentrylk (frame, this, inode, this->name, NULL, + data_lock); + } +unlock: + afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL, + locked_on); +out: + return ret; +} + +int +afr_selfheal_locked_inspect (call_frame_t *frame, xlator_t *this, uuid_t gfid, + inode_t **inode, + gf_boolean_t *data_selfheal, + gf_boolean_t *metadata_selfheal, + gf_boolean_t *entry_selfheal) +{ + int ret = -1; + gf_boolean_t dsh = _gf_false; + gf_boolean_t msh = _gf_false; + gf_boolean_t esh = _gf_false; + + ret = afr_selfheal_unlocked_inspect (frame, this, gfid, inode, + &dsh, &msh, &esh); + if (ret) + goto out; + + /* For every heal type hold locks and check if it indeed needs heal */ + + if (msh) { + msh = _gf_false; + ret = afr_selfheal_locked_metadata_inspect (frame, this, + *inode, &msh); + if (msh || ret < 0) + goto out; + } + + if (dsh) { + dsh = _gf_false; + ret = afr_selfheal_locked_data_inspect (frame, this, *inode, + &dsh); + if (dsh || ret < 0) + goto out; + } + + if (esh) { + esh = _gf_false; + ret = afr_selfheal_locked_entry_inspect (frame, this, *inode, + &esh); + } + +out: + if (entry_selfheal) + *entry_selfheal = esh; + if (data_selfheal) + *data_selfheal = dsh; + if (metadata_selfheal) + *metadata_selfheal = msh; + return ret; +} + +static xlator_t* +_get_afr_ancestor (xlator_t *xl) +{ + if (!xl || !xl->parents) + return NULL; + + while (xl->parents) { + xl = xl->parents->xlator; + if (!xl) + break; + if (strcmp (xl->type, "cluster/replicate") == 0) + return xl; + } + + return NULL; +} + +static int +glfsh_process_entries (xlator_t *xl, fd_t *fd, gf_dirent_t *entries, + off_t *offset, uint64_t *num_entries) +{ + gf_dirent_t *entry = NULL; + gf_dirent_t *tmp = NULL; + int ret = 0; + char *path = NULL; + uuid_t gfid = {0}; + inode_t *inode = NULL; + call_frame_t *frame = NULL; + xlator_t *this = NULL; + gf_boolean_t data_selfheal = _gf_false; + gf_boolean_t metadata_selfheal = _gf_false; + gf_boolean_t entry_selfheal = _gf_false; + + this = THIS; + frame = afr_frame_create (this); + if (!frame) { + ret = -1; + goto out; + } + + list_for_each_entry_safe (entry, tmp, &entries->list, list) { + *offset = entry->d_off; + if ((strcmp (entry->d_name, ".") == 0) || + (strcmp (entry->d_name, "..") == 0)) + continue; + + uuid_clear (gfid); + GF_FREE (path); + path = NULL; + if (inode) { + inode_forget (inode, 1); + inode_unref (inode); + inode = NULL; + } + + AFR_STACK_RESET (frame); + uuid_parse (entry->d_name, gfid); + ret = afr_selfheal_locked_inspect (frame, this, gfid, &inode, + &data_selfheal, + &metadata_selfheal, + &entry_selfheal); + if (ret == 0) { + if (!entry_selfheal && !metadata_selfheal && + !data_selfheal) + continue; + } + + ret = afr_shd_gfid_to_path (this, xl, gfid, &path); + + if (ret == -ENOENT || ret == -ESTALE) { + afr_shd_index_purge (xl, fd->inode, entry->d_name); + ret = 0; + continue; + } + + (*num_entries)++; + printf ("%s\n", path ? path : uuid_utoa (inode->gfid)); + } + ret = 0; +out: + if (inode) { + inode_forget (inode, 1); + inode_unref (inode); + } + + GF_FREE (path); + return ret; +} + +static int +glfsh_crawl_directory (xlator_t *readdir_xl, fd_t *fd, loc_t *loc) +{ + off_t offset = 0; + gf_dirent_t entries; + int ret = 0; + gf_boolean_t free_entries = _gf_false; + uint64_t num_entries = 0; + + INIT_LIST_HEAD (&entries.list); + + while (1) { + ret = syncop_readdir (readdir_xl, fd, 131072, offset, &entries); + if (ret <= 0) + break; + ret = 0; + free_entries = _gf_true; + + if (list_empty (&entries.list)) + goto out; + + ret = glfsh_process_entries (readdir_xl, fd, &entries, &offset, + &num_entries); + if (ret < 0) + goto out; + + gf_dirent_free (&entries); + free_entries = _gf_false; + } + ret = 0; +out: + if (free_entries) + gf_dirent_free (&entries); + if (ret < 0) { + printf ("Failed to complete gathering info. " + "Number of entries so far: %"PRIu64"\n", num_entries); + } + else { + printf ("Number of entries: %"PRIu64"\n", num_entries); + } + return ret; +} + +static int +glfsh_print_brick (xlator_t *xl, loc_t *rootloc) +{ + int ret = 0; + dict_t *xattr = NULL; + char *pathinfo = NULL; + char *brick_start = NULL; + char *brick_end = NULL; + + ret = syncop_getxattr (xl, rootloc, &xattr, GF_XATTR_PATHINFO_KEY); + if (ret < 0) + goto out; + + ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &pathinfo); + if (ret < 0) + goto out; + + brick_start = strchr (pathinfo, ':') + 1; + brick_end = pathinfo + strlen (pathinfo) - 1; + *brick_end = 0; + printf ("Brick %s\n", brick_start); + +out: + if (xattr) + dict_unref (xattr); + return ret; +} + +void +glfsh_print_brick_from_xl (xlator_t *xl) +{ + char *remote_host = NULL; + char *remote_subvol = NULL; + int ret = 0; + + ret = dict_get_str (xl->options, "remote-host", &remote_host); + if (ret < 0) + goto out; + + ret = dict_get_str (xl->options, "remote-subvolume", &remote_subvol); + if (ret < 0) + goto out; +out: + if (ret < 0) + printf ("Brick - Not able to get brick information\n"); + else + printf ("Brick %s:%s\n", remote_host, remote_subvol); +} + +void +glfsh_print_pending_heals (xlator_t *xl, loc_t *rootloc) +{ + int ret = 0; + loc_t dirloc = {0}; + fd_t *fd = NULL; + int32_t op_errno = 0; + + ret = glfsh_print_brick (xl, rootloc); + if (ret < 0) { + glfsh_print_brick_from_xl (xl); + printf ("Status: %s\n", strerror (-ret)); + goto out; + } + + ret = glfsh_get_index_dir_loc (rootloc, xl, &dirloc, &op_errno); + if (ret < 0) { + if (op_errno == ESTALE || op_errno == ENOENT) + printf ("Number of entries: 0\n"); + else + printf ("Status: %s\n", strerror (op_errno)); + goto out; + } + + fd = fd_anonymous (dirloc.inode); + ret = glfsh_crawl_directory (xl, fd, &dirloc); + if (fd) + fd_unref (fd); + if (ret < 0) + printf ("Failed to find entries with pending self-heal\n"); +out: + loc_wipe (&dirloc); + return; +} + +static int +glfsh_validate_replicate_volume (xlator_t *xl) +{ + xlator_t *afr_xl = NULL; + int ret = -1; + + while (xl->next) + xl = xl->next; + + while (xl) { + if (strcmp (xl->type, "protocol/client") == 0) { + afr_xl = _get_afr_ancestor (xl); + if (afr_xl) { + ret = 0; + break; + } + } + + xl = xl->prev; + } + + return ret; +} + +int +main (int argc, char **argv) +{ + glfs_t *fs = NULL; + int ret = 0; + char *volname = NULL; + xlator_t *top_subvol = NULL; + xlator_t *xl = NULL; + loc_t rootloc = {0}; + char logfilepath[PATH_MAX] = {0}; + xlator_t *old_THIS = NULL; + xlator_t *afr_xl = NULL; + + if (argc != 2) { + printf ("Usage: %s \n", argv[0]); + ret = -1; + goto out; + } + volname = argv[1]; + + fs = glfs_new (volname); + if (!fs) { + ret = -1; + printf ("Not able to initialize volume '%s'\n", volname); + goto out; + } + + ret = glfs_set_volfile_server (fs, "tcp", "localhost", 24007); + snprintf (logfilepath, sizeof (logfilepath), + DEFAULT_HEAL_LOG_FILE_DIRECTORY"/glfsheal-%s.log", volname); + ret = glfs_set_logging(fs, logfilepath, GF_LOG_INFO); + if (ret < 0) { + ret = -1; + printf ("Not able to initialize volume '%s'\n", volname); + goto out; + } + + ret = glfs_init (fs); + if (ret < 0) { + ret = -1; + if (errno == ENOENT) { + printf ("Volume %s does not exist\n", volname); + } + else { + printf ("%s: Not able to fetch volfile from " + "glusterd\n", volname); + } + goto out; + } + + sleep (2); + __glfs_entry_fs (fs); + top_subvol = glfs_active_subvol (fs); + if (!top_subvol) { + ret = -1; + if (errno == ENOTCONN) { + printf ("Volume %s is not started (Or) All the bricks " + "are not running.\n", volname); + } + else { + printf ("%s: Not able to mount the volume, %s\n", + volname, strerror (errno)); + } + goto out; + } + + ret = glfsh_validate_replicate_volume (top_subvol); + if (ret < 0) { + printf ("Volume %s is not of type replicate\n", volname); + goto out; + } + rootloc.inode = inode_ref (top_subvol->itable->root); + glfs_loc_touchup (&rootloc); + + xl = top_subvol; + while (xl->next) + xl = xl->next; + + while (xl) { + if (strcmp (xl->type, "protocol/client") == 0) { + afr_xl = _get_afr_ancestor (xl); + if (afr_xl) { + old_THIS = THIS; + THIS = afr_xl; + glfsh_print_pending_heals (xl, &rootloc); + THIS = old_THIS; + printf("\n"); + } + } + + xl = xl->prev; + } + + loc_wipe (&rootloc); + glfs_subvol_done (fs, top_subvol); + glfs_fini (fs); + + return 0; +out: + if (fs && top_subvol) + glfs_subvol_done (fs, top_subvol); + loc_wipe (&rootloc); + if (fs) + glfs_fini (fs); + + return ret; +} diff --git a/tests/basic/afr/self-heald.t b/tests/basic/afr/self-heald.t new file mode 100644 index 00000000000..8500a14d8c3 --- /dev/null +++ b/tests/basic/afr/self-heald.t @@ -0,0 +1,161 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +function disconnected_brick_count { + local vol=$1 + $CLI volume heal $vol info | grep -i transport | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5} +TEST $CLI volume set $V0 cluster.background-self-heal-count 0 +TEST $CLI volume set $V0 cluster.eager-lock off +TEST $CLI volume start $V0 +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill_brick $V0 $H0 $B0/${V0}4 +cd $M0 +HEAL_FILES=0 +for i in {1..10} +do + dd if=/dev/urandom of=f bs=1M count=10 2>/dev/null + HEAL_FILES=$(($HEAL_FILES+1)) #+1 for data/metadata self-heal of 'f' + mkdir a; cd a; + #+3 for metadata self-heal of 'a' one per subvolume of DHT + HEAL_FILES=$(($HEAL_FILES+3)) +done +#+3 represents entry sh on "/", one per subvolume of DHT? +HEAL_FILES=$(($HEAL_FILES + 3)) + +cd ~ +EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 + +#When bricks are down, it says Transport End point Not connected for them +EXPECT "3" disconnected_brick_count $V0 + +#Create some stale indices and verify that they are not counted in heal info +#TO create stale index create and delete files when one brick is down in +#replica pair. +for i in {11..20}; do echo abc > $M0/$i; done +HEAL_FILES=$(($HEAL_FILES + 10)) #count extra 10 files +EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 +#delete the files now, so that stale indices will remain. +for i in {11..20}; do rm -f $M0/$i; done +#After deleting files they should not appear in heal info +HEAL_FILES=$(($HEAL_FILES - 10)) +EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 + + +TEST ! $CLI volume heal $V0 +TEST $CLI volume set $V0 cluster.self-heal-daemon off +TEST ! $CLI volume heal $V0 +TEST ! $CLI volume heal $V0 full +TEST $CLI volume start $V0 force +TEST $CLI volume set $V0 cluster.self-heal-daemon on +EXPECT_WITHIN 20 "Y" glustershd_up_status +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 2 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 4 +TEST $CLI volume heal $V0 +sleep 5 #Until the heal-statistics command implementation +#check that this heals the contents partially +TEST [ $HEAL_FILES -gt $(afr_get_pending_heal_count $V0) ] + +TEST $CLI volume heal $V0 full +EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 + +#Test that ongoing IO is not considered as Pending heal +(dd if=/dev/zero of=$M0/file1 bs=1K 2>/dev/null 1>/dev/null)& +back_pid1=$!; +(dd if=/dev/zero of=$M0/file2 bs=1K 2>/dev/null 1>/dev/null)& +back_pid2=$!; +(dd if=/dev/zero of=$M0/file3 bs=1K 2>/dev/null 1>/dev/null)& +back_pid3=$!; +(dd if=/dev/zero of=$M0/file4 bs=1K 2>/dev/null 1>/dev/null)& +back_pid4=$!; +(dd if=/dev/zero of=$M0/file5 bs=1K 2>/dev/null 1>/dev/null)& +back_pid5=$!; +EXPECT 0 afr_get_pending_heal_count $V0 +kill -SIGTERM $back_pid1; +kill -SIGTERM $back_pid2; +kill -SIGTERM $back_pid3; +kill -SIGTERM $back_pid4; +kill -SIGTERM $back_pid5; +wait >/dev/null 2>&1; + +#Test that volume heal info reports files even when self-heal +#options are disabled +TEST touch $M0/f +TEST mkdir $M0/d +#DATA +TEST $CLI volume set $V0 cluster.data-self-heal off +EXPECT "off" volume_option $V0 cluster.data-self-heal +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill_brick $V0 $H0 $B0/${V0}4 +echo abc > $M0/f +EXPECT 1 afr_get_pending_heal_count $V0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN 20 "Y" glustershd_up_status +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 2 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 4 +TEST $CLI volume heal $V0 +EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +TEST $CLI volume set $V0 cluster.data-self-heal on + +#METADATA +TEST $CLI volume set $V0 cluster.metadata-self-heal off +EXPECT "off" volume_option $V0 cluster.metadata-self-heal +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill_brick $V0 $H0 $B0/${V0}4 +TEST chmod 777 $M0/f +EXPECT 1 afr_get_pending_heal_count $V0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN 20 "Y" glustershd_up_status +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 2 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 4 +TEST $CLI volume heal $V0 +EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +TEST $CLI volume set $V0 cluster.metadata-self-heal on + +#ENTRY +TEST $CLI volume set $V0 cluster.entry-self-heal off +EXPECT "off" volume_option $V0 cluster.entry-self-heal +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill_brick $V0 $H0 $B0/${V0}4 +TEST touch $M0/d/a +EXPECT 2 afr_get_pending_heal_count $V0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN 20 "Y" glustershd_up_status +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 0 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 2 +EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 4 +TEST $CLI volume heal $V0 +EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0 +TEST $CLI volume set $V0 cluster.entry-self-heal on + +#Negative test cases +#Fail volume does not exist case +TEST ! $CLI volume heal fail info + +#Fail volume stopped case +TEST $CLI volume stop $V0 +TEST ! $CLI volume heal $V0 info + +#Fail non-replicate volume info +TEST $CLI volume delete $V0 +TEST $CLI volume create $V0 $H0:$B0/${V0}{6} +TEST $CLI volume start $V0 +TEST ! $CLI volume heal $V0 info + +cleanup diff --git a/tests/basic/self-heald.t b/tests/basic/self-heald.t deleted file mode 100644 index 0e4d104cd13..00000000000 --- a/tests/basic/self-heald.t +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/../include.rc -. $(dirname $0)/../volume.rc - -cleanup; - -TEST glusterd -TEST pidof glusterd -TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5} -TEST $CLI volume set $V0 cluster.background-self-heal-count 0 -TEST $CLI volume set $V0 cluster.eager-lock off -TEST $CLI volume start $V0 -TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 -TEST kill_brick $V0 $H0 $B0/${V0}0 -TEST kill_brick $V0 $H0 $B0/${V0}2 -TEST kill_brick $V0 $H0 $B0/${V0}4 -cd $M0 -HEAL_FILES=0 -for i in {1..10} -do - dd if=/dev/urandom of=f bs=1024k count=10 2>/dev/null - HEAL_FILES=$(($HEAL_FILES+1)) - mkdir a; cd a; - HEAL_FILES=$(($HEAL_FILES+3)) #As many times as distribute subvols -done -HEAL_FILES=$(($HEAL_FILES + 3)) #Count the brick root dir - -cd ~ -EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0 -TEST ! $CLI volume heal $V0 -TEST $CLI volume set $V0 cluster.self-heal-daemon off -TEST ! $CLI volume heal $V0 info -TEST ! $CLI volume heal $V0 -TEST $CLI volume start $V0 force -TEST $CLI volume set $V0 cluster.self-heal-daemon on -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status -EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 -EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2 -EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 4 - -TEST $CLI volume heal $V0 full -EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0 -cleanup diff --git a/tests/bugs/bug-880898.t b/tests/bugs/bug-880898.t index 8ba35ec1a59..4b9fb50a522 100644 --- a/tests/bugs/bug-880898.t +++ b/tests/bugs/bug-880898.t @@ -17,7 +17,7 @@ do fi done -gluster volume heal $V0 info | grep "Status: self-heal-daemon is not running on $uuid"; -EXPECT "0" echo $? +#Command execution should fail reporting that the bricks are not running. +TEST ! $CLI volume heal $V0 info cleanup; diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c index 14a514beffa..6198d4cf72c 100644 --- a/xlators/cluster/afr/src/afr-self-heal-common.c +++ b/xlators/cluster/afr/src/afr-self-heal-common.c @@ -932,13 +932,14 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, if (replies[i].op_ret == -1) continue; - if (afr_is_data_set (this, replies[i].xdata)) + if (data_selfheal && afr_is_data_set (this, replies[i].xdata)) *data_selfheal = _gf_true; - if (afr_is_metadata_set (this, replies[i].xdata)) + if (metadata_selfheal && + afr_is_metadata_set (this, replies[i].xdata)) *metadata_selfheal = _gf_true; - if (afr_is_entry_set (this, replies[i].xdata)) + if (entry_selfheal && afr_is_entry_set (this, replies[i].xdata)) *entry_selfheal = _gf_true; valid_cnt ++; @@ -967,7 +968,8 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, priv->children[i]->name, uuid_utoa (replies[i].poststat.ia_gfid)); - *metadata_selfheal = _gf_true; + if (metadata_selfheal) + *metadata_selfheal = _gf_true; } if (!IA_EQUAL (first, replies[i].poststat, gid)) { @@ -978,7 +980,8 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, priv->children[i]->name, uuid_utoa (replies[i].poststat.ia_gfid)); - *metadata_selfheal = _gf_true; + if (metadata_selfheal) + *metadata_selfheal = _gf_true; } if (!IA_EQUAL (first, replies[i].poststat, prot)) { @@ -989,7 +992,8 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, priv->children[i]->name, uuid_utoa (replies[i].poststat.ia_gfid)); - *metadata_selfheal = _gf_true; + if (metadata_selfheal) + *metadata_selfheal = _gf_true; } if (IA_ISREG(first.ia_type) && @@ -1001,11 +1005,12 @@ afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, priv->children[i]->name, uuid_utoa (replies[i].poststat.ia_gfid)); - *data_selfheal = _gf_true; + if (data_selfheal) + *data_selfheal = _gf_true; } } - if (valid_cnt > 0) { + if (valid_cnt > 0 && link_inode) { *link_inode = afr_inode_link (inode, &first); if (!*link_inode) { ret = -EINVAL; diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h index f208e6bc813..bb3f0e7029f 100644 --- a/xlators/cluster/afr/src/afr-self-heal.h +++ b/xlators/cluster/afr/src/afr-self-heal.h @@ -198,4 +198,11 @@ afr_mark_active_sinks (xlator_t *this, unsigned char *sources, gf_boolean_t afr_does_witness_exist (xlator_t *this, uint64_t *witness); + +int +afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this, + uuid_t gfid, inode_t **link_inode, + gf_boolean_t *data_selfheal, + gf_boolean_t *metadata_selfheal, + gf_boolean_t *entry_selfheal); #endif /* !_AFR_SELFHEAL_H */ diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c index 10e19964ed9..a341015303c 100644 --- a/xlators/cluster/afr/src/afr-self-heald.c +++ b/xlators/cluster/afr/src/afr-self-heald.c @@ -40,8 +40,6 @@ #define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n]) #define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n]) -int afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p); - char * afr_subvol_name (xlator_t *this, int subvol) { diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h index 59f06b79cff..02b26b8061f 100644 --- a/xlators/cluster/afr/src/afr-self-heald.h +++ b/xlators/cluster/afr/src/afr-self-heald.h @@ -67,4 +67,10 @@ afr_selfheal_daemon_init (xlator_t *this); int afr_xl_op (xlator_t *this, dict_t *input, dict_t *output); +int +afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, + char **path_p); + +int +afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name); #endif /* !_AFR_SELF_HEALD_H */ -- cgit