From 36e39bf9cb4b7c712ef411d60203bf05af7e129f Mon Sep 17 00:00:00 2001 From: "Anand V. Avati" Date: Wed, 25 Mar 2009 18:08:43 +0530 Subject: Enhancements to distribute selfheal - create missing directories instead of creating linkfiles when entry missing on hashed subvol - detect cases where there are dirs and linkfiles for a name and make them all dirs Signed-off-by: Anand V. Avati --- xlators/cluster/dht/src/dht-common.c | 157 ++++++++++++++++++++++++----------- xlators/cluster/dht/src/dht-common.h | 4 + xlators/cluster/dht/src/dht-layout.c | 7 +- 3 files changed, 117 insertions(+), 51 deletions(-) (limited to 'xlators/cluster/dht') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index d7a16fa42..86ec00a6f 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -99,10 +99,6 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, else mkdir/chmod/chown and fix */ - /* TODO: assert equal hash type in xattr, local->xattr */ - - /* TODO: always ensure same subvolume is in layout->list[0] */ - ret = dht_layout_merge (this, layout, prev->this, op_ret, op_errno, xattr); @@ -117,8 +113,14 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, } is_dir = check_is_dir (inode, stbuf, xattr); - if (!is_dir) + if (!is_dir) { + gf_log (this->name, GF_LOG_WARNING, + "lookup of %s on %s returned non dir 0%o", + local->loc.path, prev->this->name, + stbuf->st_mode); + local->need_selfheal = 1; goto unlock; + } local->op_ret = 0; if (local->xattr == NULL) @@ -139,6 +141,12 @@ unlock: this_call_cnt = dht_frame_return (frame); if (is_last_call (this_call_cnt)) { + if (local->need_selfheal) { + local->need_selfheal = 0; + dht_lookup_everywhere (frame, this, &local->loc); + return 0; + } + if (local->op_ret == 0) { ret = dht_layout_normalize (this, &local->loc, layout); @@ -153,13 +161,14 @@ unlock: goto selfheal; } - inode_ctx_put (local->inode, this, (uint64_t)(long)layout); + inode_ctx_put (local->inode, this, + (uint64_t)(long)layout); if (local->st_ino) { local->stbuf.st_ino = local->st_ino; } else { gf_log (this->name, GF_LOG_WARNING, - "could not find hashed subvolume for %s", + "could not find hashed subvol for %s", local->loc.path); } } @@ -355,28 +364,38 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this, if (is_linkfile) { link_subvol = dht_linkfile_subvol (this, inode, buf, xattr); - gf_log (this->name, GF_LOG_DEBUG, + gf_log (this->name, GF_LOG_WARNING, "found on %s linkfile %s (-> %s)", subvol->name, loc->path, link_subvol ? link_subvol->name : "''"); goto unlock; - } else { - gf_log (this->name, GF_LOG_DEBUG, - "found on %s file %s", - subvol->name, loc->path); } - if (!local->cached_subvol) { - /* found one file */ - dht_stat_merge (this, &local->stbuf, buf, subvol); - local->xattr = dict_ref (xattr); - local->cached_subvol = subvol; - } else { - gf_log (this->name, GF_LOG_WARNING, - "multiple subvolumes (%s and %s atleast) have " - "file %s", local->cached_subvol->name, - subvol->name, local->loc.path); - } + if (is_dir) { + local->dir_count++; + + gf_log (this->name, GF_LOG_WARNING, + "found on %s directory %s", + subvol->name, loc->path); + } else { + local->file_count++; + + if (!local->cached_subvol) { + /* found one file */ + dht_stat_merge (this, &local->stbuf, buf, + subvol); + local->xattr = dict_ref (xattr); + local->cached_subvol = subvol; + gf_log (this->name, GF_LOG_DEBUG, + "found on %s file %s", + subvol->name, loc->path); + } else { + gf_log (this->name, GF_LOG_WARNING, + "multiple subvolumes (%s and %s) have " + "file %s", local->cached_subvol->name, + subvol->name, local->loc.path); + } + } } unlock: UNLOCK (&frame->lock); @@ -393,6 +412,20 @@ unlock: hashed_subvol = local->hashed_subvol; cached_subvol = local->cached_subvol; + if (local->file_count && local->dir_count) { + gf_log (this->name, GF_LOG_ERROR, + "path %s is both file and directory at the " + "backend. Please fix it manually", + loc->path); + DHT_STACK_UNWIND (frame, -1, EIO, NULL, NULL, NULL); + return 0; + } + + if (local->dir_count) { + dht_lookup_directory (frame, this, &local->loc); + return 0; + } + if (!cached_subvol) { DHT_STACK_UNWIND (frame, -1, ENOENT, NULL, NULL, NULL); return 0; @@ -459,12 +492,22 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie, gf_log (this->name, GF_LOG_WARNING, "lookup of %s on %s (following linkfile) failed (%s)", local->loc.path, subvol->name, strerror (op_errno)); - - dht_lookup_everywhere (frame, this, loc); - return 0; + goto err; } - /* TODO: assert type is non-dir and non-linkfile */ + if (check_is_dir (inode, stbuf, xattr)) { + gf_log (this->name, GF_LOG_WARNING, + "lookup of %s on %s (following linkfile) reached dir", + local->loc.path, subvol->name); + goto err; + } + + if (check_is_linkfile (inode, stbuf, xattr)) { + gf_log (this->name, GF_LOG_WARNING, + "lookup of %s on %s (following linkfile) reached link", + local->loc.path, subvol->name); + goto err; + } if (stbuf->st_nlink == 1) stbuf->st_mode |= S_ISVTX; @@ -486,6 +529,43 @@ out: DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf, xattr); return 0; + +err: + dht_lookup_everywhere (frame, this, loc); + + return 0; +} + + +int +dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc) +{ + int call_cnt = 0; + int i = 0; + dht_conf_t *conf = NULL; + dht_local_t *local = NULL; + + conf = this->private; + local = frame->local; + + call_cnt = conf->subvolume_cnt; + local->call_cnt = call_cnt; + + local->layout = dht_layout_new (this, conf->subvolume_cnt); + if (!local->layout) { + gf_log (this->name, GF_LOG_ERROR, + "memory allocation failed :("); + DHT_STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL, NULL); + return 0; + } + + for (i = 0; i < call_cnt; i++) { + STACK_WIND (frame, dht_lookup_dir_cbk, + conf->subvolumes[i], + conf->subvolumes[i]->fops->lookup, + &local->loc, local->xattr_req); + } + return 0; } @@ -501,9 +581,7 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, dht_conf_t *conf = NULL; dht_local_t *local = NULL; loc_t *loc = NULL; - int i = 0; call_frame_t *prev = NULL; - int call_cnt = 0; conf = this->private; @@ -529,25 +607,8 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, } if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) { - call_cnt = conf->subvolume_cnt; - local->call_cnt = call_cnt; - - local->layout = dht_layout_new (this, conf->subvolume_cnt); - if (!local->layout) { - op_ret = -1; - op_errno = ENOMEM; - gf_log (this->name, GF_LOG_ERROR, - "memory allocation failed :("); - goto out; - } - - for (i = 0; i < call_cnt; i++) { - STACK_WIND (frame, dht_lookup_dir_cbk, - conf->subvolumes[i], - conf->subvolumes[i]->fops->lookup, - &local->loc, local->xattr_req); - } - return 0; + dht_lookup_directory (frame, this, &local->loc); + return 0; } if (op_ret == -1) diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index ed154dc12..0eb57a196 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -70,6 +70,8 @@ struct dht_local { xlator_t *cached_subvol; xlator_t *hashed_subvol; char need_selfheal; + int file_count; + int dir_count; struct { fop_mknod_cbk_t linkfile_cbk; struct stat stbuf; @@ -200,6 +202,8 @@ int dht_hash_compute (int type, const char *name, uint32_t *hash_p); int dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk, xlator_t *tovol, xlator_t *fromvol, loc_t *loc); +int dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc); +int dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc); int dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk, loc_t *loc, dht_layout_t *layout); diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c index a45625876..45a7df601 100644 --- a/xlators/cluster/dht/src/dht-layout.c +++ b/xlators/cluster/dht/src/dht-layout.c @@ -514,6 +514,7 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol, int pos = -1; int ret = 0; int err = 0; + int dict_ret = 0; int32_t *disk_layout = NULL; int32_t count = -1; uint32_t start_off = -1; @@ -547,10 +548,10 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol, goto out; } - ret = dict_get_ptr (xattr, "trusted.glusterfs.dht", - VOID(&disk_layout)); + dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht", + VOID(&disk_layout)); - if (ret < 0) { + if (dict_ret < 0) { if (err == 0) { gf_log (this->name, GF_LOG_ERROR, "%s - disk layout missing", loc->path); -- cgit