summaryrefslogtreecommitdiffstats
path: root/xlators/cluster/dht/src/dht-selfheal.c
diff options
context:
space:
mode:
authorKotresh HR <khiremat@redhat.com>2017-01-03 02:35:06 -0500
committerRaghavendra G <rgowdapp@redhat.com>2017-04-26 09:00:34 +0000
commit4076b73b2f4fb3cca0737974b124f33f76f9c9c1 (patch)
treecff52055113fd04c28d5a99719036d59522a51ff /xlators/cluster/dht/src/dht-selfheal.c
parent1538c98f5e33e0794830d5153f17a96ff28c9914 (diff)
feature/dht: Directory synchronization
Design doc: https://review.gluster.org/16876 Directory creation is now synchronized with blocking inodelk of the parent on the hashed subvolume followed by the entrylk on the hashed subvolume between dht_mkdir, dht_rmdir, dht_rename_dir and lookup selfheal mkdir. To maintain internal consistency of directories across all subvols of dht, we need locks. Specifically we are interested in: 1. Consistency of layout of a directory. Only one writer should modify the layout at a time. A writer (layout setting during directory heal as part of lookup) shouldn't modify the layout while there are readers (all other fops like create, mkdir etc., which consume layout) and readers shouldn't read the layout while a writer is in progress. Readers can read the layout simultaneously. Writer takes a WRITE inodelk on the directory (whose layout is being modified) across ALL subvols. Reader takes a READ inodelk on the directory (whose layout is being read) on ANY subvol. 2. Consistency of directory namespace across subvols. The path and associated gfid should be same on all subvols. A gfid should not be associated with more than one path on any subvol. All fops that can change directory names (mkdir, rmdir, renamedir, directory creation phase in lookup-heal) takes an entrylk on hashed subvol of the directory. NOTE1: In point 2 above, since dht takes entrylk on hashed subvol of a directory, the transaction itself is a consumer of layout on parent directory. So, the transaction is a reader of parent layout and does an inodelk on parent directory just like any other layout reader. So a mkdir (dir/subdir) would: > Acquire a READ inodelk on "dir" on any subvol. > Acquire an entrylk (dir, "subdir") on hashed subvol of "subdir". > creates directory on hashed subvol and possibly on non-hashed subvols. > UNLOCK (entrylk) > UNLOCK (inodelk) NOTE2: mkdir fop while setting the layout of the directory being created is considered as a reader, but NOT a writer. The reason is for a fop which can consume the layout of a directory to come either of the following conditions has to be true: > mkdir syscall from application has to complete. In this case no need of synchronization. > A lookup issued on the directory racing with mkdir has to complete. Since layout setting by a lookup is considered as a writer, only one of either mkdir or lookup will set the layout. Code re-organization: All the lock related routines are moved to "dht-lock.c" file. New wrapper function is introduced to take blocking inodelk followed by entrylk 'dht_protect_namespace' Updates #191 Change-Id: I01569094dfbe1852de6f586475be79c1ba965a31 Signed-off-by: Kotresh HR <khiremat@redhat.com> BUG: 1443373 Reviewed-on: https://review.gluster.org/15472 NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Smoke: Gluster Build System <jenkins@build.gluster.org>
Diffstat (limited to 'xlators/cluster/dht/src/dht-selfheal.c')
-rw-r--r--xlators/cluster/dht/src/dht-selfheal.c99
1 files changed, 37 insertions, 62 deletions
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 0838a62..de9d30c 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -13,6 +13,7 @@
#include "xlator.h"
#include "dht-common.h"
#include "dht-messages.h"
+#include "dht-lock.h"
#include "glusterfs-acl.h"
#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,path) do { \
@@ -85,7 +86,13 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret,
int lock_count = 0;
local = frame->local;
- lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
+
+ /* Unlock entrylk */
+ dht_unlock_entrylk_wrapper (frame, &local->lock[0].ns.directory_ns);
+
+ /* Unlock inodelk */
+ lock_count = dht_lock_count (local->lock[0].ns.parent_layout.locks,
+ local->lock[0].ns.parent_layout.lk_count);
if (lock_count == 0)
goto done;
@@ -100,14 +107,15 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret,
goto done;
}
- lock_local->lock.locks = local->lock.locks;
- lock_local->lock.lk_count = local->lock.lk_count;
+ lock_local->lock[0].ns.parent_layout.locks = local->lock[0].ns.parent_layout.locks;
+ lock_local->lock[0].ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
- local->lock.locks = NULL;
- local->lock.lk_count = 0;
+ local->lock[0].ns.parent_layout.locks = NULL;
+ local->lock[0].ns.parent_layout.lk_count = 0;
- dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
- lock_local->lock.lk_count,
+ dht_unlock_inodelk (lock_frame,
+ lock_local->lock[0].ns.parent_layout.locks,
+ lock_local->lock[0].ns.parent_layout.lk_count,
dht_selfheal_unlock_cbk);
lock_frame = NULL;
@@ -579,7 +587,8 @@ dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
lk_array[i] = dht_lock_new (frame->this,
conf->subvolumes[i],
&local->loc, F_WRLCK,
- DHT_LAYOUT_HEAL_DOMAIN);
+ DHT_LAYOUT_HEAL_DOMAIN,
+ NULL);
if (lk_array[i] == NULL) {
gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
@@ -604,7 +613,7 @@ dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
lk_array[0] = dht_lock_new (frame->this, local->hashed_subvol,
&local->loc, F_WRLCK,
- DHT_LAYOUT_HEAL_DOMAIN);
+ DHT_LAYOUT_HEAL_DOMAIN, NULL);
if (lk_array[0] == NULL) {
gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
@@ -615,14 +624,14 @@ dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
}
}
- local->lock.locks = lk_array;
- local->lock.lk_count = count;
+ local->lock[0].layout.my_layout.locks = lk_array;
+ local->lock[0].layout.my_layout.lk_count = count;
ret = dht_blocking_inodelk (frame, lk_array, count, FAIL_ON_ANY_ERROR,
dht_selfheal_layout_lock_cbk);
if (ret < 0) {
- local->lock.locks = NULL;
- local->lock.lk_count = 0;
+ local->lock[0].layout.my_layout.locks = NULL;
+ local->lock[0].layout.my_layout.lk_count = 0;
goto err;
}
@@ -1454,8 +1463,8 @@ dht_selfheal_dir_mkdir_lock_cbk (call_frame_t *frame, void *cookie,
}
gf_msg (this->name, GF_LOG_WARNING, op_errno,
- DHT_MSG_INODE_LK_ERROR,
- "acquiring inodelk failed for %s",
+ DHT_MSG_ENTRYLK_ERROR,
+ "acquiring entrylk after inodelk failed for %s",
local->loc.path);
local->op_errno = op_errno;
@@ -1487,15 +1496,9 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
int missing_dirs = 0;
int i = 0;
int ret = -1;
- int count = 1;
dht_local_t *local = NULL;
- dht_conf_t *conf = NULL;
- xlator_t *this = NULL;
- dht_lock_t **lk_array = NULL;
local = frame->local;
- this = frame->this;
- conf = this->private;
local->selfheal.force_mkdir = force;
local->selfheal.hole_cnt = 0;
@@ -1511,44 +1514,16 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
return 0;
}
- count = conf->subvolume_cnt;
-
- /* Locking on all subvols in the mkdir phase of lookup selfheal is
- is done to synchronize with rmdir/rename.
- */
- lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
- if (lk_array == NULL)
- goto err;
-
- for (i = 0; i < count; i++) {
- lk_array[i] = dht_lock_new (frame->this,
- conf->subvolumes[i],
- &local->loc, F_WRLCK,
- DHT_LAYOUT_HEAL_DOMAIN);
- if (lk_array[i] == NULL)
- goto err;
- }
-
- local->lock.locks = lk_array;
- local->lock.lk_count = count;
-
- ret = dht_blocking_inodelk (frame, lk_array, count,
- IGNORE_ENOENT_ESTALE,
- dht_selfheal_dir_mkdir_lock_cbk);
+ local->current = &local->lock[0];
+ ret = dht_protect_namespace (frame, loc, local->hashed_subvol,
+ &local->current->ns,
+ dht_selfheal_dir_mkdir_lock_cbk);
- if (ret < 0) {
- local->lock.locks = NULL;
- local->lock.lk_count = 0;
+ if (ret < 0)
goto err;
- }
return 0;
err:
- if (lk_array != NULL) {
- dht_lock_array_free (lk_array, count);
- GF_FREE (lk_array);
- }
-
return -1;
}
@@ -2379,9 +2354,9 @@ dht_update_commit_hash_for_layout_unlock (call_frame_t *frame, xlator_t *this)
local = frame->local;
- ret = dht_unlock_inodelk (frame, local->lock.locks,
- local->lock.lk_count,
- dht_update_commit_hash_for_layout_done);
+ ret = dht_unlock_inodelk (frame, local->lock[0].layout.my_layout.locks,
+ local->lock[0].layout.my_layout.lk_count,
+ dht_update_commit_hash_for_layout_done);
if (ret < 0) {
/* preserve oldest error, just ... */
if (!local->op_ret) {
@@ -2614,19 +2589,19 @@ dht_update_commit_hash_for_layout (call_frame_t *frame)
lk_array[i] = dht_lock_new (frame->this,
conf->local_subvols[i],
&local->loc, F_WRLCK,
- DHT_LAYOUT_HEAL_DOMAIN);
+ DHT_LAYOUT_HEAL_DOMAIN, NULL);
if (lk_array[i] == NULL)
goto err;
}
- local->lock.locks = lk_array;
- local->lock.lk_count = count;
+ local->lock[0].layout.my_layout.locks = lk_array;
+ local->lock[0].layout.my_layout.lk_count = count;
ret = dht_blocking_inodelk (frame, lk_array, count, FAIL_ON_ANY_ERROR,
dht_update_commit_hash_for_layout_resume);
if (ret < 0) {
- local->lock.locks = NULL;
- local->lock.lk_count = 0;
+ local->lock[0].layout.my_layout.locks = NULL;
+ local->lock[0].layout.my_layout.lk_count = 0;
goto err;
}