path: root/xlators/cluster/dht/src
diff options
authorPoornima G <>2016-08-23 18:15:22 +0530
committerRaghavendra G <>2016-08-30 23:08:54 -0700
commit065a27948c4e0651f5bdac1703939adf34e5380e (patch)
tree8e563eb9e75337f9e36852e7ea2a95be27e359aa /xlators/cluster/dht/src
parent8f053f9d7270f1c6d50c0b3ab5d020503ceeb31a (diff)
dht, md-cache, upcall: Add invalidation of IATT when the layout changes
Issue: dht_layout is built as a part of lookup only. The layout can be modified by rebalance process. Since every IO fop is preceded by a lookup, there are very less issues of stale layout. But with enhancements of aggressive caching of stats in md-cache, the lookup will reduce and expose the stale layout issue often. Solution: Since stale layout is already an issue on dht, there is already a plan to fix this at the dht layer, but this fix is not currently planned for any release. Until this fix comes out, we can have a workaround where, the upcall will send a notification to md-cache when a layout xattr is changed. As a part of layout change notification the existing cache is invalidated and the next lookup will fetch the latest layout. This is not a foolproof solution as the window between the layout change and the next lookup(after invalidation of stat), where there will be stale layout. But until the final fix comes in, this reduces the stale layout window. Change-Id: Iacf871a38b35880c1fc0bc68fe7ce291265e71d4 BUG: 1369638 Signed-off-by: Poornima G <> Reviewed-on: NetBSD-regression: NetBSD Build System <> CentOS-regression: Gluster Build System <> Smoke: Gluster Build System <> Reviewed-by: Raghavendra G <>
Diffstat (limited to 'xlators/cluster/dht/src')
1 files changed, 23 insertions, 1 deletions
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 44b4b858ee1..87b77530e2f 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -19,6 +19,7 @@
#include "byte-order.h"
#include "glusterfs-acl.h"
#include "quota-common-utils.h"
+#include "upcall-utils.h"
#include <sys/time.h>
#include <libgen.h>
@@ -8551,6 +8552,11 @@ dht_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
call_cnt = conf->subvolume_cnt;
local->call_cnt = call_cnt;
+ if (xdata) {
+ if (dict_set_int8 (xdata, conf->xattr_name, 0) < 0)
+ goto err;
+ }
for (i = 0; i < call_cnt; i++) {
STACK_WIND (frame, dht_ipc_cbk, conf->subvolumes[i],
conf->subvolumes[i]->fops->ipc, op, xdata);
@@ -8613,6 +8619,8 @@ dht_notify (xlator_t *this, int event, void *data, ...)
dict_t *output = NULL;
va_list ap;
dht_methods_t *methods = NULL;
+ struct gf_upcall *up_data = NULL;
+ struct gf_upcall_cache_invalidation *up_ci = NULL;
conf = this->private;
GF_VALIDATE_OR_GOTO (this->name, conf, out);
@@ -8781,7 +8789,21 @@ unlock:
return ret;
+ up_data = (struct gf_upcall *)data;
+ if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+ break;
+ up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+ /* Since md-cache will be aggressively filtering lookups,
+ * the stale layout issue will be more pronounced. Hence
+ * when a layout xattr is changed by the rebalance process
+ * notify all the md-cache clients to invalidate the existing
+ * stat cache and send the lookup next time*/
+ if (up_ci->dict && dict_get (up_ci->dict, conf->xattr_name))
+ ret = dict_set_int8 (up_ci->dict, MDC_INVALIDATE_IATT , 0);
+ propagate = 1;
+ break;
propagate = 1;