quota: setting 'read-only' option in xdata to instruct DHT to not heal

When quota is enabled the quota enforcer tries to get the size of the
source directory by sending nameless lookup to quotad. But if the rename
is successful even on one subvol or the source layout has anomalies then
this nameless lookup in quotad tries to heal the directory which requires
a lock on as many subvols as it can. But src is already locked as part of
rename. For rename to proceed in brick it needs to complete a cluster-wide
lookup. But cluster-wide lookup in quotad is blocked on locks held by rename,
hence a deadlock. To avoid this quota sends an option in xdata which instructs
DHT not to heal.

Backport of http://review.gluster.org/#/c/13988/

> Change-Id: I792f9322331def0b1f4e16e88deef55d0c9f17f0
> BUG: 1252244
> Signed-off-by: Sakshi Bansal <sabansal@redhat.com>
> Reviewed-on: http://review.gluster.org/13988
> Smoke: Gluster Build System <jenkins@build.gluster.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
> Tested-by: Gluster Build System <jenkins@build.gluster.com>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

Change-Id: I792f9322331def0b1f4e16e88deef55d0c9f17f0
BUG: 1328473
Signed-off-by: Sakshi Bansal <sabansal@redhat.com>
Reviewed-on: http://review.gluster.org/14031
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 19 insertions, 2 deletions

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index dfaa623fa87..b9ed3bdf96e 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -165,6 +165,7 @@
 #define QUOTA_LIMIT_KEY "trusted.glusterfs.quota.limit-set"
 #define QUOTA_LIMIT_OBJECTS_KEY "trusted.glusterfs.quota.limit-objects"
 #define VIRTUAL_QUOTA_XATTR_CLEANUP_KEY "glusterfs.quota-xattr-cleanup"
+#define QUOTA_READ_ONLY_KEY "trusted.glusterfs.quota.read-only"
 
 /* Index xlator related */
 #define GF_XATTROP_INDEX_GFID "glusterfs.xattrop_index_gfid"
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 3421c1de657..ed884b60035 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -255,6 +255,7 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              heal_path       = 0;
         int              i               = 0;
         loc_t            loc             = {0 };
+        int8_t           is_read_only    = 0;
 
         local = discover_frame->local;
         layout = local->layout;
@@ -270,6 +271,12 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         if (!main_frame)
                 return 0;
 
+        ret = dict_get_int8 (local->xattr_req, QUOTA_READ_ONLY_KEY,
+                             &is_read_only);
+        if (ret < 0)
+                gf_msg_debug (this->name, 0, "key = %s not present in dict",
+                              QUOTA_READ_ONLY_KEY);
+
         if (local->file_count && local->dir_count) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         DHT_MSG_FILE_TYPE_MISMATCH,
@@ -316,7 +323,8 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                          * healing layout of directory otherwise we don't heal.
                          */
 
-                        if (local->inode && conf->randomize_by_gfid)
+                        if (local->inode && conf->randomize_by_gfid &&
+                            !is_read_only)
                                 goto selfheal;
                 }
 
@@ -333,7 +341,7 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                 }
         }
 
-        if (IA_ISDIR (local->stbuf.ia_type)) {
+        if (IA_ISDIR (local->stbuf.ia_type) && !is_read_only) {
                 for (i = 0; i < layout->cnt; i++) {
                        if (!source && !layout->list[i].err)
                                 source = layout->list[i].xlator;
diff --git a/xlators/features/quota/src/quotad.c b/xlators/features/quota/src/quotad.c
index 028c8047141..dc2665e9622 100644
--- a/xlators/features/quota/src/quotad.c
+++ b/xlators/features/quota/src/quotad.c
@@ -129,6 +129,14 @@ qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
                 goto out;
         }
 
+        ret = dict_set_int8 (xdata, QUOTA_READ_ONLY_KEY, 1);
+        if (ret < 0) {
+                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
+                        Q_MSG_ENOMEM, "dict set failed");
+                ret = -ENOMEM;
+                goto out;
+        }
+
         subvol = qd_find_subvol (this, volume_uuid);
         if (subvol == NULL) {
                 op_errno = EINVAL;