mgmt/glusterd: make sure quota enforcer has established connection with quotad before marking quota as enabled.

without this patch there is a window of time when quota is marked as
enabled in quota-enforcer, but connection to quotad wouldn't have been
established. Any checklimit done during this period can result in a
failed fop because of unavailability of quotad.

Change-Id: I0d509fabc434dd55ce9ec59157123524197fcc80
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
BUG: 969461
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/6572
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

5 files changed, 85 insertions, 49 deletions

diff --git a/tests/basic/quota-anon-fd-nfs.t b/tests/basic/quota-anon-fd-nfs.t
new file mode 100755
index 00000000000..be7bc35db9b
--- /dev/null
+++ b/tests/basic/quota-anon-fd-nfs.t
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fileio.rc
+
+cleanup;
+
+TESTS_EXPECTED_IN_LOOP=16
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+
+# The test makes use of inode-lru-limit to hit a scenario, where we
+# find an inode whose ancestry is not there. Following is the
+# hypothesis (which is confirmed by seeing logs indicating that
+# codepath has been executed, but not through a good understanding of
+# NFS internals).
+
+#     At the end of an fop, the reference count of an inode would be
+#     zero. The inode (and its ancestry) persists in memory only
+#     because of non-zero lookup count. These looked up inodes are put
+#     in an lru queue of size 1 (here). So, there can be at most one
+#     such inode in memory.
+
+#     NFS Server makes use of anonymous fds. So, if it cannot find
+#     valid fd, it does a nameless lookup. This gives us an inode
+#     whose ancestry is NULL. When a write happens on this inode,
+#     quota-enforcer/marker finds a NULL ancestry and asks
+#     storage/posix to build it.
+
+TEST $CLI volume set $V0 network.inode-lru-limit 1
+TEST $CLI volume set $V0 performance.nfs.write-behind off
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume quota $V0 enable
+TEST $CLI volume quota $V0 limit-usage / 1
+
+TEST mount -t nfs -o noac,soft,nolock,vers=3 $H0:/$V0 $N0
+deep=/0/1/2/3/4/5/6/7/8/9
+TEST mkdir -p $N0/$deep
+
+TEST touch $N0/$deep/file1 $N0/$deep/file2 $N0/$deep/file3 $N0/$deep/file4
+
+TEST fd_open 3 'w' "$N0/$deep/file1"
+TEST fd_open 4 'w' "$N0/$deep/file2"
+TEST fd_open 5 'w' "$N0/$deep/file3"
+TEST fd_open 6 'w' "$N0/$deep/file4"
+
+# consume all quota
+TEST ! dd if=/dev/zero of="$N0/$deep/file" bs=1MB count=1
+
+# At the end of each fop in server, reference count of the
+# inode associated with each of the file above drops to zero and hence
+# put into lru queue. Since lru-limit is set to 1, an fop next file
+# will displace the current inode from itable. This will ensure that
+# when writes happens on same fd, fd resolution results in
+# nameless lookup from server and quota_writev encounters an fd
+# associated with an inode whose parent is not present in itable.
+
+for j in $(seq 1 2); do
+    for i in $(seq 3 6); do
+        # failing writes indicate that we are enforcing quota set on /
+        # even with anonymous fds.
+        TEST_IN_LOOP ! fd_write $i "content"
+        TEST_IN_LOOP sync
+    done
+done
+
+exec 3>&-
+exec 4>&-
+exec 5>&-
+exec 6>&-
+
+$CLI volume statedump $V0 all
+
+TEST umount -l $N0
+
+cleanup;
diff --git a/tests/basic/quota-nfs-anon.t b/tests/basic/quota-nfs-anon.t
deleted file mode 100644
index 7b5ea5f28e0..00000000000
--- a/tests/basic/quota-nfs-anon.t
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
-
-cleanup;
-
-TEST glusterd
-TEST $CLI volume create $V0 $H0:$B0/${V0}{1}
-
-function volinfo_field()
-{
-    local vol=$1;
-    local field=$2;
-
-    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
-}
-
-
-## Verify volume is is created
-EXPECT "$V0" volinfo_field $V0 'Volume Name';
-EXPECT 'Created' volinfo_field $V0 'Status';
-
-
-## Start volume and verify
-TEST $CLI volume start $V0;
-EXPECT 'Started' volinfo_field $V0 'Status';
-
-TEST $CLI volume quota $V0 enable;
-
-## Mount NFS
-TEST mount -t nfs -o nolock,soft,intr $H0:/$V0 $N0;
-mkdir -p $N0/0/1
-TEST $CLI volume quota $V0 limit-usage /0/1 1GB 75%;
-
-deep=/0/1/2/3/4/5/6/7/8/9
-mkdir -p $N0/$deep
-dd if=/dev/zero of=$N0/$deep/file bs=1M count=502 &
-
-kill_brick $V0 $H0 $B0/${V0}{1}
-kill -TERM $(get_nfs_pid)
-
-$CLI volume start $V0 force;
-
-
-cleanup;
diff --git a/tests/bugs/bug-1035576.t b/tests/bugs/bug-1035576.t
index 08c8a5ea249..52d93dd87df 100644
--- a/tests/bugs/bug-1035576.t
+++ b/tests/bugs/bug-1035576.t
@@ -21,7 +21,7 @@ TEST $CLI volume set $V0 performance.read-ahead off
 TEST $CLI volume set $V0 background-self-heal-count 0
 TEST $CLI volume set $V0 self-heal-daemon off
 TEST $CLI volume quota $V0 enable
-sleep 5 # wait for brick to connect to quotad
+
 TEST kill_brick $V0 $H0 $B0/${V0}0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
 cd $M0
diff --git a/tests/bugs/bug-848251.t b/tests/bugs/bug-848251.t
index 844162283d8..a5c80b1b473 100644
--- a/tests/bugs/bug-848251.t
+++ b/tests/bugs/bug-848251.t
@@ -19,7 +19,6 @@ TEST $CLI volume quota $V0 enable;
 TEST MOUNTDIR="/tmp/$RANDOM"
 TEST mkdir $MOUNTDIR
 TEST glusterfs -s $H0 --volfile-id=$V0 $MOUNTDIR
-sleep 10
 
 function set_quota(){
         mkdir "$MOUNTDIR/$name"
diff --git a/tests/bugs/bug-990028.t b/tests/bugs/bug-990028.t
index ece7235cd96..fbf4175bea7 100755
--- a/tests/bugs/bug-990028.t
+++ b/tests/bugs/bug-990028.t
@@ -22,7 +22,6 @@ function __init()
     TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0
 
     TEST $CLI volume quota $V0 enable
-    sleep 15
 }
 
 #CASE-1