Tests portability: umount(8)

1) Avoid hangs on unmounting NFS on NetBSD

NetBSD umount(8) on a NFS mount whose server is gone will wait forever
because umount(8) calls realpath(3) and tries to access the mount before
it calls unmount(2). The non-portable, NetBSD-specific umount -R flag
prevent that behavior.

We therefore introduce UMOUNT_F, defined as "umount -f" on Linux and
"umount -f -R" on NetBSD to take care of forced unmounts, especially
in the NFS case.

2) Enforce usage of force_umount wrapper with timeout

Whenever umount is used it should be wrapped in force_umount with
tiemout handling. That saves us timing issues, and it handles the
NetBSD NFS case.

3) Cleanup kernel cache flush.

We used (cd $M0 && umount $M0 )	as a portable kernel cache flush
trick, but it does not flush everything we need on Linux. Introduce
a drop_cache() shell function that reverts to previously used
echo 3 > /proc/sys/vm/drop_caches on Linux, and keeps
(cd $M0 && umount $M0 ) on other systems.

BUG: 1129939
Change-Id: Iab1f5a023405f1f7270c42b595573702ca1eb6f3
Signed-off-by: Emmanuel Dreyfus <manu@netbsd.org>
Reviewed-on: http://review.gluster.org/11114
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

19 files changed, 48 insertions, 30 deletions

diff --git a/tests/basic/afr/arbiter.t b/tests/basic/afr/arbiter.t
index 8a983fb0577..91d8d531125 100644
--- a/tests/basic/afr/arbiter.t
+++ b/tests/basic/afr/arbiter.t
@@ -13,7 +13,7 @@ TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}
 TEST $CLI volume start $V0
 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;
 TEST ! stat $M0/.meta/graphs/active/$V0-replicate-0/options/arbiter-count
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $CLI volume stop $V0
 TEST $CLI volume delete $V0
 
@@ -66,5 +66,5 @@ EXPECT 0 afr_get_pending_heal_count $V0
 TEST cat $M0/file
 TEST getfattr -n user.name $M0/file
 TEST `echo append>> $M0/file`
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 cleanup
diff --git a/tests/basic/afr/sparse-file-self-heal.t b/tests/basic/afr/sparse-file-self-heal.t
index a362494d95e..b3b2f119702 100644
--- a/tests/basic/afr/sparse-file-self-heal.t
+++ b/tests/basic/afr/sparse-file-self-heal.t
@@ -54,7 +54,7 @@ EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0
 #If the file system of bricks is XFS and speculative preallocation is on,
 #dropping cahce should be done to free speculatively pre-allocated blocks
 #by XFS.
-( cd $M0 ; umount $M0  ) # fail but drops kernel cache
+drop_cache $M0
 
 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}')
 small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}')
@@ -124,7 +124,7 @@ EXPECT_WITHIN $HEAL_TIMEOUT "0" afr_get_pending_heal_count $V0
 #If the file system of bricks is XFS and speculative preallocation is on,
 #dropping cahce should be done to free speculatively pre-allocated blocks
 #by XFS.
-( cd $M0 ; umount $M0  ) # fail but drops kernel cache
+drop_cache $M0
 
 big_md5sum_0=$(md5sum $B0/${V0}0/big | awk '{print $1}')
 small_md5sum_0=$(md5sum $B0/${V0}0/small | awk '{print $1}')
diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t
index 773d8af3084..b07006545b9 100644
--- a/tests/basic/ec/ec.t
+++ b/tests/basic/ec/ec.t
@@ -221,7 +221,7 @@ TEST setup_perm_file $M0
 sleep 2
 
 # Unmount/remount so that create/write and truncate don't see cached data.
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $GFS -s $H0 --volfile-id $V0 $M1
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "8" ec_child_up_count $V0 0
 
@@ -235,7 +235,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status'
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
 
 # Unmount/remount again, same reason as before.
-TEST umount $M1
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1
 TEST $GFS -s $H0 --volfile-id $V0 $M0
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0
 
diff --git a/tests/basic/file-snapshot.t b/tests/basic/file-snapshot.t
index f61de379fcf..f07ce074cdf 100755
--- a/tests/basic/file-snapshot.t
+++ b/tests/basic/file-snapshot.t
@@ -39,9 +39,8 @@ echo '1234567890' > $M0/data-file2
 TEST dd if=$M0/data-file2 of=$M0/big-file conv=notrunc;
 TEST setfattr -n trusted.glusterfs.block-snapshot-create -v image2 $M0/big-file;
 
-# big-file may still be in kernel page cache, this will fail to umount
-# but it will purge vnode and therefore invalidate the cache.
-( cd $M0 && umount $M0 )
+# big-file may still be in kernel page cache
+drop_cache $M0
 
 TEST setfattr -n trusted.glusterfs.block-snapshot-goto -v image1 $M0/big-file;
 TEST dd if=$M0/big-file of=$M0/out-file1 bs=11 count=1;
diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index 2a5f7486d7f..851d8b66dad 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -110,7 +110,7 @@ sleep 12
 uuidgen >> d1/data2.txt
 
 # Check promotion on read to slow tier
-( cd $M0 ; umount -l $M0 ) # fail but drops kernel cache
+drop_cache $M0
 cat d1/data3.txt
 sleep 5
 EXPECT_WITHIN $PROMOTE_TIMEOUT "0" file_on_fast_tier d1/data2.txt
diff --git a/tests/bugs/distribute/bug-1117851.t b/tests/bugs/distribute/bug-1117851.t
index 4e35debae9d..678103869cf 100755
--- a/tests/bugs/distribute/bug-1117851.t
+++ b/tests/bugs/distribute/bug-1117851.t
@@ -85,8 +85,8 @@ TEST move_files $M1
 EXPECT_WITHIN 120 "done" cat $M0/status_0
 EXPECT_WITHIN 120 "done" cat $M1/status_1
 
-TEST umount $M0
-TEST umount $M1
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1
 TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
 TEST check_files $M0
 
diff --git a/tests/bugs/distribute/bug-1161311.t b/tests/bugs/distribute/bug-1161311.t
index d1bd47826f9..c5a7f041ac8 100755
--- a/tests/bugs/distribute/bug-1161311.t
+++ b/tests/bugs/distribute/bug-1161311.t
@@ -63,7 +63,7 @@ dd if=/dev/urandom of=$M0/dir1/FILE2 bs=64k count=10240
 TEST mv $M0/dir1/FILE2 $M0/dir1/FILE1
 
 # unmount and remount the volume
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs -s $H0 --volfile-id $V0 $M0;
 
 # Start the rebalance
diff --git a/tests/bugs/distribute/bug-1190734.t b/tests/bugs/distribute/bug-1190734.t
index 895dd771ed6..ae34ec84729 100644
--- a/tests/bugs/distribute/bug-1190734.t
+++ b/tests/bugs/distribute/bug-1190734.t
@@ -22,7 +22,7 @@ function get_file_count {
 
 function reset {
     $CLI volume stop $V0
-    umount $1
+    ${UMOUNT_F} $1
     $CLI volume delete $V0
 }
 
diff --git a/tests/bugs/geo-replication/bug-1111490.t b/tests/bugs/geo-replication/bug-1111490.t
index c29a79a492e..28291f25b9b 100644
--- a/tests/bugs/geo-replication/bug-1111490.t
+++ b/tests/bugs/geo-replication/bug-1111490.t
@@ -24,7 +24,7 @@ EXPECT "$uuid" getfattr --only-values -n glusterfs.gfid.string $M0/file0
 
 # unmount and mount again so as to start with a fresh inode table
 # or use another mount...
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 --aux-gfid-mount
 
 # touch the file again (gfid-access.py handles errno)
diff --git a/tests/bugs/glusterfs/bug-867253.t b/tests/bugs/glusterfs/bug-867253.t
index aee27c87158..41972a7731b 100644
--- a/tests/bugs/glusterfs/bug-867253.t
+++ b/tests/bugs/glusterfs/bug-867253.t
@@ -37,7 +37,7 @@ touch $M0/files{1..1000};
 # Kill a brick process
 kill -9 `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-${V0}0.pid`;
 
-( cd $M0 ; umount $M0 ) # fail but drops kernel cache
+drop_cache $M0
 
 ls -l $M0 >/dev/null;
 
@@ -48,7 +48,7 @@ TEST $CLI volume start $V0 force
 # Kill a brick process
 kill -9 `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
 
-( cd $M0 ; umount $M0 )
+drop_cache $M0
 
 ls -l $M0 >/dev/null;
 
diff --git a/tests/bugs/replicate/bug-1221481-allow-fops-on-dir-split-brain.t b/tests/bugs/replicate/bug-1221481-allow-fops-on-dir-split-brain.t
index cd4a9bad87b..f332019d893 100644
--- a/tests/bugs/replicate/bug-1221481-allow-fops-on-dir-split-brain.t
+++ b/tests/bugs/replicate/bug-1221481-allow-fops-on-dir-split-brain.t
@@ -33,5 +33,5 @@ EXPECT "6" echo $(ls | wc -l)
 TEST ! cat FILE
 TEST `echo hello>hello.txt`
 cd -
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 cleanup
diff --git a/tests/bugs/rpc/bug-954057.t b/tests/bugs/rpc/bug-954057.t
index faaf261d88b..65af274f09d 100755
--- a/tests/bugs/rpc/bug-954057.t
+++ b/tests/bugs/rpc/bug-954057.t
@@ -8,7 +8,7 @@
 # Note on re-reading $M0/new after enabling root-squash:
 # Since we have readen it once, the file is present in various caches.
 # In order to actually fail on second attempt we must:
-# 1) drop kernel cache, by ( cd $M0 ; umount $M0 )
+# 1) drop kernel cache
 # 2) make sure FUSE does not cache the entry. This is also
 #    in the kernel, but not flushed by a failed umount.
 #    Using $GFS enforces this because it sets --entry-timeout=0
@@ -33,7 +33,7 @@ TEST cat $M0/new
 
 TEST $CLI volume set $V0 performance.stat-prefetch off
 TEST $CLI volume set $V0 server.root-squash enable
-( cd $M0 ; umount $M0 ) # fails but drops kernel cache
+drop_cache $M0
 TEST ! mkdir $M0/other
 TEST mkdir $M0/nobody/other
 TEST cat $M0/file
diff --git a/tests/bugs/snapshot/bug-1162498.t b/tests/bugs/snapshot/bug-1162498.t
index afc065000d5..a97e4429ee7 100755
--- a/tests/bugs/snapshot/bug-1162498.t
+++ b/tests/bugs/snapshot/bug-1162498.t
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 . $(dirname $0)/../../snapshot.rc
 
 cleanup;
@@ -35,7 +36,7 @@ TEST $CLI volume stop $V0
 TEST $CLI snapshot restore snap2
 TEST $CLI volume start $V0
 
-umount -f $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs -s $H0 --volfile-id=$V0 $M0
 
 #Dir xyz exists in snap1
diff --git a/tests/bugs/snapshot/bug-1164613.t b/tests/bugs/snapshot/bug-1164613.t
index d7f956ce369..225234dc43e 100644
--- a/tests/bugs/snapshot/bug-1164613.t
+++ b/tests/bugs/snapshot/bug-1164613.t
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 . $(dirname $0)/../../snapshot.rc
 
 cleanup;
@@ -23,7 +24,7 @@ TEST $CLI volume set $V0 snapshot-directory snaps
 
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" STAT $M0/snaps/snaps/testfile
 
-umount -f $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 #Clean up
 TEST $CLI snapshot delete snaps
diff --git a/tests/features/unhashed-auto.t b/tests/features/unhashed-auto.t
index cba5b772106..ddebd03299d 100755
--- a/tests/features/unhashed-auto.t
+++ b/tests/features/unhashed-auto.t
@@ -63,7 +63,7 @@ EXPECT 'Started' volinfo_field $V0 'Status'
 TEST $GFS -s $H0 --volfile-id $V0 $M0
 TEST mkdir $M0/dir
 TEST touch_files
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 # Add a brick and do the fix-layout part of rebalance to update directory layouts
 # (including their directory commit hashes).
@@ -82,7 +82,7 @@ TEST setfattr -x trusted.glusterfs.dht.commithash $B0/${V0}3
 # correctly skipped the broadcast lookup that would have found them.
 TEST $GFS -s $H0 --volfile-id $V0 $M0
 TEST [ $(count_files) -ne 100 ]
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 # Do the fix-layout again to generate a new volume commit hash.
 TEST $CLI volume rebalance $V0 fix-layout start
@@ -92,7 +92,7 @@ TEST wait_for_rebalance
 # the mismatch and did the broadcast lookup this time.
 TEST $GFS -s $H0 --volfile-id $V0 $M0
 TEST [ $(count_files) -eq 100 ]
-TEST umount $M0
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
 # Do a *full* rebalance and verify that the directory commit hash changed.
 old_val=$(get_xattr $B0/${V0}1/dir)
diff --git a/tests/include.rc b/tests/include.rc
index 2c45873aec0..32d60b43348 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -35,6 +35,7 @@ H0=${H0:=`hostname`}; # hostname
 MOUNT_TYPE_FUSE="fuse.glusterfs"
 GREP_MOUNT_OPT_RO="grep (ro"
 GREP_MOUNT_OPT_RW="grep (rw"
+UMOUNT_F="umount -f"
 
 PATH=$PATH:${PWD}/tests/utils
 
@@ -46,6 +47,7 @@ NetBSD)
   MOUNT_TYPE_FUSE="puffs|perfuse|fuse.glusterfs"
   GREP_MOUNT_OPT_RO="grep (read-only"
   GREP_MOUNT_OPT_RW="grep -v (read-only"
+  UMOUNT_F="umount -f -R"
   ;;
 *)
   ;;
@@ -456,7 +458,10 @@ function cleanup()
         Linux)
                 flag="-l"
                 ;;
-        NetBSD|FreeBSD|Darwin)
+        NetBSD)
+                flag="-f -R"
+                ;;
+        FreeBSD|Darwin)
                 flag="-f"
                 ;;
         *)
diff --git a/tests/nfs.rc b/tests/nfs.rc
index 2e0241709d3..2140f311c33 100644
--- a/tests/nfs.rc
+++ b/tests/nfs.rc
@@ -65,6 +65,6 @@ function mount_nfs ()
 }
 
 function umount_nfs {
-        umount -f $1
+        ${UMOUNT_F} $1
         if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi
 }
diff --git a/tests/snapshot.rc b/tests/snapshot.rc
index f2ff047a8ea..3ee1d6c4450 100644
--- a/tests/snapshot.rc
+++ b/tests/snapshot.rc
@@ -127,7 +127,7 @@ function _cleanup_lvm() {
 function _cleanup_lvm_again() {
     local file
 
-    mount | grep $LVM_PREFIX | awk '{print $3}' | xargs -r umount -f
+    mount | grep $LVM_PREFIX | awk '{print $3}' | xargs -r ${UMOUNT_F}
 
     /sbin/vgs | grep $LVM_PREFIX | awk '{print $1}' | xargs -r vgremove -f
 
@@ -182,7 +182,7 @@ function _umount_lv() {
     local num=$1
     local l="L$num"
 
-    umount -f ${!l} 2>/dev/null || true
+    ${UMOUNT_F} ${!l} 2>/dev/null || true
     rmdir ${!l} 2>/dev/null || true
 }
 
diff --git a/tests/volume.rc b/tests/volume.rc
index 5a7d0343a14..b12ba42784e 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -435,7 +435,7 @@ function path_exists {
 }
 
 function force_umount {
-        umount -f $*
+        ${UMOUNT_F} $*
         if [ $? -eq 0 ]; then echo "Y"; else echo "N"; fi
 }
 
@@ -543,3 +543,15 @@ function get_scrubd_count {
         ps auxw | grep glusterfs | grep scrub.pid | grep -v grep | wc -l
 }
 
+
+function drop_cache() {
+	case $OSTYPE in
+	Linux)
+		echo 3 > /proc/sys/vm/drop_caches
+		;;
+	*)
+		# fail but flush caches
+		( cd $1 && umount $1 2>/dev/null )
+		;;
+	esac
+}