auditing: Sample creation & removal of filesystem entries as well as errors

Summary:
- Adds the ability for gluster to log every single CREATE and UNLINK that happens on the bricks (right before invoking sys_unlink() or open(...| O_CREAT)
- Makes it so that CREATEs and UNLINKs are not downsampled in io-stats
- This is a port of D3268156, D3778968, D3903894 & D3301527 to 3.8

Reviewed By: kvigor

Change-Id: I1bce28068c02b7d202f094094237646b4d39794b
Reviewed-on: https://review.gluster.org/18198
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

1 files changed, 71 insertions, 16 deletions

diff --git a/tests/basic/fop-sampling.t b/tests/basic/fop-sampling.t
index d22826c4224..e429fd8cb07 100644
--- a/tests/basic/fop-sampling.t
+++ b/tests/basic/fop-sampling.t
@@ -7,6 +7,7 @@
 
 BRICK_SAMPLES="$(gluster --print-logdir)/samples/glusterfsd__d_backends_${V0}0.samp"
 NFS_SAMPLES="$(gluster --print-logdir)/samples/glusterfs_nfsd_${V0}.samp"
+FUSE_SAMPLES="/var/log/glusterfs/samples/glusterfs_${V0}.samp"
 
 function check_path {
         op=$1
@@ -22,25 +23,26 @@ function check_path {
 
 function print_cnt() {
   local FOP_TYPE=$1
-  local FOP_CNT=$(grep ,${FOP_TYPE} ${BRICK_SAMPLES} | wc -l)
+  local SAMP_FILE=$2
+  local FOP_CNT=$(grep ,${FOP_TYPE} ${SAMP_FILE} | wc -l)
   echo $FOP_CNT
 }
 
 # Verify we got non-zero counts for stats/lookup/readdir
 check_samples() {
-        STAT_CNT=$(print_cnt STAT)
+        STAT_CNT=$(print_cnt STAT $BRICK_SAMPLES)
         if [ "$STAT_CNT" -le "0" ]; then
                 echo "STAT count is zero"
                 return
         fi
 
-        LOOKUP_CNT=$(print_cnt LOOKUP)
+        LOOKUP_CNT=$(print_cnt LOOKUP $BRICK_SAMPLES)
         if [ "$LOOKUP_CNT" -le "0" ]; then
                 echo "LOOKUP count is zero"
                 return
         fi
 
-        READDIR_CNT=$(print_cnt READDIR)
+        READDIR_CNT=$(print_cnt READDIR $BRICK_SAMPLES)
         if [ "$READDIR_CNT" -le "0" ]; then
                 echo "READDIR count is zero"
                 return
@@ -64,20 +66,22 @@ TEST $CLI volume start $V0
 
 >${NFS_SAMPLES}
 >${BRICK_SAMPLES}
+>${FUSE_SAMPLES}
 
 #################
 # Basic Samples #
 #################
+
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
 
 for i in {1..5}
 do
         dd if=/dev/zero of=${M0}/testfile$i bs=4k count=1
+        rm ${M0}/testfile$i
 done
 
 TEST ls -l $M0
 EXPECT_WITHIN 6 "OK" check_samples
-
 sleep 2
 
 ################################
@@ -86,17 +90,21 @@ sleep 2
 
 TEST mount_nfs $H0:$V0 $N0
 
-ls $N0 &> /dev/null
-touch $N0/file1
-stat $N0/file1 &> /dev/null
-echo "some data" > $N0/file1
-dd if=/dev/zero of=$N0/file2 bs=1M count=10 conv=fsync
-dd if=/dev/zero of=$N0/file1 bs=1M count=1
-cat $N0/file2 &> /dev/null
-mkdir -p $N0/dir1
-rmdir $N0/dir1
-rm $N0/file1
-rm $N0/file2
+>$FUSE_SAMPLES
+
+for dir in "$N0" "$M0"; do
+  ls $dir &> /dev/null
+  touch $dir/file1
+  stat $dir/file1 &> /dev/null
+  echo "some data" > $dir/file1
+  dd if=/dev/zero of=$dir/file2 bs=1M count=10 conv=fsync
+  dd if=/dev/zero of=$dir/file1 bs=1M count=1
+  cat $dir/file2 &> /dev/null
+  mkdir -p $dir/dir1
+  rmdir $dir/dir1
+  rm $dir/file1
+  rm $dir/file2
+done;
 
 EXPECT_WITHIN 10 "Y" check_path CREATE /file1 $BRICK_SAMPLES
 EXPECT_WITHIN 10 "Y" check_path LOOKUP /file1 $BRICK_SAMPLES
@@ -126,4 +134,51 @@ EXPECT_WITHIN 10 "Y" check_path RMDIR /dir1 $NFS_SAMPLES
 EXPECT_WITHIN 10 "Y" check_path UNLINK /file1 $NFS_SAMPLES
 EXPECT_WITHIN 10 "Y" check_path UNLINK /file2 $NFS_SAMPLES
 
+
+EXPECT_WITHIN 10 "Y" check_path CREATE /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path LOOKUP /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path OPEN /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path SETATTR /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path WRITE /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path FLUSH /file2 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path FSYNC /file2 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path OPEN /file2 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path READ /file2 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path TRUNCATE /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path MKDIR /dir1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path RMDIR /dir1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path UNLINK /file1 $FUSE_SAMPLES
+EXPECT_WITHIN 10 "Y" check_path UNLINK /file2 $FUSE_SAMPLES
+
+######################
+# Errors in samples  #
+#####################
+
+# With a very low sample rate, we should still audit creates & unlinks 1:1
+TEST $CLI volume set $V0 diagnostics.fop-sample-interval 1000
+TEST $CLI volume set $V0 diagnostics.fop-sample-enable-audit on
+
+>${NFS_SAMPLES}
+>${BRICK_SAMPLES}
+>${FUSE_SAMPLES}
+
+mkdir -pv $M0/1/2/3/4
+touch $M0/1/2/3/4/{a,b,c}
+dd if=/dev/zero of=$M0/1/2/3/4/d bs=1k count=10
+dd if=/dev/zero of=$M0/1/2/3/4/d bs=1M count=10
+rm -rfv $M0/*
+sleep 6
+
+TEST grep "MKDIR.*/1/2/3/4" $FUSE_SAMPLES
+TEST grep "CREATE.*/1/2/3/4" $FUSE_SAMPLES
+TEST grep "RMDIR.*/1/2" $FUSE_SAMPLES
+TEST grep "UNLINK.*/1/2/3/4/a" $FUSE_SAMPLES
+TEST grep "TRUNCATE.*/1/2/3/4/d" $FUSE_SAMPLES
+
+TEST [ $(print_cnt MKDIR $FUSE_SAMPLES) -eq "4" ]
+TEST [ $(print_cnt CREATE $FUSE_SAMPLES) -eq "4" ]
+TEST [ $(print_cnt RMDIR $FUSE_SAMPLES) -eq "4" ]
+TEST [ $(print_cnt UNLINK $FUSE_SAMPLES) -eq "4" ]
+TEST [ $(print_cnt TRUNCATE $FUSE_SAMPLES) -eq "1" ]
+
 cleanup;