tests: Enable geo-rep test cases

This patch re-enables the geo-rep test cases.
Along with it does following optimizations.

1. Use EXPECT_WITHIN instead of sleep
2. Clean up geo-rep ssh key after test
3. Changes to gverify.sh and S56glusterd-geo-rep-create-post.sh
   to use the given ssh identity file for geo-rep create
4. Make gluster-command-dir configurable and introduce
   slave-gluster-command-dir which points the parent directory
   of gluster binaries in master and slave respectively.

Change-Id: Ia7696278d9dd3ba04224dcd7c3564088ca970b04
BUG: 1480491
Signed-off-by: Kotresh HR <khiremat@redhat.com>

1 files changed, 92 insertions, 77 deletions

diff --git a/tests/geo-rep/georep-basic-dr-rsync.t b/tests/geo-rep/georep-basic-dr-rsync.t
index 39da524f639..683506684de 100644
--- a/tests/geo-rep/georep-basic-dr-rsync.t
+++ b/tests/geo-rep/georep-basic-dr-rsync.t
@@ -3,6 +3,15 @@
 . $(dirname $0)/../include.rc
 . $(dirname $0)/../volume.rc
 . $(dirname $0)/../geo-rep.rc
+. $(dirname $0)/../env.rc
+
+AREQUAL_PATH=$(dirname $0)/../utils
+test "`uname -s`" != "Linux" && {
+    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
+    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
+    CFLAGS="$CFLAGS -lintl";
+}
+build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
 
 ### Basic Tests with Distribute Replicate volumes
 
@@ -15,7 +24,8 @@ TEST pidof glusterd
 ##Variables
 GEOREP_CLI="$CLI volume geo-replication"
 master=$GMV0
-slave=${H0}::${GSV0}
+SH0="127.0.0.1"
+slave=${SH0}::${GSV0}
 num_active=2
 num_passive=2
 master_mnt=$M0
@@ -30,14 +40,14 @@ TEST $CLI volume create $GMV0 replica 2 $H0:$B0/${GMV0}{1,2,3,4};
 TEST $CLI volume start $GMV0
 
 ##create_and_start_slave_volume
-TEST $CLI volume create $GSV0 replica 2 $H0:$B0/${GSV0}{1,2,3,4};   #5
+TEST $CLI volume create $GSV0 replica 2 $H0:$B0/${GSV0}{1,2,3,4};
 TEST $CLI volume start $GSV0
 
 ##Create, start and mount meta_volume
 TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3};
 TEST $CLI volume start $META_VOL
 TEST mkdir -p $META_MNT
-TEST glusterfs -s $H0 --volfile-id $META_VOL $META_MNT     #10
+TEST glusterfs -s $H0 --volfile-id $META_VOL $META_MNT
 
 ##Mount master
 TEST glusterfs -s $H0 --volfile-id $GMV0 $M0
@@ -52,98 +62,97 @@ TEST glusterfs -s $H0 --volfile-id $GSV0 $M1
 #Check Hybrid Crawl
 TEST create_data "hybrid"
 TEST create_georep_session $master $slave
-EXPECT 4 check_status_num_rows "Created"             #15
+EXPECT_WITHIN $GEO_REP_TIMEOUT 4 check_status_num_rows "Created"
+
+#Config gluster-command-dir
+TEST $GEOREP_CLI $master $slave config gluster-command-dir ${GLUSTER_CMD_DIR}
+
+#Config gluster-command-dir
+TEST $GEOREP_CLI $master $slave config slave-gluster-command-dir ${GLUSTER_CMD_DIR}
 
 #Enable_metavolume
 TEST $GEOREP_CLI $master $slave config use_meta_volume true
 
+#Set changelog roll-over time to 3 secs
+TEST $CLI volume set $GMV0 changelog.rollover-time 3
+
 #Start_georep
 TEST $GEOREP_CLI $master $slave start
 
-sleep 10
-EXPECT 2 check_status_num_rows "Active"
-EXPECT 2 check_status_num_rows "Passive"
-
-#DATA_TESTS HYBRID
-sleep 15
-TEST regular_file_ok ${slave_mnt}/hybrid_f1         #20
-TEST directory_ok ${slave_mnt}/$hybrid_d1
-TEST rename_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
-TEST rename_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
-TEST symlink_ok hybrid_f1 ${slave_mnt}/hybrid_sl1
-TEST hardlink_file_ok ${slave_mnt}/hybrid_f1 ${slave_mnt}/hybrid_hl1 #25
-TEST unlink_ok ${slave_mnt}/hybrid_f2
-TEST unlink_ok ${slave_mnt}/hybrid_d2
-TEST data_ok ${slave_mnt}/hybrid_f1 "HelloWorld!"
-TEST chown_file_ok ${slave_mnt}/hybrid_chown_f1
-
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Active"
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Passive"
+
+#data_tests "hybrid"
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 regular_file_ok ${slave_mnt}/hybrid_f1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/$hybrid_d1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_f3 ${slave_mnt}/hybrid_f4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/hybrid_d3 ${slave_mnt}/hybrid_d4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 symlink_ok hybrid_f1 ${slave_mnt}/hybrid_sl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_file_ok ${slave_mnt}/hybrid_f1 ${slave_mnt}/hybrid_hl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/hybrid_f2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/hybrid_d2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 data_ok ${slave_mnt}/hybrid_f1 "HelloWorld!"
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 chown_file_ok ${slave_mnt}/hybrid_chown_f1
 
 #Check History Crawl.
-TEST $GEOREP_CLI $master $slave stop                #30
+TEST $GEOREP_CLI $master $slave stop
 TEST create_data "history"
 TEST $GEOREP_CLI $master $slave start
-sleep 10
-EXPECT 2 check_status_num_rows "Active"
-EXPECT 2 check_status_num_rows "Passive"
-
-
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Active"
+EXPECT_WITHIN $GEO_REP_TIMEOUT  2 check_status_num_rows "Passive"
 
 #data_tests "history"
-sleep 15
-TEST regular_file_ok ${slave_mnt}/history_f1        #35
-TEST directory_ok ${slave_mnt}/history_d1
-TEST rename_ok ${slave_mnt}/history_f3 ${slave_mnt}/history_f4
-TEST rename_ok ${slave_mnt}/history_d3 ${slave_mnt}/history_d4
-TEST symlink_ok history_f1 ${slave_mnt}/history_sl1
-TEST hardlink_file_ok ${slave_mnt}/history_f1 ${slave_mnt}/history_hl1   #40
-TEST unlink_ok ${slave_mnt}/history_f2
-TEST unlink_ok ${slave_mnt}/history_d2
-TEST data_ok ${slave_mnt}/history_f1 "HelloWorld!"
-TEST chown_file_ok ${slave_mnt}/history_chown_f1
-
-#Check History Crawl.
-TEST create_data "changelog"                        #45
-sleep 15
-TEST check_status "Changelog Crawl"
-
-#data_tests "changelog"
-sleep 15
-TEST regular_file_ok ${slave_mnt}/changelog_f1
-TEST directory_ok ${slave_mnt}/changelog_d1
-TEST rename_ok ${slave_mnt}/changelog_f3 ${slave_mnt}/changelog_f4
-TEST rename_ok ${slave_mnt}/changelog_d3 ${slave_mnt}/changelog_d4      #50
-TEST symlink_ok changelog_f1 ${slave_mnt}/changelog_sl1
-TEST hardlink_file_ok ${slave_mnt}/changelog_f1 ${slave_mnt}/changelog_hl1
-TEST unlink_ok ${slave_mnt}/changelog_f2
-TEST unlink_ok ${slave_mnt}/changelog_d2
-TEST data_ok ${slave_mnt}/changelog_f1 "HelloWorld!"                    #55
-TEST chown_file_ok ${slave_mnt}/changelog_chown_f1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 regular_file_ok ${slave_mnt}/history_f1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/history_d1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/history_f3 ${slave_mnt}/history_f4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/history_d3 ${slave_mnt}/history_d4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 symlink_ok history_f1 ${slave_mnt}/history_sl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_file_ok ${slave_mnt}/history_f1 ${slave_mnt}/history_hl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/history_f2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/history_d2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 data_ok ${slave_mnt}/history_f1 "HelloWorld!"
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 chown_file_ok ${slave_mnt}/history_chown_f1
+
+#Check Changelog Crawl.
+EXPECT_WITHIN $GEO_REP_TIMEOUT 2 check_status_num_rows "Changelog Crawl"
+TEST create_data "changelog"
 
 # logrotate test
-logrotate_simulate logrotate_test_file 2
-logrotate_simulate logrotate_test_file 2
-logrotate_simulate logrotate_test_file 2
-logrotate_simulate logrotate_test_file 2
-sleep 15
-EXPECT 0 check_status_num_rows "Faulty"
+logrotate_file=${master_mnt}/logrotate/lg_test_file
+TEST mkdir -p ${master_mnt}/logrotate
+logrotate_simulate $logrotate_file 2
+logrotate_simulate $logrotate_file 2
+logrotate_simulate $logrotate_file 2
+logrotate_simulate $logrotate_file 2
 
 # CREATE + RENAME
-create_rename create_rename_test_file
-sleep 15
-TEST $GEOREP_CLI $master $slave stop
-sleep 5
-TEST $GEOREP_CLI $master $slave start
-sleep 15
-TEST create_rename_ok create_rename_test_file                                #58
+create_rename ${master_mnt}/rename_test_file
 
 # hard-link rename
-hardlink_rename hardlink_rename_test_file
-sleep 15
-TEST $GEOREP_CLI $master $slave stop
-sleep 5
-TEST $GEOREP_CLI $master $slave start
-sleep 15
-TEST hardlink_rename_ok hardlink_rename_test_file
+hardlink_rename ${master_mnt}/hardlink_rename_test_file
+
+#SYNC CHECK
+#data_tests "changelog"
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 regular_file_ok ${slave_mnt}/changelog_f1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/changelog_d1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/changelog_f3 ${slave_mnt}/changelog_f4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 rename_ok ${slave_mnt}/changelog_d3 ${slave_mnt}/changelog_d4
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 symlink_ok changelog_f1 ${slave_mnt}/changelog_sl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_file_ok ${slave_mnt}/changelog_f1 ${slave_mnt}/changelog_hl1
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/changelog_f2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 1 unlink_ok ${slave_mnt}/changelog_d2
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 data_ok ${slave_mnt}/changelog_f1 "HelloWorld!"
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 chown_file_ok ${slave_mnt}/changelog_chown_f1
+
+#logrotate
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 directory_ok ${slave_mnt}/logrotate
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 arequal_checksum ${master_mnt}/logrotate ${slave_mnt}/logrotate
+
+#CREATE+RENAME
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 create_rename_ok ${slave_mnt}/create_rename_test_file
+
+#hardlink rename
+EXPECT_WITHIN $GEO_REP_TIMEOUT 0 hardlink_rename_ok ${slave_mnt}/hardlink_rename_test_file
 
 #Stop Geo-rep
 TEST $GEOREP_CLI $master $slave stop
@@ -151,6 +160,12 @@ TEST $GEOREP_CLI $master $slave stop
 #Delete Geo-rep
 TEST $GEOREP_CLI $master $slave delete
 
+#Cleanup are-equal binary
+TEST rm $AREQUAL_PATH/arequal-checksum
+
+#Cleanup authorized keys
+sed -i '/^command=.*SSH_ORIGINAL_COMMAND#.*/d' ~/.ssh/authorized_keys
+sed -i '/^command=.*gsyncd.*/d' ~/.ssh/authorized_keys
+
 cleanup;
-#G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=000000
 #G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=000000