tests: Enable geo-rep test cases

This patch re-enables the geo-rep test cases.
Along with it does following optimizations.

1. Use EXPECT_WITHIN instead of sleep
2. Clean up geo-rep ssh key after test
3. Changes to gverify.sh and S56glusterd-geo-rep-create-post.sh
   to use the given ssh identity file for geo-rep create
4. Make gluster-command-dir configurable and introduce
   slave-gluster-command-dir which points the parent directory
   of gluster binaries in master and slave respectively.

Change-Id: Ia7696278d9dd3ba04224dcd7c3564088ca970b04
BUG: 1480491
Signed-off-by: Kotresh HR <khiremat@redhat.com>

4 files changed, 33 insertions, 10 deletions

diff --git a/geo-replication/gsyncd.conf.in b/geo-replication/gsyncd.conf.in
index ecdeab9377e..53cc76b842a 100644
--- a/geo-replication/gsyncd.conf.in
+++ b/geo-replication/gsyncd.conf.in
@@ -117,7 +117,11 @@ help=If SSH keys are not secured with gsyncd prefix then use this configuration
 
 [gluster-command-dir]
 value=/usr/local/sbin/
-help=Directory where Gluster binary exists
+help=Directory where Gluster binaries exist on master
+
+[slave-gluster-command-dir]
+value=/usr/local/sbin/
+help=Directory where Gluster binaries exist on slave
 
 [gluster-params]
 value = aux-gfid-mount acl
diff --git a/geo-replication/src/gverify.sh b/geo-replication/src/gverify.sh
index 9b1328aa631..8eae8469ff0 100755
--- a/geo-replication/src/gverify.sh
+++ b/geo-replication/src/gverify.sh
@@ -11,11 +11,19 @@ slave_log_file=`gluster --print-logdir`/geo-replication-slaves/slave.log
 
 function SSHM()
 {
-    ssh -p ${SSH_PORT} -q \
-	-oPasswordAuthentication=no \
-	-oStrictHostKeyChecking=no \
-	-oControlMaster=yes \
-	"$@";
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    fi
 }
 
 function get_inode_num()
@@ -172,7 +180,12 @@ function main()
         exit 1;
     fi;
 
-    ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    fi
+
     if [ $? -ne 0 ]; then
         echo "FORCE_BLOCKER|Passwordless ssh login has not been setup with $3 for user $2." > $log_file
         exit 1;
diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py
index edecf076a03..04ceb435bf7 100644
--- a/geo-replication/syncdaemon/gsyncd.py
+++ b/geo-replication/syncdaemon/gsyncd.py
@@ -117,6 +117,8 @@ def main():
     p.add_argument("--slave-log-level", help="Slave Gsyncd Log level")
     p.add_argument("--slave-gluster-log-level",
                    help="Slave Gluster mount Log level")
+    p.add_argument("--slave-gluster-command-dir",
+                   help="Directory where Gluster binaries exist on slave")
     p.add_argument("--slave-access-mount", action="store_true",
                    help="Do not lazy umount the slave volume")
 
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index ab0484d5f5e..2d48a28c145 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -807,8 +807,10 @@ class Mounter(object):
 
     @classmethod
     def get_glusterprog(cls):
-        return os.path.join(gconf.get("gluster-command-dir"),
-                            cls.glusterprog)
+        gluster_cmd_dir = gconf.get("gluster-command-dir")
+        if rconf.args.subcmd == "slave":
+            gluster_cmd_dir = gconf.get("slave-gluster-command-dir")
+        return os.path.join(gluster_cmd_dir, cls.glusterprog)
 
     def umount_l(self, d):
         """perform lazy umount"""
@@ -1371,7 +1373,9 @@ class SSH(object):
                 '--slave-timeout', str(gconf.get("slave-timeout")),
                 '--slave-log-level', gconf.get("slave-log-level"),
                 '--slave-gluster-log-level',
-                gconf.get("slave-gluster-log-level")]
+                gconf.get("slave-gluster-log-level"),
+                '--slave-gluster-command-dir',
+                gconf.get("slave-gluster-command-dir")]
 
         if gconf.get("slave-access-mount"):
             args_to_slave.append('--slave-access-mount')