diff options
Diffstat (limited to 'tests/functional/common')
3 files changed, 196 insertions, 84 deletions
diff --git a/tests/functional/common/heketi/test_volume_multi_req.py b/tests/functional/common/heketi/test_volume_multi_req.py index 7b723e7c..03658fae 100644 --- a/tests/functional/common/heketi/test_volume_multi_req.py +++ b/tests/functional/common/heketi/test_volume_multi_req.py @@ -227,7 +227,9 @@ class TestVolumeMultiReq(HeketiClientSetupBaseClass): # deploy a temporary storage class sc = build_storage_class( name=tname, - resturl=self.heketi_server_url) + resturl=self.heketi_server_url, + restuser=self.heketi_cli_user, + restuserkey=self.heketi_cli_key) with temp_config(ocp_node, sc) as tmpfn: oc_create(ocp_node, tmpfn) self.addCleanup(delete_storageclass, ocp_node, tname) @@ -286,7 +288,9 @@ class TestVolumeMultiReq(HeketiClientSetupBaseClass): # deploy a temporary storage class sc = build_storage_class( name=tname, - resturl=self.heketi_server_url) + resturl=self.heketi_server_url, + restuser=self.heketi_cli_user, + restuserkey=self.heketi_cli_key) with temp_config(ocp_node, sc) as tmpfn: oc_create(ocp_node, tmpfn) self.addCleanup(delete_storageclass, ocp_node, tname) @@ -339,7 +343,9 @@ class TestVolumeMultiReq(HeketiClientSetupBaseClass): # deploy a temporary storage class sc = build_storage_class( name=tname, - resturl=self.heketi_server_url) + resturl=self.heketi_server_url, + restuser=self.heketi_cli_user, + restuserkey=self.heketi_cli_key) with temp_config(ocp_node, sc) as tmpfn: oc_create(ocp_node, tmpfn) self.addCleanup(delete_storageclass, ocp_node, tname) diff --git a/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py b/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py index 3c01427e..933033ae 100644 --- a/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py +++ b/tests/functional/common/provisioning/test_dynamic_provisioning_block_p0_cases.py @@ -5,6 +5,11 @@ from cnslibs.common.dynamic_provisioning import ( get_pvc_status, verify_pod_status_running) from cnslibs.cns.cns_baseclass import CnsGlusterBlockBaseClass +from cnslibs.common.exceptions import ( + ConfigError, + ExecutionError) +from cnslibs.common.heketi_ops import ( + export_heketi_cli_server) from cnslibs.common.openshift_ops import ( get_ocp_gluster_pod_names, oc_create, @@ -12,7 +17,7 @@ from cnslibs.common.openshift_ops import ( oc_rsh) from cnslibs.common.waiter import Waiter from glusto.core import Glusto as g -import time +import unittest class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): @@ -23,19 +28,6 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): def test_dynamic_provisioning_glusterblock(self): g.log.info("test_dynamic_provisioning_glusterblock") storage_class = self.cns_storage_class['storage_class2'] - cmd = "export HEKETI_CLI_SERVER=%s" % storage_class['resturl'] - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cmd = ("export HEKETI_CLI_SERVER=%s && heketi-cli cluster list " - "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1" % ( - storage_class['resturl'])) - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cluster_id = out.strip().split("\n")[0] - sc_name = storage_class['name'] - pvc_name1 = "mongodb1-block" cmd = ("oc get svc | grep heketi | grep -v endpoints " "| awk '{print $2}'") ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") @@ -43,6 +35,21 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): cmd, self.ocp_master_node[0])) heketi_cluster_ip = out.strip().split("\n")[0] resturl_block = "http://%s:8080" % heketi_cluster_ip + if not export_heketi_cli_server( + self.heketi_client_node, + heketi_cli_server=resturl_block, + heketi_cli_user=self.heketi_cli_user, + heketi_cli_key=self.heketi_cli_key): + raise ExecutionError("Failed to export heketi cli server on %s" + % self.heketi_client_node) + cmd = ("heketi-cli cluster list " + "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1") + ret, out, err = g.run(self.ocp_client[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_client[0])) + cluster_id = out.strip().split("\n")[0] + sc_name = storage_class['name'] + pvc_name1 = "mongodb1-block" ret = create_storage_class_file( self.ocp_master_node[0], sc_name, @@ -64,7 +71,7 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -112,7 +119,6 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): oc_delete(self.ocp_master_node[0], 'pod', pod_name) ret = verify_pod_status_running(self.ocp_master_node[0], pvc_name1) - ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") self.assertTrue(ret, "verify mongodb pod status as running failed") cmd = ("oc get pods | grep %s | grep -v deploy " "| awk {'print $1'}") % pvc_name1 @@ -129,22 +135,10 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) + @unittest.skip("skiping heketi-pod failure testcase") def test_dynamic_provisioning_glusterblock_heketipod_failure(self): g.log.info("test_dynamic_provisioning_glusterblock_Heketipod_Failure") storage_class = self.cns_storage_class['storage_class2'] - cmd = "export HEKETI_CLI_SERVER=%s" % storage_class['resturl'] - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cmd = ("export HEKETI_CLI_SERVER=%s && heketi-cli cluster list " - "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1") % ( - storage_class['resturl']) - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cluster_id = out.strip().split("\n")[0] - sc_name = storage_class['name'] - pvc_name2 = "mongodb2-block" cmd = ("oc get svc | grep heketi | grep -v endpoints " "| awk '{print $2}'") ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") @@ -152,6 +146,21 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): cmd, self.ocp_master_node[0])) heketi_cluster_ip = out.strip().split("\n")[0] resturl_block = "http://%s:8080" % heketi_cluster_ip + if not export_heketi_cli_server( + self.heketi_client_node, + heketi_cli_server=resturl_block, + heketi_cli_user=self.heketi_cli_user, + heketi_cli_key=self.heketi_cli_key): + raise ExecutionError("Failed to export heketi cli server on %s" + % self.heketi_client_node) + cmd = ("heketi-cli cluster list " + "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1") + ret, out, err = g.run(self.ocp_client[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_client[0])) + cluster_id = out.strip().split("\n")[0] + sc_name = storage_class['name'] + pvc_name2 = "mongodb2-block" ret = create_storage_class_file( self.ocp_master_node[0], sc_name, @@ -173,7 +182,7 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -227,30 +236,29 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): cmd, self.ocp_master_node[0])) ret = verify_pod_status_running(self.ocp_master_node[0], "heketi") self.assertTrue(ret, "verify heketi pod status as running failed") - oc_delete(self.ocp_master_node[0], 'sc', sc_name) cmd = ("oc get svc | grep heketi | grep -v endpoints " "| awk '{print $2}'") ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) - heketi_cluster_ip = out.strip().split("\n")[0] - resturl_block = "http://%s:8080" % heketi_cluster_ip - ret = create_storage_class_file( - self.ocp_master_node[0], - sc_name, - resturl_block, - storage_class['provisioner'], - restuser=storage_class['restuser'], - restsecretnamespace=storage_class['restsecretnamespace'], - restsecretname=storage_class['restsecretname'], - hacount=storage_class['hacount'], - clusterids=cluster_id) - self.assertTrue(ret, "creation of storage-class file failed") - provisioner_name = storage_class['provisioner'].split("/") - file_path = "/%s-%s-storage-class.yaml" % ( - sc_name, provisioner_name[1]) - oc_create(self.ocp_master_node[0], file_path) - for w in Waiter(300, 30): + heketi_cluster_new_ip = out.strip().split("\n")[0] + if heketi_cluster_new_ip != heketi_cluster_ip: + oc_delete(self.ocp_master_node[0], 'sc', sc_name) + resturl_block = "http://%s:8080" % heketi_cluster_new_ip + ret = create_storage_class_file( + self.ocp_master_node[0], + sc_name, + resturl_block, + storage_class['provisioner'], + restuser=storage_class['restuser'], + secretnamespace=storage_class['secretnamespace'], + secretname=storage_class['secretname']) + self.assertTrue(ret, "creation of storage-class file failed") + provisioner_name = storage_class['provisioner'].split("/") + file_path = "/%s-%s-storage-class.yaml" % ( + sc_name, provisioner_name[1]) + oc_create(self.ocp_master_node[0], file_path) + for w in Waiter(600, 30): ret, status = get_pvc_status(self.ocp_master_node[0], pvc_name3) self.assertTrue(ret, "failed to get pvc status of %s" % ( @@ -273,6 +281,12 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): pvc_name3) self.assertTrue(ret, "verify %s pod status as " "running failed" % pvc_name3) + cmd = ("oc get pods | grep %s | grep -v deploy " + "|awk {'print $1'}") % pvc_name3 + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + pod_name = out.strip().split("\n")[0] cmd = ("dd if=/dev/urandom of=/var/lib/mongodb/data/file " "bs=1K count=100") ret, out, err = oc_rsh(self.ocp_master_node[0], pod_name, cmd) @@ -282,19 +296,6 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): def test_dynamic_provisioning_glusterblock_glusterpod_failure(self): g.log.info("test_dynamic_provisioning_glusterblock_Glusterpod_Failure") storage_class = self.cns_storage_class['storage_class2'] - cmd = "export HEKETI_CLI_SERVER=%s" % storage_class['resturl'] - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cmd = ("export HEKETI_CLI_SERVER=%s && heketi-cli cluster list " - "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1") % ( - storage_class['resturl']) - ret, out, err = g.run(self.ocp_client[0], cmd, "root") - self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( - cmd, self.ocp_client[0])) - cluster_id = out.strip().split("\n")[0] - sc_name = storage_class['name'] - pvc_name4 = "mongodb-4-block" cmd = ("oc get svc | grep heketi | grep -v endpoints " "| awk '{print $2}'") ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") @@ -302,6 +303,21 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): cmd, self.ocp_master_node[0])) heketi_cluster_ip = out.strip().split("\n")[0] resturl_block = "http://%s:8080" % heketi_cluster_ip + if not export_heketi_cli_server( + self.heketi_client_node, + heketi_cli_server=resturl_block, + heketi_cli_user=self.heketi_cli_user, + heketi_cli_key=self.heketi_cli_key): + raise ExecutionError("Failed to export heketi cli server on %s" + % self.heketi_client_node) + cmd = ("heketi-cli cluster list " + "| grep Id | cut -d ':' -f 2 | cut -d '[' -f 1") + ret, out, err = g.run(self.ocp_client[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_client[0])) + cluster_id = out.strip().split("\n")[0] + sc_name = storage_class['name'] + pvc_name4 = "mongodb-4-block" ret = create_storage_class_file( self.ocp_master_node[0], sc_name, @@ -323,7 +339,7 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -366,10 +382,21 @@ class TestDynamicProvisioningBlockP0(CnsGlusterBlockBaseClass): cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_node_name) - ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + for w in Waiter(600, 30): + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + new_gluster_pod_name = out.strip().split("\n")[0].strip() + if ret == 0 and not new_gluster_pod_name: + continue + else: + break + if w.expired: + error_msg = "exceeded timeout, new gluster pod not created" + g.log.error(error_msg) + raise ExecutionError(error_msg) self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) new_gluster_pod_name = out.strip().split("\n")[0].strip() + g.log.info("new gluster pod name is %s" % new_gluster_pod_name) ret = verify_pod_status_running(self.ocp_master_node[0], new_gluster_pod_name) self.assertTrue(ret, "verify %s pod status as running " diff --git a/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py b/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py index 9ae0e987..0167573f 100644 --- a/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py +++ b/tests/functional/common/provisioning/test_dynamic_provisioning_p0_cases.py @@ -4,14 +4,20 @@ from cnslibs.common.dynamic_provisioning import ( create_storage_class_file, get_pvc_status, verify_pod_status_running) +from cnslibs.cns.cns_baseclass import ( + CnsBaseClass, + CnsSetupBaseClass) +from cnslibs.common.exceptions import ( + ConfigError, + ExecutionError) from cnslibs.common.openshift_ops import ( get_ocp_gluster_pod_names, - oc_rsh) -from cnslibs.cns.cns_baseclass import CnsBaseClass -from cnslibs.common.openshift_ops import ( oc_create, - oc_delete) + oc_delete, + oc_rsh) +from cnslibs.common.waiter import Waiter from glusto.core import Glusto as g +import unittest class TestDynamicProvisioningP0(CnsBaseClass): @@ -19,16 +25,22 @@ class TestDynamicProvisioningP0(CnsBaseClass): Class that contain P0 dynamic provisioning test cases for glusterfile volume ''' - def test_dynamic_provisioning_glusterfile(self): g.log.info("test_dynamic_provisioning_glusterfile") storage_class = self.cns_storage_class['storage_class1'] sc_name = storage_class['name'] pvc_name1 = "mongodb1" + cmd = ("oc get svc | grep heketi | grep -v endpoints " + "| awk '{print $2}'") + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + heketi_cluster_ip = out.strip().split("\n")[0] + resturl = "http://%s:8080" % heketi_cluster_ip ret = create_storage_class_file( self.ocp_master_node[0], sc_name, - storage_class['resturl'], + resturl, storage_class['provisioner'], restuser=storage_class['restuser'], secretnamespace=storage_class['secretnamespace'], @@ -44,7 +56,7 @@ class TestDynamicProvisioningP0(CnsBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -77,7 +89,6 @@ class TestDynamicProvisioningP0(CnsBaseClass): oc_delete(self.ocp_master_node[0], 'pod', pod_name) ret = verify_pod_status_running(self.ocp_master_node[0], pvc_name1) - ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") self.assertTrue(ret, "verify mongodb pod status as running failed") cmd = ("oc get pods | grep %s | grep -v deploy " "| awk {'print $1'}") % pvc_name1 @@ -94,15 +105,23 @@ class TestDynamicProvisioningP0(CnsBaseClass): self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) + @unittest.skip("skiping heketi-pod failure testcase") def test_dynamic_provisioning_glusterfile_heketipod_failure(self): g.log.info("test_dynamic_provisioning_glusterfile_Heketipod_Failure") storage_class = self.cns_storage_class['storage_class1'] sc_name = storage_class['name'] pvc_name2 = "mongodb2" + cmd = ("oc get svc | grep heketi | grep -v endpoints " + "| awk '{print $2}'") + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + heketi_cluster_ip = out.strip().split("\n")[0] + resturl = "http://%s:8080" % heketi_cluster_ip ret = create_storage_class_file( self.ocp_master_node[0], sc_name, - storage_class['resturl'], + resturl, storage_class['provisioner'], restuser=storage_class['restuser'], secretnamespace=storage_class['secretnamespace'], @@ -118,7 +137,7 @@ class TestDynamicProvisioningP0(CnsBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -172,15 +191,57 @@ class TestDynamicProvisioningP0(CnsBaseClass): cmd, self.ocp_master_node[0])) ret = verify_pod_status_running(self.ocp_master_node[0], "heketi") self.assertTrue(ret, "verify heketi pod status as running failed") - ret, status = get_pvc_status(self.ocp_master_node[0], + cmd = ("oc get svc | grep heketi | grep -v endpoints " + "| awk '{print $2}'") + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + heketi_cluster_new_ip = out.strip().split("\n")[0] + if heketi_cluster_new_ip != heketi_cluster_ip: + oc_delete(self.ocp_master_node[0], 'sc', sc_name) + resturl = "http://%s:8080" % heketi_cluster_ip + ret = create_storage_class_file( + self.ocp_master_node[0], + sc_name, + resturl, + storage_class['provisioner'], + restuser=storage_class['restuser'], + secretnamespace=storage_class['secretnamespace'], + secretname=storage_class['secretname']) + self.assertTrue(ret, "creation of storage-class file failed") + provisioner_name = storage_class['provisioner'].split("/") + file_path = "/%s-%s-storage-class.yaml" % ( + sc_name, provisioner_name[1]) + oc_create(self.ocp_master_node[0], file_path) + for w in Waiter(600, 30): + ret, status = get_pvc_status(self.ocp_master_node[0], pvc_name3) - self.assertTrue(ret, "failed to get pvc status of %s" % pvc_name3) + self.assertTrue(ret, "failed to get pvc status of %s" % ( + pvc_name3)) + if status != "Bound": + g.log.info("pvc status of %s is not in Bound state," + " sleeping for 30 sec" % pvc_name3) + continue + else: + break + if w.expired: + error_msg = ("exceeded timeout 300 sec, pvc %s not in" + " Bound state" % pvc_name3) + g.log.error(error_msg) + raise ExecutionError(error_msg) self.assertEqual(status, "Bound", "pvc status of %s " - "is not in Bound state" % pvc_name3) + "is not in Bound state, its state is %s" % ( + pvc_name3, status)) ret = verify_pod_status_running(self.ocp_master_node[0], pvc_name3) self.assertTrue(ret, "verify %s pod status " "as running failed" % pvc_name3) + cmd = ("oc get pods | grep %s | grep -v deploy " + "|awk {'print $1'}") % pvc_name3 + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + pod_name = out.strip().split("\n")[0] cmd = ("dd if=/dev/urandom of=/var/lib/mongodb/data/file " "bs=1K count=100") ret, out, err = oc_rsh(self.ocp_master_node[0], pod_name, cmd) @@ -192,10 +253,17 @@ class TestDynamicProvisioningP0(CnsBaseClass): storage_class = self.cns_storage_class['storage_class1'] sc_name = storage_class['name'] pvc_name4 = "mongodb4" + cmd = ("oc get svc | grep heketi | grep -v endpoints " + "| awk '{print $2}'") + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( + cmd, self.ocp_master_node[0])) + heketi_cluster_ip = out.strip().split("\n")[0] + resturl = "http://%s:8080" % heketi_cluster_ip ret = create_storage_class_file( self.ocp_master_node[0], sc_name, - storage_class['resturl'], + resturl, storage_class['provisioner'], restuser=storage_class['restuser'], secretnamespace=storage_class['secretnamespace'], @@ -211,7 +279,7 @@ class TestDynamicProvisioningP0(CnsBaseClass): ret = create_secret_file(self.ocp_master_node[0], secret['secret_name'], secret['namespace'], - secret['data_key'], + self.secret_data_key, secret['type']) self.assertTrue(ret, "creation of heketi-secret file failed") oc_create(self.ocp_master_node[0], @@ -245,7 +313,7 @@ class TestDynamicProvisioningP0(CnsBaseClass): g.log.info("gluster_pod_list - %s" % gluster_pod_list) gluster_pod_name = gluster_pod_list[0] cmd = ("oc get pods -o wide | grep %s | grep -v deploy " - "|awk '{print $7}'") % gluster_pod_name + "| awk '{print $7}'") % gluster_pod_name ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) @@ -254,10 +322,21 @@ class TestDynamicProvisioningP0(CnsBaseClass): cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_node_name) - ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + for w in Waiter(600, 30): + ret, out, err = g.run(self.ocp_master_node[0], cmd, "root") + new_gluster_pod_name = out.strip().split("\n")[0].strip() + if ret == 0 and not new_gluster_pod_name: + continue + else: + break + if w.expired: + error_msg = "exceeded timeout, new gluster pod not created" + g.log.error(error_msg) + raise ExecutionError(error_msg) self.assertEqual(ret, 0, "failed to execute command %s on %s" % ( cmd, self.ocp_master_node[0])) new_gluster_pod_name = out.strip().split("\n")[0].strip() + g.log.info("new gluster pod name is %s" % new_gluster_pod_name) ret = verify_pod_status_running(self.ocp_master_node[0], new_gluster_pod_name) self.assertTrue(ret, "verify %s pod status as running " |