Fix broken dynamic_provisioning tests

Fix 2 autotests, remove skip decorator for them.
And rework logic of mongodb template uploading for mongodb pod,
so it takes couple of seconds instead of minutes.

Change-Id: Ib2b09364ae435b9784b76c2f2581c197128f9649

3 files changed, 42 insertions, 20 deletions

diff --git a/cns-libs/cnslibs/cns/cns_baseclass.py b/cns-libs/cnslibs/cns/cns_baseclass.py
index 6f8cb4b9..3df23fe1 100644
--- a/cns-libs/cnslibs/cns/cns_baseclass.py
+++ b/cns-libs/cnslibs/cns/cns_baseclass.py
@@ -65,6 +65,7 @@ class CnsBaseClass(unittest.TestCase):
 
         # Initializes heketi config variables
         heketi_config = g.config['cns']['heketi_config']
+        cls.heketi_dc_name = heketi_config['heketi_dc_name']
         cls.heketi_service_name = heketi_config['heketi_service_name']
         cls.heketi_client_node = heketi_config['heketi_client_node']
         cls.heketi_server_url = heketi_config['heketi_server_url']
diff --git a/cns-libs/cnslibs/common/dynamic_provisioning.py b/cns-libs/cnslibs/common/dynamic_provisioning.py
index 7c1d0168..6285239a 100644
--- a/cns-libs/cnslibs/common/dynamic_provisioning.py
+++ b/cns-libs/cnslibs/common/dynamic_provisioning.py
@@ -1,6 +1,7 @@
 from collections import OrderedDict
 import json
 import os
+import tempfile
 
 from glusto.core import Glusto as g
 from glustolibs.misc.misc_libs import upload_scripts
@@ -267,35 +268,39 @@ def create_mongodb_pod(hostname, pvc_name, pvc_size, sc_name):
      Returns: True if successfull,
               False otherwise
     '''
-    ret = upload_scripts(hostname,
-                         os.path.join(TEMPLATE_DIR, "mongodb-template.json"),
-                         "/tmp/app-templates", "root")
-    if not ret:
-        g.log.error("Failed to upload mongodp template to %s" % hostname)
-        return False
+    template_path = os.path.join(TEMPLATE_DIR, "mongodb-template.json")
+    with open(template_path, 'r') as template_f:
+        data = json.load(template_f, object_pairs_hook=OrderedDict)
+    data['objects'][1]['metadata']['annotations'][
+        'volume.beta.kubernetes.io/storage-class'] = sc_name
+
+    tmp_fd, tmp_path = tempfile.mkstemp(
+        prefix='cns-automation-mongodb-pvcname-%s-' % pvc_name, suffix='.json')
+    dst_dir = '/tmp'
+    dst_path = os.path.join(dst_dir, os.path.basename(tmp_path))
+    try:
+        with os.fdopen(tmp_fd, 'w') as tmp_f:
+            json.dump(
+                data, tmp_f, sort_keys=False, indent=4, ensure_ascii=False)
+        if not upload_scripts(hostname, tmp_path, dst_dir, "root"):
+            g.log.error("Failed to upload mongodp template to %s" % hostname)
+            return False
+    finally:
+        os.remove(tmp_path)
+
     try:
         conn = g.rpyc_get_connection(hostname, user="root")
         if conn is None:
             g.log.error("Failed to get rpyc connection of node %s"
                         % hostname)
             return False
-        with conn.builtin.open(
-                '/tmp/app-templates/mongodb-template.json', 'r') as data_file:
-            data = json.load(data_file, object_pairs_hook=OrderedDict)
-        data['objects'][1]['metadata']['annotations'][
-            'volume.beta.kubernetes.io/storage-class'] = sc_name
-        with conn.builtin.open('/%s.json' % pvc_name, 'w') as data_file:
-            json.dump(data, data_file, sort_keys=False,
-                      indent=4, ensure_ascii=False)
-        cmd = ("oc new-app /%s.json --param=DATABASE_SERVICE_NAME=%s "
+        cmd = ("oc new-app %s --param=DATABASE_SERVICE_NAME=%s "
                "--param=VOLUME_CAPACITY=%sGi") % (
-                   pvc_name, pvc_name, pvc_size)
+                   dst_path, pvc_name, pvc_size)
         ret, out, err = g.run(hostname, cmd, "root")
         if ret != 0:
-            g.log.error("failed to execute cmd %s on %s" % (
-                            cmd, hostname))
+            g.log.error("failed to execute cmd %s on %s" % (cmd, hostname))
             return False
-
     except Exception as err:
         g.log.error("failed to create mongodb pod %s" % err)
         return False
diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py
index 3d3dd061..84edfdd6 100644
--- a/cns-libs/cnslibs/common/openshift_ops.py
+++ b/cns-libs/cnslibs/common/openshift_ops.py
@@ -6,9 +6,11 @@ Various utility functions for interacting with OCP/OpenShift.
 import re
 import types
 
+from glusto.core import Glusto as g
 import yaml
 
-from glusto.core import Glusto as g
+from cnslibs.common import exceptions
+from cnslibs.common import waiter
 
 
 PODS_WIDE_RE = re.compile(
@@ -291,3 +293,17 @@ def create_namespace(hostname, namespace):
         return True
     g.log.error("failed to create namespace %s" % namespace)
     return False
+
+
+def wait_for_resource_absence(ocp_node, rtype, name,
+                              interval=10, timeout=120):
+    for w in waiter.Waiter(timeout=timeout, interval=interval):
+        try:
+            oc_get_yaml(ocp_node, rtype, name, raise_on_error=True)
+        except AssertionError:
+            return
+    if w.expired:
+        error_msg = "%s '%s' still exists after waiting for it %d seconds" % (
+            rtype, name, timeout)
+        g.log.error(error_msg)
+        raise exceptions.ExecutionError(error_msg)