summaryrefslogtreecommitdiffstats
path: root/openshift-storage-libs
diff options
context:
space:
mode:
authorValerii Ponomarov <vponomar@redhat.com>2019-06-24 01:29:41 +0530
committervponomar <vponomar@redhat.com>2019-07-11 09:13:40 +0000
commit32724906d1fd5fd09759e1daf927594461c26d5e (patch)
treeb6744bd1f0b1364e43810c343a873f6022e3e575 /openshift-storage-libs
parent1ef65c168d3b45a5af5d263ff3403263810a42d8 (diff)
Make Heketi commands run on a Heketi POD when main command fails
It happens that heketi client located out of the Heketi POD may fail not reaching the server side. So, add back-up approach where we run Heketi commands on a Heketi POD when main commands fail. Change-Id: Ie6ae5be82082f34426f9288b02575e3abd4940f5
Diffstat (limited to 'openshift-storage-libs')
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/heketi_ops.py421
1 files changed, 126 insertions, 295 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
index f8d4175..7266445 100644
--- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
@@ -6,10 +6,12 @@ except ImportError:
import json
import re
+import time
from glusto.core import Glusto as g
+import six
-from openshiftstoragelibs import exceptions
+from openshiftstoragelibs import command
from openshiftstoragelibs import heketi_version
from openshiftstoragelibs.utils import parse_prometheus_data
@@ -18,6 +20,53 @@ HEKETI_COMMAND_TIMEOUT = g.config.get("common", {}).get(
"heketi_command_timeout", 120)
TIMEOUT_PREFIX = "timeout %s " % HEKETI_COMMAND_TIMEOUT
+MASTER_NODE = list(g.config["ocp_servers"]["master"].keys())[0]
+HEKETI_DC = g.config.get("cns", g.config.get("openshift"))[
+ "heketi_config"]["heketi_dc_name"]
+GET_HEKETI_PODNAME_CMD = (
+ "oc get pods -l deploymentconfig=%s -o=custom-columns=:.metadata.name "
+ "--no-headers" % HEKETI_DC
+)
+
+
+def cmd_run_on_heketi_pod(cmd, raise_on_error=True):
+ """Autodetect Heketi podname and run specified command on it."""
+ heketi_podname = command.cmd_run(
+ cmd=GET_HEKETI_PODNAME_CMD, hostname=MASTER_NODE).strip()
+ # NOTE(vponomar): we redefine '--server' option which is provided
+ # as part of the 'cmd' var.
+ assert heketi_podname.strip(), (
+ "Heketi POD not found on '%s' node using following command: \n%s" % (
+ MASTER_NODE, GET_HEKETI_PODNAME_CMD))
+ if '--server=' in cmd and 'heketi-cli' in cmd:
+ cmd_with_podname_prefix = (
+ "oc exec %s -- %s --server=http://localhost:8080" % (
+ heketi_podname, cmd))
+ else:
+ cmd_with_podname_prefix = "oc exec %s -- %s" % (heketi_podname, cmd)
+ result = command.cmd_run(
+ cmd=cmd_with_podname_prefix, hostname=MASTER_NODE,
+ raise_on_error=raise_on_error)
+ return result
+
+
+def heketi_cmd_run(hostname, cmd, raise_on_error=True):
+ """Run Heketi client command from a node backing up with Heketi pod CLI."""
+ try:
+ out = command.cmd_run(
+ cmd=cmd, hostname=hostname, raise_on_error=raise_on_error)
+ except Exception as e:
+ g.log.error(
+ 'Failed to run "%s" command on the "%s" host. '
+ 'Got following error:\n%s' % (cmd, hostname, e))
+ if ('connection refused' in six.text_type(e).lower() or
+ 'operation timed out' in six.text_type(e).lower()):
+ time.sleep(1)
+ out = cmd_run_on_heketi_pod(cmd, raise_on_error=raise_on_error)
+ else:
+ raise
+ return out
+
def _set_heketi_global_flags(heketi_server_url, **kwargs):
"""Helper function to set heketi-cli global flags."""
@@ -44,7 +93,7 @@ def _set_heketi_global_flags(heketi_server_url, **kwargs):
def heketi_volume_create(heketi_client_node, heketi_server_url, size,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Creates heketi volume with the given user options.
Args:
@@ -75,9 +124,8 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
Returns:
dict: volume create info on success, only cli option is specified
without --json option, then it returns raw string output.
- Tuple (ret, out, err): if raw_cli_output is True.
Raises:
- exceptions.ExecutionError when error occurs and raw_cli_output is False
+ exceptions.ExecutionError when error occurs
Example:
heketi_volume_create(heketi_client_node, heketi_server_url, size)
@@ -134,8 +182,6 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
"--secret %s" % kwargs.get("secret") if kwargs.get("secret") else "")
user_arg = "--user %s" % kwargs.get("user") if kwargs.get("user") else ""
- err_msg = "Failed to create volume. "
-
cmd = ("heketi-cli -s %s volume create --size=%s %s %s %s %s %s %s "
"%s %s %s %s %s %s %s %s %s %s" % (
heketi_server_url, str(size), block_arg, clusters_arg,
@@ -145,21 +191,14 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
persistent_volume_file_arg, redundancy_arg, replica_arg,
snapshot_factor_arg, json_arg, secret_arg, user_arg))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- err_msg += "Out: %s \n Err: %s" % (out, err)
- g.log.error(err_msg)
- raise exceptions.ExecutionError(err_msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi volume info command.
Args:
@@ -175,8 +214,6 @@ def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id,
Returns:
dict: volume info on success
- False: in case of failure
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -191,24 +228,14 @@ def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id,
cmd = "heketi-cli -s %s volume info %s %s %s %s" % (
heketi_server_url, volume_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
-
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id,
- expand_size, raw_cli_output=False, **kwargs):
+ expand_size, **kwargs):
"""Executes heketi volume expand command.
Args:
@@ -226,7 +253,6 @@ def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id,
Returns:
dict: volume expand info on success, only cli option is specified
without --json option, then it returns raw string output.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -244,24 +270,14 @@ def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id,
heketi_server_url, volume_id, expand_size, json_arg,
admin_key, user))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
- raw_cli_output=False, raise_on_error=True, **kwargs):
+ raise_on_error=True, **kwargs):
"""Executes heketi volume delete command.
Args:
@@ -279,10 +295,9 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
Returns:
str: volume delete command output on success
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
- exceptions.ExecutionError when error occurs and raw_cli_output is False
+ exceptions.ExecutionError when error occurs
Example:
heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id)
@@ -290,25 +305,16 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
heketi_server_url, **kwargs)
- err_msg = "Failed to delete '%s' volume. " % volume_id
cmd = "heketi-cli -s %s volume delete %s %s %s %s" % (
heketi_server_url, volume_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- err_msg += "Out: %s, \nErr: %s" % (out, err)
- g.log.error(err_msg)
- if raise_on_error:
- raise exceptions.ExecutionError(err_msg)
+ out = heketi_cmd_run(
+ heketi_client_node, cmd, raise_on_error=raise_on_error)
return out
-def heketi_volume_list(heketi_client_node, heketi_server_url,
- raw_cli_output=False, **kwargs):
+def heketi_volume_list(heketi_client_node, heketi_server_url, **kwargs):
"""Executes heketi volume list command.
Args:
@@ -324,7 +330,6 @@ def heketi_volume_list(heketi_client_node, heketi_server_url,
Returns:
dict: volume list with --json on success, if cli option is specified
without --json option or with url, it returns raw string output.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -339,23 +344,13 @@ def heketi_volume_list(heketi_client_node, heketi_server_url,
cmd = "heketi-cli -s %s volume list %s %s %s" % (
heketi_server_url, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
-def heketi_topology_info(heketi_client_node, heketi_server_url,
- raw_cli_output=False, **kwargs):
+def heketi_topology_info(heketi_client_node, heketi_server_url, **kwargs):
"""Executes heketi topology info command.
Args:
@@ -371,7 +366,6 @@ def heketi_topology_info(heketi_client_node, heketi_server_url,
Returns:
dict: topology info if --json option is specified. If only cli option
is specified, raw command output is returned on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -386,16 +380,7 @@ def heketi_topology_info(heketi_client_node, heketi_server_url,
cmd = "heketi-cli -s %s topology info %s %s %s" % (
heketi_server_url, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -427,14 +412,20 @@ def hello_heketi(heketi_client_node, heketi_server_url, **kwargs):
heketi_server_url, **kwargs)
cmd = "curl --max-time 10 %s/hello" % heketi_server_url
- ret, out, err = g.run(heketi_client_node, cmd)
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ try:
+ command.cmd_run(cmd=cmd, hostname=heketi_client_node)
+ except Exception as e:
+ g.log.error(
+ 'Failed to run "%s" command on the "%s" host. '
+ 'Got following error:\n%s' % (cmd, heketi_client_node, e))
+ if ('connection refused' in six.text_type(e).lower() or
+ 'operation timed out' in six.text_type(e).lower()):
+ time.sleep(1)
+ cmd_run_on_heketi_pod(
+ "curl --max-time 10 http://localhost:8080/hello")
+ else:
+ raise
return True
@@ -470,14 +461,7 @@ def heketi_cluster_delete(heketi_client_node, heketi_server_url, cluster_id,
cmd = "heketi-cli -s %s cluster delete %s %s %s %s" % (
heketi_server_url, cluster_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
@@ -513,14 +497,7 @@ def heketi_cluster_info(heketi_client_node, heketi_server_url, cluster_id,
cmd = "heketi-cli -s %s cluster info %s %s %s %s" % (
heketi_server_url, cluster_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -556,21 +533,14 @@ def heketi_cluster_list(heketi_client_node, heketi_server_url, **kwargs):
cmd = "heketi-cli -s %s cluster list %s %s %s" % (
heketi_server_url, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
def heketi_device_add(heketi_client_node, heketi_server_url, device_name,
- node_id, raw_cli_output=False, **kwargs):
+ node_id, **kwargs):
"""Executes heketi device add command.
Args:
@@ -587,7 +557,6 @@ def heketi_device_add(heketi_client_node, heketi_server_url, device_name,
Returns:
str: heketi device add command output on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -603,21 +572,12 @@ def heketi_device_add(heketi_client_node, heketi_server_url, device_name,
cmd = "heketi-cli -s %s device add --name=%s --node=%s %s %s %s" % (
heketi_server_url, device_name, node_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
def heketi_device_delete(heketi_client_node, heketi_server_url, device_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi device delete command.
Args:
@@ -633,7 +593,6 @@ def heketi_device_delete(heketi_client_node, heketi_server_url, device_id,
Returns:
str: heketi device delete command output on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -648,21 +607,12 @@ def heketi_device_delete(heketi_client_node, heketi_server_url, device_id,
cmd = "heketi-cli -s %s device delete %s %s %s %s" % (
heketi_server_url, device_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
def heketi_device_disable(heketi_client_node, heketi_server_url, device_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi device disable command.
Args:
@@ -678,7 +628,6 @@ def heketi_device_disable(heketi_client_node, heketi_server_url, device_id,
Returns:
str: heketi device disable command output on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -692,22 +641,12 @@ def heketi_device_disable(heketi_client_node, heketi_server_url, device_id,
cmd = "heketi-cli -s %s device disable %s %s %s %s" % (
heketi_server_url, device_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
-
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
def heketi_device_enable(heketi_client_node, heketi_server_url, device_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi device enable command.
Args:
@@ -723,7 +662,6 @@ def heketi_device_enable(heketi_client_node, heketi_server_url, device_id,
Returns:
str: heketi device enable command output on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -737,22 +675,12 @@ def heketi_device_enable(heketi_client_node, heketi_server_url, device_id,
cmd = "heketi-cli -s %s device enable %s %s %s %s" % (
heketi_server_url, device_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
-
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
def heketi_device_info(heketi_client_node, heketi_server_url, device_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi device info command.
Args:
@@ -769,7 +697,6 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id,
Returns:
Str: device info as raw CLI output if "json" arg is not provided.
Dict: device info parsed to dict if "json" arg is provided.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -784,17 +711,7 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id,
cmd = "heketi-cli -s %s device info %s %s %s %s" % (
heketi_server_url, device_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
-
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
device_info = json.loads(out)
return device_info
@@ -803,7 +720,7 @@ def heketi_device_info(heketi_client_node, heketi_server_url, device_id,
def heketi_device_remove(heketi_client_node, heketi_server_url, device_id,
- raw_cli_output=False, **kwargs):
+ **kwargs):
"""Executes heketi device remove command.
Args:
@@ -819,7 +736,6 @@ def heketi_device_remove(heketi_client_node, heketi_server_url, device_id,
Returns:
str: heketi device remove command output on success.
- Tuple (ret, out, err): if raw_cli_output is True
Raises:
exceptions.ExecutionError: if command fails.
@@ -834,17 +750,7 @@ def heketi_device_remove(heketi_client_node, heketi_server_url, device_id,
cmd = "heketi-cli -s %s device remove %s %s %s %s" % (
heketi_server_url, device_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if raw_cli_output:
- return ret, out, err
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
-
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
@@ -879,14 +785,7 @@ def heketi_node_delete(heketi_client_node, heketi_server_url, node_id,
cmd = "heketi-cli -s %s node delete %s %s %s %s" % (
heketi_server_url, node_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
@@ -921,14 +820,7 @@ def heketi_node_disable(heketi_client_node, heketi_server_url, node_id,
cmd = "heketi-cli -s %s node disable %s %s %s %s" % (
heketi_server_url, node_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
@@ -963,14 +855,7 @@ def heketi_node_enable(heketi_client_node, heketi_server_url, node_id,
cmd = "heketi-cli -s %s node enable %s %s %s %s" % (
heketi_server_url, node_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
return out
@@ -1005,14 +890,7 @@ def heketi_node_info(heketi_client_node, heketi_server_url, node_id, **kwargs):
cmd = "heketi-cli -s %s node info %s %s %s %s" % (
heketi_server_url, node_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -1038,14 +916,7 @@ def heketi_node_list(heketi_client_node, heketi_server_url,
cmd = "heketi-cli -s %s node list %s %s %s" % (
heketi_server_url, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
heketi_node_id_list = []
for line in out.strip().split("\n"):
@@ -1087,14 +958,7 @@ def heketi_blockvolume_info(heketi_client_node, heketi_server_url,
cmd = "heketi-cli -s %s blockvolume info %s %s %s %s" % (
heketi_server_url, block_volume_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -1163,14 +1027,7 @@ def heketi_blockvolume_create(heketi_client_node, heketi_server_url, size,
clusters_arg, ha_arg, name_arg, name_arg,
admin_key, user, json_arg))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -1206,18 +1063,12 @@ def heketi_blockvolume_delete(heketi_client_node, heketi_server_url,
heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
heketi_server_url, **kwargs)
- err_msg = "Failed to delete '%s' volume. " % block_volume_id
cmd = "heketi-cli -s %s blockvolume delete %s %s %s %s" % (
heketi_server_url, block_volume_id, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- err_msg += "Out: %s, \nErr: %s" % (out, err)
- g.log.error(err_msg)
- if raise_on_error:
- raise exceptions.ExecutionError(err_msg)
+ out = heketi_cmd_run(
+ heketi_client_node, cmd, raise_on_error=raise_on_error)
return out
@@ -1252,14 +1103,7 @@ def heketi_blockvolume_list(heketi_client_node, heketi_server_url, **kwargs):
cmd = "heketi-cli -s %s blockvolume list %s %s %s" % (
heketi_server_url, json_arg, admin_key, user)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, heketi_client_node, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(heketi_client_node, cmd)
if json_arg:
return json.loads(out)
return out
@@ -1296,14 +1140,7 @@ def verify_volume_name_prefix(hostname, prefix, namespace, pvc_name,
cmd = "heketi-cli -s %s volume list %s %s %s | grep %s" % (
heketi_server_url, json_arg, admin_key, user, heketi_vol_name_prefix)
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(hostname, cmd, "root")
-
- if ret != 0:
- msg = (
- "Failed to execute '%s' command on '%s' node with following "
- "error: %s" % (cmd, hostname, err))
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+ out = heketi_cmd_run(hostname, cmd)
output = out.strip()
g.log.info("heketi volume with volnameprefix present %s" % output)
return True
@@ -1347,14 +1184,9 @@ def set_tags(heketi_client_node, heketi_server_url, source, source_id, tag,
cmd = ("heketi-cli -s %s %s settags %s %s %s %s" %
(heketi_server_url, source, source_id, tag, user, secret))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if not ret:
- g.log.info("Tagging of %s to %s is successful" % (source, tag))
- return True
-
- g.log.error(err)
- raise exceptions.ExecutionError(err)
+ heketi_cmd_run(heketi_client_node, cmd)
+ g.log.info("Tagging of %s to %s is successful" % (source, tag))
+ return True
def set_arbiter_tag(heketi_client_node, heketi_server_url, source,
@@ -1437,14 +1269,9 @@ def rm_tags(heketi_client_node, heketi_server_url, source, source_id, tag,
cmd = ("heketi-cli -s %s %s rmtags %s %s %s %s" %
(heketi_server_url, source, source_id, tag, user, secret))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if not ret:
- g.log.info("Removal of %s tag from %s is successful." % (tag, source))
- return True
-
- g.log.error(err)
- raise exceptions.ExecutionError(err)
+ heketi_cmd_run(heketi_client_node, cmd)
+ g.log.info("Removal of %s tag from %s is successful." % (tag, source))
+ return True
def rm_arbiter_tag(heketi_client_node, heketi_server_url, source, source_id,
@@ -1508,11 +1335,21 @@ def get_heketi_metrics(heketi_client_node, heketi_server_url,
raise NotImplementedError(msg)
cmd = "curl --max-time 10 %s/metrics" % heketi_server_url
- ret, out, err = g.run(heketi_client_node, cmd)
- if ret != 0:
- msg = "failed to get Heketi metrics with following error: %s" % err
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
+
+ try:
+ out = command.cmd_run(cmd=cmd, hostname=heketi_client_node)
+ except Exception as e:
+ g.log.error(
+ 'Failed to run "%s" command on the "%s" host. '
+ 'Got following error:\n%s' % (cmd, heketi_client_node, e))
+ if ('connection refused' in six.text_type(e).lower() or
+ 'operation timed out' in six.text_type(e).lower()):
+ time.sleep(1)
+ out = cmd_run_on_heketi_pod(
+ "curl --max-time 10 http://localhost:8080/metrics")
+ else:
+ raise
+
if prometheus_format:
return out.strip()
return parse_prometheus_data(out)
@@ -1546,13 +1383,7 @@ def heketi_examine_gluster(heketi_client_node, heketi_server_url):
cmd = ("heketi-cli server state examine gluster -s %s %s %s"
% (heketi_server_url, user, secret))
cmd = TIMEOUT_PREFIX + cmd
- ret, out, err = g.run(heketi_client_node, cmd)
-
- if ret != 0:
- msg = "failed to examine gluster with following error: %s" % err
- g.log.error(msg)
- raise exceptions.ExecutionError(msg)
-
+ out = heketi_cmd_run(heketi_client_node, cmd)
return json.loads(out)