summaryrefslogtreecommitdiffstats
path: root/openshift-storage-libs
diff options
context:
space:
mode:
authorValerii Ponomarov <vponomar@redhat.com>2019-06-27 13:10:22 +0530
committervponomar <vponomar@redhat.com>2019-07-16 12:09:56 +0000
commit1f3bb5b08d05f5711af842edc3dbf924acc14bf5 (patch)
tree49c11000d9c5254f0b0a9ac62f4809b01e099e46 /openshift-storage-libs
parent32724906d1fd5fd09759e1daf927594461c26d5e (diff)
Fix logic for recreation of SSH connections
Before, in case we had some constant networking issue trying to connect to a host, we were falling into the endless loop of connection recreation. The error we got after it is the 'recursion limit exceeded'. It is unexpected. So, change this logic making it avoid recursion and do just one connection recreation attempt which is useful to fix broken connections. Change-Id: Id808edbad7e6d69ad58a75bfbae176fddb173d18
Diffstat (limited to 'openshift-storage-libs')
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/__init__.py60
1 files changed, 51 insertions, 9 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/__init__.py b/openshift-storage-libs/openshiftstoragelibs/__init__.py
index 8d4d25c..5ca0f5f 100644
--- a/openshift-storage-libs/openshiftstoragelibs/__init__.py
+++ b/openshift-storage-libs/openshiftstoragelibs/__init__.py
@@ -1,5 +1,7 @@
from glusto.core import Glusto
-from six import add_metaclass
+import six
+
+from openshiftstoragelibs import exceptions
def monkeypatch_class(name, bases, namespace):
@@ -11,15 +13,55 @@ def monkeypatch_class(name, bases, namespace):
return base
-@add_metaclass(monkeypatch_class)
+@six.add_metaclass(monkeypatch_class)
class MonkeyPatchedGlusto(Glusto):
@classmethod
- def _get_ssh_connection(cls, host, user=None):
- ssh = super(MonkeyPatchedGlusto, cls)._get_ssh_connection(
- host=host, user=user)
+ def _wrapper_for_get_ssh_connection(cls, host, user=None, recreate=False):
+ if recreate and "%s@%s" % (user, host) in cls._ssh_connections:
+ cls.ssh_close_connection(host=host, user=user)
+ ssh = cls._get_ssh_connection(host, user)
if not ssh:
- super(MonkeyPatchedGlusto, cls).ssh_close_connection(
- host=host, user=user)
- ssh = super(MonkeyPatchedGlusto, cls)._get_ssh_connection(
- host=host, user=user)
+ if "%s@%s" % (user, host) in cls._ssh_connections:
+ cls.ssh_close_connection(host=host, user=user)
+ ssh = cls._get_ssh_connection(host=host, user=user)
+ if not ssh:
+ raise exceptions.ExecutionError(
+ "Failed to establish SSH connection to the '%s' host "
+ "using '%s' user. Plese check availability of the "
+ "hostname and make sure it has passwordless "
+ "connection from your host." % (host, user))
return ssh
+
+ @classmethod
+ def run(cls, host, command, user=None, log_level=None):
+ """Wrapper for original "run" method fixing broken connections."""
+ if not user:
+ user = cls.user
+
+ ctlpersist = ''
+ if cls.use_controlpersist:
+ ctlpersist = " (cp)"
+
+ # output command
+ cls.log.info("%s@%s%s: %s" % (user, host, ctlpersist, command))
+
+ # run the command
+ ssh = cls._wrapper_for_get_ssh_connection(host, user, False)
+ try:
+ proc = ssh.popen(command, universal_newlines=True)
+ except Exception as e:
+ err_msg = (
+ "Failed to establish SSH connection: %s" % six.text_type(e))
+ cls.log.error(err_msg)
+ ssh = cls._wrapper_for_get_ssh_connection(host, user, True)
+ proc = ssh.popen(command, universal_newlines=True)
+
+ stdout, stderr = proc.communicate()
+ retcode = proc.returncode
+
+ # output command results
+ identifier = "%s@%s" % (user, host)
+ cls._log_results(identifier, retcode, stdout, stderr,
+ log_level=log_level)
+
+ return (retcode, stdout, stderr)