diff options
| -rwxr-xr-x | libs/connect/ssh.py | 35 | ||||
| -rw-r--r-- | libs/utils/clientutils.py | 203 | ||||
| -rw-r--r-- | libs/utils/hostutils.py | 419 | ||||
| -rw-r--r-- | libs/utils/serverutils.py | 112 | ||||
| -rw-r--r-- | libs/utils/validate.py | 138 |
5 files changed, 737 insertions, 170 deletions
diff --git a/libs/connect/ssh.py b/libs/connect/ssh.py index a419c31..e4540d5 100755 --- a/libs/connect/ssh.py +++ b/libs/connect/ssh.py @@ -7,8 +7,8 @@ import time from atfglobals import GlobalObj class SshConnection(): - - def __init__(self): + + def __init__(self): self._connection = paramiko.SSHClient() def connect(self, host, user, password): @@ -17,14 +17,14 @@ class SshConnection(): SSH to Server "host" as User "user" Parameter: - host: Server IP Address + host: Server IP Address user: Login Username password: Login password Return: Success: 0 Failure: 1 - """ + """ logger = GlobalObj.getLoggerObj() self._connection.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -43,22 +43,22 @@ class SshConnection(): except paramiko.SSHException: logger.error("SSHException: Unknown server " + host) - return 1 + return 1 return 0 def close(self): """ Objective: - Close SSH Connections + Close SSH Connections """ self._connection.close() - return + return def executecommand(self, command, commandInput=None): """ Objective: - Execute Command "comamnd" + Execute Command "comamnd" Parameters: command: command to execute @@ -73,14 +73,23 @@ class SshConnection(): output["stdoutdata"] = None output["stderrdata"] = None exit_status_ready_flag = True - + try: transport = self._connection.get_transport() channel = transport.open_session() channel.exec_command(command) # Adding sleep to get the correct exit_status. - time.sleep(5) - exit_status_ready_flag = channel.exit_status_ready() + timeout = 60 + sleeptime = 5 + while timeout: + time.sleep(sleeptime) + exit_status_ready_flag = channel.exit_status_ready() + if not exit_status_ready_flag: + timeout -=1 + continue + else: + break + if not exit_status_ready_flag: stdin = channel.makefile("wb") @@ -91,7 +100,7 @@ class SshConnection(): after executing comamnd for command completion") stdin.write("\n") return output - + stdout = channel.makefile("rb") stderr = channel.makefile_stderr("rb") exit_status = channel.recv_exit_status() @@ -104,5 +113,3 @@ class SshConnection(): logger.error("Unable to Execute Command: " + command) return output - - diff --git a/libs/utils/clientutils.py b/libs/utils/clientutils.py index 4de2998..347e159 100644 --- a/libs/utils/clientutils.py +++ b/libs/utils/clientutils.py @@ -12,7 +12,6 @@ Supported Wrappers :- import atfutils import hostutils from atfglobals import GlobalObj -import pdb def umount(mountkey): """unmounts a mountpoint @@ -20,22 +19,22 @@ def umount(mountkey): Parameters: mountkey : name given to a mount as specified in testenv.cfg file. Ex:-"mount1" - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ logger = GlobalObj.getLoggerObj() base_command = "umount" env = GlobalObj.getTestenvObj() cm = GlobalObj.getConnectionsManagerObj() - + mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. '%s' not defined in TestEnvironment" % mountkey) return 1 - + clientkey = mount_obj.client client_connection = cm.getConnection(clientkey) if not client_connection: @@ -56,7 +55,7 @@ def umount(mountkey): else: logger.error("Unable to umount %s" % mountkey) - + return return_status def umountall(): @@ -65,10 +64,10 @@ def umountall(): Parameters: None - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ env = GlobalObj.getTestenvObj() failure_flag = False @@ -90,10 +89,10 @@ def mount(mountkey): Parameters: mountkey : name given to a mount as specified in testenv.cfg file. Ex:-"mount1" - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ logger = GlobalObj.getLoggerObj() base_command = "mount" @@ -101,13 +100,13 @@ def mount(mountkey): cm = GlobalObj.getConnectionsManagerObj() command = [base_command] options = [] - + mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. %s not defined in TestEnvironment" % mountkey) return 1 - + clientkey = mount_obj.client client_connection = cm.getConnection(clientkey) if not client_connection: @@ -128,10 +127,10 @@ def mount(mountkey): if mount_obj.options: options.extend([mount_obj.option]) options.extend([device, mount_obj.dir]) - + command.extend(options) command = ' '.join(command) - + logger.debug('%s: Executing Command: %s' % (clientkey, command)) output = client_connection.executecommand(command) return_status = atfutils.assert_success(output['exitstatus']) @@ -140,14 +139,14 @@ def mount(mountkey): return return_status def mountall(): - """mounts a filesystem for all mounts specified in testenv.cfg file. + """mounts a filesystem for all mounts specified in testenv.cfg file. Parameters: None - + Returns: Success : 0 - Failure : 1` + Failure : 1` """ env = GlobalObj.getTestenvObj() @@ -159,134 +158,82 @@ def mountall(): return 0 +def md5sum_of_mount(mountkey): + """ + Parameter: mount (tye: string) + Returns: output of arequal-checksum command execution(type:dict) + Key : Value of the Output + exitstatus: exit status of the arequal-checksum command on mount + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + mount_obj = env.getMount(mountkey) + if not mount_obj: + logger.error("InValid Mount. %s not defined in TestEnvironment" + % mountkey) + output["exitstatus"] = 1 + return output + + clientkey = mount_obj.client + path = mount_obj.dir + output = hostutils.md5sum(clientkey, path) + return output + +def md5sum_of_mounts(mounts): + """ + Description: + Calculate md5sum of mounts using arequal-checksum + + Parameters: mounts (type: List) + + Returns: md5sums of all the mounts (type: dict) + Keys: mounts + Value: ouput (type:dict) + exitstatus: exit status of the arequal-checksum command on mount + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + md5sums = {} + for mountkey in mounts: + output = md5sum_of_mount(mountkey) + md5sums[mountkey] = output + + return md5sums def execute_on_mount(mountkey, command, commandInput=None): """ """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + logger = GlobalObj.getLoggerObj() env = GlobalObj.getTestenvObj() mount_obj = env.getMount(mountkey) if not mount_obj: logger.error("InValid Mount. %s not defined in TestEnvironment" % mountkey) - return 1 - - + output["exitstatus"] = 1 + return output + + clientkey = mount_obj.client mountdir = mount_obj.dir command = "cd " + mountdir + " ;" + command output = hostutils.execute_command(clientkey, command, commandInput) return output - + __all__ = ['execute_on_mount', + 'md5sum_of_mounts', 'umount', 'umountall', 'mount', 'mountall'] - -##def umountall(clientkey): -## """ -## """ -## base_command = "umount " -## env = GlobalObj.get_testenv_obj() -## cm = GlobalObj.get_connectionsmanager_obj() -## client_obj = env.getclient(clientkey) -## mountdir = client_obj.mountdir -## volume = client_obj.device -## client = cm.getconnection(clientkey) -## -## mountpoints = [] -## success_flag = False -## failure_flag = False -## command = "mount | grep " + mountdir -## output = client.executecommand(command) -## if not output["exitstatus"]: -## for data in output["stdoutdata"]: -## mountpoints.append(data.split(" ")[2]) -## -## for mountpoint in mountpoints: -## command = base_command + mountpoint -## output = client.executecommand(command) -## return_code = utils.assert_success(output['exitstatus']) -## if return_code: -## failure_flag = True -## else: -## success_flag = True -## continue -## -## if failure_flag: -## return 1 -## -## mountpoints = [] -## success_flag = False -## failure_flag = False -## command = "mount | grep " + volume -## output = client.executecommand(command) -## if not output["exitstatus"]: -## for data in output["stdoutdata"]: -## mountpoints.append(data.split(" ")[2]) -## -## for mountpoint in mountpoints: -## command = base_command + mountpoint -## output = client.executecommand(command) -## return_code = utils.assert_success(output['exitstatus']) -## if return_code: -## failure_flag = True -## else: -## success_flag = True -## continue -## -## if failure_flag: -## return 1 -## -## return 0 - -##def cd_mount(mountkey): -## """ -## """ -## env = GlobalObj.getTestenvObj() -## mount_obj = env.getMount(mountkey) -## if not mount_obj: -## print "InValid Mount. %s not defined in TestEnvironment" % mountkey -## return 1 -## -## clientkey = mount_obj.client -## dirpath = mount_obj.dir -## return_status = hostutils.cd(clientkey, dirpath) -## return return_status -## -##def cd_allmounts(): -## """ -## """ -## env = GlobalObj.getTestenvObj() -## mounts_keys = env.getMountsKeys() -## for mountkey in mounts_keys: -## return_status = cd_mount(mountkey) -## if return_status: -## return return_status -## -## return 0 - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/libs/utils/hostutils.py b/libs/utils/hostutils.py index e6f7189..c7ac7f2 100644 --- a/libs/utils/hostutils.py +++ b/libs/utils/hostutils.py @@ -13,7 +13,7 @@ import re from collections import namedtuple import atfutils from atfglobals import GlobalObj -import pdb +import os system_dirs = re.compile('(/bin|/boot|/dev|/etc|/lib|/mnt|/net|/opt|/root|/sbin|/usr|/var|/sys)\/?$') @@ -27,15 +27,15 @@ def cd(hostkey, dirpath): if not host_connection: logger.error("SSH Connection Not established to host '%s' " % hostkey) return 1 - + command = ' '.join([base_command, dirpath]) logger.debug('%s: Executing Command: %s' % (hostkey, command)) output = host_connection.executecommand(command) return_status = atfutils.assert_success(output['exitstatus']) atfutils.print_stdout(output['stdoutdata']) atfutils.print_stderr(output['stderrdata']) - return return_status - + return return_status + def rmdir(hostkey, dirpath): """ """ @@ -45,7 +45,7 @@ def rmdir(hostkey, dirpath): if system_dirs.match(dirpath): logger.error("System Directiories cannot be deleted") return 1 - + else: host_connection = cm.getConnection(hostkey) if not host_connection: @@ -60,7 +60,7 @@ def rmdir(hostkey, dirpath): atfutils.print_stdout(output['stdoutdata']) atfutils.print_stderr(output['stderrdata']) return return_status - + def mkdir(hostkey, dirpath): """ """ @@ -92,7 +92,7 @@ def mkfs(hostkey, device, fstype=None): """ logger = GlobalObj.getLoggerObj() base_command = "mkfs" - cm = GlobalObj.getConnectionsManagerObj() + cm = GlobalObj.getConnectionsManagerObj() host_connection = cm.getConnection(hostkey) command = [base_command] options = [] @@ -103,9 +103,9 @@ def mkfs(hostkey, device, fstype=None): if fstype is None: fstype = "xfs" - + options.extend(["-t", fstype, "-f", device]) - + command.extend(options) command = ' '.join(command) logger.debug('%s: Executing Command: %s' % (hostkey, command)) @@ -115,13 +115,48 @@ def mkfs(hostkey, device, fstype=None): atfutils.print_stderr(output['stderrdata']) return return_status +def md5sum(hostkey, path): + """ + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + + base_command1 = "rm -rf" + base_command2 = "arequal-checksum" + landfill_dir = os.path.join(path, ".landfill") + + cm = GlobalObj.getConnectionsManagerObj() + host_connection = cm.getConnection(hostkey) + if not host_connection: + logger.error("SSH connection to host '%s' has not been established" + % hostkey) + output["exitstatus"] = 1 + return output + + command1 = ' '.join([base_command1, landfill_dir]) + command2 = ' '.join([base_command2, path]) + command = ';'.join([command1, command2]) + output = host_connection.executecommand(command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + if output['stdoutdata'] is None or (not output['stdoutdata']): + output['stdoutdata'] = "" + + else: + output['stdoutdata'] = str(output['stdoutdata']) + + return output + def find_mountpoints(hostkey, device): """ """ logger = GlobalObj.getLoggerObj() - base_command = "mount | grep " + base_command = "mount | grep " cm = GlobalObj.getConnectionsManagerObj() - + host_connection = cm.getConnection(hostkey) if not host_connection: logger.error("SSH connection to host '%s' has not been established" @@ -141,13 +176,18 @@ def find_mountpoints(hostkey, device): def execute_command(hostkey, command, commandInput=None): """ """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None logger = GlobalObj.getLoggerObj() cm = GlobalObj.getConnectionsManagerObj() host_connection = cm.getConnection(hostkey) if not host_connection: logger.error("SSH Connection Not established to host '%s' " % hostkey) - return 1 + output["exitstatus"] = 1 + return output new_command = _substitute_value_for_variables(command) logger.debug('%s: Executing Command: %s' % (hostkey, command)) @@ -157,6 +197,7 @@ def execute_command(hostkey, command, commandInput=None): return output + def _substitute_value_for_variables(command): """ """ @@ -178,14 +219,14 @@ def _substitute_value_for_variables(command): "name", "option"]) variables_to_replace = [] - + variables = pattern_for_variables.findall(command) - if not variables: + if not variables: return new_command - + else: active_volume = env.getActiveVolume() - + for variable in variables: if variable not in variables_to_replace: name, option = (variable.strip("<>")).split(".") @@ -224,11 +265,353 @@ def _substitute_value_for_variables(command): new_command = "" return new_command - return new_command + return new_command + +def gluster_install_tar(version): + """ + """ + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + cm = GlobalObj.getConnectionsManagerObj() + down_path = ''.join(env.getGlusterDownloadPaths()) + + if system_dirs.match(down_path): + logger.error("System Directiories cannot be created") + return 1 + + if not down_path[-1] is '/': + down_path= down_path+'/' + + install_path = "/opt/gusterfs/"+version+"/" + + host_keys = env.getHostsKeys() + for hostkey in host_keys: + host_connection = cm.getConnection(hostkey) + if not host_connection: + logger.error("SSH Connection not established to host'%s'" + %hostkey) + return 1 + """ + stopping all the gluster processes + """ + kill_command = "killall glusterd && killall glusterfsd && killall glusterfs" + logger.debug('%s: Executing command : %s' %(hostkey, kill_command)) + output = host_connection.executecommand(kill_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + Cleaning /etc/glusterd, usr/local/sbin and /usr/sbin + """ + remove_command = "rm -rf /etc/glusterd/ && rm -rf /usr/local/sbin/gluster* && rm -rf /usr/sbin/gluster* " + logger.debug('%s: Executing command : %s' %(hostkey, remove_command)) + output = host_connection.executecommand(remove_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + checking for rpm installation of gluster and removing it. + """ + cleanup_command = "rpm -qa | grep gluster | xargs rpm -e" + logger.debug('%s: Executing command : %s' %(hostkey, cleanup_command)) + output = host_connection.executecommand(cleanup_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + mkdir_command = ["mkdir -p",down_path] + mkdir_command =' '.join(mkdir_command) + logger.debug('%s: Executing command : %s' %(hostkey, mkdir_command)) + output = host_connection.executecommand(mkdir_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + download_url= "http://bits.gluster.com/pub/gluster/glusterfs/src/glusterfs-"+version+".tar.gz" + """ + changing directory to the download path + """ + chdir = 'cd '+down_path+' && ' + wget_command = 'wget '+download_url + download_command = chdir + wget_command + logger.debug('%s: Executing command:%s'%(hostkey, download_command)) + output = host_connection.executecommand(download_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + extract = 'tar -xzf glusterfs-'+version+'.tar.gz' + extract_command = chdir + extract + logger.debug('%s: Executing command : %s'%(hostkey, extract_command)) + output = host_connection.executecommand(extract_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + """ + changing directory to the glusterfs directory + """ + chdir = 'cd '+down_path+'glusterfs-'+version+' && ' + configure = "./autogen.sh && ./configure --prefix="+install_path+" CFLAGS=\"-g -O0 -DDEBUG\" " + configure_command = chdir + configure + logger.debug('%s: Executing command : %s' %(hostkey, configure_command)) + output = host_connection.executecommand(configure_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + make = "make && make install" + make_command = chdir + make + logger.debug('%s: Executing command : %s'%(hostkey, make_command)) + output = host_connection.executecommand(make_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + """ + """ + symlink_command = "ln -s "+install_path+"sbin/gluster /usr/sbin/gluster && ln -s "+install_path+"sbin/glusterd /usr/sbin/glusterd && ln -s "+install_path+"sbin/glusterfsd /usr/sbin/glusterfsd && ln -s "+install_path+"sbin/glusterfs /usr/sbin/glusterfs" + logger.debug('%s: Executing command : %s'%(hostkey, symlink_command)) + output = host_connection.executecommand(symlink_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + + return 0 + +def gluster_install_git(branch): + """ + """ + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + cm = GlobalObj.getConnectionsManagerObj() + down_path = ''.join(env.getGlusterDownloadPaths()) + + if system_dirs.match(down_path): + logger.error("System Directiories cannot be created") + return 1 + + if not down_path[-1] is '/': + down_path= down_path+'/' + + install_path = "/opt/gusterfs/"+branch+'/' + + host_keys = env.getHostsKeys() + for hostkey in host_keys: + host_connection = cm.getConnection(hostkey) + if not host_connection: + logger.error("SSH Connection not established to host'%s'" + %hostkey) + return 1 + """ + stopping all the gluster processes + """ + kill_command = "killall glusterd && killall glusterfsd && killall glusterfs" + logger.debug('%s: Executing command : %s' %(hostkey, kill_command)) + output = host_connection.executecommand(kill_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + Cleaning /etc/glusterd , /usr/local/sbin , /usr/sbin/ + and previous git directory + """ + remove_command = "rm -rf /etc/glusterd/ && rm -rf "+down_path+"glusterfs.git && rm -rf /usr/local/sbin/gluster* && rm -rf /usr/sbin/gluster*" + logger.debug('%s: Executing command : %s' %(hostkey, remove_command)) + output = host_connection.executecommand(remove_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + checking for rpm installation of gluster and removing it. + """ + cleanup_command = "rpm -qa | grep gluster | xargs rpm -e" + logger.debug('%s: Executing command : %s' %(hostkey, cleanup_command)) + output = host_connection.executecommand(cleanup_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + mkdir_command = ["mkdir -p",down_path] + mkdir_command =' '.join(mkdir_command) + logger.debug('%s: Executing command : %s' %(hostkey, mkdir_command)) + output = host_connection.executecommand(mkdir_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + git_url ='https://github.com/gluster/glusterfs.git' + + """ + changing directory to the download path + """ + chdir = 'cd '+down_path+' && ' + clone_command = ' git clone '+git_url+" "+"glusterfs.git" + git_command = chdir + clone_command + + logger.debug('%s: Executing command:%s'%(hostkey, git_command)) + output = host_connection.executecommand(git_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + """ + changing directory to the glusterfs directory and checking out + to the branch + """ + chdir = 'cd '+down_path+'glusterfs.git'+' && ' + checkout = "git checkout "+branch+' && ' + git_pull = "git pull" + git_command = chdir+ checkout + git_pull + logger.debug('%s: Executing command : %s' %(hostkey, git_command)) + output = host_connection.executecommand(git_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + configure = "./autogen.sh && ./configure --prefix="+install_path+" CFLAGS=\"-g -O0 -DDEBUG -lefence\" " + configure_command = chdir + configure + logger.debug('%s: Executing command : %s' %(hostkey, configure_command)) + output = host_connection.executecommand(configure_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + make = "make && make install" + make_command = chdir + make + logger.debug('%s: Executing command : %s'%(hostkey, make_command)) + output = host_connection.executecommand(make_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + """ + """ + symlink_command = "ln -s "+install_path+"sbin/gluster /usr/sbin/gluster && ln -s "+install_path+"sbin/glusterd /usr/sbin/glusterd && ln -s "+install_path+"sbin/glusterfsd /usr/sbin/glusterfsd && ln -s "+install_path+"sbin/glusterfs /usr/sbin/glusterfs" + logger.debug('%s: Executing command : %s'%(hostkey, symlink_command)) + output = host_connection.executecommand(symlink_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + return 0 + +def gluster_install_rpm(version, *components): + """ + """ + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + cm = GlobalObj.getConnectionsManagerObj() + down_path = ''.join(env.getGlusterDownloadPaths()) + + if system_dirs.match(down_path): + logger.error("System Directiories cannot be created") + return 1 + + if not down_path[-1] is '/': + down_path= down_path+'/' + rpm_path = down_path+"glusterfs.rpm" + + host_keys = env.getHostsKeys() + for hostkey in host_keys: + host_connection = cm.getConnection(hostkey) + if not host_connection: + logger.error("SSH Connection not established to host'%s'" + %hostkey) + return 1 + + """ + stopping all the gluster processes + """ + kill_command = "killall glusterd && killall glusterfsd && killall glusterfs" + logger.debug('%s: Executing command : %s' %(hostkey, kill_command)) + output = host_connection.executecommand(kill_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + deleting /etc/glusterd + """ + remove_command = "rm -rf /etc/glusterd/ && rm -rf /usr/sbin/gluster* && rm -rf /usr/local/sbin/gluster*" + logger.debug('%s: Executing command : %s' %(hostkey, remove_command)) + output = host_connection.executecommand(remove_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + """ + checking for rpm installation of gluster and removing it. + """ + cleanup_command = "rpm -qa | grep gluster | xargs rpm -e" + logger.debug('%s: Executing command : %s' %(hostkey, cleanup_command)) + output = host_connection.executecommand(cleanup_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + + mkdir_command = ["mkdir -p",rpm_path] + mkdir_command =' '.join(mkdir_command) + logger.debug('%s: Executing command : %s' %(hostkey, mkdir_command)) + output = host_connection.executecommand(mkdir_command) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + for component in components: + component_url = "http://bits.gluster.com/pub/gluster/glusterfs/"+version+"/x86_64/glusterfs-"+component+"-"+version+"-1.x86_64.rpm" + chdir = 'cd '+rpm_path+' && ' + wget_command = "wget "+component_url + down_command = chdir + wget_command + + logger.debug('%s: Executing command:%s'%(hostkey, down_command)) + output = host_connection.executecommand(down_command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + + install_command = "rpm -ivh glusterfs-"+component+"-"+version+"-1.x86_64.rpm --nodeps" + + command = chdir + install_command + + logger.debug('%s: Executing command:%s'%(hostkey, command)) + output = host_connection.executecommand(command) + return_status = atfutils.assert_success(output['exitstatus']) + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + if return_status: + return 1 + + return 0 + + + + __all__ = ['cd', 'rmdir', 'mkdir', 'mkfs', + 'md5sum', 'find_mountpoints', - 'execute_command'] + 'execute_command', + 'gluster_install_tar', + 'gluster_install_git', + 'gluster_install_rpm'] diff --git a/libs/utils/serverutils.py b/libs/utils/serverutils.py index 068dd17..10fe830 100644 --- a/libs/utils/serverutils.py +++ b/libs/utils/serverutils.py @@ -3,6 +3,98 @@ import re import hostutils from atfglobals import GlobalObj +import atfutils + +def md5sum_of_brick(brickkey): + """ + Parameter: brick (tye: string) + Returns: output of arequal-checksum command execution(type:dict) + Key : Value of the Output + exitstatus: exit status of the arequal-checksum command on brick + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + logger = GlobalObj.getLoggerObj() + env = GlobalObj.getTestenvObj() + + raw_brick_obj = env.getRawBrick(brickkey) + if not raw_brick_obj: + logger.error("InValid Brick. %s not defined in TestEnvironment" % + brickkey) + output["exitstatus"] = 1 + return output + + else: + serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] + + brick_obj = env.getBrick(brickkey) + if not brick_obj: + logger.error("InValid Brick. %s not defined in TestEnvironment" + % brickkey) + output["exitstatus"] = 1 + return output + else: + brick_path = brick_obj.path + + output = hostutils.md5sum(serverkey, brick_path) + return output + +def md5sum_of_bricks(bricks): + """ + Description: + Calculate md5sum of bricks using arequal-checksum + + Parameters: bricks (type: List) + + Returns: md5sums of all the bricks (type: dict) + Keys: bricks + Value: ouput (type:dict) + exitstatus: exit status of the arequal-checksum command on brick + stdoutdata: stdout data of arequal-checksum command execution + stderrdata: stderr data of arequal-checksum command execution + """ + md5sums = {} + for brickkey in bricks: + output = md5sum_of_brick(brickkey) + md5sums[brickkey] = output + + return md5sums + +def get_gfid_on_brick(brickkey, filename="."): + """ + """ + output = {} + output["exitstatus"] = None + output["stdoutdata"] = None + output["stderrdata"] = None + base_command = "getfattr -n 'trusted.gfid' -e hex" + command = ' '.join([base_command, filename]) + output = execute_on_brick(brickkey, command) + if atfutils.assert_success(output['exitstatus']): + atfutils.print_stdout(output['stdoutdata']) + atfutils.print_stderr(output['stderrdata']) + + elif output['stdoutdata'] is None or (not output['stdoutdata']): + output['stdoutdata'] = "" + + else: + output['stdoutdata'] = str(output['stdoutdata']) + + return output + +def get_gfid_on_bricks(bricks, filename="."): + """ + """ + gfid_on_bricks = {} + for brickkey in bricks: + output = get_gfid_on_brick(brickkey, filename) + gfid_on_bricks[brickkey] = output + + return gfid_on_bricks def execute_on_brick(brickkey, command, commandInput=None): """ @@ -11,7 +103,7 @@ def execute_on_brick(brickkey, command, commandInput=None): output["exitstatus"] = None output["stdoutdata"] = None output["stderrdata"] = None - + logger = GlobalObj.getLoggerObj() env = GlobalObj.getTestenvObj() @@ -19,23 +111,23 @@ def execute_on_brick(brickkey, command, commandInput=None): if not raw_brick_obj: logger.error("InValid Brick. %s not defined in TestEnvironment" % brickkey) + output["exitstatus"] = 1 return output - serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] - + else: + serverkey = re.split("\.", raw_brick_obj.hostname, maxsplit=1)[0] + brick_obj = env.getBrick(brickkey) if not brick_obj: logger.error("InValid Brick. %s not defined in TestEnvironment" % brickkey) + output["exitstatus"] = 1 return output - exportdirpath = brick_obj.path + else: + exportdirpath = brick_obj.path command = "cd " + exportdirpath + ";" + command output = hostutils.execute_command(serverkey, command, commandInput) return output -__all__ = ['execute_on_brick'] - - - - - +__all__ = ['execute_on_brick', + 'md5sum_of_bricks'] diff --git a/libs/utils/validate.py b/libs/utils/validate.py new file mode 100644 index 0000000..2cdfa45 --- /dev/null +++ b/libs/utils/validate.py @@ -0,0 +1,138 @@ +import operator +import atfutils +import serverutils +import clientutils +from atfglobals import GlobalObj +import re + +def assert_success_of_outputs(outputs): + """ + """ + for key in outputs.keys(): + output = outputs[key] + if atfutils.assert_success(output["exitstatus"]): + return 1 + return 0 + +def match_md5sum(sum1, sum2): + """ + Parameters: + sum1: md5sum1 (type: string) + sum2: md5sum2 (type: string) + + Description: + Compares md5sum1 with md5sum2. + If they are equal, return True. else False + """ + md5sum_match_status = False + md5sum_match_status = operator.eq(sum1, sum2) + return md5sum_match_status + +def match_md5sums(md5sums): + """ + Parameters: + md5sums: MD5Sums of files + (type: dict where key: mountkey, value: md5sum (type:string)) + + Description: + Compares md5sums of files. + If the md5sum of all mounts matches returns True else return False. + """ + logger = GlobalObj.getLoggerObj() + md5sum_match_status = False + compare_key, output = md5sums.popitem() + compare_value = output["stdoutdata"] + + if not md5sums: + md5sum_match_status = True + else: + for key in md5sums.keys(): + output = md5sums[key] + value = output["stdoutdata"] + md5sum_match_status = match_md5sum(compare_value, value) + if md5sum_match_status is False: + logger.error("Md5sum of %s didn't match with Md5sum of %s" % + (key, compare_key)) + return md5sum_match_status + + return md5sum_match_status + +def match_gfid(gfid1, gfid2): + """ + """ + gfid_match_status = False + gfid_match_status = operator.eq(gfid1, gfid2) + return gfid_match_status + +def match_gfids(gfid_on_bricks): + """ + """ + gfid_match_status = False + compare_key, output = gfid_on_bricks.popitem() + compare_value = output["stdoutdata"] + + if not gfid_on_bricks: + gfid_match_status = True + else: + for key in gfid_on_bricks.keys(): + output = gfid_on_bricks[key] + value = output["stdoutdata"] + gfid_match_status = match_gfid(compare_value, value) + if gfid_match_status is False: + logger.error("gfid of %s didn't match gfid of %s" % + compare_key, key) + return gfid_match_status + + return gfid_match_status + +def validate_on_bricks(bricks, command, expected_status, + expected_output, stream="stdout"): + """ + Parameters: + bricks : list of bricks + command : command to execute on each brick + expected_status : return status to expect + expected_output : output to expect + """ + for brick in bricks: + output = serverutils.execute_on_brick(brick, command) + validate_status = atfutils.validate_output(output, expected_status, + expected_output, stream) + if validate_status is not 0: + return validate_status + + return 0 + +def validate_md5sums(mounts, bricks): + """ + """ + mounts_md5sums = clientutils.md5sum_of_mounts(mounts) + bricks_md5sums = serverutils.md5sum_of_bricks(bricks) + + md5sums = mounts_md5sums + md5sums.update(bricks_md5sums) + + assert_success_status = assert_success_of_outputs(md5sums) + if assert_success_status is not 0: + return assert_success_status + + md5sum_match_status = match_md5sums(md5sums) + if md5sum_match_status is False: + return 1 + else: + return 0 + +def validate_gfids(bricks, filename="."): + """ + """ + gfid_on_bricks = serverutils.get_gfid_on_bricks(bricks, filename) + + assert_success_status = assert_success_of_outputs(gfid_on_bricks) + if assert_success_status is not 0: + return assert_success_status + + gfid_match_status = match_gfids(gfid_on_bricks) + if gfid_match_status is False: + return 1 + else: + return 0 |