distaf: Added bitrot_ops and lib utils

Added library functions for gluster bitrot operations and added 
functions in lib_utils.py which are generic across all components 

Change-Id: I877ded038c9f4c1e7aa1a15b035fcd7ebb0da21f 
BUG: 1332885 
Signed-off-by: Arthy Loganathan <aloganat@redhat.com>
Change-Id: I877ded038c9f4c1e7aa1a15b035fcd7ebb0da21f
Reviewed-on: http://review.gluster.org/14209
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: ShwethaHPanduranga
Smoke: Gluster Build System <jenkins@build.gluster.org>

2 files changed, 989 insertions, 0 deletions

diff --git a/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/bitrot_ops.py b/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/bitrot_ops.py
new file mode 100644
index 00000000000..671dd762c76
--- /dev/null
+++ b/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/bitrot_ops.py
@@ -0,0 +1,683 @@
+#!/usr/bin/env python
+#  This file is part of DiSTAF
+#  Copyright (C) 2015-2016  Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+    Description: Library for gluster bitrot operations.
+"""
+
+from distaf.util import tc
+from distaflibs.gluster.volume_ops import get_volume_option, get_volume_status
+from distaflibs.gluster.lib_utils import (get_pathinfo,
+                                          calculate_checksum,
+                                          get_extended_attributes_info)
+import time
+import re
+
+try:
+    import xml.etree.cElementTree as etree
+except ImportError:
+    import xml.etree.ElementTree as etree
+
+
+# Global variables
+SCRUBBER_TIMEOUT = 100
+
+
+def enable_bitrot(volname, mnode=None):
+    """Enables bitrot for given volume
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        enable_bitrot(testvol)
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s enable" % volname
+    return tc.run(mnode, cmd)
+
+
+def disable_bitrot(volname, mnode=None):
+    """Disables bitrot for given volume
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        disable_bitrot(testvol)
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s disable" % volname
+    return tc.run(mnode, cmd)
+
+
+def is_bitrot_enabled(volname, mnode=None):
+    """Checks if bitrot is enabled in given volume
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        is_bitrot_enabled(testvol)
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    output = get_volume_option(volname, "features.bitrot", mnode)
+    if output is None:
+        return False
+
+    tc.logger.info("Bitrot Status in volume %s: %s"
+                   % (volname, output["features.bitrot"]))
+    if output["features.bitrot"] != 'on':
+        return False
+
+    return True
+
+
+def is_file_signed(filename, mountpoint, expected_file_version=None):
+    """Verifies if the given file is signed
+
+    Args:
+        filename (str): relative path of filename to be verified
+        mountpoint (str): mount point of the file. If mount type is
+            nfs or cifs, then mount the volume with gluster mount and
+            give the gluster mount path for this parameter.
+
+    Kwargs:
+        expected_file_version (str): file version to check with getfattr output
+            If this option is set, this function
+            will check file versioning as part of signing verification.
+            If this option is set to None, function will not check
+            for file versioning. Defaults to None.
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        is_file_signed('file1', "/mnt/glusterfs", expected_file_version='2')
+    """
+
+    filename_mnt = mountpoint + "/" + filename
+
+    # Getting file path in the rhs node
+    file_location = get_pathinfo(filename_mnt)
+    if file_location is None:
+        tc.logger.error("Failed to get backend file path in is_file_signed()")
+        return False
+
+    path_info = file_location[0].split(':')
+
+    expected_file_signature = (calculate_checksum([path_info[1]],
+                                                  mnode=path_info[0])
+                               [path_info[1]])
+
+    attr_info = get_extended_attributes_info([path_info[1]],
+                                             mnode=path_info[0])
+    if attr_info is None:
+        tc.logger.error("Failed to get attribute info in is_file_signed()")
+        return False
+
+    file_signature = attr_info[path_info[1]]['trusted.bit-rot.signature']
+
+    if expected_file_version is not None:
+        expected_file_version = ('{0:02d}'.format(int(
+                                 expected_file_version))).ljust(16, '0')
+        actual_signature_file_version = re.findall('.{16}',
+                                                   file_signature[4:]).pop(0)
+
+        # Verifying file version after signing
+        if actual_signature_file_version != expected_file_version:
+            tc.logger.error("File version mismatch in signature.Filename: %s ."
+                            "Expected file version: %s.Actual file version: %s"
+                            % (filename, expected_file_version,
+                               actual_signature_file_version))
+            return False
+
+    actual_file_signature = ''.join(re.findall('.{16}',
+                                               file_signature[4:])[1:])
+
+    # Verifying file signature
+    if actual_file_signature != expected_file_signature:
+        tc.logger.error("File signature mismatch. File name: %s . Expected "
+                        "file signature: %s. Actual file signature: %s"
+                        % (filename, expected_file_signature,
+                           actual_file_signature))
+        return False
+    return True
+
+
+def is_file_bad(filename, mnode):
+    """Verifies if scrubber identifies bad file
+    Args:
+        filename (str): absolute path of the file in mnode
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        is_file_bad("/bricks/file1", "abc.xyz.com")
+    """
+    ret = True
+    count = 0
+    flag = 0
+    while (count < SCRUBBER_TIMEOUT):
+        attr_info = get_extended_attributes_info([filename], mnode=mnode)
+        if attr_info is None:
+            ret = False
+
+        if 'trusted.bit-rot.bad-file' in attr_info[filename]:
+            flag = 1
+            break
+
+        time.sleep(10)
+        count = count + 10
+    if not flag:
+        tc.logger.error("Scrubber failed to identify bad file")
+        ret = False
+
+    return ret
+
+
+def bring_down_bitd(mnode=None):
+    """Brings down bitd process
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        bring_down_bitd()
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    kill_cmd = ("pid=`cat /var/lib/glusterd/bitd/run/bitd.pid` && "
+                "kill -15 $pid || kill -9 $pid")
+    ret, _, _ = tc.run(mnode, kill_cmd)
+    if ret != 0:
+        tc.logger.error("Unable to kill the bitd for %s"
+                        % mnode)
+        return False
+    else:
+        return True
+
+
+def bring_down_scrub_process(mnode=None):
+    """Brings down scrub process
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        bring_down_scrub_process()
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    kill_cmd = ("pid=`cat /var/lib/glusterd/scrub/run/scrub.pid` && "
+                "kill -15 $pid || kill -9 $pid")
+
+    ret, _, _ = tc.run(mnode, kill_cmd)
+    if ret != 0:
+        tc.logger.error("Unable to kill the scrub process for %s"
+                        % mnode)
+        return False
+    else:
+        return True
+
+
+def set_scrub_throttle(volname, mnode=None, type='lazy'):
+    """Sets scrub throttle
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+        type (str): throttling type (lazy|normal|aggressive)
+            Defaults to 'lazy'
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        set_scrub_throttle(testvol)
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub-throttle %s" % (volname, type)
+    return tc.run(mnode, cmd)
+
+
+def set_scrub_frequency(volname, mnode=None, type='biweekly'):
+    """Sets scrub frequency
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+        type (str): frequency type (hourly|daily|weekly|biweekly|monthly)
+            Defaults to 'biweekly'
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        set_scrub_frequency(testvol)
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub-frequency %s" % (volname, type)
+    return tc.run(mnode, cmd)
+
+
+def pause_scrub(volname, mnode=None):
+    """Pauses scrub
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        pause_scrub(testvol)
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub pause" % volname
+    return tc.run(mnode, cmd)
+
+
+def resume_scrub(volname, mnode=None):
+    """Resumes scrub
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        resume_scrub(testvol)
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub resume" % volname
+    return tc.run(mnode, cmd)
+
+
+def get_bitd_pid(mnode=None):
+    """Gets bitd process id for the given node
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        str: pid of the bitd process on success
+        NoneType: None if command execution fails, errors.
+
+    Example:
+        get_bitd_pid()
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = ("cat /var/lib/glusterd/bitd/run/bitd.pid")
+    ret, out, _ = tc.run(mnode, cmd)
+    if ret != 0:
+        tc.logger.error("Unable to get bitd pid for %s"
+                        % mnode)
+        return None
+
+    return out.strip("\n")
+
+
+def get_scrub_process_pid(mnode=None):
+    """Gets scrub process id for the given node
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        str: pid of the scrub process on success
+        NoneType: None if command execution fails, errors.
+
+    Example:
+        get_scrub_process_pid()
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = ("cat /var/lib/glusterd/scrub/run/scrub.pid")
+    ret, out, _ = tc.run(mnode, cmd)
+    if ret != 0:
+        tc.logger.error("Unable to get scrub pid for %s"
+                        % mnode)
+        return None
+
+    return out.strip("\n")
+
+
+def is_bitd_running(volname, mnode=None):
+    """Checks if bitd is running on the given node
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        is_bitd_running("testvol")
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    vol_status = get_volume_status(volname=volname, mnode=mnode)
+    if vol_status is None:
+        tc.logger.error("Failed to get volume status in isBitdRunning()")
+        return False
+
+    if 'Bitrot Daemon' not in vol_status[volname]['localhost']:
+        tc.logger.error("Bitrot is not enabled in volume %s"
+                        % volname)
+        return False
+
+    bitd_status = vol_status[volname]['localhost']['Bitrot Daemon']['status']
+    if bitd_status != '1':
+        tc.logger.error("Bitrot Daemon is not running in node %s"
+                        % mnode)
+        return False
+    return True
+
+
+def is_scrub_process_running(volname, mnode=None):
+    """Checks if scrub process is running on the given node
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        True on success, False otherwise
+
+    Example:
+        is_scrub_process_running("testvol")
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    vol_status = get_volume_status(volname=volname, mnode=mnode)
+    if vol_status is None:
+        tc.logger.error("Failed to get volume status in "
+                        "isScrubProcessRunning()")
+        return False
+
+    if 'Scrubber Daemon' not in vol_status[volname]['localhost']:
+        tc.logger.error("Bitrot is not enabled in volume %s"
+                        % volname)
+        return False
+
+    bitd_status = vol_status[volname]['localhost']['Scrubber Daemon']['status']
+    if bitd_status != '1':
+        tc.logger.error("Scrubber Daemon is not running in node %s"
+                        % mnode)
+        return False
+    return True
+
+
+def scrub_status(volname, mnode=None):
+    """Executes gluster bitrot scrub status command
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        tuple: Tuple containing three elements (ret, out, err).
+            The first element 'ret' is of type 'int' and is the return value
+            of command execution.
+
+            The second element 'out' is of type 'str' and is the stdout value
+            of the command execution.
+
+            The third element 'err' is of type 'str' and is the stderr value
+            of the command execution.
+
+    Example:
+        scrub_status(testvol)
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub status" % volname
+    return tc.run(mnode, cmd)
+
+
+def get_scrub_status(volname, mnode=None):
+    """Parse the output of gluster bitrot scrub status command
+
+    Args:
+        volname (str): volume name
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+            If None, defaults to servers[0].
+
+    Returns:
+        dict: scrub status in dict format
+        NoneType: None if command execution fails, errors.
+
+    Example:
+        >>>get_scrub_status(testvol)
+        {'State of scrub': 'Active', 'Bitrot error log location':
+        '/var/log/glusterfs/bitd.log', 'Scrub impact': 'aggressive',
+        'Scrub frequency': 'hourly', 'status_info': {'localhost':
+        {'Duration of last scrub (D:M:H:M:S)': '0:0:0:0', 'corrupted_gfid':
+        ['475ca13f-577f-460c-a5d7-ea18bb0e7779'], 'Error count': '1',
+        'Last completed scrub time': '2016-06-21 12:46:19',
+        'Number of Skipped files': '0', 'Number of Scrubbed files': '0'},
+        '10.70.47.118': {'Duration of last scrub (D:M:H:M:S)': '0:0:0:1',
+        'corrupted_gfid': ['19e62b26-5942-4867-a2f6-e354cd166da9',
+        'fab55c36-0580-4d11-9ac0-d8e4e51f39a0'], 'Error count': '2',
+        'Last completed scrub time': '2016-06-21 12:46:03',
+        'Number of Skipped files': '0', 'Number of Scrubbed files': '2'}},
+        'Volume name': 'testvol', 'Scrubber error log location':
+        '/var/log/glusterfs/scrub.log'}
+    """
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = "gluster volume bitrot %s scrub status" % volname
+    ret, out, err = tc.run(mnode, cmd)
+    if ret != 0:
+        tc.logger.error("Unable to get scrub status for volume %s"
+                        % volname)
+        return False
+
+    match = re.search('(.*?)(==.*==.*)', out, re.S)
+    if match is None:
+        tc.logger.error("Mismatch in regex. Scrub status raw output is not"
+                        " in expected format")
+        return False
+    info = match.group(2).replace('\n\n', '\n')
+
+    if "Corrupted object's [GFID]" in info:
+        info = info.replace("Corrupted object's [GFID]:\n",
+                            "Corrupted object's [GFID]:")
+        regex = 'Node(?:(?!Node).)*?Corrupted object.*?:.*?\n+='
+        temp_list = re.findall(regex, info, re.S)
+        corrupt_list = []
+        for node in temp_list:
+            tmp_reg = ('Node: (\S+)\n.*Error count.*'
+                       + 'Corrupted object.*?:(.*)\n=.*')
+            m = re.search(tmp_reg, node, re.S)
+            if m is None:
+                tc.logger.error("Mismatch in cli output when bad file"
+                                "is identified")
+                return False
+            corrupt_list.append(m.groups())
+    else:
+        corrupt_list = []
+    info_list = re.findall('Node:.*?\n.*:.*\n.*:.*\n.*:.*\n.*:.*\n.*:.*\n+',
+                           info)
+    temp_list = []
+    for item in info_list:
+        item = item.replace('\n\n', '')
+        temp_list.append(item)
+
+    tmp_dict1 = {}
+    for item in temp_list:
+        tmp = item.split('\n')
+        tmp_0 = tmp[0].split(':')
+        tmp.pop(0)
+        tmp_dict = {}
+        for tmp_item in tmp[:-1]:
+            tmp_1 = tmp_item.split(': ')
+            tmp_dict[tmp_1[0].strip(' ')] = tmp_1[1].strip(' ')
+        tmp_dict1[tmp_0[1].strip(' ')] = tmp_dict
+    status_dict = {}
+    for item in match.group(1).split('\n\n')[:-1]:
+        elmt = item.split(':')
+        tmp_elmt = elmt[1].strip(' ').strip('\n')
+        status_dict[elmt[0].strip(' ').strip('\n')] = tmp_elmt
+
+    status_dict['status_info'] = tmp_dict1
+    for elmt in corrupt_list:
+        if elmt[0].strip(' ') in status_dict['status_info'].keys():
+            val = elmt[1].split('\n')
+            val = filter(None, val)
+            gfid = "corrupted_gfid"
+            status_dict['status_info'][elmt[0].strip(' ')][gfid] = val
+    return status_dict
diff --git a/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py b/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py
new file mode 100644
index 00000000000..36db4de47b3
--- /dev/null
+++ b/tests/distaf/distaf_libs/distaflibs-gluster/distaflibs/gluster/lib_utils.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+#  This file is part of DiSTAF
+#  Copyright (C) 2015-2016  Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+    Description: Helper library for gluster modules.
+"""
+
+from distaf.util import tc
+from distaflibs.gluster.volume_ops import get_volume_info
+from distaflibs.gluster.mount_ops import mount_volume, umount_volume
+import re
+import time
+
+try:
+    import xml.etree.cElementTree as etree
+except ImportError:
+    import xml.etree.ElementTree as etree
+
+
+def append_string_to_file(mnode, filename, str_to_add_in_file):
+    """Appends the given string in the file.
+
+    Example:
+        append_string_to_file("abc.def.com", "/var/log/messages",
+                           "test_1_string")
+
+    Args:
+        mnode (str): Node on which cmd has to be executed.
+        filename (str): absolute file path to append the string
+        str_to_add_in_file (str): string to be added in the file,
+            which is used as a start and stop string for parsing
+            the file in search_pattern_in_file().
+
+    Returns:
+        True, on success, False otherwise
+    """
+    try:
+        conn = tc.get_connection(mnode, 'root')
+        if conn == -1:
+            tc.logger.error("Unable to get connection to 'root' of node %s"
+                            " in append_string_to_file()" % mnode)
+            return False
+
+        with conn.builtin.open(filename, 'a') as _filehandle:
+            _filehandle.write(str_to_add_in_file)
+
+        return True
+    except:
+        tc.logger.error("Exception occured while adding string to "
+                        "file %s in append_string_to_file()" % filename)
+        return False
+    finally:
+        conn.close()
+
+
+def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse,
+                           end_str_to_parse):
+    """checks if the given search pattern exists in the file
+       in between 'start_str_to_parse' and 'end_str_to_parse' string.
+
+    Example:
+        search_pattern = r'.*scrub.*'
+        search_log("abc.def.com", search_pattern, "/var/log/messages",
+                    "start_pattern", "end_pattern")
+
+    Args:
+        mnode (str): Node on which cmd has to be executed.
+        search_pattern (str): regex string to be matched in the file
+        filename (str): absolute file path to search given string
+        start_str_to_parse (str): this will be as start string in the
+            file from which this method will check
+            if the given search string is present.
+        end_str_to_parse (str): this will be as end string in the
+            file whithin which this method will check
+            if the given search string is present.
+
+    Returns:
+        True, if search_pattern is present in the file
+        False, otherwise
+    """
+
+    cmd = ("awk '{a[NR]=$0}/" + start_str_to_parse + "/{s=NR}/" +
+           end_str_to_parse + "/{e=NR}END{for(i=s;i<=e;++i)print a[i]}' "
+           + filename)
+
+    ret, out, err = tc.run(mnode, cmd)
+    if ret != 0:
+        tc.logger.error("Failed to match start and end pattern in file"
+                        % filename)
+        return False
+
+    if not re.search(search_pattern, str(out), re.S):
+        tc.logger.error("file %s did not have the expected message"
+                        % filename)
+        return False
+
+    return True
+
+
+def calculate_checksum(file_list, chksum_type='sha256sum', mnode=None):
+    """This module calculates given checksum for the given file list
+
+    Example:
+        calculate_checksum([file1, file2])
+
+    Args:
+        file_list (list): absolute file names for which checksum
+            to be calculated
+
+    Kwargs:
+        mnode (str): Node on which cmd has to be executed.
+        chksum_type (str): type of the checksum algorithm.
+            Defaults to sha256sum
+
+    Returns:
+        NoneType: None if command execution fails, parse errors.
+        dict: checksum value for each file in the given file list
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    cmd = chksum_type + " %s" % ' '.join(file_list)
+    ret = tc.run(mnode, cmd)
+    if ret[0] != 0:
+        tc.logger.error("Failed to execute checksum command in server %s"
+                        % mnode)
+        return None
+
+    checksum_dict = {}
+    for line in ret[1].split('\n')[:-1]:
+        match = re.search(r'^(\S+)\s+(\S+)', line.strip())
+        if match is None:
+            tc.logger.error("checksum output is not in"
+                            "expected format")
+            return None
+
+        checksum_dict[match.group(2)] = match.group(1)
+
+    return checksum_dict
+
+
+def get_extended_attributes_info(file_list, encoding='hex', attr_name='',
+                                 mnode=None):
+    """This module gets extended attribute info for the given file list
+
+    Example:
+        get_extended_attributes_info([file1, file2])
+
+    Args:
+        file_list (list): absolute file names for which extended
+            attributes to be fetched
+
+    Kwargs:
+        encoding (str): encoding format
+        attr_name (str): extended attribute name
+        mnode (str): Node on which cmd has to be executed.
+
+    Returns:
+        NoneType: None if command execution fails, parse errors.
+        dict: extended attribute for each file in the given file list
+    """
+
+    if mnode is None:
+        mnode = tc.servers[0]
+
+    if attr_name == '':
+        cmd = "getfattr -d -m . -e %s %s" % (encoding, ' '.join(file_list))
+    else:
+        cmd = "getfattr -d -m . -n %s %s" % (attr_name, ' '.join(file_list))
+
+    ret = tc.run(mnode, cmd)
+    if ret[0] != 0:
+        tc.logger.error("Failed to execute getfattr command in server %s"
+                        % mnode)
+        return None
+
+    attr_dict = {}
+    for each_attr in ret[1].split('\n\n')[:-1]:
+        for line in each_attr.split('\n'):
+            if line.startswith('#'):
+                match = re.search(r'.*file:\s(\S+).*', line)
+                if match is None:
+                    tc.logger.error("getfattr output is not in "
+                                    "expected format")
+                    return None
+                key = "/" + match.group(1)
+                attr_dict[key] = {}
+            else:
+                output = line.split('=')
+                attr_dict[key][output[0]] = output[1]
+    return attr_dict
+
+
+def get_pathinfo(filename, volname, client=None):
+    """This module gets filepath of the given file in gluster server.
+
+    Example:
+        get_pathinfo("file1", "testvol")
+
+    Args:
+        filename (str): relative path of file
+        volname (str): volume name
+
+    Kwargs:
+        client (str): client on which cmd has to be executed.
+
+    Returns:
+        NoneType: None if command execution fails, parse errors.
+        list: file path for the given file in gluster server
+    """
+
+    if client is None:
+        client = tc.clients[0]
+
+    server = get_volume_info(volname)[volname]['bricks'][0].split(':')[0]
+    mount_point = '/mnt/tmp_fuse'
+
+    #Performing glusterfs mount because only with glusterfs mount
+    #the file location in gluster server can be identified from client
+    #machine
+    ret, _, _ = mount_volume(volname, mtype='glusterfs',
+                             mpoint=mount_point,
+                             mserver=server,
+                             mclient=client)
+    if ret != 0:
+        tc.logger.error("Failed to do gluster mount on volume %s to fetch"
+                        "pathinfo from client %s"
+                        % (volname, client))
+        return None
+
+    filename = mount_point + '/' + filename
+    attr_name = 'trusted.glusterfs.pathinfo'
+    output = get_extended_attributes_info([filename],
+                                          attr_name=attr_name,
+                                          mnode=client)
+    if output is None:
+        tc.logger.error("Failed to get path info for %s" % filename)
+        return None
+
+    pathinfo = output[filename][attr_name]
+
+    umount_volume(client, mount_point)
+
+    return re.findall(".*?POSIX.*?:(\S+)\>", pathinfo)
+
+
+def list_files(dir_path, parse_str="", mnode=None):
+    """This module list files from the given file path
+
+    Example:
+        list_files("/root/dir1/")
+
+    Args:
+        dir_path (str): directory path name
+
+    Kwargs:
+        parse_str (str): sub string of the filename to be fetched
+        mnode (str): Node on which cmd has to be executed.
+
+    Returns:
+        NoneType: None if command execution fails, parse errors.
+        list: files with absolute name
+    """
+    if mnode is None:
+        mnode = tc.clients[0]
+
+    try:
+        conn = tc.get_connection(mnode, 'root')
+        if conn == -1:
+            tc.logger.error("Unable to get connection to 'root' of node %s"
+                            % mnode)
+            return None
+
+        filepaths = []
+        for root, directories, files in conn.modules.os.walk(dir_path):
+            for filename in files:
+                if parse_str != "":
+                    if parse_str in filename:
+                        filepath = conn.modules.os.path.join(root, filename)
+                        filepaths.append(filepath)
+                else:
+                    filepath = conn.modules.os.path.join(root, filename)
+                    filepaths.append(filepath)
+        return filepaths
+    except:
+        tc.logger.error("Exception occured in list_files()")
+        return None
+
+    finally:
+        conn.close()