initial dht libs

* glusterfile.py - helper for gluster client and backend files.
* glusterdir.py - helper for gluster client and backend dirs.
* brickdir.py - helper for collection and hashing of brickdirs (from pathinfo data).
* layout.py - base class for simple DHT layout validation.
* dht_test_util.py - utility module to walk a directory tree and run tests against files.
* constants.py - definitions for constants used in DHT libraries.
* exceptions.py - definitions for exceptions raised in DHT libraries.

Change-Id: I44770a822e0ec79561b3aa048e555320f622116a
Signed-off-by: Jonathan Holloway <jholloway@redhat.com>

1 files changed, 710 insertions, 0 deletions

diff --git a/glustolibs-gluster/glustolibs/gluster/glusterfile.py b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
new file mode 100644
index 000000000..fcc10f25f
--- /dev/null
+++ b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
@@ -0,0 +1,710 @@
+#!/usr/bin/env python
+#  Copyright (C) 2018 Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+"""Description: Module for library gluster file class and related functions.
+
+A GlusterFile is a file object that exists on the client and backend brick.
+This module provides low-level functions and a GlusterFile class to maintain
+state and manage properties of a file in both locations.
+"""
+
+import ctypes
+import os
+import re
+
+from glusto.core import Glusto as g
+
+from glustolibs.gluster.layout import Layout
+
+
+def calculate_hash(host, filename):
+    """ Function to import DHT Hash library.
+
+    Args:
+        filename (str): the name of the file
+
+    Returns:
+        An integer representation of the hash
+    """
+    # TODO: For testcases specifically testing hashing routine
+    #        consider using a baseline external Davies-Meyer hash_value.c
+    #        Creating comparison hash from same library we are testing
+    #        may not be best practice here. (Holloway)
+    try:
+        # Check if libglusterfs.so.0 is available locally
+        glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0")
+        g.log.debug("Library libglusterfs.so.0 loaded locally")
+    except OSError:
+        conn = g.get_connection(host)
+        glusterfs = \
+            conn.modules.ctypes.cdll.LoadLibrary("libglusterfs.so.0")
+        g.log.debug("Library libglusterfs.so.0 loaded via rpyc")
+
+    computed_hash = \
+        ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, len(filename)))
+    # conn.close()
+
+    return int(computed_hash.value)
+
+
+def get_mountpoint(host, fqpath):
+    """Retrieve the mountpoint under a file
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        The mountpoint on success. None on fail.
+    """
+    command = "df -P %s | awk 'END{print $NF}'" % fqpath
+    rcode, rout, rerr = g.run(host, command)
+    if rcode == 0:
+        return rout.strip()
+
+    g.log.error("Get mountpoint failed: %s" % rerr)
+    return None
+
+
+def get_fattr(host, fqpath, fattr):
+    """getfattr for filepath on remote system
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+        fattr (str): name of the fattr to retrieve
+
+    Returns:
+        getfattr result on success. None on fail.
+    """
+    command = ("getfattr --absolute-names --only-values -n '%s' %s" %
+               (fattr, fqpath))
+    rcode, rout, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return rout.strip()
+
+    g.log.error('getfattr failed: %s' % rerr)
+    return None
+
+
+def get_fattr_list(host, fqpath):
+    """List of xattr for filepath on remote system.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        Dictionary of xattrs on success. None on fail.
+    """
+    command = "getfattr --absolute-names -d -m - %s" % fqpath
+    rcode, rout, rerr = g.run(host, command)
+
+    if rcode == 0:
+        xattr_list = {}
+        for xattr_string in rout.strip().split('\n'):
+            xattr = xattr_string.split('=', 1)
+            if len(xattr) > 1:
+                key, value = xattr
+                xattr_list[key] = value
+
+        return xattr_list
+
+    g.log.error('getfattr failed: %s' % rerr)
+    return None
+
+
+def set_fattr(host, fqpath, fattr, value):
+    """setfattr for filepath on remote system
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+        fattr (str): The name of the fattr to retrieve.
+
+    Returns:
+        setfattr result on success. errorcode on fail
+    """
+    command = 'setfattr -n %s -v %s %s' % (fattr, value, fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('setfattr failed: %s', rerr)
+    return False
+
+
+def delete_fattr(host, fqpath, fattr):
+    """remove fattr for filepath on remote system
+
+    Args:
+        host (str): hostname/ip of remote system
+        fqpath (str): the fully qualified path of the file
+        fattr (str): name of the fattr to delete
+
+    Returns:
+        setfattr result on success. errorcode on fail
+    """
+    command = 'setfattr -x %s %s' % (fattr, fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('setfattr -x failed: %s' % rerr)
+    return False
+
+
+def file_exists(host, fqpath):
+    """Check if file exists at path on host
+
+    Args:
+        host (str): hostname or ip of system
+        filename (str): fully qualified path of file
+
+    Returns:
+        True if file exists. False if file does not exist
+    """
+    command = "ls -ld %s" % fqpath
+    rcode, _, rerr = g.run(host, command)
+    if rcode == 0:
+        return True
+
+    g.log.error('File does not exist: %s', rerr)
+    return False
+
+
+def get_md5sum(host, fqpath):
+    """Get the md5 checksum for the file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        The md5sum of the file on success. None on fail.
+    """
+    command = "md5sum %s" % fqpath
+    rcode, rout, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return rout.strip()
+
+    g.log.error('md5sum failed: %s' % rerr)
+    return None
+
+
+def get_file_stat(host, fqpath):
+    """Get file stat information about a file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        A dictionary of file stat data. None on fail.
+    """
+    statformat = '%F:%n:%i:%a:%s:%h:%u:%g:%U:%G'
+    command = "stat -c '%s' %s" % (statformat, fqpath)
+    rcode, rout, rerr = g.run(host, command)
+    if rcode == 0:
+        stat_data = {}
+        stat_string = rout.strip()
+        (filetype, filename, inode,
+         access, size, links,
+         uid, gid, username, groupname) = stat_string.split(":")
+
+        stat_data['filetype'] = filetype
+        stat_data['filename'] = filename
+        stat_data["inode"] = inode
+        stat_data["access"] = access
+        stat_data["size"] = size
+        stat_data["links"] = links
+        stat_data["username"] = username
+        stat_data["groupname"] = groupname
+        stat_data["uid"] = uid
+        stat_data["gid"] = gid
+
+        return stat_data
+
+    g.log.error("Could not stat file %s: %s" % (fqpath, rerr))
+    return None
+
+
+def set_file_permissions(host, fqpath, perms):
+    """Set permissions on a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+        perms (str): A permissions string as passed to chmod.
+
+    Returns:
+        True on success. False on fail.
+    """
+    command = "chmod %s %s" % (perms, fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('chmod failed: %s' % rerr)
+    return False
+
+
+def set_file_owner(host, fqpath, user):
+    """Set file owner for  a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+        user (str): The user owning the file.
+
+    Returns:
+        True on success. False on fail.
+    """
+    command = "chown %s %s" % (user, fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('chown failed: %s' % rerr)
+    return False
+
+
+def set_file_group(host, fqpath, group):
+    """Set file group for  a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+        group (str): The group owning the file.
+
+    Returns:
+        True on success. False on fail.
+    """
+    command = "chgrp %s %s" % (group, fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('chgrp failed: %s' % rerr)
+    return False
+
+
+def move_file(host, source_fqpath, dest_fqpath):
+    """Move a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        source_fqpath (str): The fully-qualified path to the file to move.
+        dest_fqpath (str): The fully-qualified path to the new file location.
+
+    Returns:
+        True on success. False on fail.
+    """
+    command = "mv %s %s" % (source_fqpath, dest_fqpath)
+    rcode, _, rerr = g.run(host, command)
+
+    if rcode == 0:
+        return True
+
+    g.log.error('mv failed: %s' % rerr)
+    return False
+
+
+def remove_file(host, fqpath, force=False):
+    """Removes a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        True on success. False on fail.
+    """
+    command_list = ['rm']
+    if force:
+        command_list.append('-f')
+    command_list.append(fqpath)
+    rcode, _, rerr = g.run(host, ' '.join(command_list))
+
+    if rcode == 0:
+        return True
+
+    g.log.error('Remove file failed: %s' % rerr)
+    return False
+
+
+def get_pathinfo(host, fqpath):
+    """Get pathinfo for a remote file.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        A dictionary of pathinfo data for a remote file. None on fail.
+    """
+    pathinfo = {}
+    pathinfo['raw'] = get_fattr(host, fqpath, 'trusted.glusterfs.pathinfo')
+    pathinfo['brickdir_paths'] = re.findall(".*?POSIX.*?:(\S+)\>",
+                                            pathinfo['raw'])
+
+    return pathinfo
+
+
+def is_linkto_file(host, fqpath):
+    """Test if file is a dht linkto file.
+    To return True, file must...
+    1. be of file type 'sticky empty'
+    2. have size of 0
+    3. have the glusterfs.dht.linkto xattr set.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        True or False
+    """
+    command = 'file %s' % fqpath
+    rcode, rout, _ = g.run(host, command)
+
+    if rcode == 0:
+        if 'sticky empty' in rout.strip():
+            stat = get_file_stat(host, fqpath)
+            if int(stat['size']) == 0:
+                # xattr = get_fattr(host, fqpath,
+                #                  'trusted.glusterfs.dht.linkto')
+                xattr = get_dht_linkto_xattr(host, fqpath)
+                if xattr is not None:
+                    return True
+
+    return False
+
+
+def get_dht_linkto_xattr(host, fqpath):
+    """Get the glusterfs.dht.linkto xattr for a file on a brick.
+
+    Args:
+        host (str): The hostname/ip of the remote system.
+        fqpath (str): The fully-qualified path to the file.
+
+    Returns:
+        Return value of get_fattr trusted.glusterfs.dht.linkto call.
+    """
+    linkto_xattr = get_fattr(host, fqpath, 'trusted.glusterfs.dht.linkto')
+
+    return linkto_xattr
+
+
+class GlusterFile(object):
+    """Class to handle files specific to Gluster (client and backend)"""
+    def __init__(self, host, fqpath):
+        self._host = host
+        self._fqpath = fqpath
+
+        self._mountpoint = None
+        self._calculated_hash = None
+        self._pathinfo = None
+        self._parent_dir_pathinfo = None
+        self._parent_dir_layout = None
+
+        self._previous_fqpath = None
+
+    @property
+    def host(self):
+        """str: the hostname/ip of the client system hosting the file."""
+        return self._host
+
+    @property
+    def fqpath(self):
+        """str: the fully-qualified path of the file on the client system."""
+        return self._fqpath
+
+    @property
+    def relative_path(self):
+        """str: the relative path from the mountpoint of the file."""
+        return os.path.relpath(self._fqpath, self.mountpoint)
+
+    @property
+    def basename(self):
+        """str: the name of the file with directories stripped from string."""
+        return os.path.basename(self._fqpath)
+
+    @property
+    def parent_dir(self):
+        """str: the full-qualified path of the file's parent directory."""
+        return os.path.dirname(self._fqpath)
+
+    @property
+    def mountpoint(self):
+        """str: the fully-qualified path of the mountpoint under the file."""
+        if self._mountpoint is None:
+            self._mountpoint = get_mountpoint(self._host, self._fqpath)
+
+        return self._mountpoint
+
+    @property
+    def pathinfo(self):
+        """dict: a dictionary of path_info-related values"""
+        self._pathinfo = get_pathinfo(self._host, self._fqpath)
+
+        return self._pathinfo
+
+    @property
+    def parent_dir_pathinfo(self):
+        """"dict: a dictionary of path_info-related values for the parent dir
+                    of the file's fqpath.
+        """
+        parent_dir_pathinfo = get_pathinfo(self._host, self.parent_dir)
+
+        return parent_dir_pathinfo
+
+    @property
+    def exists_on_client(self):
+        """bool: Does the file exists on the client?"""
+        ret = file_exists(self._host, self._fqpath)
+
+        if ret:
+            return True
+
+        return False
+
+    @property
+    def exists_on_bricks(self):
+        """bool: Does the file exist on the backend bricks?"""
+        flag = 0
+        for brickdir_path in self.pathinfo['brickdir_paths']:
+            (host, fqpath) = brickdir_path.split(':')
+            if not file_exists(host, fqpath):
+                flag = flag | 1
+
+        if flag == 0:
+            return True
+
+        return False
+
+    @property
+    def exists_on_hashed_bricks(self):
+        """bool: Does the file exist on the hashed bricks as expected?"""
+        # TODO: inject check for linkto and data files
+        flag = 0
+        for brickdir_path in self.hashed_bricks:
+            (host, fqpath) = brickdir_path.split(':')
+            if not file_exists(host, fqpath):
+                flag = flag | 1
+
+        if flag == 0:
+            return True
+
+        return False
+
+    @property
+    def exists_on_cached_bricks(self):
+        """bool: Does the file exist on the cached bricks as expected?
+
+            This currently is redundant as the cache list is currently
+            created by searching bricks for the file. This will be more
+            useful when the cached brick list is compiled by following the
+            subvolume info provided in the linkto xattr.
+        """
+        flag = 0
+        for brickdir_path in self.cached_bricks:
+            (host, fqpath) = brickdir_path.split(':')
+            if not file_exists(host, fqpath):
+                flag = flag | 1
+
+        if flag == 0:
+            return True
+
+        return False
+
+    @property
+    def exists(self):
+        """bool: does the file exist on both client and backend bricks"""
+        return (self.exists_on_client, self.exists_on_bricks)
+
+    @property
+    def stat_on_client(self):
+        """dict: a dictionary of stat data"""
+        return get_file_stat(self._host, self._fqpath)
+
+    @property
+    def stat_on_bricks(self):
+        """dict: a dictionary of stat dictionaries for the file on bricks"""
+        file_stats = {}
+        for brickdir_path in self.pathinfo['brickdir_paths']:
+            (host, fqpath) = brickdir_path.split(':')
+            file_stats[brickdir_path] = get_file_stat(host, fqpath)
+
+        return file_stats
+
+    @property
+    def stat(self):
+        """list: a list of the stat dictionary data for client and bricks."""
+        return (self.stat_on_client, self.stat_on_bricks)
+
+    @property
+    def md5sum_on_client(self):
+        """str: the md5sum for the file on the client"""
+        return get_md5sum(self._host, self._fqpath)
+
+    @property
+    def md5sum_on_bricks(self):
+        """dict: a dictionary of md5sums for the file on bricks"""
+        # TODO: handle dispersed ???
+        file_md5s = {}
+        for brickdir_path in self.pathinfo['brickdir_paths']:
+            (host, fqpath) = brickdir_path.split(':')
+            file_md5s[brickdir_path] = get_md5sum(host, fqpath)
+
+        return file_md5s
+
+    @property
+    def md5sum(self):
+        """list: a list of client and brick md5sum data"""
+        return (self.md5sum_on_client, self.md5sum_on_bricks)
+
+    @property
+    def calculated_hash(self):
+        """str: the computed hash of the file using libglusterfs"""
+        if self._calculated_hash is None:
+            self._calculated_hash = calculate_hash(self._host, self.basename)
+
+        return self._calculated_hash
+
+    @property
+    def parent_dir_layout(self):
+        """obj: Layout instance of the file's parent directory"""
+        if self._parent_dir_layout is None:
+            layout = Layout(self.parent_dir_pathinfo)
+            self._parent_dir_layout = layout
+        else:
+            layout = self._parent_dir_layout
+
+        return layout
+
+    @property
+    def hashed_bricks(self):
+        """list: the list of bricks matching with hashrange surrounding hash"""
+        brickpaths = []
+        for brickdir in self.parent_dir_layout.brickdirs:
+            low = brickdir.hashrange_low
+            high = brickdir.hashrange_high
+            if low < self.calculated_hash < high:
+                brickpaths.append(brickdir.path)
+                g.log.debug("%s: %d - %d - %d" % (brickdir.path,
+                                                  brickdir.hashrange_low,
+                                                  self.calculated_hash,
+                                                  brickdir.hashrange_high))
+
+        return brickpaths
+
+    @property
+    def cached_bricks(self):
+        """list: the list of bricks with the cached file(s)"""
+        # TODO: build list from subvolume in glusterfs.dht.linkto xattr
+        brickpaths = []
+        for brickdir in self.parent_dir_layout.brickdirs:
+            fqpath = os.path.join(brickdir.fqpath, self.basename)
+            if file_exists(brickdir.host, fqpath):
+                if not is_linkto_file(brickdir.host, fqpath):
+                    brickpaths.append(brickdir.path)
+
+        return brickpaths
+
+    def move(self, dest_fqpath):
+        """Move the file to a new location and store previous fqpath.
+
+        Args:
+            dest_fqpath (str): The fully-qualified destination path.
+
+        Returns:
+            True on success. False on fail.
+        """
+        ret = move_file(self._host, self._fqpath, dest_fqpath)
+
+        if ret:
+            # TODO: change this to use a setter/getter for heavy lifting once
+            #        and can reset everything from one place
+            self._previous_fqpath = self._fqpath
+            self._fqpath = dest_fqpath
+
+            return True
+
+        return False
+
+    def create(self):
+        """Creates a simple file via copy for testing purposes.
+            Also creates parent directories if they don't exist.
+        Args:
+            None
+
+        Returns:
+            True on success. False on failure.
+        """
+        if not self.exists_on_client:
+            command = "mkdir -p %s" % self.parent_dir
+            rcode, _, _ = g.run(self._host, command)
+            if rcode != 0:
+                return False
+            command = "cp /etc/inittab %s" % self._fqpath
+            rcode, _, _ = g.run(self._host, command)
+            if rcode == 0:
+                return True
+
+        return False
+
+    def get_xattr(self, xattr):
+        """Get the xattr for the file instance.
+
+        Args:
+            xattr (str): The file attribute to get from file.
+
+        Returns:
+            Result of get_fattr function.
+        """
+        return get_fattr(self._host, self._fqpath, xattr)
+
+    def set_xattr(self, xattr, value):
+        """Set the specified xattr for the file instance.
+
+        Args:
+            xattr (str): The attribute to set on the file.
+            value (str): the value for the attribute.
+
+        Returns:
+            Return of set_fattr function.
+        """
+        return set_fattr(self._host, self._fqpath, xattr, value)
+
+    def delete_xattr(self, xattr):
+        """Delete the specified xattr for the file instance.
+
+        Args:
+            xattr (str): The attribute to delete.
+
+        Returns:
+            Return of delete_fattr function.
+        """
+        return delete_fattr(self._host, self._fqpath, xattr)