summaryrefslogtreecommitdiffstats
path: root/glustolibs-gluster/glustolibs/gluster/glusterfile.py
diff options
context:
space:
mode:
authorJonathan Holloway <jholloway@redhat.com>2018-01-23 02:02:38 +0100
committerNigel Babu <nigelb@redhat.com>2018-02-26 10:42:31 +0000
commitab7a71cc4b2862d267c3e6fae67c711e51abca77 (patch)
tree42060d76949e5c8bb4736c2ecdf1ecf68eed6df5 /glustolibs-gluster/glustolibs/gluster/glusterfile.py
parent1374783418db58f6f4fe153d83082549bf2d000f (diff)
initial dht libs
* glusterfile.py - helper for gluster client and backend files. * glusterdir.py - helper for gluster client and backend dirs. * brickdir.py - helper for collection and hashing of brickdirs (from pathinfo data). * layout.py - base class for simple DHT layout validation. * dht_test_util.py - utility module to walk a directory tree and run tests against files. * constants.py - definitions for constants used in DHT libraries. * exceptions.py - definitions for exceptions raised in DHT libraries. Change-Id: I44770a822e0ec79561b3aa048e555320f622116a Signed-off-by: Jonathan Holloway <jholloway@redhat.com>
Diffstat (limited to 'glustolibs-gluster/glustolibs/gluster/glusterfile.py')
-rw-r--r--glustolibs-gluster/glustolibs/gluster/glusterfile.py710
1 files changed, 710 insertions, 0 deletions
diff --git a/glustolibs-gluster/glustolibs/gluster/glusterfile.py b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
new file mode 100644
index 0000000..fcc10f2
--- /dev/null
+++ b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
@@ -0,0 +1,710 @@
+#!/usr/bin/env python
+# Copyright (C) 2018 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+"""Description: Module for library gluster file class and related functions.
+
+A GlusterFile is a file object that exists on the client and backend brick.
+This module provides low-level functions and a GlusterFile class to maintain
+state and manage properties of a file in both locations.
+"""
+
+import ctypes
+import os
+import re
+
+from glusto.core import Glusto as g
+
+from glustolibs.gluster.layout import Layout
+
+
+def calculate_hash(host, filename):
+ """ Function to import DHT Hash library.
+
+ Args:
+ filename (str): the name of the file
+
+ Returns:
+ An integer representation of the hash
+ """
+ # TODO: For testcases specifically testing hashing routine
+ # consider using a baseline external Davies-Meyer hash_value.c
+ # Creating comparison hash from same library we are testing
+ # may not be best practice here. (Holloway)
+ try:
+ # Check if libglusterfs.so.0 is available locally
+ glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0")
+ g.log.debug("Library libglusterfs.so.0 loaded locally")
+ except OSError:
+ conn = g.get_connection(host)
+ glusterfs = \
+ conn.modules.ctypes.cdll.LoadLibrary("libglusterfs.so.0")
+ g.log.debug("Library libglusterfs.so.0 loaded via rpyc")
+
+ computed_hash = \
+ ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, len(filename)))
+ # conn.close()
+
+ return int(computed_hash.value)
+
+
+def get_mountpoint(host, fqpath):
+ """Retrieve the mountpoint under a file
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ The mountpoint on success. None on fail.
+ """
+ command = "df -P %s | awk 'END{print $NF}'" % fqpath
+ rcode, rout, rerr = g.run(host, command)
+ if rcode == 0:
+ return rout.strip()
+
+ g.log.error("Get mountpoint failed: %s" % rerr)
+ return None
+
+
+def get_fattr(host, fqpath, fattr):
+ """getfattr for filepath on remote system
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+ fattr (str): name of the fattr to retrieve
+
+ Returns:
+ getfattr result on success. None on fail.
+ """
+ command = ("getfattr --absolute-names --only-values -n '%s' %s" %
+ (fattr, fqpath))
+ rcode, rout, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return rout.strip()
+
+ g.log.error('getfattr failed: %s' % rerr)
+ return None
+
+
+def get_fattr_list(host, fqpath):
+ """List of xattr for filepath on remote system.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ Dictionary of xattrs on success. None on fail.
+ """
+ command = "getfattr --absolute-names -d -m - %s" % fqpath
+ rcode, rout, rerr = g.run(host, command)
+
+ if rcode == 0:
+ xattr_list = {}
+ for xattr_string in rout.strip().split('\n'):
+ xattr = xattr_string.split('=', 1)
+ if len(xattr) > 1:
+ key, value = xattr
+ xattr_list[key] = value
+
+ return xattr_list
+
+ g.log.error('getfattr failed: %s' % rerr)
+ return None
+
+
+def set_fattr(host, fqpath, fattr, value):
+ """setfattr for filepath on remote system
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+ fattr (str): The name of the fattr to retrieve.
+
+ Returns:
+ setfattr result on success. errorcode on fail
+ """
+ command = 'setfattr -n %s -v %s %s' % (fattr, value, fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('setfattr failed: %s', rerr)
+ return False
+
+
+def delete_fattr(host, fqpath, fattr):
+ """remove fattr for filepath on remote system
+
+ Args:
+ host (str): hostname/ip of remote system
+ fqpath (str): the fully qualified path of the file
+ fattr (str): name of the fattr to delete
+
+ Returns:
+ setfattr result on success. errorcode on fail
+ """
+ command = 'setfattr -x %s %s' % (fattr, fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('setfattr -x failed: %s' % rerr)
+ return False
+
+
+def file_exists(host, fqpath):
+ """Check if file exists at path on host
+
+ Args:
+ host (str): hostname or ip of system
+ filename (str): fully qualified path of file
+
+ Returns:
+ True if file exists. False if file does not exist
+ """
+ command = "ls -ld %s" % fqpath
+ rcode, _, rerr = g.run(host, command)
+ if rcode == 0:
+ return True
+
+ g.log.error('File does not exist: %s', rerr)
+ return False
+
+
+def get_md5sum(host, fqpath):
+ """Get the md5 checksum for the file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ The md5sum of the file on success. None on fail.
+ """
+ command = "md5sum %s" % fqpath
+ rcode, rout, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return rout.strip()
+
+ g.log.error('md5sum failed: %s' % rerr)
+ return None
+
+
+def get_file_stat(host, fqpath):
+ """Get file stat information about a file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ A dictionary of file stat data. None on fail.
+ """
+ statformat = '%F:%n:%i:%a:%s:%h:%u:%g:%U:%G'
+ command = "stat -c '%s' %s" % (statformat, fqpath)
+ rcode, rout, rerr = g.run(host, command)
+ if rcode == 0:
+ stat_data = {}
+ stat_string = rout.strip()
+ (filetype, filename, inode,
+ access, size, links,
+ uid, gid, username, groupname) = stat_string.split(":")
+
+ stat_data['filetype'] = filetype
+ stat_data['filename'] = filename
+ stat_data["inode"] = inode
+ stat_data["access"] = access
+ stat_data["size"] = size
+ stat_data["links"] = links
+ stat_data["username"] = username
+ stat_data["groupname"] = groupname
+ stat_data["uid"] = uid
+ stat_data["gid"] = gid
+
+ return stat_data
+
+ g.log.error("Could not stat file %s: %s" % (fqpath, rerr))
+ return None
+
+
+def set_file_permissions(host, fqpath, perms):
+ """Set permissions on a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+ perms (str): A permissions string as passed to chmod.
+
+ Returns:
+ True on success. False on fail.
+ """
+ command = "chmod %s %s" % (perms, fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('chmod failed: %s' % rerr)
+ return False
+
+
+def set_file_owner(host, fqpath, user):
+ """Set file owner for a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+ user (str): The user owning the file.
+
+ Returns:
+ True on success. False on fail.
+ """
+ command = "chown %s %s" % (user, fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('chown failed: %s' % rerr)
+ return False
+
+
+def set_file_group(host, fqpath, group):
+ """Set file group for a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+ group (str): The group owning the file.
+
+ Returns:
+ True on success. False on fail.
+ """
+ command = "chgrp %s %s" % (group, fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('chgrp failed: %s' % rerr)
+ return False
+
+
+def move_file(host, source_fqpath, dest_fqpath):
+ """Move a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ source_fqpath (str): The fully-qualified path to the file to move.
+ dest_fqpath (str): The fully-qualified path to the new file location.
+
+ Returns:
+ True on success. False on fail.
+ """
+ command = "mv %s %s" % (source_fqpath, dest_fqpath)
+ rcode, _, rerr = g.run(host, command)
+
+ if rcode == 0:
+ return True
+
+ g.log.error('mv failed: %s' % rerr)
+ return False
+
+
+def remove_file(host, fqpath, force=False):
+ """Removes a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ True on success. False on fail.
+ """
+ command_list = ['rm']
+ if force:
+ command_list.append('-f')
+ command_list.append(fqpath)
+ rcode, _, rerr = g.run(host, ' '.join(command_list))
+
+ if rcode == 0:
+ return True
+
+ g.log.error('Remove file failed: %s' % rerr)
+ return False
+
+
+def get_pathinfo(host, fqpath):
+ """Get pathinfo for a remote file.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ A dictionary of pathinfo data for a remote file. None on fail.
+ """
+ pathinfo = {}
+ pathinfo['raw'] = get_fattr(host, fqpath, 'trusted.glusterfs.pathinfo')
+ pathinfo['brickdir_paths'] = re.findall(".*?POSIX.*?:(\S+)\>",
+ pathinfo['raw'])
+
+ return pathinfo
+
+
+def is_linkto_file(host, fqpath):
+ """Test if file is a dht linkto file.
+ To return True, file must...
+ 1. be of file type 'sticky empty'
+ 2. have size of 0
+ 3. have the glusterfs.dht.linkto xattr set.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ True or False
+ """
+ command = 'file %s' % fqpath
+ rcode, rout, _ = g.run(host, command)
+
+ if rcode == 0:
+ if 'sticky empty' in rout.strip():
+ stat = get_file_stat(host, fqpath)
+ if int(stat['size']) == 0:
+ # xattr = get_fattr(host, fqpath,
+ # 'trusted.glusterfs.dht.linkto')
+ xattr = get_dht_linkto_xattr(host, fqpath)
+ if xattr is not None:
+ return True
+
+ return False
+
+
+def get_dht_linkto_xattr(host, fqpath):
+ """Get the glusterfs.dht.linkto xattr for a file on a brick.
+
+ Args:
+ host (str): The hostname/ip of the remote system.
+ fqpath (str): The fully-qualified path to the file.
+
+ Returns:
+ Return value of get_fattr trusted.glusterfs.dht.linkto call.
+ """
+ linkto_xattr = get_fattr(host, fqpath, 'trusted.glusterfs.dht.linkto')
+
+ return linkto_xattr
+
+
+class GlusterFile(object):
+ """Class to handle files specific to Gluster (client and backend)"""
+ def __init__(self, host, fqpath):
+ self._host = host
+ self._fqpath = fqpath
+
+ self._mountpoint = None
+ self._calculated_hash = None
+ self._pathinfo = None
+ self._parent_dir_pathinfo = None
+ self._parent_dir_layout = None
+
+ self._previous_fqpath = None
+
+ @property
+ def host(self):
+ """str: the hostname/ip of the client system hosting the file."""
+ return self._host
+
+ @property
+ def fqpath(self):
+ """str: the fully-qualified path of the file on the client system."""
+ return self._fqpath
+
+ @property
+ def relative_path(self):
+ """str: the relative path from the mountpoint of the file."""
+ return os.path.relpath(self._fqpath, self.mountpoint)
+
+ @property
+ def basename(self):
+ """str: the name of the file with directories stripped from string."""
+ return os.path.basename(self._fqpath)
+
+ @property
+ def parent_dir(self):
+ """str: the full-qualified path of the file's parent directory."""
+ return os.path.dirname(self._fqpath)
+
+ @property
+ def mountpoint(self):
+ """str: the fully-qualified path of the mountpoint under the file."""
+ if self._mountpoint is None:
+ self._mountpoint = get_mountpoint(self._host, self._fqpath)
+
+ return self._mountpoint
+
+ @property
+ def pathinfo(self):
+ """dict: a dictionary of path_info-related values"""
+ self._pathinfo = get_pathinfo(self._host, self._fqpath)
+
+ return self._pathinfo
+
+ @property
+ def parent_dir_pathinfo(self):
+ """"dict: a dictionary of path_info-related values for the parent dir
+ of the file's fqpath.
+ """
+ parent_dir_pathinfo = get_pathinfo(self._host, self.parent_dir)
+
+ return parent_dir_pathinfo
+
+ @property
+ def exists_on_client(self):
+ """bool: Does the file exists on the client?"""
+ ret = file_exists(self._host, self._fqpath)
+
+ if ret:
+ return True
+
+ return False
+
+ @property
+ def exists_on_bricks(self):
+ """bool: Does the file exist on the backend bricks?"""
+ flag = 0
+ for brickdir_path in self.pathinfo['brickdir_paths']:
+ (host, fqpath) = brickdir_path.split(':')
+ if not file_exists(host, fqpath):
+ flag = flag | 1
+
+ if flag == 0:
+ return True
+
+ return False
+
+ @property
+ def exists_on_hashed_bricks(self):
+ """bool: Does the file exist on the hashed bricks as expected?"""
+ # TODO: inject check for linkto and data files
+ flag = 0
+ for brickdir_path in self.hashed_bricks:
+ (host, fqpath) = brickdir_path.split(':')
+ if not file_exists(host, fqpath):
+ flag = flag | 1
+
+ if flag == 0:
+ return True
+
+ return False
+
+ @property
+ def exists_on_cached_bricks(self):
+ """bool: Does the file exist on the cached bricks as expected?
+
+ This currently is redundant as the cache list is currently
+ created by searching bricks for the file. This will be more
+ useful when the cached brick list is compiled by following the
+ subvolume info provided in the linkto xattr.
+ """
+ flag = 0
+ for brickdir_path in self.cached_bricks:
+ (host, fqpath) = brickdir_path.split(':')
+ if not file_exists(host, fqpath):
+ flag = flag | 1
+
+ if flag == 0:
+ return True
+
+ return False
+
+ @property
+ def exists(self):
+ """bool: does the file exist on both client and backend bricks"""
+ return (self.exists_on_client, self.exists_on_bricks)
+
+ @property
+ def stat_on_client(self):
+ """dict: a dictionary of stat data"""
+ return get_file_stat(self._host, self._fqpath)
+
+ @property
+ def stat_on_bricks(self):
+ """dict: a dictionary of stat dictionaries for the file on bricks"""
+ file_stats = {}
+ for brickdir_path in self.pathinfo['brickdir_paths']:
+ (host, fqpath) = brickdir_path.split(':')
+ file_stats[brickdir_path] = get_file_stat(host, fqpath)
+
+ return file_stats
+
+ @property
+ def stat(self):
+ """list: a list of the stat dictionary data for client and bricks."""
+ return (self.stat_on_client, self.stat_on_bricks)
+
+ @property
+ def md5sum_on_client(self):
+ """str: the md5sum for the file on the client"""
+ return get_md5sum(self._host, self._fqpath)
+
+ @property
+ def md5sum_on_bricks(self):
+ """dict: a dictionary of md5sums for the file on bricks"""
+ # TODO: handle dispersed ???
+ file_md5s = {}
+ for brickdir_path in self.pathinfo['brickdir_paths']:
+ (host, fqpath) = brickdir_path.split(':')
+ file_md5s[brickdir_path] = get_md5sum(host, fqpath)
+
+ return file_md5s
+
+ @property
+ def md5sum(self):
+ """list: a list of client and brick md5sum data"""
+ return (self.md5sum_on_client, self.md5sum_on_bricks)
+
+ @property
+ def calculated_hash(self):
+ """str: the computed hash of the file using libglusterfs"""
+ if self._calculated_hash is None:
+ self._calculated_hash = calculate_hash(self._host, self.basename)
+
+ return self._calculated_hash
+
+ @property
+ def parent_dir_layout(self):
+ """obj: Layout instance of the file's parent directory"""
+ if self._parent_dir_layout is None:
+ layout = Layout(self.parent_dir_pathinfo)
+ self._parent_dir_layout = layout
+ else:
+ layout = self._parent_dir_layout
+
+ return layout
+
+ @property
+ def hashed_bricks(self):
+ """list: the list of bricks matching with hashrange surrounding hash"""
+ brickpaths = []
+ for brickdir in self.parent_dir_layout.brickdirs:
+ low = brickdir.hashrange_low
+ high = brickdir.hashrange_high
+ if low < self.calculated_hash < high:
+ brickpaths.append(brickdir.path)
+ g.log.debug("%s: %d - %d - %d" % (brickdir.path,
+ brickdir.hashrange_low,
+ self.calculated_hash,
+ brickdir.hashrange_high))
+
+ return brickpaths
+
+ @property
+ def cached_bricks(self):
+ """list: the list of bricks with the cached file(s)"""
+ # TODO: build list from subvolume in glusterfs.dht.linkto xattr
+ brickpaths = []
+ for brickdir in self.parent_dir_layout.brickdirs:
+ fqpath = os.path.join(brickdir.fqpath, self.basename)
+ if file_exists(brickdir.host, fqpath):
+ if not is_linkto_file(brickdir.host, fqpath):
+ brickpaths.append(brickdir.path)
+
+ return brickpaths
+
+ def move(self, dest_fqpath):
+ """Move the file to a new location and store previous fqpath.
+
+ Args:
+ dest_fqpath (str): The fully-qualified destination path.
+
+ Returns:
+ True on success. False on fail.
+ """
+ ret = move_file(self._host, self._fqpath, dest_fqpath)
+
+ if ret:
+ # TODO: change this to use a setter/getter for heavy lifting once
+ # and can reset everything from one place
+ self._previous_fqpath = self._fqpath
+ self._fqpath = dest_fqpath
+
+ return True
+
+ return False
+
+ def create(self):
+ """Creates a simple file via copy for testing purposes.
+ Also creates parent directories if they don't exist.
+ Args:
+ None
+
+ Returns:
+ True on success. False on failure.
+ """
+ if not self.exists_on_client:
+ command = "mkdir -p %s" % self.parent_dir
+ rcode, _, _ = g.run(self._host, command)
+ if rcode != 0:
+ return False
+ command = "cp /etc/inittab %s" % self._fqpath
+ rcode, _, _ = g.run(self._host, command)
+ if rcode == 0:
+ return True
+
+ return False
+
+ def get_xattr(self, xattr):
+ """Get the xattr for the file instance.
+
+ Args:
+ xattr (str): The file attribute to get from file.
+
+ Returns:
+ Result of get_fattr function.
+ """
+ return get_fattr(self._host, self._fqpath, xattr)
+
+ def set_xattr(self, xattr, value):
+ """Set the specified xattr for the file instance.
+
+ Args:
+ xattr (str): The attribute to set on the file.
+ value (str): the value for the attribute.
+
+ Returns:
+ Return of set_fattr function.
+ """
+ return set_fattr(self._host, self._fqpath, xattr, value)
+
+ def delete_xattr(self, xattr):
+ """Delete the specified xattr for the file instance.
+
+ Args:
+ xattr (str): The attribute to delete.
+
+ Returns:
+ Return of delete_fattr function.
+ """
+ return delete_fattr(self._host, self._fqpath, xattr)