From 655b0d2793386d2059b9c682e931035a83619917 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 10 Aug 2016 15:28:48 +0530 Subject: Move source files into gfapi/ dir Currently, many source files are directly placed under gluster/ dir: gluster/exceptions.py gluster/gfapi.py gluster/utils.py When multiple packages (RPMs) are sharing the same gluster namespace, these source files will conflict if there are source files with same names provided by other projects. Fix: Move all source files in gluster/* to gluster/gfapi/* Note that this patch does not break how existing users import gfapi. Change-Id: Idf9d07eefafe8333215d6c61201c97c982565ba9 Signed-off-by: Prashanth Pai --- gluster/api.py | 512 --------- gluster/exceptions.py | 21 - gluster/gfapi.py | 1736 ------------------------------ gluster/gfapi/__init__.py | 14 + gluster/gfapi/api.py | 512 +++++++++ gluster/gfapi/exceptions.py | 21 + gluster/gfapi/gfapi.py | 1734 +++++++++++++++++++++++++++++ gluster/gfapi/utils.py | 57 + gluster/utils.py | 57 - setup.py | 7 +- test/functional/libgfapi-python-tests.py | 4 +- test/unit/gluster/test_gfapi.py | 14 +- test/unit/gluster/test_utils.py | 4 +- 13 files changed, 2353 insertions(+), 2340 deletions(-) delete mode 100755 gluster/api.py delete mode 100644 gluster/exceptions.py delete mode 100755 gluster/gfapi.py create mode 100644 gluster/gfapi/__init__.py create mode 100644 gluster/gfapi/api.py create mode 100644 gluster/gfapi/exceptions.py create mode 100644 gluster/gfapi/gfapi.py create mode 100644 gluster/gfapi/utils.py delete mode 100644 gluster/utils.py diff --git a/gluster/api.py b/gluster/api.py deleted file mode 100755 index c440de6..0000000 --- a/gluster/api.py +++ /dev/null @@ -1,512 +0,0 @@ -# Copyright (c) 2016 Red Hat, Inc. -# -# This file is part of libgfapi-python project which is a -# subproject of GlusterFS ( www.gluster.org) -# -# This file is licensed to you under your choice of the GNU Lesser -# General Public License, version 3 or any later version (LGPLv3 or -# later), or the GNU General Public License, version 2 (GPLv2), in all -# cases as published by the Free Software Foundation. - -import ctypes -from ctypes.util import find_library -from ctypes import sizeof - - -# LD_LIBRARY_PATH is not looked up by ctypes.util.find_library() -so_file_name = find_library("gfapi") - -if so_file_name is None: - for name in ["libgfapi.so.0", "libgfapi.so"]: - try: - ctypes.CDLL(name, ctypes.RTLD_GLOBAL, use_errno=True) - except OSError: - pass - else: - so_file_name = name - break - if so_file_name is None: - # The .so file cannot be found (or loaded) - # May be you need to run ldconfig command - raise Exception("libgfapi.so not found") - -# Looks like ctypes is having trouble with dependencies, so just force them to -# load with RTLD_GLOBAL until I figure that out. -try: - client = ctypes.CDLL(so_file_name, ctypes.RTLD_GLOBAL, use_errno=True) - # The above statement "may" fail with OSError on some systems if - # libgfapi.so is located in /usr/local/lib/. This happens when glusterfs - # is installed from source. Refer to: http://bugs.python.org/issue18502 -except OSError: - raise ImportError("ctypes.CDLL() cannot load {0}. You might want to set " - "LD_LIBRARY_PATH env variable".format(so_file_name)) - -# c_ssize_t was only added in py27. Manually backporting it here for py26 -try: - import ctypes.c_ssize_t -except ImportError: - if sizeof(ctypes.c_uint) == sizeof(ctypes.c_void_p): - setattr(ctypes, 'c_ssize_t', ctypes.c_int) - elif sizeof(ctypes.c_ulong) == sizeof(ctypes.c_void_p): - setattr(ctypes, 'c_ssize_t', ctypes.c_long) - elif sizeof(ctypes.c_ulonglong) == sizeof(ctypes.c_void_p): - setattr(ctypes, 'c_ssize_t', ctypes.c_longlong) - - -# Wow, the Linux kernel folks really play nasty games with this structure. If -# you look at the man page for stat(2) and then at this definition you'll note -# two discrepancies. First, we seem to have st_nlink and st_mode reversed. In -# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit -# they're in the man-page order. Even uglier, the man page makes no mention of -# the *nsec fields, but they are very much present and if they're not included -# then we get memory corruption because libgfapi has a structure definition -# that's longer than ours and they overwrite some random bit of memory after -# the space we allocated. Yes, that's all very disgusting, and I'm still not -# sure this will really work on 32-bit because all of the field types are so -# obfuscated behind macros and feature checks. - - -class Stat (ctypes.Structure): - _fields_ = [ - ("st_dev", ctypes.c_ulong), - ("st_ino", ctypes.c_ulong), - ("st_nlink", ctypes.c_ulong), - ("st_mode", ctypes.c_uint), - ("st_uid", ctypes.c_uint), - ("st_gid", ctypes.c_uint), - ("st_rdev", ctypes.c_ulong), - ("st_size", ctypes.c_ulong), - ("st_blksize", ctypes.c_ulong), - ("st_blocks", ctypes.c_ulong), - ("st_atime", ctypes.c_ulong), - ("st_atimensec", ctypes.c_ulong), - ("st_mtime", ctypes.c_ulong), - ("st_mtimensec", ctypes.c_ulong), - ("st_ctime", ctypes.c_ulong), - ("st_ctimensec", ctypes.c_ulong), - ] - - -class Statvfs (ctypes.Structure): - _fields_ = [ - ("f_bsize", ctypes.c_ulong), - ("f_frsize", ctypes.c_ulong), - ("f_blocks", ctypes.c_ulong), - ("f_bfree", ctypes.c_ulong), - ("f_bavail", ctypes.c_ulong), - ("f_files", ctypes.c_ulong), - ("f_ffree", ctypes.c_ulong), - ("f_favail", ctypes.c_ulong), - ("f_fsid", ctypes.c_ulong), - ("f_flag", ctypes.c_ulong), - ("f_namemax", ctypes.c_ulong), - ("__f_spare", ctypes.c_int * 6), - ] - - -class Dirent (ctypes.Structure): - _fields_ = [ - ("d_ino", ctypes.c_ulong), - ("d_off", ctypes.c_ulong), - ("d_reclen", ctypes.c_ushort), - ("d_type", ctypes.c_char), - ("d_name", ctypes.c_char * 256), - ] - - -class Timespec (ctypes.Structure): - _fields_ = [ - ('tv_sec', ctypes.c_long), - ('tv_nsec', ctypes.c_long) - ] - - -# Here is the reference card of libgfapi library exported -# apis with its different versions. -# -# GFAPI_3.4.0 { -# glfs_new; -# glfs_set_volfile; -# glfs_set_volfile_server; -# glfs_set_logging; -# glfs_init; -# glfs_fini; -# glfs_open; -# glfs_creat; -# glfs_close; -# glfs_from_glfd; -# glfs_set_xlator_option; -# glfs_read; -# glfs_write; -# glfs_read_async; -# glfs_write_async; -# glfs_readv; -# glfs_writev; -# glfs_readv_async; -# glfs_writev_async; -# glfs_pread; -# glfs_pwrite; -# glfs_pread_async; -# glfs_pwrite_async; -# glfs_preadv; -# glfs_pwritev; -# glfs_preadv_async; -# glfs_pwritev_async; -# glfs_lseek; -# glfs_truncate; -# glfs_ftruncate; -# glfs_ftruncate_async; -# glfs_lstat; -# glfs_stat; -# glfs_fstat; -# glfs_fsync; -# glfs_fsync_async; -# glfs_fdatasync; -# glfs_fdatasync_async; -# glfs_access; -# glfs_symlink; -# glfs_readlink; -# glfs_mknod; -# glfs_mkdir; -# glfs_unlink; -# glfs_rmdir; -# glfs_rename; -# glfs_link; -# glfs_opendir; -# glfs_readdir_r; -# glfs_readdirplus_r; -# glfs_telldir; -# glfs_seekdir; -# glfs_closedir; -# glfs_statvfs; -# glfs_chmod; -# glfs_fchmod; -# glfs_chown; -# glfs_lchown; -# glfs_fchown; -# glfs_utimens; -# glfs_lutimens; -# glfs_futimens; -# glfs_getxattr; -# glfs_lgetxattr; -# glfs_fgetxattr; -# glfs_listxattr; -# glfs_llistxattr; -# glfs_flistxattr; -# glfs_setxattr; -# glfs_lsetxattr; -# glfs_fsetxattr; -# glfs_removexattr; -# glfs_lremovexattr; -# glfs_fremovexattr; -# glfs_getcwd; -# glfs_chdir; -# glfs_fchdir; -# glfs_realpath; -# glfs_posix_lock; -# glfs_dup; -# -# } -# -# GFAPI_3.4.2 { -# glfs_setfsuid; -# glfs_setfsgid; -# glfs_setfsgroups; -# glfs_h_lookupat; -# glfs_h_creat; -# glfs_h_mkdir; -# glfs_h_mknod; -# glfs_h_symlink; -# glfs_h_unlink; -# glfs_h_close; -# glfs_h_truncate; -# glfs_h_stat; -# glfs_h_getattrs; -# glfs_h_setattrs; -# glfs_h_readlink; -# glfs_h_link; -# glfs_h_rename; -# glfs_h_extract_handle; -# glfs_h_create_from_handle; -# glfs_h_opendir; -# glfs_h_open; -# } -# -# GFAPI_3.5.0 { -# -# glfs_get_volumeid; -# glfs_readdir; -# glfs_readdirplus; -# glfs_fallocate; -# glfs_discard; -# glfs_discard_async; -# glfs_zerofill; -# glfs_zerofill_async; -# glfs_caller_specific_init; -# glfs_h_setxattrs; -# -# } -# -# GFAPI_3.5.1 { -# -# glfs_unset_volfile_server; -# glfs_h_getxattrs; -# glfs_h_removexattrs; -# -# } -# -# GFAPI_3.6.0 { -# -# glfs_get_volfile; -# glfs_h_access; -# -# } -# - -def gfapi_prototype(method_name, restype, *argtypes): - """ - Create a named foreign function belonging to gfapi - - :param method_name: Name of the foreign function - :param restype: resulting type - :param argtypes: arguments that represent argument types - :returns: foreign function of gfapi library - """ - # use_errno=True ensures that errno is exposed by ctypes.get_errno() - return ctypes.CFUNCTYPE(restype, *argtypes, use_errno=True)( - (method_name, client) - ) - -# Define function prototypes for the wrapper functions. - -glfs_init = gfapi_prototype('glfs_init', ctypes.c_int, ctypes.c_void_p) - -glfs_statvfs = gfapi_prototype('glfs_statvfs', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_void_p) - -glfs_new = gfapi_prototype('glfs_new', ctypes.c_void_p, ctypes.c_char_p) - -glfs_set_volfile_server = gfapi_prototype( - 'glfs_set_volfile_server', ctypes.c_int, - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int) - -glfs_set_logging = gfapi_prototype('glfs_set_logging', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_int) - -glfs_fini = gfapi_prototype('glfs_fini', ctypes.c_int, - ctypes.c_void_p) - -glfs_creat = gfapi_prototype('glfs_creat', ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_char_p, ctypes.c_int, ctypes.c_uint) - -glfs_open = gfapi_prototype('glfs_open', ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_char_p, ctypes.c_int) - -glfs_close = gfapi_prototype('glfs_close', ctypes.c_int, - ctypes.c_void_p) - -glfs_lstat = gfapi_prototype('glfs_lstat', ctypes.c_int, - ctypes.c_void_p, ctypes.c_char_p, - ctypes.POINTER(Stat)) - -glfs_stat = gfapi_prototype('glfs_stat', ctypes.c_int, - ctypes.c_void_p, ctypes.c_char_p, - ctypes.POINTER(Stat)) - -glfs_fstat = gfapi_prototype('glfs_fstat', ctypes.c_int, - ctypes.c_void_p, ctypes.POINTER(Stat)) - -glfs_chmod = gfapi_prototype('glfs_chmod', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_ushort) - -glfs_fchmod = gfapi_prototype('glfs_fchmod', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_ushort) - -glfs_chown = gfapi_prototype('glfs_chown', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_uint, - ctypes.c_uint) - -glfs_lchown = gfapi_prototype('glfs_lchown', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_uint, - ctypes.c_uint) - -glfs_fchown = gfapi_prototype('glfs_fchown', ctypes.c_int, - ctypes.c_void_p, ctypes.c_uint, - ctypes.c_uint) - -glfs_dup = gfapi_prototype('glfs_dup', ctypes.c_void_p, - ctypes.c_void_p) - -glfs_fdatasync = gfapi_prototype('glfs_fdatasync', ctypes.c_int, - ctypes.c_void_p) - -glfs_fsync = gfapi_prototype('glfs_fsync', ctypes.c_int, - ctypes.c_void_p) - -glfs_lseek = gfapi_prototype('glfs_lseek', ctypes.c_int, - ctypes.c_void_p, ctypes.c_int, - ctypes.c_int) - -glfs_read = gfapi_prototype('glfs_read', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int) - -glfs_write = gfapi_prototype('glfs_write', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int) - -glfs_getxattr = gfapi_prototype('glfs_getxattr', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t) - -glfs_listxattr = gfapi_prototype('glfs_listxattr', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t) - -glfs_removexattr = gfapi_prototype('glfs_removexattr', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p) - -glfs_setxattr = gfapi_prototype('glfs_setxattr', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int) - -glfs_rename = gfapi_prototype('glfs_rename', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p) - -glfs_link = gfapi_prototype('glfs_link', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p) - -glfs_symlink = gfapi_prototype('glfs_symlink', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p) - -glfs_unlink = gfapi_prototype('glfs_unlink', ctypes.c_int, - ctypes.c_void_p, ctypes.c_char_p) - -glfs_readdir_r = gfapi_prototype('glfs_readdir_r', ctypes.c_int, - ctypes.c_void_p, - ctypes.POINTER(Dirent), - ctypes.POINTER(ctypes.POINTER(Dirent))) - -glfs_readdirplus_r = gfapi_prototype('glfs_readdirplus_r', ctypes.c_int, - ctypes.c_void_p, - ctypes.POINTER(Stat), - ctypes.POINTER(Dirent), - ctypes.POINTER(ctypes.POINTER(Dirent))) - -glfs_closedir = gfapi_prototype('glfs_closedir', ctypes.c_int, - ctypes.c_void_p) - -glfs_mkdir = gfapi_prototype('glfs_mkdir', ctypes.c_int, - ctypes.c_void_p, ctypes.c_char_p, - ctypes.c_ushort) - -glfs_opendir = gfapi_prototype('glfs_opendir', ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_char_p) - -glfs_rmdir = gfapi_prototype('glfs_rmdir', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p) - -glfs_setfsuid = gfapi_prototype('glfs_setfsuid', ctypes.c_int, - ctypes.c_uint) - -glfs_setfsgid = gfapi_prototype('glfs_setfsgid', ctypes.c_int, - ctypes.c_uint) - -glfs_ftruncate = gfapi_prototype('glfs_ftruncate', ctypes.c_int, - ctypes.c_void_p, ctypes.c_int) - -glfs_fgetxattr = gfapi_prototype('glfs_fgetxattr', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t) - -glfs_fremovexattr = gfapi_prototype('glfs_fremovexattr', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p) - -glfs_fsetxattr = gfapi_prototype('glfs_fsetxattr', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int) - -glfs_flistxattr = gfapi_prototype('glfs_flistxattr', ctypes.c_ssize_t, - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t) - -glfs_access = gfapi_prototype('glfs_access', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_int) - -glfs_readlink = gfapi_prototype('glfs_readlink', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_size_t) - -glfs_chdir = gfapi_prototype('glfs_chdir', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p) - -glfs_getcwd = gfapi_prototype('glfs_getcwd', ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_size_t) - -glfs_fallocate = gfapi_prototype('glfs_fallocate', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_int, - ctypes.c_size_t) - -glfs_discard = gfapi_prototype('glfs_discard', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_int, - ctypes.c_size_t) - -glfs_zerofill = gfapi_prototype('glfs_zerofill', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_int, - ctypes.c_size_t) - -glfs_utimens = gfapi_prototype('glfs_utimens', ctypes.c_int, - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.POINTER(Timespec)) diff --git a/gluster/exceptions.py b/gluster/exceptions.py deleted file mode 100644 index 05496f0..0000000 --- a/gluster/exceptions.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2016 Red Hat, Inc. -# -# This file is part of libgfapi-python project which is a -# subproject of GlusterFS ( www.gluster.org) -# -# This file is licensed to you under your choice of the GNU Lesser -# General Public License, version 3 or any later version (LGPLv3 or -# later), or the GNU General Public License, version 2 (GPLv2), in all -# cases as published by the Free Software Foundation. - - -class LibgfapiException(Exception): - pass - - -class VolumeNotMounted(LibgfapiException): - pass - - -class Error(EnvironmentError): - pass diff --git a/gluster/gfapi.py b/gluster/gfapi.py deleted file mode 100755 index 222f7a2..0000000 --- a/gluster/gfapi.py +++ /dev/null @@ -1,1736 +0,0 @@ -# Copyright (c) 2016 Red Hat, Inc. -# -# This file is part of libgfapi-python project which is a -# subproject of GlusterFS ( www.gluster.org) -# -# This file is licensed to you under your choice of the GNU Lesser -# General Public License, version 3 or any later version (LGPLv3 or -# later), or the GNU General Public License, version 2 (GPLv2), in all -# cases as published by the Free Software Foundation. - -import ctypes -import os -import math -import time -import stat -import errno -from collections import Iterator - -from gluster import api -from gluster.exceptions import LibgfapiException, Error -from gluster.utils import validate_mount, validate_glfd - -__version__ = '1.0' - -# TODO: Move this utils.py -python_mode_to_os_flags = {} - - -def _populate_mode_to_flags_dict(): - # http://pubs.opengroup.org/onlinepubs/9699919799/functions/fopen.html - for mode in ['r', 'rb']: - python_mode_to_os_flags[mode] = os.O_RDONLY - for mode in ['w', 'wb']: - python_mode_to_os_flags[mode] = os.O_WRONLY | os.O_CREAT | os.O_TRUNC - for mode in ['a', 'ab']: - python_mode_to_os_flags[mode] = os.O_WRONLY | os.O_CREAT | os.O_APPEND - for mode in ['r+', 'rb+', 'r+b']: - python_mode_to_os_flags[mode] = os.O_RDWR - for mode in ['w+', 'wb+', 'w+b']: - python_mode_to_os_flags[mode] = os.O_RDWR | os.O_CREAT | os.O_TRUNC - for mode in ['a+', 'ab+', 'a+b']: - python_mode_to_os_flags[mode] = os.O_RDWR | os.O_CREAT | os.O_APPEND - -_populate_mode_to_flags_dict() - - -class File(object): - - def __init__(self, fd, path=None, mode=None): - """ - Create a File object equivalent to Python's built-in File object. - - :param fd: glfd pointer - :param path: Path of the file. This is optional. - :param mode: The I/O mode of the file. - """ - self.fd = fd - self.originalpath = path - self._mode = mode - self._validate_init() - - def __enter__(self): - # __enter__ should only be called within the context - # of a 'with' statement when opening a file through - # Volume.fopen() - self._validate_init() - return self - - def __exit__(self, type, value, tb): - if self.fd: - self.close() - - def _validate_init(self): - if self.fd is None: - raise ValueError("I/O operation on invalid fd") - elif not isinstance(self.fd, int): - raise ValueError("I/O operation on invalid fd") - - @property - def fileno(self): - """ - Return the internal file descriptor (glfd) that is used by the - underlying implementation to request I/O operations. - """ - return self.fd - - @property - def mode(self): - """ - The I/O mode for the file. If the file was created using the - Volume.fopen() function, this will be the value of the mode - parameter. This is a read-only attribute. - """ - return self._mode - - @property - def name(self): - """ - If the file object was created using Volume.fopen(), - the name of the file. - """ - return self.originalpath - - @property - def closed(self): - """ - Bool indicating the current state of the file object. This is a - read-only attribute; the close() method changes the value. - """ - return not self.fd - - @validate_glfd - def close(self): - """ - Close the file. A closed file cannot be read or written any more. - - :raises: OSError on failure - """ - ret = api.glfs_close(self.fd) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - self.fd = None - - @validate_glfd - def discard(self, offset, length): - """ - This is similar to UNMAP command that is used to return the - unused/freed blocks back to the storage system. In this - implementation, fallocate with FALLOC_FL_PUNCH_HOLE is used to - eventually release the blocks to the filesystem. If the brick has - been mounted with '-o discard' option, then the discard request - will eventually reach the SCSI storage if the storage device - supports UNMAP. - - :param offset: Starting offset - :param len: Length or size in bytes to discard - :raises: OSError on failure - """ - ret = api.glfs_discard(self.fd, offset, length) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def dup(self): - """ - Return a duplicate of File object. This duplicate File class instance - encapsulates a duplicate glfd obtained by invoking glfs_dup(). - - :raises: OSError on failure - """ - dupfd = api.glfs_dup(self.fd) - if not dupfd: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return File(dupfd, self.originalpath) - - @validate_glfd - def fallocate(self, mode, offset, length): - """ - This is a Linux-specific sys call, unlike posix_fallocate() - - Allows the caller to directly manipulate the allocated disk space for - the file for the byte range starting at offset and continuing for - length bytes. - - :param mode: Operation to be performed on the given range - :param offset: Starting offset - :param length: Size in bytes, starting at offset - :raises: OSError on failure - """ - ret = api.glfs_fallocate(self.fd, mode, offset, length) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fchmod(self, mode): - """ - Change this file's mode - - :param mode: new mode - :raises: OSError on failure - """ - ret = api.glfs_fchmod(self.fd, mode) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fchown(self, uid, gid): - """ - Change this file's owner and group id - - :param uid: new user id for file - :param gid: new group id for file - :raises: OSError on failure - """ - ret = api.glfs_fchown(self.fd, uid, gid) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fdatasync(self): - """ - Flush buffer cache pages pertaining to data, but not the ones - pertaining to metadata. - - :raises: OSError on failure - """ - ret = api.glfs_fdatasync(self.fd) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fgetsize(self): - """ - Return the size of a file, as reported by fstat() - - :returns: the size of the file in bytes - """ - return self.fstat().st_size - - @validate_glfd - def fgetxattr(self, key, size=0): - """ - Retrieve the value of the extended attribute identified by key - for the file. - - :param key: Key of extended attribute - :param size: If size is specified as zero, we first determine the - size of xattr and then allocate a buffer accordingly. - If size is non-zero, it is assumed the caller knows - the size of xattr. - :returns: Value of extended attribute corresponding to key specified. - :raises: OSError on failure - """ - if size == 0: - size = api.glfs_fgetxattr(self.fd, key, None, size) - if size < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - buf = ctypes.create_string_buffer(size) - rc = api.glfs_fgetxattr(self.fd, key, buf, size) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return buf.value[:rc] - - @validate_glfd - def flistxattr(self, size=0): - """ - Retrieve list of extended attributes for the file. - - :param size: If size is specified as zero, we first determine the - size of list and then allocate a buffer accordingly. - If size is non-zero, it is assumed the caller knows - the size of the list. - :returns: List of extended attributes. - :raises: OSError on failure - """ - if size == 0: - size = api.glfs_flistxattr(self.fd, None, 0) - if size < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - buf = ctypes.create_string_buffer(size) - rc = api.glfs_flistxattr(self.fd, buf, size) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - xattrs = [] - # Parsing character by character is ugly, but it seems like the - # easiest way to deal with the "strings separated by NUL in one - # buffer" format. - i = 0 - while i < rc: - new_xa = buf.raw[i] - i += 1 - while i < rc: - next_char = buf.raw[i] - i += 1 - if next_char == '\0': - xattrs.append(new_xa) - break - new_xa += next_char - xattrs.sort() - return xattrs - - @validate_glfd - def fsetxattr(self, key, value, flags=0): - """ - Set extended attribute of file. - - :param key: The key of extended attribute. - :param value: The valiue of extended attribute. - :param flags: Possible values are 0 (default), 1 and 2. - If 0 - xattr will be created if it does not exist, or - the value will be replaced if the xattr exists. If 1 - - it performs a pure create, which fails if the named - attribute already exists. If 2 - it performs a pure - replace operation, which fails if the named attribute - does not already exist. - :raises: OSError on failure - """ - ret = api.glfs_fsetxattr(self.fd, key, value, len(value), flags) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fremovexattr(self, key): - """ - Remove a extended attribute of the file. - - :param key: The key of extended attribute. - :raises: OSError on failure - """ - ret = api.glfs_fremovexattr(self.fd, key) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def fstat(self): - """ - Returns Stat object for this file. - - :return: Returns the stat information of the file. - :raises: OSError on failure - """ - s = api.Stat() - rc = api.glfs_fstat(self.fd, ctypes.byref(s)) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return s - - @validate_glfd - def fsync(self): - """ - Flush buffer cache pages pertaining to data and metadata. - - :raises: OSError on failure - """ - ret = api.glfs_fsync(self.fd) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def ftruncate(self, length): - """ - Truncated the file to a size of length bytes. - - If the file previously was larger than this size, the extra data is - lost. If the file previously was shorter, it is extended, and the - extended part reads as null bytes. - - :param length: Length to truncate the file to in bytes. - :raises: OSError on failure - """ - ret = api.glfs_ftruncate(self.fd, length) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def lseek(self, pos, how): - """ - Set the read/write offset position of this file. - The new position is defined by 'pos' relative to 'how' - - :param pos: sets new offset position according to 'how' - :param how: SEEK_SET, sets offset position 'pos' bytes relative to - beginning of file, SEEK_CUR, the position is set relative - to the current position and SEEK_END sets the position - relative to the end of the file. - :returns: the new offset position - :raises: OSError on failure - """ - ret = api.glfs_lseek(self.fd, pos, how) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return ret - - @validate_glfd - def read(self, size=-1): - """ - Read at most size bytes from the file. - - :param buflen: length of read buffer. If less than 0, then whole - file is read. Default is -1. - :returns: buffer of 'size' length - :raises: OSError on failure - """ - if size < 0: - size = self.fgetsize() - rbuf = ctypes.create_string_buffer(size) - ret = api.glfs_read(self.fd, rbuf, size, 0) - if ret > 0: - # In python 2.x, read() always returns a string. It's really upto - # the consumer to decode this string into whatever encoding it was - # written with. - return rbuf.raw[:ret] - elif ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_glfd - def readinto(self, buf): - """ - Read up to len(buf) bytes into buf which must be a bytearray. - (buf cannot be a string as strings are immutable in python) - - This method is useful when you have to read a large file over - multiple read calls. While read() allocates a buffer every time - it's invoked, readinto() copies data to an already allocated - buffer passed to it. - - :returns: the number of bytes read (0 for EOF). - :raises: OSError on failure - """ - if type(buf) is bytearray: - buf_ptr = (ctypes.c_ubyte * len(buf)).from_buffer(buf) - else: - # TODO: Allow reading other types such as array.array - raise TypeError("buffer must of type bytearray") - nread = api.glfs_read(self.fd, buf_ptr, len(buf_ptr), 0) - if nread < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return nread - - @validate_glfd - def write(self, data, flags=0): - """ - Write data to the file. - - :param data: The data to be written to file. - :returns: The size in bytes actually written - :raises: OSError on failure - """ - # creating a ctypes.c_ubyte buffer to handle converting bytearray - # to the required C data type - - if type(data) is bytearray: - buf = (ctypes.c_ubyte * len(data)).from_buffer(data) - else: - buf = data - ret = api.glfs_write(self.fd, buf, len(buf), flags) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return ret - - @validate_glfd - def zerofill(self, offset, length): - """ - Fill 'length' number of bytes with zeroes starting from 'offset'. - - :param offset: Start at offset location - :param length: Size/length in bytes - :raises: OSError on failure - """ - ret = api.glfs_zerofill(self.fd, offset, length) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - -class Dir(Iterator): - - def __init__(self, fd, readdirplus=False): - # Add a reference so the module-level variable "api" doesn't - # get yanked out from under us (see comment above File def'n). - self._api = api - self.fd = fd - self.readdirplus = readdirplus - self.cursor = ctypes.POINTER(api.Dirent)() - - def __del__(self): - self._api.glfs_closedir(self.fd) - self._api = None - - def next(self): - entry = api.Dirent() - entry.d_reclen = 256 - - if self.readdirplus: - stat_info = api.Stat() - ret = api.glfs_readdirplus_r(self.fd, ctypes.byref(stat_info), - ctypes.byref(entry), - ctypes.byref(self.cursor)) - else: - ret = api.glfs_readdir_r(self.fd, ctypes.byref(entry), - ctypes.byref(self.cursor)) - - if ret != 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - if (not self.cursor) or (not self.cursor.contents): - # Reached end of directory stream - raise StopIteration - - if self.readdirplus: - return (entry, stat_info) - else: - return entry - - -class DirEntry(object): - """ - Object yielded by scandir() to expose the file path and other file - attributes of a directory entry. scandir() will provide stat information - without making additional calls. DirEntry instances are not intended to be - stored in long-lived data structures; if you know the file metadata has - changed or if a long time has elapsed since calling scandir(), call - Volume.stat(entry.path) to fetch up-to-date information. - - DirEntry() instances from Python 3.5 have follow_symlinks set to True by - default. In this implementation, follow_symlinks is set to False by - default as it incurs an additional stat call over network. - """ - - __slots__ = ('_name', '_vol', '_lstat', '_stat', '_path') - - def __init__(self, vol, scandir_path, name, lstat): - self._name = name - self._vol = vol - self._lstat = lstat - self._stat = None - self._path = os.path.join(scandir_path, name) - - @property - def name(self): - """ - The entry's base filename, relative to the scandir() path argument. - """ - return self._name - - @property - def path(self): - """ - The entry's full path name: equivalent to os.path.join(scandir_path, - entry.name) where scandir_path is the scandir() path argument. The path - is only absolute if the scandir() path argument was absolute. - """ - return self._path - - def stat(self, follow_symlinks=False): - """ - Returns information equivalent of a lstat() system call on the entry. - This does not follow symlinks by default. - """ - if follow_symlinks: - if self._stat is None: - if self.is_symlink(): - self._stat = self._vol.stat(self.path) - else: - self._stat = self._lstat - return self._stat - else: - return self._lstat - - def is_dir(self, follow_symlinks=False): - """ - Return True if this entry is a directory; return False if the entry is - any other kind of file, or if it doesn't exist anymore. - """ - if follow_symlinks and self.is_symlink(): - try: - st = self.stat(follow_symlinks=follow_symlinks) - except OSError as err: - if err.errno != errno.ENOENT: - raise - return False - else: - return stat.S_ISDIR(st.st_mode) - else: - return stat.S_ISDIR(self._lstat.st_mode) - - def is_file(self, follow_symlinks=False): - """ - Return True if this entry is a file; return False if the entry is a - directory or other non-file entry, or if it doesn't exist anymore. - """ - if follow_symlinks and self.is_symlink(): - try: - st = self.stat(follow_symlinks=follow_symlinks) - except OSError as err: - if err.errno != errno.ENOENT: - raise - return False - else: - return stat.S_ISREG(st.st_mode) - else: - return stat.S_ISREG(self._lstat.st_mode) - - def is_symlink(self): - """ - Return True if this entry is a symbolic link (even if broken); return - False if the entry points to a directory or any kind of file, or if it - doesn't exist anymore. - """ - return stat.S_ISLNK(self._lstat.st_mode) - - def inode(self): - """ - Return the inode number of the entry. - """ - return self._lstat.st_ino - - def __str__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) - - __repr__ = __str__ - - -class Volume(object): - - def __init__(self, host, volname, - proto="tcp", port=24007, log_file=None, log_level=7): - """ - Create a Volume object instance. - - :param host: Host with glusterd management daemon running. - :param volname: Name of GlusterFS volume to be mounted and used. - :param proto: Transport protocol to be used to connect to management - daemon. Permitted values are "tcp" and "rdma". - :param port: Port number where gluster management daemon is listening. - :param log_file: Path to log file. When this is set to None, a new - logfile will be created in default log directory - i.e /var/log/glusterfs - :param log_level: Integer specifying the degree of verbosity. - Higher the value, more verbose the logging. - - """ - # TODO: Provide an interface where user can specify volfile directly - # instead of providing host and other details. This is helpful in cases - # where user wants to load some non default xlator on client side. For - # example, aux-gfid-mount or mount volume as read-only. - - # Add a reference so the module-level variable "api" doesn't - # get yanked out from under us (see comment above File def'n). - self._api = api - - self._mounted = False - self.fs = None - self.log_file = log_file - self.log_level = log_level - - if None in (volname, host): - # TODO: Validate host based on regex for IP/FQDN. - raise LibgfapiException("Host and Volume name should not be None.") - if proto not in ('tcp', 'rdma'): - raise LibgfapiException("Invalid protocol specified.") - if not isinstance(port, (int, long)): - raise LibgfapiException("Invalid port specified.") - - self.host = host - self.volname = volname - self.protocol = proto - self.port = port - - @property - def mounted(self): - """ - Read-only attribute that returns True if the volume is mounted. - The value of the attribute is internally changed on invoking - mount() and umount() functions. - """ - return self._mounted - - def mount(self): - """ - Mount a GlusterFS volume for use. - - :raises: LibgfapiException on failure - """ - if self.fs and self._mounted: - # Already mounted - return - - self.fs = api.glfs_new(self.volname) - if not self.fs: - err = ctypes.get_errno() - raise LibgfapiException("glfs_new(%s) failed: %s" % - (self.volname, os.strerror(err))) - - ret = api.glfs_set_volfile_server(self.fs, self.protocol, - self.host, self.port) - if ret < 0: - err = ctypes.get_errno() - raise LibgfapiException("glfs_set_volfile_server(%s, %s, %s, " - "%s) failed: %s" % (self.fs, self.protocol, - self.host, self.port, - os.strerror(err))) - - self.set_logging(self.log_file, self.log_level) - - if self.fs and not self._mounted: - ret = api.glfs_init(self.fs) - if ret < 0: - err = ctypes.get_errno() - raise LibgfapiException("glfs_init(%s) failed: %s" % - (self.fs, os.strerror(err))) - else: - self._mounted = True - - def umount(self): - """ - Unmount a mounted GlusterFS volume. - - Provides users a way to free resources instead of just waiting for - python garbage collector to call __del__() at some point later. - - :raises: LibgfapiException on failure - """ - if self.fs: - ret = self._api.glfs_fini(self.fs) - if ret < 0: - err = ctypes.get_errno() - raise LibgfapiException("glfs_fini(%s) failed: %s" % - (self.fs, os.strerror(err))) - else: - # Succeeded. Protect against multiple umount() calls. - self._mounted = False - self.fs = None - - def __del__(self): - try: - self.umount() - except LibgfapiException: - pass - - def set_logging(self, log_file, log_level): - """ - Set logging parameters. Can be invoked either before or after - invoking mount(). - - When invoked before mount(), the preferred log file and log level - choices are recorded and then later enforced internally as part of - mount() - - When invoked at any point after mount(), the change in log file - and log level is instantaneous. - - :param log_file: Path of log file. - If set to "/dev/null", nothing will be logged. - If set to None, a new logfile will be created in - default log directory (/var/log/glusterfs) - :param log_level: Integer specifying the degree of verbosity. - Higher the value, more verbose the logging. - """ - if self.fs: - ret = api.glfs_set_logging(self.fs, self.log_file, self.log_level) - if ret < 0: - err = ctypes.get_errno() - raise LibgfapiException("glfs_set_logging(%s, %s) failed: %s" % - (self.log_file, self.log_level, - os.strerror(err))) - self.log_file = log_file - self.log_level = log_level - - @validate_mount - def access(self, path, mode): - """ - Use the real uid/gid to test for access to path. - - :param path: Path to be checked. - :param mode: mode should be F_OK to test the existence of path, or - it can be the inclusive OR of one or more of R_OK, W_OK, - and X_OK to test permissions - :returns: True if access is allowed, False if not - """ - ret = api.glfs_access(self.fs, path, mode) - if ret == 0: - return True - else: - return False - - @validate_mount - def chdir(self, path): - """ - Change the current working directory to the given path. - - :param path: Path to change current working directory to - :raises: OSError on failure - """ - ret = api.glfs_chdir(self.fs, path) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def chmod(self, path, mode): - """ - Change mode of path - - :param path: the item to be modified - :param mode: new mode - :raises: OSError on failure - """ - ret = api.glfs_chmod(self.fs, path, mode) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def chown(self, path, uid, gid): - """ - Change owner and group id of path - - :param path: the path to be modified - :param uid: new user id for path - :param gid: new group id for path - :raises: OSError on failure - """ - ret = api.glfs_chown(self.fs, path, uid, gid) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - def exists(self, path): - """ - Returns True if path refers to an existing path. Returns False for - broken symbolic links. This function may return False if permission is - not granted to execute stat() on the requested file, even if the path - physically exists. - """ - try: - self.stat(path) - except OSError: - return False - return True - - def getatime(self, path): - """ - Returns the last access time as reported by stat() - """ - return self.stat(path).st_atime - - def getctime(self, path): - """ - Returns the time when changes were made to the path as reported by - stat(). This time is updated when changes are made to the file or - dir's inode or the contents of the file - """ - return self.stat(path).st_ctime - - @validate_mount - def getcwd(self): - """ - Returns current working directory. - """ - PATH_MAX = 4096 - buf = ctypes.create_string_buffer(PATH_MAX) - ret = api.glfs_getcwd(self.fs, buf, PATH_MAX) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return buf.value - - def getmtime(self, path): - """ - Returns the time when changes were made to the content of the path - as reported by stat() - """ - return self.stat(path).st_mtime - - def getsize(self, path): - """ - Return the size of a file in bytes, reported by stat() - """ - return self.stat(path).st_size - - @validate_mount - def getxattr(self, path, key, size=0): - """ - Retrieve the value of the extended attribute identified by key - for path specified. - - :param path: Path to file or directory - :param key: Key of extended attribute - :param size: If size is specified as zero, we first determine the - size of xattr and then allocate a buffer accordingly. - If size is non-zero, it is assumed the caller knows - the size of xattr. - :returns: Value of extended attribute corresponding to key specified. - :raises: OSError on failure - """ - if size == 0: - size = api.glfs_getxattr(self.fs, path, key, None, 0) - if size < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - buf = ctypes.create_string_buffer(size) - rc = api.glfs_getxattr(self.fs, path, key, buf, size) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return buf.value[:rc] - - def isdir(self, path): - """ - Returns True if path is an existing directory. Returns False on all - failure cases including when path does not exist. - """ - try: - s = self.stat(path) - except OSError: - return False - return stat.S_ISDIR(s.st_mode) - - def isfile(self, path): - """ - Return True if path is an existing regular file. Returns False on all - failure cases including when path does not exist. - """ - try: - s = self.stat(path) - except OSError: - return False - return stat.S_ISREG(s.st_mode) - - def islink(self, path): - """ - Return True if path refers to a directory entry that is a symbolic - link. Returns False on all failure cases including when path does - not exist. - """ - try: - s = self.lstat(path) - except OSError: - return False - return stat.S_ISLNK(s.st_mode) - - def listdir(self, path): - """ - Return a list containing the names of the entries in the directory - given by path. The list is in arbitrary order. It does not include - the special entries '.' and '..' even if they are present in the - directory. - - :param path: Path to directory - :raises: OSError on failure - :returns: List of names of directory entries - """ - dir_list = [] - for entry in self.opendir(path): - if not isinstance(entry, api.Dirent): - break - name = entry.d_name[:entry.d_reclen] - if name not in (".", ".."): - dir_list.append(name) - return dir_list - - def listdir_with_stat(self, path): - """ - Return a list containing the name and stat of the entries in the - directory given by path. The list is in arbitrary order. It does - not include the special entries '.' and '..' even if they are present - in the directory. - - :param path: Path to directory - :raises: OSError on failure - :returns: A list of tuple. The tuple is of the form (name, stat) where - name is a string indicating name of the directory entry and - stat contains stat info of the entry. - """ - # List of tuple. Each tuple is of the form (name, stat) - entries_with_stat = [] - for (entry, stat_info) in self.opendir(path, readdirplus=True): - if not (isinstance(entry, api.Dirent) and - isinstance(stat_info, api.Stat)): - break - name = entry.d_name[:entry.d_reclen] - if name not in (".", ".."): - entries_with_stat.append((name, stat_info)) - return entries_with_stat - - def scandir(self, path): - """ - Return an iterator of :class:`DirEntry` objects corresponding to the - entries in the directory given by path. The entries are yielded in - arbitrary order, and the special entries '.' and '..' are not - included. - - Using scandir() instead of listdir() can significantly increase the - performance of code that also needs file type or file attribute - information, because :class:`DirEntry` objects expose this - information. - - scandir() provides same functionality as listdir_with_stat() except - that scandir() does not return a list and is an iterator. Hence scandir - is less memory intensive on large directories. - - :param path: Path to directory - :raises: OSError on failure - :yields: Instance of :class:`DirEntry` class. - """ - for (entry, lstat) in self.opendir(path, readdirplus=True): - name = entry.d_name[:entry.d_reclen] - if name not in (".", ".."): - yield DirEntry(self, path, name, lstat) - - @validate_mount - def listxattr(self, path, size=0): - """ - Retrieve list of extended attribute keys for the specified path. - - :param path: Path to file or directory. - :param size: If size is specified as zero, we first determine the - size of list and then allocate a buffer accordingly. - If size is non-zero, it is assumed the caller knows - the size of the list. - :returns: List of extended attribute keys. - :raises: OSError on failure - """ - if size == 0: - size = api.glfs_listxattr(self.fs, path, None, 0) - if size < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - buf = ctypes.create_string_buffer(size) - rc = api.glfs_listxattr(self.fs, path, buf, size) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - xattrs = [] - # Parsing character by character is ugly, but it seems like the - # easiest way to deal with the "strings separated by NUL in one - # buffer" format. - i = 0 - while i < rc: - new_xa = buf.raw[i] - i += 1 - while i < rc: - next_char = buf.raw[i] - i += 1 - if next_char == '\0': - xattrs.append(new_xa) - break - new_xa += next_char - xattrs.sort() - return xattrs - - @validate_mount - def lstat(self, path): - """ - Return stat information of path. If path is a symbolic link, then it - returns information about the link itself, not the file that it refers - to. - - :raises: OSError on failure - """ - s = api.Stat() - rc = api.glfs_lstat(self.fs, path, ctypes.byref(s)) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return s - - def makedirs(self, path, mode=0777): - """ - Recursive directory creation function. Like mkdir(), but makes all - intermediate-level directories needed to contain the leaf directory. - The default mode is 0777 (octal). - - :raises: OSError if the leaf directory already exists or cannot be - created. Can also raise OSError if creation of any non-leaf - directories fails. - """ - head, tail = os.path.split(path) - if not tail: - head, tail = os.path.split(head) - if head and tail and not self.exists(head): - try: - self.makedirs(head, mode) - except OSError as err: - if err.errno != errno.EEXIST: - raise - if tail == os.curdir: - return - self.mkdir(path, mode) - - @validate_mount - def mkdir(self, path, mode=0777): - """ - Create a directory named path with numeric mode mode. - The default mode is 0777 (octal). - - :raises: OSError on failure - """ - ret = api.glfs_mkdir(self.fs, path, mode) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def fopen(self, path, mode='r'): - """ - Similar to Python's built-in File object returned by Python's open() - - Unlike Python's open(), fopen() provided here is only for convenience - and it does NOT invoke glibc's fopen and does NOT do any kind of - I/O bufferring as of today. - - The most commonly-used values of mode are 'r' for reading, 'w' for - writing (truncating the file if it already exists), and 'a' for - appending. If mode is omitted, it defaults to 'r'. - - Modes 'r+', 'w+' and 'a+' open the file for updating (reading and - writing); note that 'w+' truncates the file. - - Append 'b' to the mode to open the file in binary mode but this has - no effect as of today. - - :param path: Path of file to be opened - :param mode: Mode to open the file with. This is a string. - :returns: an instance of File class - :raises: OSError on failure to create/open file. - TypeError and ValueError if mode is invalid. - """ - if not isinstance(mode, basestring): - raise TypeError("Mode must be a string") - try: - flags = python_mode_to_os_flags[mode] - except KeyError: - raise ValueError("Invalid mode") - else: - if (os.O_CREAT & flags) == os.O_CREAT: - fd = api.glfs_creat(self.fs, path, flags, 0666) - else: - fd = api.glfs_open(self.fs, path, flags) - if not fd: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return File(fd, path=path, mode=mode) - - @validate_mount - def open(self, path, flags, mode=0777): - """ - Similar to Python's os.open() - - As of today, the only way to consume the raw glfd returned is by - passing it to File class. - - :param path: Path of file to be opened - :param flags: Integer which flags must include one of the following - access modes: os.O_RDONLY, os.O_WRONLY, or os.O_RDWR. - :param mode: specifies the permissions to use in case a new - file is created. The default mode is 0777 (octal) - :returns: the raw glfd (pointer to memory in C, number in python) - :raises: OSError on failure to create/open file. - TypeError if flags is not an integer. - """ - if not isinstance(flags, int): - raise TypeError("flags must evaluate to an integer") - - if (os.O_CREAT & flags) == os.O_CREAT: - fd = api.glfs_creat(self.fs, path, flags, mode) - else: - fd = api.glfs_open(self.fs, path, flags) - if not fd: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - return fd - - @validate_mount - def opendir(self, path, readdirplus=False): - """ - Open a directory. - - :param path: Path to the directory - :param readdirplus: Enable readdirplus which will also fetch stat - information for each entry of directory. - :returns: Returns a instance of Dir class - :raises: OSError on failure - """ - fd = api.glfs_opendir(self.fs, path) - if not fd: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return Dir(fd, readdirplus) - - @validate_mount - def readlink(self, path): - """ - Return a string representing the path to which the symbolic link - points. The result may be either an absolute or relative pathname. - - :param path: Path of symbolic link - :returns: Contents of symlink - :raises: OSError on failure - """ - PATH_MAX = 4096 - buf = ctypes.create_string_buffer(PATH_MAX) - ret = api.glfs_readlink(self.fs, path, buf, PATH_MAX) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return buf.value[:ret] - - def remove(self, path): - """ - Remove (delete) the file path. If path is a directory, OSError - is raised. This is identical to the unlink() function. - - :raises: OSError on failure - """ - return self.unlink(path) - - @validate_mount - def removexattr(self, path, key): - """ - Remove a extended attribute of the path. - - :param path: Path to the file or directory. - :param key: The key of extended attribute. - :raises: OSError on failure - """ - ret = api.glfs_removexattr(self.fs, path, key) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def rename(self, src, dst): - """ - Rename the file or directory from src to dst. If dst is a directory, - OSError will be raised. If dst exists and is a file, it will be - replaced silently if the user has permission. - - :raises: OSError on failure - """ - ret = api.glfs_rename(self.fs, src, dst) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def rmdir(self, path): - """ - Remove (delete) the directory path. Only works when the directory is - empty, otherwise, OSError is raised. In order to remove whole - directory trees, rmtree() can be used. - - :raises: OSError on failure - """ - ret = api.glfs_rmdir(self.fs, path) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - def rmtree(self, path, ignore_errors=False, onerror=None): - """ - Delete a whole directory tree structure. Raises OSError - if path is a symbolic link. - - :param path: Directory tree to remove - :param ignore_errors: If True, errors are ignored - :param onerror: If set, it is called to handle the error with arguments - (func, path, exc) where func is the function that - raised the error, path is the argument that caused it - to fail; and exc is the exception that was raised. - If ignore_errors is False and onerror is None, an - exception is raised - :raises: OSError on failure if onerror is None - """ - if ignore_errors: - def onerror(*args): - pass - elif onerror is None: - def onerror(*args): - raise - if self.islink(path): - raise OSError("Cannot call rmtree on a symbolic link") - - try: - for entry in self.scandir(path): - fullname = os.path.join(path, entry.name) - if entry.is_dir(follow_symlinks=False): - self.rmtree(fullname, ignore_errors, onerror) - else: - try: - self.unlink(fullname) - except OSError as e: - onerror(self.unlink, fullname, e) - except OSError as e: - # self.scandir() is not a list and is a true iterator, it can - # raise an exception and blow-up. The try-except block here is to - # handle it gracefully and return. - onerror(self.scandir, path, e) - - try: - self.rmdir(path) - except OSError as e: - onerror(self.rmdir, path, e) - - def setfsuid(self, uid): - """ - setfsuid() changes the value of the caller's filesystem user ID-the - user ID that the Linux kernel uses to check for all accesses to the - filesystem. - - :raises: OSError on failure - """ - ret = api.glfs_setfsuid(uid) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - def setfsgid(self, gid): - """ - setfsgid() changes the value of the caller's filesystem group ID-the - group ID that the Linux kernel uses to check for all accesses to the - filesystem. - - :raises: OSError on failure - """ - ret = api.glfs_setfsgid(gid) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def setxattr(self, path, key, value, flags=0): - """ - Set extended attribute of the path. - - :param path: Path to file or directory. - :param key: The key of extended attribute. - :param value: The valiue of extended attribute. - :param flags: Possible values are 0 (default), 1 and 2. - If 0 - xattr will be created if it does not exist, or - the value will be replaced if the xattr exists. If 1 - - it performs a pure create, which fails if the named - attribute already exists. If 2 - it performs a pure - replace operation, which fails if the named attribute - does not already exist. - - :raises: OSError on failure - """ - ret = api.glfs_setxattr(self.fs, path, key, value, len(value), flags) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def stat(self, path): - """ - Returns stat information of path. - - :raises: OSError on failure - """ - s = api.Stat() - rc = api.glfs_stat(self.fs, path, ctypes.byref(s)) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return s - - @validate_mount - def statvfs(self, path): - """ - Returns information about a mounted glusterfs volume. path is the - pathname of any file within the mounted filesystem. - - :returns: An object whose attributes describe the filesystem on the - given path, and correspond to the members of the statvfs - structure, namely: f_bsize, f_frsize, f_blocks, f_bfree, - f_bavail, f_files, f_ffree, f_favail, f_fsid, f_flag, - and f_namemax. - - :raises: OSError on failure - """ - s = api.Statvfs() - rc = api.glfs_statvfs(self.fs, path, ctypes.byref(s)) - if rc < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return s - - @validate_mount - def link(self, source, link_name): - """ - Create a hard link pointing to source named link_name. - - :raises: OSError on failure - """ - ret = api.glfs_link(self.fs, source, link_name) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def symlink(self, source, link_name): - """ - Create a symbolic link 'link_name' which points to 'source' - - :raises: OSError on failure - """ - ret = api.glfs_symlink(self.fs, source, link_name) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def unlink(self, path): - """ - Delete the file 'path' - - :raises: OSError on failure - """ - ret = api.glfs_unlink(self.fs, path) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - @validate_mount - def utime(self, path, times): - """ - Set the access and modified times of the file specified by path. If - times is None, then the file's access and modified times are set to - the current time. (The effect is similar to running the Unix program - touch on the path.) Otherwise, times must be a 2-tuple of numbers, - of the form (atime, mtime) which is used to set the access and - modified times, respectively. - - - :raises: OSError on failure to change time. - TypeError if invalid times is passed. - """ - if times is None: - now = time.time() - times = (now, now) - else: - if type(times) is not tuple or len(times) != 2: - raise TypeError("utime() arg 2 must be a tuple (atime, mtime)") - - timespec_array = (api.Timespec * 2)() - - # Set atime - decimal, whole = math.modf(times[0]) - timespec_array[0].tv_sec = int(whole) - timespec_array[0].tv_nsec = int(decimal * 1e9) - - # Set mtime - decimal, whole = math.modf(times[1]) - timespec_array[1].tv_sec = int(whole) - timespec_array[1].tv_nsec = int(decimal * 1e9) - - ret = api.glfs_utimens(self.fs, path, timespec_array) - if ret < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - - def walk(self, top, topdown=True, onerror=None, followlinks=False): - """ - Generate the file names in a directory tree by walking the tree either - top-down or bottom-up. - - Slight difference in behaviour in comparison to os.walk(): - When os.walk() is called with 'followlinks=False' (default), symlinks - to directories are included in the 'dirnames' list. When Volume.walk() - is called with 'followlinks=False' (default), symlinks to directories - are included in 'filenames' list. This is NOT a bug. - http://python.6.x6.nabble.com/os-walk-with-followlinks-False-td3559133.html - - :param top: Directory path to walk - :param topdown: If topdown is True or not specified, the triple for a - directory is generated before the triples for any of - its subdirectories. If topdown is False, the triple - for a directory is generated after the triples for all - of its subdirectories. - :param onerror: If optional argument onerror is specified, it should be - a function; it will be called with one argument, an - OSError instance. It can report the error to continue - with the walk, or raise exception to abort the walk. - :param followlinks: Set followlinks to True to visit directories - pointed to by symlinks. - :raises: OSError on failure if onerror is None - :yields: a 3-tuple (dirpath, dirnames, filenames) where dirpath is a - string, the path to the directory. dirnames is a list of the - names of the subdirectories in dirpath (excluding '.' and - '..'). filenames is a list of the names of the non-directory - files in dirpath. - """ - dirs = [] # List of DirEntry objects - nondirs = [] # List of names (strings) - - try: - for entry in self.scandir(top): - if entry.is_dir(follow_symlinks=followlinks): - dirs.append(entry) - else: - nondirs.append(entry.name) - except OSError as err: - # self.scandir() is not a list and is a true iterator, it can - # raise an exception and blow-up. The try-except block here is to - # handle it gracefully and return. - if onerror is not None: - onerror(err) - return - - if topdown: - yield top, [d.name for d in dirs], nondirs - - for directory in dirs: - # NOTE: Both is_dir() and is_symlink() can be true for the same - # path when follow_symlinks is set to True - if followlinks or not directory.is_symlink(): - new_path = os.path.join(top, directory.name) - for x in self.walk(new_path, topdown, onerror, followlinks): - yield x - - if not topdown: - yield top, [d.name for d in dirs], nondirs - - def samefile(self, path1, path2): - """ - Return True if both pathname arguments refer to the same file or - directory (as indicated by device number and inode number). Raise an - exception if a stat() call on either pathname fails. - - :param path1: Path to one file - :param path2: Path to another file - :raises: OSError if stat() fails - """ - s1 = self.stat(path1) - s2 = self.stat(path2) - return s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev - - @classmethod - def copyfileobj(self, fsrc, fdst, length=128 * 1024): - """ - Copy the contents of the file-like object fsrc to the file-like object - fdst. The integer length, if given, is the buffer size. Note that if - the current file position of the fsrc object is not 0, only the - contents from the current file position to the end of the file will be - copied. - - :param fsrc: Source file object - :param fdst: Destination file object - :param length: Size of buffer in bytes to be used in copying - :raises: OSError on failure - """ - buf = bytearray(length) - while True: - nread = fsrc.readinto(buf) - if not nread or nread <= 0: - break - if nread == length: - # Entire buffer is filled, do not slice. - fdst.write(buf) - else: - # TODO: - # Use memoryview to avoid internal copy done on slicing. - fdst.write(buf[0:nread]) - - def copyfile(self, src, dst): - """ - Copy the contents (no metadata) of the file named src to a file named - dst. dst must be the complete target file name. If src and dst are - the same, Error is raised. The destination location must be writable. - If dst already exists, it will be replaced. Special files such as - character or block devices and pipes cannot be copied with this - function. src and dst are path names given as strings. - - :param src: Path of source file - :param dst: Path of destination file - :raises: Error if src and dst file are same file. - OSError on failure to read/write. - """ - _samefile = False - try: - _samefile = self.samefile(src, dst) - except OSError: - # Dst file need not exist. - pass - - if _samefile: - raise Error("`%s` and `%s` are the same file" % (src, dst)) - - with self.fopen(src, 'rb') as fsrc: - with self.fopen(dst, 'wb') as fdst: - self.copyfileobj(fsrc, fdst) - - def copymode(self, src, dst): - """ - Copy the permission bits from src to dst. The file contents, owner, - and group are unaffected. src and dst are path names given as strings. - - :param src: Path of source file - :param dst: Path of destination file - :raises: OSError on failure. - """ - st = self.stat(src) - mode = stat.S_IMODE(st.st_mode) - self.chmod(dst, mode) - - def copystat(self, src, dst): - """ - Copy the permission bits, last access time, last modification time, - and flags from src to dst. The file contents, owner, and group are - unaffected. src and dst are path names given as strings. - - :param src: Path of source file - :param dst: Path of destination file - :raises: OSError on failure. - """ - st = self.stat(src) - mode = stat.S_IMODE(st.st_mode) - self.utime(dst, (st.st_atime, st.st_mtime)) - self.chmod(dst, mode) - # TODO: Handle st_flags on FreeBSD - - def copy(self, src, dst): - """ - Copy data and mode bits ("cp src dst") - - Copy the file src to the file or directory dst. If dst is a directory, - a file with the same basename as src is created (or overwritten) in - the directory specified. Permission bits are copied. src and dst are - path names given as strings. - - :param src: Path of source file - :param dst: Path of destination file or directory - :raises: OSError on failure - """ - if self.isdir(dst): - dst = os.path.join(dst, os.path.basename(src)) - self.copyfile(src, dst) - self.copymode(src, dst) - - def copy2(self, src, dst): - """ - Similar to copy(), but metadata is copied as well - in fact, this is - just copy() followed by copystat(). This is similar to the Unix command - cp -p. - - The destination may be a directory. - - :param src: Path of source file - :param dst: Path of destination file or directory - :raises: OSError on failure - """ - if self.isdir(dst): - dst = os.path.join(dst, os.path.basename(src)) - self.copyfile(src, dst) - self.copystat(src, dst) - - def copytree(self, src, dst, symlinks=False, ignore=None): - """ - Recursively copy a directory tree using copy2(). - - The destination directory must not already exist. - If exception(s) occur, an Error is raised with a list of reasons. - - If the optional symlinks flag is true, symbolic links in the - source tree result in symbolic links in the destination tree; if - it is false, the contents of the files pointed to by symbolic - links are copied. - - The optional ignore argument is a callable. If given, it - is called with the 'src' parameter, which is the directory - being visited by copytree(), and 'names' which is the list of - 'src' contents, as returned by os.listdir(): - - callable(src, names) -> ignored_names - - Since copytree() is called recursively, the callable will be - called once for each directory that is copied. It returns a - list of names relative to the 'src' directory that should - not be copied. - """ - def _isdir(path, statinfo, follow_symlinks=False): - if stat.S_ISDIR(statinfo.st_mode): - return True - if follow_symlinks and stat.S_ISLNK(statinfo.st_mode): - return self.isdir(path) - return False - - # Can't used scandir() here to support ignored_names functionality - names_with_stat = self.listdir_with_stat(src) - if ignore is not None: - ignored_names = ignore(src, [n for n, s in names_with_stat]) - else: - ignored_names = set() - - self.makedirs(dst) - errors = [] - for (name, st) in names_with_stat: - if name in ignored_names: - continue - srcpath = os.path.join(src, name) - dstpath = os.path.join(dst, name) - try: - if symlinks and stat.S_ISLNK(st.st_mode): - linkto = self.readlink(srcpath) - self.symlink(linkto, dstpath) - # shutil's copytree() calls os.path.isdir() which will return - # true even if it's a symlink pointing to a dir. Mimicking the - # same behaviour here with _isdir() - elif _isdir(srcpath, st, follow_symlinks=not symlinks): - self.copytree(srcpath, dstpath, symlinks) - else: - # The following is equivalent of copy2(). copy2() is not - # invoked directly to avoid multiple duplicate stat calls. - with self.fopen(srcpath, 'rb') as fsrc: - with self.fopen(dstpath, 'wb') as fdst: - self.copyfileobj(fsrc, fdst) - self.utime(dstpath, (st.st_atime, st.st_mtime)) - self.chmod(dstpath, stat.S_IMODE(st.st_mode)) - except (Error, EnvironmentError, OSError) as why: - errors.append((srcpath, dstpath, str(why))) - - try: - self.copystat(src, dst) - except OSError as why: - errors.append((src, dst, str(why))) - - if errors: - raise Error(errors) diff --git a/gluster/gfapi/__init__.py b/gluster/gfapi/__init__.py new file mode 100644 index 0000000..8d99bf2 --- /dev/null +++ b/gluster/gfapi/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of libgfapi-python project which is a +# subproject of GlusterFS ( www.gluster.org) +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +__version__ = '1.0' + +from gfapi import File, Dir, DirEntry, Volume +__all__ = ['File', 'Dir', 'DirEntry', 'Volume'] diff --git a/gluster/gfapi/api.py b/gluster/gfapi/api.py new file mode 100644 index 0000000..c440de6 --- /dev/null +++ b/gluster/gfapi/api.py @@ -0,0 +1,512 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of libgfapi-python project which is a +# subproject of GlusterFS ( www.gluster.org) +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +import ctypes +from ctypes.util import find_library +from ctypes import sizeof + + +# LD_LIBRARY_PATH is not looked up by ctypes.util.find_library() +so_file_name = find_library("gfapi") + +if so_file_name is None: + for name in ["libgfapi.so.0", "libgfapi.so"]: + try: + ctypes.CDLL(name, ctypes.RTLD_GLOBAL, use_errno=True) + except OSError: + pass + else: + so_file_name = name + break + if so_file_name is None: + # The .so file cannot be found (or loaded) + # May be you need to run ldconfig command + raise Exception("libgfapi.so not found") + +# Looks like ctypes is having trouble with dependencies, so just force them to +# load with RTLD_GLOBAL until I figure that out. +try: + client = ctypes.CDLL(so_file_name, ctypes.RTLD_GLOBAL, use_errno=True) + # The above statement "may" fail with OSError on some systems if + # libgfapi.so is located in /usr/local/lib/. This happens when glusterfs + # is installed from source. Refer to: http://bugs.python.org/issue18502 +except OSError: + raise ImportError("ctypes.CDLL() cannot load {0}. You might want to set " + "LD_LIBRARY_PATH env variable".format(so_file_name)) + +# c_ssize_t was only added in py27. Manually backporting it here for py26 +try: + import ctypes.c_ssize_t +except ImportError: + if sizeof(ctypes.c_uint) == sizeof(ctypes.c_void_p): + setattr(ctypes, 'c_ssize_t', ctypes.c_int) + elif sizeof(ctypes.c_ulong) == sizeof(ctypes.c_void_p): + setattr(ctypes, 'c_ssize_t', ctypes.c_long) + elif sizeof(ctypes.c_ulonglong) == sizeof(ctypes.c_void_p): + setattr(ctypes, 'c_ssize_t', ctypes.c_longlong) + + +# Wow, the Linux kernel folks really play nasty games with this structure. If +# you look at the man page for stat(2) and then at this definition you'll note +# two discrepancies. First, we seem to have st_nlink and st_mode reversed. In +# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit +# they're in the man-page order. Even uglier, the man page makes no mention of +# the *nsec fields, but they are very much present and if they're not included +# then we get memory corruption because libgfapi has a structure definition +# that's longer than ours and they overwrite some random bit of memory after +# the space we allocated. Yes, that's all very disgusting, and I'm still not +# sure this will really work on 32-bit because all of the field types are so +# obfuscated behind macros and feature checks. + + +class Stat (ctypes.Structure): + _fields_ = [ + ("st_dev", ctypes.c_ulong), + ("st_ino", ctypes.c_ulong), + ("st_nlink", ctypes.c_ulong), + ("st_mode", ctypes.c_uint), + ("st_uid", ctypes.c_uint), + ("st_gid", ctypes.c_uint), + ("st_rdev", ctypes.c_ulong), + ("st_size", ctypes.c_ulong), + ("st_blksize", ctypes.c_ulong), + ("st_blocks", ctypes.c_ulong), + ("st_atime", ctypes.c_ulong), + ("st_atimensec", ctypes.c_ulong), + ("st_mtime", ctypes.c_ulong), + ("st_mtimensec", ctypes.c_ulong), + ("st_ctime", ctypes.c_ulong), + ("st_ctimensec", ctypes.c_ulong), + ] + + +class Statvfs (ctypes.Structure): + _fields_ = [ + ("f_bsize", ctypes.c_ulong), + ("f_frsize", ctypes.c_ulong), + ("f_blocks", ctypes.c_ulong), + ("f_bfree", ctypes.c_ulong), + ("f_bavail", ctypes.c_ulong), + ("f_files", ctypes.c_ulong), + ("f_ffree", ctypes.c_ulong), + ("f_favail", ctypes.c_ulong), + ("f_fsid", ctypes.c_ulong), + ("f_flag", ctypes.c_ulong), + ("f_namemax", ctypes.c_ulong), + ("__f_spare", ctypes.c_int * 6), + ] + + +class Dirent (ctypes.Structure): + _fields_ = [ + ("d_ino", ctypes.c_ulong), + ("d_off", ctypes.c_ulong), + ("d_reclen", ctypes.c_ushort), + ("d_type", ctypes.c_char), + ("d_name", ctypes.c_char * 256), + ] + + +class Timespec (ctypes.Structure): + _fields_ = [ + ('tv_sec', ctypes.c_long), + ('tv_nsec', ctypes.c_long) + ] + + +# Here is the reference card of libgfapi library exported +# apis with its different versions. +# +# GFAPI_3.4.0 { +# glfs_new; +# glfs_set_volfile; +# glfs_set_volfile_server; +# glfs_set_logging; +# glfs_init; +# glfs_fini; +# glfs_open; +# glfs_creat; +# glfs_close; +# glfs_from_glfd; +# glfs_set_xlator_option; +# glfs_read; +# glfs_write; +# glfs_read_async; +# glfs_write_async; +# glfs_readv; +# glfs_writev; +# glfs_readv_async; +# glfs_writev_async; +# glfs_pread; +# glfs_pwrite; +# glfs_pread_async; +# glfs_pwrite_async; +# glfs_preadv; +# glfs_pwritev; +# glfs_preadv_async; +# glfs_pwritev_async; +# glfs_lseek; +# glfs_truncate; +# glfs_ftruncate; +# glfs_ftruncate_async; +# glfs_lstat; +# glfs_stat; +# glfs_fstat; +# glfs_fsync; +# glfs_fsync_async; +# glfs_fdatasync; +# glfs_fdatasync_async; +# glfs_access; +# glfs_symlink; +# glfs_readlink; +# glfs_mknod; +# glfs_mkdir; +# glfs_unlink; +# glfs_rmdir; +# glfs_rename; +# glfs_link; +# glfs_opendir; +# glfs_readdir_r; +# glfs_readdirplus_r; +# glfs_telldir; +# glfs_seekdir; +# glfs_closedir; +# glfs_statvfs; +# glfs_chmod; +# glfs_fchmod; +# glfs_chown; +# glfs_lchown; +# glfs_fchown; +# glfs_utimens; +# glfs_lutimens; +# glfs_futimens; +# glfs_getxattr; +# glfs_lgetxattr; +# glfs_fgetxattr; +# glfs_listxattr; +# glfs_llistxattr; +# glfs_flistxattr; +# glfs_setxattr; +# glfs_lsetxattr; +# glfs_fsetxattr; +# glfs_removexattr; +# glfs_lremovexattr; +# glfs_fremovexattr; +# glfs_getcwd; +# glfs_chdir; +# glfs_fchdir; +# glfs_realpath; +# glfs_posix_lock; +# glfs_dup; +# +# } +# +# GFAPI_3.4.2 { +# glfs_setfsuid; +# glfs_setfsgid; +# glfs_setfsgroups; +# glfs_h_lookupat; +# glfs_h_creat; +# glfs_h_mkdir; +# glfs_h_mknod; +# glfs_h_symlink; +# glfs_h_unlink; +# glfs_h_close; +# glfs_h_truncate; +# glfs_h_stat; +# glfs_h_getattrs; +# glfs_h_setattrs; +# glfs_h_readlink; +# glfs_h_link; +# glfs_h_rename; +# glfs_h_extract_handle; +# glfs_h_create_from_handle; +# glfs_h_opendir; +# glfs_h_open; +# } +# +# GFAPI_3.5.0 { +# +# glfs_get_volumeid; +# glfs_readdir; +# glfs_readdirplus; +# glfs_fallocate; +# glfs_discard; +# glfs_discard_async; +# glfs_zerofill; +# glfs_zerofill_async; +# glfs_caller_specific_init; +# glfs_h_setxattrs; +# +# } +# +# GFAPI_3.5.1 { +# +# glfs_unset_volfile_server; +# glfs_h_getxattrs; +# glfs_h_removexattrs; +# +# } +# +# GFAPI_3.6.0 { +# +# glfs_get_volfile; +# glfs_h_access; +# +# } +# + +def gfapi_prototype(method_name, restype, *argtypes): + """ + Create a named foreign function belonging to gfapi + + :param method_name: Name of the foreign function + :param restype: resulting type + :param argtypes: arguments that represent argument types + :returns: foreign function of gfapi library + """ + # use_errno=True ensures that errno is exposed by ctypes.get_errno() + return ctypes.CFUNCTYPE(restype, *argtypes, use_errno=True)( + (method_name, client) + ) + +# Define function prototypes for the wrapper functions. + +glfs_init = gfapi_prototype('glfs_init', ctypes.c_int, ctypes.c_void_p) + +glfs_statvfs = gfapi_prototype('glfs_statvfs', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_void_p) + +glfs_new = gfapi_prototype('glfs_new', ctypes.c_void_p, ctypes.c_char_p) + +glfs_set_volfile_server = gfapi_prototype( + 'glfs_set_volfile_server', ctypes.c_int, + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int) + +glfs_set_logging = gfapi_prototype('glfs_set_logging', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_int) + +glfs_fini = gfapi_prototype('glfs_fini', ctypes.c_int, + ctypes.c_void_p) + +glfs_creat = gfapi_prototype('glfs_creat', ctypes.c_void_p, ctypes.c_void_p, + ctypes.c_char_p, ctypes.c_int, ctypes.c_uint) + +glfs_open = gfapi_prototype('glfs_open', ctypes.c_void_p, ctypes.c_void_p, + ctypes.c_char_p, ctypes.c_int) + +glfs_close = gfapi_prototype('glfs_close', ctypes.c_int, + ctypes.c_void_p) + +glfs_lstat = gfapi_prototype('glfs_lstat', ctypes.c_int, + ctypes.c_void_p, ctypes.c_char_p, + ctypes.POINTER(Stat)) + +glfs_stat = gfapi_prototype('glfs_stat', ctypes.c_int, + ctypes.c_void_p, ctypes.c_char_p, + ctypes.POINTER(Stat)) + +glfs_fstat = gfapi_prototype('glfs_fstat', ctypes.c_int, + ctypes.c_void_p, ctypes.POINTER(Stat)) + +glfs_chmod = gfapi_prototype('glfs_chmod', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_ushort) + +glfs_fchmod = gfapi_prototype('glfs_fchmod', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_ushort) + +glfs_chown = gfapi_prototype('glfs_chown', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_uint, + ctypes.c_uint) + +glfs_lchown = gfapi_prototype('glfs_lchown', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_uint, + ctypes.c_uint) + +glfs_fchown = gfapi_prototype('glfs_fchown', ctypes.c_int, + ctypes.c_void_p, ctypes.c_uint, + ctypes.c_uint) + +glfs_dup = gfapi_prototype('glfs_dup', ctypes.c_void_p, + ctypes.c_void_p) + +glfs_fdatasync = gfapi_prototype('glfs_fdatasync', ctypes.c_int, + ctypes.c_void_p) + +glfs_fsync = gfapi_prototype('glfs_fsync', ctypes.c_int, + ctypes.c_void_p) + +glfs_lseek = gfapi_prototype('glfs_lseek', ctypes.c_int, + ctypes.c_void_p, ctypes.c_int, + ctypes.c_int) + +glfs_read = gfapi_prototype('glfs_read', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int) + +glfs_write = gfapi_prototype('glfs_write', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int) + +glfs_getxattr = gfapi_prototype('glfs_getxattr', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t) + +glfs_listxattr = gfapi_prototype('glfs_listxattr', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t) + +glfs_removexattr = gfapi_prototype('glfs_removexattr', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p) + +glfs_setxattr = gfapi_prototype('glfs_setxattr', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int) + +glfs_rename = gfapi_prototype('glfs_rename', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p) + +glfs_link = gfapi_prototype('glfs_link', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p) + +glfs_symlink = gfapi_prototype('glfs_symlink', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p) + +glfs_unlink = gfapi_prototype('glfs_unlink', ctypes.c_int, + ctypes.c_void_p, ctypes.c_char_p) + +glfs_readdir_r = gfapi_prototype('glfs_readdir_r', ctypes.c_int, + ctypes.c_void_p, + ctypes.POINTER(Dirent), + ctypes.POINTER(ctypes.POINTER(Dirent))) + +glfs_readdirplus_r = gfapi_prototype('glfs_readdirplus_r', ctypes.c_int, + ctypes.c_void_p, + ctypes.POINTER(Stat), + ctypes.POINTER(Dirent), + ctypes.POINTER(ctypes.POINTER(Dirent))) + +glfs_closedir = gfapi_prototype('glfs_closedir', ctypes.c_int, + ctypes.c_void_p) + +glfs_mkdir = gfapi_prototype('glfs_mkdir', ctypes.c_int, + ctypes.c_void_p, ctypes.c_char_p, + ctypes.c_ushort) + +glfs_opendir = gfapi_prototype('glfs_opendir', ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_char_p) + +glfs_rmdir = gfapi_prototype('glfs_rmdir', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p) + +glfs_setfsuid = gfapi_prototype('glfs_setfsuid', ctypes.c_int, + ctypes.c_uint) + +glfs_setfsgid = gfapi_prototype('glfs_setfsgid', ctypes.c_int, + ctypes.c_uint) + +glfs_ftruncate = gfapi_prototype('glfs_ftruncate', ctypes.c_int, + ctypes.c_void_p, ctypes.c_int) + +glfs_fgetxattr = gfapi_prototype('glfs_fgetxattr', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t) + +glfs_fremovexattr = gfapi_prototype('glfs_fremovexattr', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p) + +glfs_fsetxattr = gfapi_prototype('glfs_fsetxattr', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int) + +glfs_flistxattr = gfapi_prototype('glfs_flistxattr', ctypes.c_ssize_t, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t) + +glfs_access = gfapi_prototype('glfs_access', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_int) + +glfs_readlink = gfapi_prototype('glfs_readlink', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_size_t) + +glfs_chdir = gfapi_prototype('glfs_chdir', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p) + +glfs_getcwd = gfapi_prototype('glfs_getcwd', ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t) + +glfs_fallocate = gfapi_prototype('glfs_fallocate', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_size_t) + +glfs_discard = gfapi_prototype('glfs_discard', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_size_t) + +glfs_zerofill = gfapi_prototype('glfs_zerofill', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_size_t) + +glfs_utimens = gfapi_prototype('glfs_utimens', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.POINTER(Timespec)) diff --git a/gluster/gfapi/exceptions.py b/gluster/gfapi/exceptions.py new file mode 100644 index 0000000..05496f0 --- /dev/null +++ b/gluster/gfapi/exceptions.py @@ -0,0 +1,21 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of libgfapi-python project which is a +# subproject of GlusterFS ( www.gluster.org) +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +class LibgfapiException(Exception): + pass + + +class VolumeNotMounted(LibgfapiException): + pass + + +class Error(EnvironmentError): + pass diff --git a/gluster/gfapi/gfapi.py b/gluster/gfapi/gfapi.py new file mode 100644 index 0000000..89d8388 --- /dev/null +++ b/gluster/gfapi/gfapi.py @@ -0,0 +1,1734 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of libgfapi-python project which is a +# subproject of GlusterFS ( www.gluster.org) +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +import ctypes +import os +import math +import time +import stat +import errno +from collections import Iterator + +from gluster.gfapi import api +from gluster.gfapi.exceptions import LibgfapiException, Error +from gluster.gfapi.utils import validate_mount, validate_glfd + +# TODO: Move this utils.py +python_mode_to_os_flags = {} + + +def _populate_mode_to_flags_dict(): + # http://pubs.opengroup.org/onlinepubs/9699919799/functions/fopen.html + for mode in ['r', 'rb']: + python_mode_to_os_flags[mode] = os.O_RDONLY + for mode in ['w', 'wb']: + python_mode_to_os_flags[mode] = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + for mode in ['a', 'ab']: + python_mode_to_os_flags[mode] = os.O_WRONLY | os.O_CREAT | os.O_APPEND + for mode in ['r+', 'rb+', 'r+b']: + python_mode_to_os_flags[mode] = os.O_RDWR + for mode in ['w+', 'wb+', 'w+b']: + python_mode_to_os_flags[mode] = os.O_RDWR | os.O_CREAT | os.O_TRUNC + for mode in ['a+', 'ab+', 'a+b']: + python_mode_to_os_flags[mode] = os.O_RDWR | os.O_CREAT | os.O_APPEND + +_populate_mode_to_flags_dict() + + +class File(object): + + def __init__(self, fd, path=None, mode=None): + """ + Create a File object equivalent to Python's built-in File object. + + :param fd: glfd pointer + :param path: Path of the file. This is optional. + :param mode: The I/O mode of the file. + """ + self.fd = fd + self.originalpath = path + self._mode = mode + self._validate_init() + + def __enter__(self): + # __enter__ should only be called within the context + # of a 'with' statement when opening a file through + # Volume.fopen() + self._validate_init() + return self + + def __exit__(self, type, value, tb): + if self.fd: + self.close() + + def _validate_init(self): + if self.fd is None: + raise ValueError("I/O operation on invalid fd") + elif not isinstance(self.fd, int): + raise ValueError("I/O operation on invalid fd") + + @property + def fileno(self): + """ + Return the internal file descriptor (glfd) that is used by the + underlying implementation to request I/O operations. + """ + return self.fd + + @property + def mode(self): + """ + The I/O mode for the file. If the file was created using the + Volume.fopen() function, this will be the value of the mode + parameter. This is a read-only attribute. + """ + return self._mode + + @property + def name(self): + """ + If the file object was created using Volume.fopen(), + the name of the file. + """ + return self.originalpath + + @property + def closed(self): + """ + Bool indicating the current state of the file object. This is a + read-only attribute; the close() method changes the value. + """ + return not self.fd + + @validate_glfd + def close(self): + """ + Close the file. A closed file cannot be read or written any more. + + :raises: OSError on failure + """ + ret = api.glfs_close(self.fd) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + self.fd = None + + @validate_glfd + def discard(self, offset, length): + """ + This is similar to UNMAP command that is used to return the + unused/freed blocks back to the storage system. In this + implementation, fallocate with FALLOC_FL_PUNCH_HOLE is used to + eventually release the blocks to the filesystem. If the brick has + been mounted with '-o discard' option, then the discard request + will eventually reach the SCSI storage if the storage device + supports UNMAP. + + :param offset: Starting offset + :param len: Length or size in bytes to discard + :raises: OSError on failure + """ + ret = api.glfs_discard(self.fd, offset, length) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def dup(self): + """ + Return a duplicate of File object. This duplicate File class instance + encapsulates a duplicate glfd obtained by invoking glfs_dup(). + + :raises: OSError on failure + """ + dupfd = api.glfs_dup(self.fd) + if not dupfd: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return File(dupfd, self.originalpath) + + @validate_glfd + def fallocate(self, mode, offset, length): + """ + This is a Linux-specific sys call, unlike posix_fallocate() + + Allows the caller to directly manipulate the allocated disk space for + the file for the byte range starting at offset and continuing for + length bytes. + + :param mode: Operation to be performed on the given range + :param offset: Starting offset + :param length: Size in bytes, starting at offset + :raises: OSError on failure + """ + ret = api.glfs_fallocate(self.fd, mode, offset, length) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fchmod(self, mode): + """ + Change this file's mode + + :param mode: new mode + :raises: OSError on failure + """ + ret = api.glfs_fchmod(self.fd, mode) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fchown(self, uid, gid): + """ + Change this file's owner and group id + + :param uid: new user id for file + :param gid: new group id for file + :raises: OSError on failure + """ + ret = api.glfs_fchown(self.fd, uid, gid) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fdatasync(self): + """ + Flush buffer cache pages pertaining to data, but not the ones + pertaining to metadata. + + :raises: OSError on failure + """ + ret = api.glfs_fdatasync(self.fd) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fgetsize(self): + """ + Return the size of a file, as reported by fstat() + + :returns: the size of the file in bytes + """ + return self.fstat().st_size + + @validate_glfd + def fgetxattr(self, key, size=0): + """ + Retrieve the value of the extended attribute identified by key + for the file. + + :param key: Key of extended attribute + :param size: If size is specified as zero, we first determine the + size of xattr and then allocate a buffer accordingly. + If size is non-zero, it is assumed the caller knows + the size of xattr. + :returns: Value of extended attribute corresponding to key specified. + :raises: OSError on failure + """ + if size == 0: + size = api.glfs_fgetxattr(self.fd, key, None, size) + if size < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + buf = ctypes.create_string_buffer(size) + rc = api.glfs_fgetxattr(self.fd, key, buf, size) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return buf.value[:rc] + + @validate_glfd + def flistxattr(self, size=0): + """ + Retrieve list of extended attributes for the file. + + :param size: If size is specified as zero, we first determine the + size of list and then allocate a buffer accordingly. + If size is non-zero, it is assumed the caller knows + the size of the list. + :returns: List of extended attributes. + :raises: OSError on failure + """ + if size == 0: + size = api.glfs_flistxattr(self.fd, None, 0) + if size < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + buf = ctypes.create_string_buffer(size) + rc = api.glfs_flistxattr(self.fd, buf, size) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + xattrs = [] + # Parsing character by character is ugly, but it seems like the + # easiest way to deal with the "strings separated by NUL in one + # buffer" format. + i = 0 + while i < rc: + new_xa = buf.raw[i] + i += 1 + while i < rc: + next_char = buf.raw[i] + i += 1 + if next_char == '\0': + xattrs.append(new_xa) + break + new_xa += next_char + xattrs.sort() + return xattrs + + @validate_glfd + def fsetxattr(self, key, value, flags=0): + """ + Set extended attribute of file. + + :param key: The key of extended attribute. + :param value: The valiue of extended attribute. + :param flags: Possible values are 0 (default), 1 and 2. + If 0 - xattr will be created if it does not exist, or + the value will be replaced if the xattr exists. If 1 - + it performs a pure create, which fails if the named + attribute already exists. If 2 - it performs a pure + replace operation, which fails if the named attribute + does not already exist. + :raises: OSError on failure + """ + ret = api.glfs_fsetxattr(self.fd, key, value, len(value), flags) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fremovexattr(self, key): + """ + Remove a extended attribute of the file. + + :param key: The key of extended attribute. + :raises: OSError on failure + """ + ret = api.glfs_fremovexattr(self.fd, key) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def fstat(self): + """ + Returns Stat object for this file. + + :return: Returns the stat information of the file. + :raises: OSError on failure + """ + s = api.Stat() + rc = api.glfs_fstat(self.fd, ctypes.byref(s)) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return s + + @validate_glfd + def fsync(self): + """ + Flush buffer cache pages pertaining to data and metadata. + + :raises: OSError on failure + """ + ret = api.glfs_fsync(self.fd) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def ftruncate(self, length): + """ + Truncated the file to a size of length bytes. + + If the file previously was larger than this size, the extra data is + lost. If the file previously was shorter, it is extended, and the + extended part reads as null bytes. + + :param length: Length to truncate the file to in bytes. + :raises: OSError on failure + """ + ret = api.glfs_ftruncate(self.fd, length) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def lseek(self, pos, how): + """ + Set the read/write offset position of this file. + The new position is defined by 'pos' relative to 'how' + + :param pos: sets new offset position according to 'how' + :param how: SEEK_SET, sets offset position 'pos' bytes relative to + beginning of file, SEEK_CUR, the position is set relative + to the current position and SEEK_END sets the position + relative to the end of the file. + :returns: the new offset position + :raises: OSError on failure + """ + ret = api.glfs_lseek(self.fd, pos, how) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return ret + + @validate_glfd + def read(self, size=-1): + """ + Read at most size bytes from the file. + + :param buflen: length of read buffer. If less than 0, then whole + file is read. Default is -1. + :returns: buffer of 'size' length + :raises: OSError on failure + """ + if size < 0: + size = self.fgetsize() + rbuf = ctypes.create_string_buffer(size) + ret = api.glfs_read(self.fd, rbuf, size, 0) + if ret > 0: + # In python 2.x, read() always returns a string. It's really upto + # the consumer to decode this string into whatever encoding it was + # written with. + return rbuf.raw[:ret] + elif ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_glfd + def readinto(self, buf): + """ + Read up to len(buf) bytes into buf which must be a bytearray. + (buf cannot be a string as strings are immutable in python) + + This method is useful when you have to read a large file over + multiple read calls. While read() allocates a buffer every time + it's invoked, readinto() copies data to an already allocated + buffer passed to it. + + :returns: the number of bytes read (0 for EOF). + :raises: OSError on failure + """ + if type(buf) is bytearray: + buf_ptr = (ctypes.c_ubyte * len(buf)).from_buffer(buf) + else: + # TODO: Allow reading other types such as array.array + raise TypeError("buffer must of type bytearray") + nread = api.glfs_read(self.fd, buf_ptr, len(buf_ptr), 0) + if nread < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return nread + + @validate_glfd + def write(self, data, flags=0): + """ + Write data to the file. + + :param data: The data to be written to file. + :returns: The size in bytes actually written + :raises: OSError on failure + """ + # creating a ctypes.c_ubyte buffer to handle converting bytearray + # to the required C data type + + if type(data) is bytearray: + buf = (ctypes.c_ubyte * len(data)).from_buffer(data) + else: + buf = data + ret = api.glfs_write(self.fd, buf, len(buf), flags) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return ret + + @validate_glfd + def zerofill(self, offset, length): + """ + Fill 'length' number of bytes with zeroes starting from 'offset'. + + :param offset: Start at offset location + :param length: Size/length in bytes + :raises: OSError on failure + """ + ret = api.glfs_zerofill(self.fd, offset, length) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + +class Dir(Iterator): + + def __init__(self, fd, readdirplus=False): + # Add a reference so the module-level variable "api" doesn't + # get yanked out from under us (see comment above File def'n). + self._api = api + self.fd = fd + self.readdirplus = readdirplus + self.cursor = ctypes.POINTER(api.Dirent)() + + def __del__(self): + self._api.glfs_closedir(self.fd) + self._api = None + + def next(self): + entry = api.Dirent() + entry.d_reclen = 256 + + if self.readdirplus: + stat_info = api.Stat() + ret = api.glfs_readdirplus_r(self.fd, ctypes.byref(stat_info), + ctypes.byref(entry), + ctypes.byref(self.cursor)) + else: + ret = api.glfs_readdir_r(self.fd, ctypes.byref(entry), + ctypes.byref(self.cursor)) + + if ret != 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + if (not self.cursor) or (not self.cursor.contents): + # Reached end of directory stream + raise StopIteration + + if self.readdirplus: + return (entry, stat_info) + else: + return entry + + +class DirEntry(object): + """ + Object yielded by scandir() to expose the file path and other file + attributes of a directory entry. scandir() will provide stat information + without making additional calls. DirEntry instances are not intended to be + stored in long-lived data structures; if you know the file metadata has + changed or if a long time has elapsed since calling scandir(), call + Volume.stat(entry.path) to fetch up-to-date information. + + DirEntry() instances from Python 3.5 have follow_symlinks set to True by + default. In this implementation, follow_symlinks is set to False by + default as it incurs an additional stat call over network. + """ + + __slots__ = ('_name', '_vol', '_lstat', '_stat', '_path') + + def __init__(self, vol, scandir_path, name, lstat): + self._name = name + self._vol = vol + self._lstat = lstat + self._stat = None + self._path = os.path.join(scandir_path, name) + + @property + def name(self): + """ + The entry's base filename, relative to the scandir() path argument. + """ + return self._name + + @property + def path(self): + """ + The entry's full path name: equivalent to os.path.join(scandir_path, + entry.name) where scandir_path is the scandir() path argument. The path + is only absolute if the scandir() path argument was absolute. + """ + return self._path + + def stat(self, follow_symlinks=False): + """ + Returns information equivalent of a lstat() system call on the entry. + This does not follow symlinks by default. + """ + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + self._stat = self._vol.stat(self.path) + else: + self._stat = self._lstat + return self._stat + else: + return self._lstat + + def is_dir(self, follow_symlinks=False): + """ + Return True if this entry is a directory; return False if the entry is + any other kind of file, or if it doesn't exist anymore. + """ + if follow_symlinks and self.is_symlink(): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as err: + if err.errno != errno.ENOENT: + raise + return False + else: + return stat.S_ISDIR(st.st_mode) + else: + return stat.S_ISDIR(self._lstat.st_mode) + + def is_file(self, follow_symlinks=False): + """ + Return True if this entry is a file; return False if the entry is a + directory or other non-file entry, or if it doesn't exist anymore. + """ + if follow_symlinks and self.is_symlink(): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as err: + if err.errno != errno.ENOENT: + raise + return False + else: + return stat.S_ISREG(st.st_mode) + else: + return stat.S_ISREG(self._lstat.st_mode) + + def is_symlink(self): + """ + Return True if this entry is a symbolic link (even if broken); return + False if the entry points to a directory or any kind of file, or if it + doesn't exist anymore. + """ + return stat.S_ISLNK(self._lstat.st_mode) + + def inode(self): + """ + Return the inode number of the entry. + """ + return self._lstat.st_ino + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + +class Volume(object): + + def __init__(self, host, volname, + proto="tcp", port=24007, log_file=None, log_level=7): + """ + Create a Volume object instance. + + :param host: Host with glusterd management daemon running. + :param volname: Name of GlusterFS volume to be mounted and used. + :param proto: Transport protocol to be used to connect to management + daemon. Permitted values are "tcp" and "rdma". + :param port: Port number where gluster management daemon is listening. + :param log_file: Path to log file. When this is set to None, a new + logfile will be created in default log directory + i.e /var/log/glusterfs + :param log_level: Integer specifying the degree of verbosity. + Higher the value, more verbose the logging. + + """ + # TODO: Provide an interface where user can specify volfile directly + # instead of providing host and other details. This is helpful in cases + # where user wants to load some non default xlator on client side. For + # example, aux-gfid-mount or mount volume as read-only. + + # Add a reference so the module-level variable "api" doesn't + # get yanked out from under us (see comment above File def'n). + self._api = api + + self._mounted = False + self.fs = None + self.log_file = log_file + self.log_level = log_level + + if None in (volname, host): + # TODO: Validate host based on regex for IP/FQDN. + raise LibgfapiException("Host and Volume name should not be None.") + if proto not in ('tcp', 'rdma'): + raise LibgfapiException("Invalid protocol specified.") + if not isinstance(port, (int, long)): + raise LibgfapiException("Invalid port specified.") + + self.host = host + self.volname = volname + self.protocol = proto + self.port = port + + @property + def mounted(self): + """ + Read-only attribute that returns True if the volume is mounted. + The value of the attribute is internally changed on invoking + mount() and umount() functions. + """ + return self._mounted + + def mount(self): + """ + Mount a GlusterFS volume for use. + + :raises: LibgfapiException on failure + """ + if self.fs and self._mounted: + # Already mounted + return + + self.fs = api.glfs_new(self.volname) + if not self.fs: + err = ctypes.get_errno() + raise LibgfapiException("glfs_new(%s) failed: %s" % + (self.volname, os.strerror(err))) + + ret = api.glfs_set_volfile_server(self.fs, self.protocol, + self.host, self.port) + if ret < 0: + err = ctypes.get_errno() + raise LibgfapiException("glfs_set_volfile_server(%s, %s, %s, " + "%s) failed: %s" % (self.fs, self.protocol, + self.host, self.port, + os.strerror(err))) + + self.set_logging(self.log_file, self.log_level) + + if self.fs and not self._mounted: + ret = api.glfs_init(self.fs) + if ret < 0: + err = ctypes.get_errno() + raise LibgfapiException("glfs_init(%s) failed: %s" % + (self.fs, os.strerror(err))) + else: + self._mounted = True + + def umount(self): + """ + Unmount a mounted GlusterFS volume. + + Provides users a way to free resources instead of just waiting for + python garbage collector to call __del__() at some point later. + + :raises: LibgfapiException on failure + """ + if self.fs: + ret = self._api.glfs_fini(self.fs) + if ret < 0: + err = ctypes.get_errno() + raise LibgfapiException("glfs_fini(%s) failed: %s" % + (self.fs, os.strerror(err))) + else: + # Succeeded. Protect against multiple umount() calls. + self._mounted = False + self.fs = None + + def __del__(self): + try: + self.umount() + except LibgfapiException: + pass + + def set_logging(self, log_file, log_level): + """ + Set logging parameters. Can be invoked either before or after + invoking mount(). + + When invoked before mount(), the preferred log file and log level + choices are recorded and then later enforced internally as part of + mount() + + When invoked at any point after mount(), the change in log file + and log level is instantaneous. + + :param log_file: Path of log file. + If set to "/dev/null", nothing will be logged. + If set to None, a new logfile will be created in + default log directory (/var/log/glusterfs) + :param log_level: Integer specifying the degree of verbosity. + Higher the value, more verbose the logging. + """ + if self.fs: + ret = api.glfs_set_logging(self.fs, self.log_file, self.log_level) + if ret < 0: + err = ctypes.get_errno() + raise LibgfapiException("glfs_set_logging(%s, %s) failed: %s" % + (self.log_file, self.log_level, + os.strerror(err))) + self.log_file = log_file + self.log_level = log_level + + @validate_mount + def access(self, path, mode): + """ + Use the real uid/gid to test for access to path. + + :param path: Path to be checked. + :param mode: mode should be F_OK to test the existence of path, or + it can be the inclusive OR of one or more of R_OK, W_OK, + and X_OK to test permissions + :returns: True if access is allowed, False if not + """ + ret = api.glfs_access(self.fs, path, mode) + if ret == 0: + return True + else: + return False + + @validate_mount + def chdir(self, path): + """ + Change the current working directory to the given path. + + :param path: Path to change current working directory to + :raises: OSError on failure + """ + ret = api.glfs_chdir(self.fs, path) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def chmod(self, path, mode): + """ + Change mode of path + + :param path: the item to be modified + :param mode: new mode + :raises: OSError on failure + """ + ret = api.glfs_chmod(self.fs, path, mode) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def chown(self, path, uid, gid): + """ + Change owner and group id of path + + :param path: the path to be modified + :param uid: new user id for path + :param gid: new group id for path + :raises: OSError on failure + """ + ret = api.glfs_chown(self.fs, path, uid, gid) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + def exists(self, path): + """ + Returns True if path refers to an existing path. Returns False for + broken symbolic links. This function may return False if permission is + not granted to execute stat() on the requested file, even if the path + physically exists. + """ + try: + self.stat(path) + except OSError: + return False + return True + + def getatime(self, path): + """ + Returns the last access time as reported by stat() + """ + return self.stat(path).st_atime + + def getctime(self, path): + """ + Returns the time when changes were made to the path as reported by + stat(). This time is updated when changes are made to the file or + dir's inode or the contents of the file + """ + return self.stat(path).st_ctime + + @validate_mount + def getcwd(self): + """ + Returns current working directory. + """ + PATH_MAX = 4096 + buf = ctypes.create_string_buffer(PATH_MAX) + ret = api.glfs_getcwd(self.fs, buf, PATH_MAX) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return buf.value + + def getmtime(self, path): + """ + Returns the time when changes were made to the content of the path + as reported by stat() + """ + return self.stat(path).st_mtime + + def getsize(self, path): + """ + Return the size of a file in bytes, reported by stat() + """ + return self.stat(path).st_size + + @validate_mount + def getxattr(self, path, key, size=0): + """ + Retrieve the value of the extended attribute identified by key + for path specified. + + :param path: Path to file or directory + :param key: Key of extended attribute + :param size: If size is specified as zero, we first determine the + size of xattr and then allocate a buffer accordingly. + If size is non-zero, it is assumed the caller knows + the size of xattr. + :returns: Value of extended attribute corresponding to key specified. + :raises: OSError on failure + """ + if size == 0: + size = api.glfs_getxattr(self.fs, path, key, None, 0) + if size < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + buf = ctypes.create_string_buffer(size) + rc = api.glfs_getxattr(self.fs, path, key, buf, size) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return buf.value[:rc] + + def isdir(self, path): + """ + Returns True if path is an existing directory. Returns False on all + failure cases including when path does not exist. + """ + try: + s = self.stat(path) + except OSError: + return False + return stat.S_ISDIR(s.st_mode) + + def isfile(self, path): + """ + Return True if path is an existing regular file. Returns False on all + failure cases including when path does not exist. + """ + try: + s = self.stat(path) + except OSError: + return False + return stat.S_ISREG(s.st_mode) + + def islink(self, path): + """ + Return True if path refers to a directory entry that is a symbolic + link. Returns False on all failure cases including when path does + not exist. + """ + try: + s = self.lstat(path) + except OSError: + return False + return stat.S_ISLNK(s.st_mode) + + def listdir(self, path): + """ + Return a list containing the names of the entries in the directory + given by path. The list is in arbitrary order. It does not include + the special entries '.' and '..' even if they are present in the + directory. + + :param path: Path to directory + :raises: OSError on failure + :returns: List of names of directory entries + """ + dir_list = [] + for entry in self.opendir(path): + if not isinstance(entry, api.Dirent): + break + name = entry.d_name[:entry.d_reclen] + if name not in (".", ".."): + dir_list.append(name) + return dir_list + + def listdir_with_stat(self, path): + """ + Return a list containing the name and stat of the entries in the + directory given by path. The list is in arbitrary order. It does + not include the special entries '.' and '..' even if they are present + in the directory. + + :param path: Path to directory + :raises: OSError on failure + :returns: A list of tuple. The tuple is of the form (name, stat) where + name is a string indicating name of the directory entry and + stat contains stat info of the entry. + """ + # List of tuple. Each tuple is of the form (name, stat) + entries_with_stat = [] + for (entry, stat_info) in self.opendir(path, readdirplus=True): + if not (isinstance(entry, api.Dirent) and + isinstance(stat_info, api.Stat)): + break + name = entry.d_name[:entry.d_reclen] + if name not in (".", ".."): + entries_with_stat.append((name, stat_info)) + return entries_with_stat + + def scandir(self, path): + """ + Return an iterator of :class:`DirEntry` objects corresponding to the + entries in the directory given by path. The entries are yielded in + arbitrary order, and the special entries '.' and '..' are not + included. + + Using scandir() instead of listdir() can significantly increase the + performance of code that also needs file type or file attribute + information, because :class:`DirEntry` objects expose this + information. + + scandir() provides same functionality as listdir_with_stat() except + that scandir() does not return a list and is an iterator. Hence scandir + is less memory intensive on large directories. + + :param path: Path to directory + :raises: OSError on failure + :yields: Instance of :class:`DirEntry` class. + """ + for (entry, lstat) in self.opendir(path, readdirplus=True): + name = entry.d_name[:entry.d_reclen] + if name not in (".", ".."): + yield DirEntry(self, path, name, lstat) + + @validate_mount + def listxattr(self, path, size=0): + """ + Retrieve list of extended attribute keys for the specified path. + + :param path: Path to file or directory. + :param size: If size is specified as zero, we first determine the + size of list and then allocate a buffer accordingly. + If size is non-zero, it is assumed the caller knows + the size of the list. + :returns: List of extended attribute keys. + :raises: OSError on failure + """ + if size == 0: + size = api.glfs_listxattr(self.fs, path, None, 0) + if size < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + buf = ctypes.create_string_buffer(size) + rc = api.glfs_listxattr(self.fs, path, buf, size) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + xattrs = [] + # Parsing character by character is ugly, but it seems like the + # easiest way to deal with the "strings separated by NUL in one + # buffer" format. + i = 0 + while i < rc: + new_xa = buf.raw[i] + i += 1 + while i < rc: + next_char = buf.raw[i] + i += 1 + if next_char == '\0': + xattrs.append(new_xa) + break + new_xa += next_char + xattrs.sort() + return xattrs + + @validate_mount + def lstat(self, path): + """ + Return stat information of path. If path is a symbolic link, then it + returns information about the link itself, not the file that it refers + to. + + :raises: OSError on failure + """ + s = api.Stat() + rc = api.glfs_lstat(self.fs, path, ctypes.byref(s)) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return s + + def makedirs(self, path, mode=0777): + """ + Recursive directory creation function. Like mkdir(), but makes all + intermediate-level directories needed to contain the leaf directory. + The default mode is 0777 (octal). + + :raises: OSError if the leaf directory already exists or cannot be + created. Can also raise OSError if creation of any non-leaf + directories fails. + """ + head, tail = os.path.split(path) + if not tail: + head, tail = os.path.split(head) + if head and tail and not self.exists(head): + try: + self.makedirs(head, mode) + except OSError as err: + if err.errno != errno.EEXIST: + raise + if tail == os.curdir: + return + self.mkdir(path, mode) + + @validate_mount + def mkdir(self, path, mode=0777): + """ + Create a directory named path with numeric mode mode. + The default mode is 0777 (octal). + + :raises: OSError on failure + """ + ret = api.glfs_mkdir(self.fs, path, mode) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def fopen(self, path, mode='r'): + """ + Similar to Python's built-in File object returned by Python's open() + + Unlike Python's open(), fopen() provided here is only for convenience + and it does NOT invoke glibc's fopen and does NOT do any kind of + I/O bufferring as of today. + + The most commonly-used values of mode are 'r' for reading, 'w' for + writing (truncating the file if it already exists), and 'a' for + appending. If mode is omitted, it defaults to 'r'. + + Modes 'r+', 'w+' and 'a+' open the file for updating (reading and + writing); note that 'w+' truncates the file. + + Append 'b' to the mode to open the file in binary mode but this has + no effect as of today. + + :param path: Path of file to be opened + :param mode: Mode to open the file with. This is a string. + :returns: an instance of File class + :raises: OSError on failure to create/open file. + TypeError and ValueError if mode is invalid. + """ + if not isinstance(mode, basestring): + raise TypeError("Mode must be a string") + try: + flags = python_mode_to_os_flags[mode] + except KeyError: + raise ValueError("Invalid mode") + else: + if (os.O_CREAT & flags) == os.O_CREAT: + fd = api.glfs_creat(self.fs, path, flags, 0666) + else: + fd = api.glfs_open(self.fs, path, flags) + if not fd: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return File(fd, path=path, mode=mode) + + @validate_mount + def open(self, path, flags, mode=0777): + """ + Similar to Python's os.open() + + As of today, the only way to consume the raw glfd returned is by + passing it to File class. + + :param path: Path of file to be opened + :param flags: Integer which flags must include one of the following + access modes: os.O_RDONLY, os.O_WRONLY, or os.O_RDWR. + :param mode: specifies the permissions to use in case a new + file is created. The default mode is 0777 (octal) + :returns: the raw glfd (pointer to memory in C, number in python) + :raises: OSError on failure to create/open file. + TypeError if flags is not an integer. + """ + if not isinstance(flags, int): + raise TypeError("flags must evaluate to an integer") + + if (os.O_CREAT & flags) == os.O_CREAT: + fd = api.glfs_creat(self.fs, path, flags, mode) + else: + fd = api.glfs_open(self.fs, path, flags) + if not fd: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + return fd + + @validate_mount + def opendir(self, path, readdirplus=False): + """ + Open a directory. + + :param path: Path to the directory + :param readdirplus: Enable readdirplus which will also fetch stat + information for each entry of directory. + :returns: Returns a instance of Dir class + :raises: OSError on failure + """ + fd = api.glfs_opendir(self.fs, path) + if not fd: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return Dir(fd, readdirplus) + + @validate_mount + def readlink(self, path): + """ + Return a string representing the path to which the symbolic link + points. The result may be either an absolute or relative pathname. + + :param path: Path of symbolic link + :returns: Contents of symlink + :raises: OSError on failure + """ + PATH_MAX = 4096 + buf = ctypes.create_string_buffer(PATH_MAX) + ret = api.glfs_readlink(self.fs, path, buf, PATH_MAX) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return buf.value[:ret] + + def remove(self, path): + """ + Remove (delete) the file path. If path is a directory, OSError + is raised. This is identical to the unlink() function. + + :raises: OSError on failure + """ + return self.unlink(path) + + @validate_mount + def removexattr(self, path, key): + """ + Remove a extended attribute of the path. + + :param path: Path to the file or directory. + :param key: The key of extended attribute. + :raises: OSError on failure + """ + ret = api.glfs_removexattr(self.fs, path, key) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def rename(self, src, dst): + """ + Rename the file or directory from src to dst. If dst is a directory, + OSError will be raised. If dst exists and is a file, it will be + replaced silently if the user has permission. + + :raises: OSError on failure + """ + ret = api.glfs_rename(self.fs, src, dst) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def rmdir(self, path): + """ + Remove (delete) the directory path. Only works when the directory is + empty, otherwise, OSError is raised. In order to remove whole + directory trees, rmtree() can be used. + + :raises: OSError on failure + """ + ret = api.glfs_rmdir(self.fs, path) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + def rmtree(self, path, ignore_errors=False, onerror=None): + """ + Delete a whole directory tree structure. Raises OSError + if path is a symbolic link. + + :param path: Directory tree to remove + :param ignore_errors: If True, errors are ignored + :param onerror: If set, it is called to handle the error with arguments + (func, path, exc) where func is the function that + raised the error, path is the argument that caused it + to fail; and exc is the exception that was raised. + If ignore_errors is False and onerror is None, an + exception is raised + :raises: OSError on failure if onerror is None + """ + if ignore_errors: + def onerror(*args): + pass + elif onerror is None: + def onerror(*args): + raise + if self.islink(path): + raise OSError("Cannot call rmtree on a symbolic link") + + try: + for entry in self.scandir(path): + fullname = os.path.join(path, entry.name) + if entry.is_dir(follow_symlinks=False): + self.rmtree(fullname, ignore_errors, onerror) + else: + try: + self.unlink(fullname) + except OSError as e: + onerror(self.unlink, fullname, e) + except OSError as e: + # self.scandir() is not a list and is a true iterator, it can + # raise an exception and blow-up. The try-except block here is to + # handle it gracefully and return. + onerror(self.scandir, path, e) + + try: + self.rmdir(path) + except OSError as e: + onerror(self.rmdir, path, e) + + def setfsuid(self, uid): + """ + setfsuid() changes the value of the caller's filesystem user ID-the + user ID that the Linux kernel uses to check for all accesses to the + filesystem. + + :raises: OSError on failure + """ + ret = api.glfs_setfsuid(uid) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + def setfsgid(self, gid): + """ + setfsgid() changes the value of the caller's filesystem group ID-the + group ID that the Linux kernel uses to check for all accesses to the + filesystem. + + :raises: OSError on failure + """ + ret = api.glfs_setfsgid(gid) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def setxattr(self, path, key, value, flags=0): + """ + Set extended attribute of the path. + + :param path: Path to file or directory. + :param key: The key of extended attribute. + :param value: The valiue of extended attribute. + :param flags: Possible values are 0 (default), 1 and 2. + If 0 - xattr will be created if it does not exist, or + the value will be replaced if the xattr exists. If 1 - + it performs a pure create, which fails if the named + attribute already exists. If 2 - it performs a pure + replace operation, which fails if the named attribute + does not already exist. + + :raises: OSError on failure + """ + ret = api.glfs_setxattr(self.fs, path, key, value, len(value), flags) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def stat(self, path): + """ + Returns stat information of path. + + :raises: OSError on failure + """ + s = api.Stat() + rc = api.glfs_stat(self.fs, path, ctypes.byref(s)) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return s + + @validate_mount + def statvfs(self, path): + """ + Returns information about a mounted glusterfs volume. path is the + pathname of any file within the mounted filesystem. + + :returns: An object whose attributes describe the filesystem on the + given path, and correspond to the members of the statvfs + structure, namely: f_bsize, f_frsize, f_blocks, f_bfree, + f_bavail, f_files, f_ffree, f_favail, f_fsid, f_flag, + and f_namemax. + + :raises: OSError on failure + """ + s = api.Statvfs() + rc = api.glfs_statvfs(self.fs, path, ctypes.byref(s)) + if rc < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return s + + @validate_mount + def link(self, source, link_name): + """ + Create a hard link pointing to source named link_name. + + :raises: OSError on failure + """ + ret = api.glfs_link(self.fs, source, link_name) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def symlink(self, source, link_name): + """ + Create a symbolic link 'link_name' which points to 'source' + + :raises: OSError on failure + """ + ret = api.glfs_symlink(self.fs, source, link_name) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def unlink(self, path): + """ + Delete the file 'path' + + :raises: OSError on failure + """ + ret = api.glfs_unlink(self.fs, path) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + @validate_mount + def utime(self, path, times): + """ + Set the access and modified times of the file specified by path. If + times is None, then the file's access and modified times are set to + the current time. (The effect is similar to running the Unix program + touch on the path.) Otherwise, times must be a 2-tuple of numbers, + of the form (atime, mtime) which is used to set the access and + modified times, respectively. + + + :raises: OSError on failure to change time. + TypeError if invalid times is passed. + """ + if times is None: + now = time.time() + times = (now, now) + else: + if type(times) is not tuple or len(times) != 2: + raise TypeError("utime() arg 2 must be a tuple (atime, mtime)") + + timespec_array = (api.Timespec * 2)() + + # Set atime + decimal, whole = math.modf(times[0]) + timespec_array[0].tv_sec = int(whole) + timespec_array[0].tv_nsec = int(decimal * 1e9) + + # Set mtime + decimal, whole = math.modf(times[1]) + timespec_array[1].tv_sec = int(whole) + timespec_array[1].tv_nsec = int(decimal * 1e9) + + ret = api.glfs_utimens(self.fs, path, timespec_array) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + def walk(self, top, topdown=True, onerror=None, followlinks=False): + """ + Generate the file names in a directory tree by walking the tree either + top-down or bottom-up. + + Slight difference in behaviour in comparison to os.walk(): + When os.walk() is called with 'followlinks=False' (default), symlinks + to directories are included in the 'dirnames' list. When Volume.walk() + is called with 'followlinks=False' (default), symlinks to directories + are included in 'filenames' list. This is NOT a bug. + http://python.6.x6.nabble.com/os-walk-with-followlinks-False-td3559133.html + + :param top: Directory path to walk + :param topdown: If topdown is True or not specified, the triple for a + directory is generated before the triples for any of + its subdirectories. If topdown is False, the triple + for a directory is generated after the triples for all + of its subdirectories. + :param onerror: If optional argument onerror is specified, it should be + a function; it will be called with one argument, an + OSError instance. It can report the error to continue + with the walk, or raise exception to abort the walk. + :param followlinks: Set followlinks to True to visit directories + pointed to by symlinks. + :raises: OSError on failure if onerror is None + :yields: a 3-tuple (dirpath, dirnames, filenames) where dirpath is a + string, the path to the directory. dirnames is a list of the + names of the subdirectories in dirpath (excluding '.' and + '..'). filenames is a list of the names of the non-directory + files in dirpath. + """ + dirs = [] # List of DirEntry objects + nondirs = [] # List of names (strings) + + try: + for entry in self.scandir(top): + if entry.is_dir(follow_symlinks=followlinks): + dirs.append(entry) + else: + nondirs.append(entry.name) + except OSError as err: + # self.scandir() is not a list and is a true iterator, it can + # raise an exception and blow-up. The try-except block here is to + # handle it gracefully and return. + if onerror is not None: + onerror(err) + return + + if topdown: + yield top, [d.name for d in dirs], nondirs + + for directory in dirs: + # NOTE: Both is_dir() and is_symlink() can be true for the same + # path when follow_symlinks is set to True + if followlinks or not directory.is_symlink(): + new_path = os.path.join(top, directory.name) + for x in self.walk(new_path, topdown, onerror, followlinks): + yield x + + if not topdown: + yield top, [d.name for d in dirs], nondirs + + def samefile(self, path1, path2): + """ + Return True if both pathname arguments refer to the same file or + directory (as indicated by device number and inode number). Raise an + exception if a stat() call on either pathname fails. + + :param path1: Path to one file + :param path2: Path to another file + :raises: OSError if stat() fails + """ + s1 = self.stat(path1) + s2 = self.stat(path2) + return s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev + + @classmethod + def copyfileobj(self, fsrc, fdst, length=128 * 1024): + """ + Copy the contents of the file-like object fsrc to the file-like object + fdst. The integer length, if given, is the buffer size. Note that if + the current file position of the fsrc object is not 0, only the + contents from the current file position to the end of the file will be + copied. + + :param fsrc: Source file object + :param fdst: Destination file object + :param length: Size of buffer in bytes to be used in copying + :raises: OSError on failure + """ + buf = bytearray(length) + while True: + nread = fsrc.readinto(buf) + if not nread or nread <= 0: + break + if nread == length: + # Entire buffer is filled, do not slice. + fdst.write(buf) + else: + # TODO: + # Use memoryview to avoid internal copy done on slicing. + fdst.write(buf[0:nread]) + + def copyfile(self, src, dst): + """ + Copy the contents (no metadata) of the file named src to a file named + dst. dst must be the complete target file name. If src and dst are + the same, Error is raised. The destination location must be writable. + If dst already exists, it will be replaced. Special files such as + character or block devices and pipes cannot be copied with this + function. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: Error if src and dst file are same file. + OSError on failure to read/write. + """ + _samefile = False + try: + _samefile = self.samefile(src, dst) + except OSError: + # Dst file need not exist. + pass + + if _samefile: + raise Error("`%s` and `%s` are the same file" % (src, dst)) + + with self.fopen(src, 'rb') as fsrc: + with self.fopen(dst, 'wb') as fdst: + self.copyfileobj(fsrc, fdst) + + def copymode(self, src, dst): + """ + Copy the permission bits from src to dst. The file contents, owner, + and group are unaffected. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: OSError on failure. + """ + st = self.stat(src) + mode = stat.S_IMODE(st.st_mode) + self.chmod(dst, mode) + + def copystat(self, src, dst): + """ + Copy the permission bits, last access time, last modification time, + and flags from src to dst. The file contents, owner, and group are + unaffected. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: OSError on failure. + """ + st = self.stat(src) + mode = stat.S_IMODE(st.st_mode) + self.utime(dst, (st.st_atime, st.st_mtime)) + self.chmod(dst, mode) + # TODO: Handle st_flags on FreeBSD + + def copy(self, src, dst): + """ + Copy data and mode bits ("cp src dst") + + Copy the file src to the file or directory dst. If dst is a directory, + a file with the same basename as src is created (or overwritten) in + the directory specified. Permission bits are copied. src and dst are + path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file or directory + :raises: OSError on failure + """ + if self.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + self.copyfile(src, dst) + self.copymode(src, dst) + + def copy2(self, src, dst): + """ + Similar to copy(), but metadata is copied as well - in fact, this is + just copy() followed by copystat(). This is similar to the Unix command + cp -p. + + The destination may be a directory. + + :param src: Path of source file + :param dst: Path of destination file or directory + :raises: OSError on failure + """ + if self.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + self.copyfile(src, dst) + self.copystat(src, dst) + + def copytree(self, src, dst, symlinks=False, ignore=None): + """ + Recursively copy a directory tree using copy2(). + + The destination directory must not already exist. + If exception(s) occur, an Error is raised with a list of reasons. + + If the optional symlinks flag is true, symbolic links in the + source tree result in symbolic links in the destination tree; if + it is false, the contents of the files pointed to by symbolic + links are copied. + + The optional ignore argument is a callable. If given, it + is called with the 'src' parameter, which is the directory + being visited by copytree(), and 'names' which is the list of + 'src' contents, as returned by os.listdir(): + + callable(src, names) -> ignored_names + + Since copytree() is called recursively, the callable will be + called once for each directory that is copied. It returns a + list of names relative to the 'src' directory that should + not be copied. + """ + def _isdir(path, statinfo, follow_symlinks=False): + if stat.S_ISDIR(statinfo.st_mode): + return True + if follow_symlinks and stat.S_ISLNK(statinfo.st_mode): + return self.isdir(path) + return False + + # Can't used scandir() here to support ignored_names functionality + names_with_stat = self.listdir_with_stat(src) + if ignore is not None: + ignored_names = ignore(src, [n for n, s in names_with_stat]) + else: + ignored_names = set() + + self.makedirs(dst) + errors = [] + for (name, st) in names_with_stat: + if name in ignored_names: + continue + srcpath = os.path.join(src, name) + dstpath = os.path.join(dst, name) + try: + if symlinks and stat.S_ISLNK(st.st_mode): + linkto = self.readlink(srcpath) + self.symlink(linkto, dstpath) + # shutil's copytree() calls os.path.isdir() which will return + # true even if it's a symlink pointing to a dir. Mimicking the + # same behaviour here with _isdir() + elif _isdir(srcpath, st, follow_symlinks=not symlinks): + self.copytree(srcpath, dstpath, symlinks) + else: + # The following is equivalent of copy2(). copy2() is not + # invoked directly to avoid multiple duplicate stat calls. + with self.fopen(srcpath, 'rb') as fsrc: + with self.fopen(dstpath, 'wb') as fdst: + self.copyfileobj(fsrc, fdst) + self.utime(dstpath, (st.st_atime, st.st_mtime)) + self.chmod(dstpath, stat.S_IMODE(st.st_mode)) + except (Error, EnvironmentError, OSError) as why: + errors.append((srcpath, dstpath, str(why))) + + try: + self.copystat(src, dst) + except OSError as why: + errors.append((src, dst, str(why))) + + if errors: + raise Error(errors) diff --git a/gluster/gfapi/utils.py b/gluster/gfapi/utils.py new file mode 100644 index 0000000..a556a41 --- /dev/null +++ b/gluster/gfapi/utils.py @@ -0,0 +1,57 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of libgfapi-python project which is a +# subproject of GlusterFS ( www.gluster.org) +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +import os +import errno +from functools import wraps +from gluster.gfapi.exceptions import VolumeNotMounted + + +def validate_mount(func): + """ + Decorator to assert that volume is initialized and mounted before any + further I/O calls are invoked by methods. + + :param func: method to be decorated and checked. + """ + def _exception(volname): + raise VolumeNotMounted('Volume "%s" not mounted.' % (volname)) + + @wraps(func) + def wrapper(*args, **kwargs): + self = args[0] + if self.fs and self._mounted: + return func(*args, **kwargs) + else: + return _exception(self.volname) + wrapper.__wrapped__ = func + + return wrapper + + +def validate_glfd(func): + """ + Decorator to assert that glfd is valid. + + :param func: method to be decorated and checked. + """ + def _exception(): + raise OSError(errno.EBADF, os.strerror(errno.EBADF)) + + @wraps(func) + def wrapper(*args, **kwargs): + self = args[0] + if self.fd: + return func(*args, **kwargs) + else: + return _exception() + wrapper.__wrapped__ = func + + return wrapper diff --git a/gluster/utils.py b/gluster/utils.py deleted file mode 100644 index bc55184..0000000 --- a/gluster/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2016 Red Hat, Inc. -# -# This file is part of libgfapi-python project which is a -# subproject of GlusterFS ( www.gluster.org) -# -# This file is licensed to you under your choice of the GNU Lesser -# General Public License, version 3 or any later version (LGPLv3 or -# later), or the GNU General Public License, version 2 (GPLv2), in all -# cases as published by the Free Software Foundation. - -import os -import errno -from functools import wraps -from gluster.exceptions import VolumeNotMounted - - -def validate_mount(func): - """ - Decorator to assert that volume is initialized and mounted before any - further I/O calls are invoked by methods. - - :param func: method to be decorated and checked. - """ - def _exception(volname): - raise VolumeNotMounted('Volume "%s" not mounted.' % (volname)) - - @wraps(func) - def wrapper(*args, **kwargs): - self = args[0] - if self.fs and self._mounted: - return func(*args, **kwargs) - else: - return _exception(self.volname) - wrapper.__wrapped__ = func - - return wrapper - - -def validate_glfd(func): - """ - Decorator to assert that glfd is valid. - - :param func: method to be decorated and checked. - """ - def _exception(): - raise OSError(errno.EBADF, os.strerror(errno.EBADF)) - - @wraps(func) - def wrapper(*args, **kwargs): - self = args[0] - if self.fd: - return func(*args, **kwargs) - else: - return _exception() - wrapper.__wrapped__ = func - - return wrapper diff --git a/setup.py b/setup.py index 912959b..26871d7 100644 --- a/setup.py +++ b/setup.py @@ -12,11 +12,12 @@ import os import re -from setuptools import setup +from setuptools import setup, find_packages # Get version without importing. -gfapi_file_path = os.path.join(os.path.dirname(__file__), 'gluster/gfapi.py') +gfapi_file_path = os.path.join(os.path.dirname(__file__), + 'gluster/gfapi/__init__.py') with open(gfapi_file_path) as f: for line in f: match = re.match(r"__version__.*'([0-9.]+)'", line) @@ -36,7 +37,7 @@ setup( author='Red Hat, Inc.', author_email='gluster-devel@gluster.org', url='http://www.gluster.org', - packages=['gluster', ], + packages=find_packages(exclude=['test*']), test_suite='nose.collector', classifiers=[ 'Development Status :: 5 - Production/Stable' diff --git a/test/functional/libgfapi-python-tests.py b/test/functional/libgfapi-python-tests.py index 8c62685..cf269d3 100644 --- a/test/functional/libgfapi-python-tests.py +++ b/test/functional/libgfapi-python-tests.py @@ -19,9 +19,9 @@ from test import get_test_config from ConfigParser import NoSectionError, NoOptionError from uuid import uuid4 -from gluster.api import Stat +from gluster.gfapi.api import Stat from gluster.gfapi import File, Volume, DirEntry -from gluster.exceptions import LibgfapiException, Error +from gluster.gfapi.exceptions import LibgfapiException, Error config = get_test_config() if config: diff --git a/test/unit/gluster/test_gfapi.py b/test/unit/gluster/test_gfapi.py index 15ce061..4b58597 100644 --- a/test/unit/gluster/test_gfapi.py +++ b/test/unit/gluster/test_gfapi.py @@ -18,8 +18,8 @@ import math import errno from gluster.gfapi import File, Dir, Volume, DirEntry -from gluster import api -from gluster.exceptions import LibgfapiException +from gluster.gfapi import api +from gluster.gfapi.exceptions import LibgfapiException from nose import SkipTest from mock import Mock, MagicMock, patch from contextlib import nested @@ -483,7 +483,7 @@ class TestVolume(unittest.TestCase): mock_glfs_creat = Mock() mock_glfs_creat.return_value = 2 - with patch("gluster.api.glfs_creat", mock_glfs_creat): + with patch("gluster.gfapi.api.glfs_creat", mock_glfs_creat): with File(self.vol.open("file.txt", os.O_CREAT, 0644)) as f: self.assertTrue(isinstance(f, File)) self.assertEqual(mock_glfs_creat.call_count, 1) @@ -849,7 +849,7 @@ class TestVolume(unittest.TestCase): mock_glfs_open = Mock() mock_glfs_open.return_value = 2 - with patch("gluster.api.glfs_open", mock_glfs_open): + with patch("gluster.gfapi.api.glfs_open", mock_glfs_open): with File(self.vol.open("file.txt", os.O_WRONLY)) as f: self.assertTrue(isinstance(f, File)) self.assertEqual(mock_glfs_open.call_count, 1) @@ -864,14 +864,14 @@ class TestVolume(unittest.TestCase): with self.vol.open("file.txt", os.O_WRONLY) as fd: self.assertEqual(fd, None) - with patch("gluster.api.glfs_open", mock_glfs_open): + with patch("gluster.gfapi.api.glfs_open", mock_glfs_open): self.assertRaises(OSError, assert_open) def test_open_direct_success(self): mock_glfs_open = Mock() mock_glfs_open.return_value = 2 - with patch("gluster.api.glfs_open", mock_glfs_open): + with patch("gluster.gfapi.api.glfs_open", mock_glfs_open): f = File(self.vol.open("file.txt", os.O_WRONLY)) self.assertTrue(isinstance(f, File)) self.assertEqual(mock_glfs_open.call_count, 1) @@ -882,7 +882,7 @@ class TestVolume(unittest.TestCase): mock_glfs_open = Mock() mock_glfs_open.return_value = None - with patch("gluster.api.glfs_open", mock_glfs_open): + with patch("gluster.gfapi.api.glfs_open", mock_glfs_open): self.assertRaises(OSError, self.vol.open, "file.txt", os.O_RDONLY) def test_opendir_success(self): diff --git a/test/unit/gluster/test_utils.py b/test/unit/gluster/test_utils.py index 446bcd9..e58e225 100644 --- a/test/unit/gluster/test_utils.py +++ b/test/unit/gluster/test_utils.py @@ -10,8 +10,8 @@ import unittest -from gluster import utils -from gluster.exceptions import VolumeNotMounted +from gluster.gfapi import utils +from gluster.gfapi.exceptions import VolumeNotMounted class TestUtils(unittest.TestCase): -- cgit