diff options
Diffstat (limited to 'gluster/swift/common')
| -rw-r--r-- | gluster/swift/common/fs_utils.py | 99 | ||||
| -rw-r--r-- | gluster/swift/common/utils.py | 62 |
2 files changed, 68 insertions, 93 deletions
diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py index 8b26fd0..199984a 100644 --- a/gluster/swift/common/fs_utils.py +++ b/gluster/swift/common/fs_utils.py @@ -18,42 +18,39 @@ import os import errno import stat import random -import os.path as os_path # noqa +import ctypes +import os.path as _os_path from eventlet import sleep +from swift.common.utils import load_libc_function from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ - NotDirectoryError, GlusterFileSystemOSError, GlusterFileSystemIOError + NotDirectoryError, GlusterFileSystemOSError -class Fake_file(object): - def __init__(self, path): - self.path = path - - def tell(self): - return 0 - - def read(self, count): - return None - - def fileno(self): - return -1 - - def close(self): - pass +os_path = _os_path def do_walk(*args, **kwargs): return os.walk(*args, **kwargs) -def do_write(fd, msg): +def do_write(fd, buf): try: - cnt = os.write(fd, msg) + cnt = os.write(fd, buf) except OSError as err: raise GlusterFileSystemOSError( err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd)) return cnt +def do_read(fd, n): + try: + buf = os.read(fd, n) + except OSError as err: + raise GlusterFileSystemOSError( + err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd)) + return buf + + def do_ismount(path): """ Test whether a path is a mount point. @@ -203,37 +200,21 @@ def do_fstat(fd): def do_open(path, flags, **kwargs): - if isinstance(flags, int): - try: - fd = os.open(path, flags, **kwargs) - except OSError as err: - raise GlusterFileSystemOSError( - err.errno, '%s, os.open("%s", %x, %r)' % ( - err.strerror, path, flags, kwargs)) - return fd - else: - try: - fp = open(path, flags, **kwargs) - except IOError as err: - raise GlusterFileSystemIOError( - err.errno, '%s, open("%s", %s, %r)' % ( - err.strerror, path, flags, kwargs)) - return fp + try: + fd = os.open(path, flags, **kwargs) + except OSError as err: + raise GlusterFileSystemOSError( + err.errno, '%s, os.open("%s", %x, %r)' % ( + err.strerror, path, flags, kwargs)) + return fd def do_close(fd): - if isinstance(fd, file) or isinstance(fd, Fake_file): - try: - fd.close() - except IOError as err: - raise GlusterFileSystemIOError( - err.errno, '%s, os.close(%s)' % (err.strerror, fd)) - else: - try: - os.close(fd) - except OSError as err: - raise GlusterFileSystemOSError( - err.errno, '%s, os.close(%s)' % (err.strerror, fd)) + try: + os.close(fd) + except OSError as err: + raise GlusterFileSystemOSError( + err.errno, '%s, os.close(%s)' % (err.strerror, fd)) def do_unlink(path, log=True): @@ -268,9 +249,31 @@ def do_fsync(fd): def do_fdatasync(fd): try: os.fdatasync(fd) + except AttributeError: + do_fsync(fd) except OSError as err: raise GlusterFileSystemOSError( - err.errno, '%s, os.fdatasync("%s")' % (err.strerror, fd)) + err.errno, '%s, os.fsync("%s")' % (err.strerror, fd)) + + +_posix_fadvise = None + + +def do_fadvise64(fd, offset, length): + global _posix_fadvise + if _posix_fadvise is None: + _posix_fadvise = load_libc_function('posix_fadvise64') + # 4 means "POSIX_FADV_DONTNEED" + _posix_fadvise(fd, ctypes.c_uint64(offset), + ctypes.c_uint64(length), 4) + + +def do_lseek(fd, pos, how): + try: + os.lseek(fd, pos, how) + except OSError as err: + raise GlusterFileSystemOSError( + err.errno, '%s, os.fsync("%s")' % (err.strerror, fd)) def mkdirs(path): diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index 5d4b6a4..6773836 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -17,15 +17,13 @@ import os import stat import errno import xattr -import random import logging from hashlib import md5 from eventlet import sleep import cPickle as pickle -from swift.common.utils import normalize_timestamp from gluster.swift.common.exceptions import GlusterFileSystemIOError -from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \ - do_stat, do_fstat, do_listdir, do_walk, do_rmdir +from gluster.swift.common.fs_utils import os_path, do_stat, do_listdir, \ + do_walk, do_rmdir, do_fstat from gluster.swift.common import Glusterfs X_CONTENT_TYPE = 'Content-Type' @@ -56,6 +54,21 @@ PICKLE_PROTOCOL = 2 CHUNK_SIZE = 65536 +def normalize_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx.xxxxx (10.5) format. + + Note that timestamps using values greater than or equal to November 20th, + 2286 at 17:46 UTC will use 11 digits to represent the number of + seconds. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return "%016.05f" % (float(timestamp)) + + def read_metadata(path_or_fd): """ Helper function to read the pickled metadata from a File/Directory. @@ -207,7 +220,6 @@ def validate_account(metadata): def validate_object(metadata): if not metadata: - logging.warn('validate_object: No metadata') return False if X_TIMESTAMP not in metadata.keys() or \ @@ -451,38 +463,6 @@ def create_account_metadata(acc_path): return rmd -def write_pickle(obj, dest, tmp=None, pickle_protocol=0): - """ - Ensure that a pickle file gets written to disk. The file is first written - to a tmp file location in the destination directory path, ensured it is - synced to disk, then moved to its final destination name. - - This version takes advantage of Gluster's dot-prefix-dot-suffix naming - where the a file named ".thefile.name.9a7aasv" is hashed to the same - Gluster node as "thefile.name". This ensures the renaming of a temp file - once written does not move it to another Gluster node. - - :param obj: python object to be pickled - :param dest: path of final destination file - :param tmp: path to tmp to use, defaults to None (ignored) - :param pickle_protocol: protocol to pickle the obj with, defaults to 0 - """ - dirname = os.path.dirname(dest) - basename = os.path.basename(dest) - tmpname = '.' + basename + '.' + \ - md5(basename + str(random.random())).hexdigest() - tmppath = os.path.join(dirname, tmpname) - with open(tmppath, 'wb') as fo: - pickle.dump(obj, fo, pickle_protocol) - # TODO: This flush() method call turns into a flush() system call - # We'll need to wrap this as well, but we would do this by writing - #a context manager for our own open() method which returns an object - # in fo which makes the gluster API call. - fo.flush() - do_fsync(fo) - do_rename(tmppath, dest) - - # The following dir_xxx calls should definitely be replaced # with a Metadata class to encapsulate their implementation. # :FIXME: For now we have them as functions, but we should @@ -557,11 +537,3 @@ def rmobjdir(dir_path): raise else: return True - - -# Over-ride Swift's utils.write_pickle with ours -# -# FIXME: Is this even invoked anymore given we don't perform container or -# account updates? -import swift.common.utils -swift.common.utils.write_pickle = write_pickle |