Rebase to lastest OpenStack Swift DiskFile API

As of October 28th, 2013, we rebase to OpenStack Swift master (commit
4bfe674) to pick up the lastest officially supported DiskFile API
changes. We use a snapshot of OpenStack Swift stored in the
gluster-swift launchpad downloads area so that we can deliberately
rebase at our own pace.

With this refactoring, all the code for handling I/O is wholly
contained in the swift tree for object operations. This will allow us
to use a different fs_utils implementation in the future (for example,
one based on a yet-to-be-implemented python bindings over libgfapi).
This also means the "Fake_file" class has been removed entirely.

Change-Id: I767983f88c59786e30b6c64da16d1cb6ab3c3e7f
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/5993
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>

2 files changed, 68 insertions, 93 deletions

diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py
index 8b26fd0..199984a 100644
--- a/gluster/swift/common/fs_utils.py
+++ b/gluster/swift/common/fs_utils.py
@@ -18,42 +18,39 @@ import os
 import errno
 import stat
 import random
-import os.path as os_path    # noqa
+import ctypes
+import os.path as _os_path
 from eventlet import sleep
+from swift.common.utils import load_libc_function
 from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
-    NotDirectoryError, GlusterFileSystemOSError, GlusterFileSystemIOError
+    NotDirectoryError, GlusterFileSystemOSError
 
 
-class Fake_file(object):
-    def __init__(self, path):
-        self.path = path
-
-    def tell(self):
-        return 0
-
-    def read(self, count):
-        return None
-
-    def fileno(self):
-        return -1
-
-    def close(self):
-        pass
+os_path = _os_path
 
 
 def do_walk(*args, **kwargs):
     return os.walk(*args, **kwargs)
 
 
-def do_write(fd, msg):
+def do_write(fd, buf):
     try:
-        cnt = os.write(fd, msg)
+        cnt = os.write(fd, buf)
     except OSError as err:
         raise GlusterFileSystemOSError(
             err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd))
     return cnt
 
 
+def do_read(fd, n):
+    try:
+        buf = os.read(fd, n)
+    except OSError as err:
+        raise GlusterFileSystemOSError(
+            err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd))
+    return buf
+
+
 def do_ismount(path):
     """
     Test whether a path is a mount point.
@@ -203,37 +200,21 @@ def do_fstat(fd):
 
 
 def do_open(path, flags, **kwargs):
-    if isinstance(flags, int):
-        try:
-            fd = os.open(path, flags, **kwargs)
-        except OSError as err:
-            raise GlusterFileSystemOSError(
-                err.errno, '%s, os.open("%s", %x, %r)' % (
-                    err.strerror, path, flags, kwargs))
-        return fd
-    else:
-        try:
-            fp = open(path, flags, **kwargs)
-        except IOError as err:
-            raise GlusterFileSystemIOError(
-                err.errno, '%s, open("%s", %s, %r)' % (
-                    err.strerror, path, flags, kwargs))
-        return fp
+    try:
+        fd = os.open(path, flags, **kwargs)
+    except OSError as err:
+        raise GlusterFileSystemOSError(
+            err.errno, '%s, os.open("%s", %x, %r)' % (
+                err.strerror, path, flags, kwargs))
+    return fd
 
 
 def do_close(fd):
-    if isinstance(fd, file) or isinstance(fd, Fake_file):
-        try:
-            fd.close()
-        except IOError as err:
-            raise GlusterFileSystemIOError(
-                err.errno, '%s, os.close(%s)' % (err.strerror, fd))
-    else:
-        try:
-            os.close(fd)
-        except OSError as err:
-            raise GlusterFileSystemOSError(
-                err.errno, '%s, os.close(%s)' % (err.strerror, fd))
+    try:
+        os.close(fd)
+    except OSError as err:
+        raise GlusterFileSystemOSError(
+            err.errno, '%s, os.close(%s)' % (err.strerror, fd))
 
 
 def do_unlink(path, log=True):
@@ -268,9 +249,31 @@ def do_fsync(fd):
 def do_fdatasync(fd):
     try:
         os.fdatasync(fd)
+    except AttributeError:
+        do_fsync(fd)
     except OSError as err:
         raise GlusterFileSystemOSError(
-            err.errno, '%s, os.fdatasync("%s")' % (err.strerror, fd))
+            err.errno, '%s, os.fsync("%s")' % (err.strerror, fd))
+
+
+_posix_fadvise = None
+
+
+def do_fadvise64(fd, offset, length):
+    global _posix_fadvise
+    if _posix_fadvise is None:
+        _posix_fadvise = load_libc_function('posix_fadvise64')
+    # 4 means "POSIX_FADV_DONTNEED"
+    _posix_fadvise(fd, ctypes.c_uint64(offset),
+                   ctypes.c_uint64(length), 4)
+
+
+def do_lseek(fd, pos, how):
+    try:
+        os.lseek(fd, pos, how)
+    except OSError as err:
+        raise GlusterFileSystemOSError(
+            err.errno, '%s, os.fsync("%s")' % (err.strerror, fd))
 
 
 def mkdirs(path):
diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index 5d4b6a4..6773836 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -17,15 +17,13 @@ import os
 import stat
 import errno
 import xattr
-import random
 import logging
 from hashlib import md5
 from eventlet import sleep
 import cPickle as pickle
-from swift.common.utils import normalize_timestamp
 from gluster.swift.common.exceptions import GlusterFileSystemIOError
-from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \
-    do_stat, do_fstat, do_listdir, do_walk, do_rmdir
+from gluster.swift.common.fs_utils import os_path, do_stat, do_listdir, \
+    do_walk, do_rmdir, do_fstat
 from gluster.swift.common import Glusterfs
 
 X_CONTENT_TYPE = 'Content-Type'
@@ -56,6 +54,21 @@ PICKLE_PROTOCOL = 2
 CHUNK_SIZE = 65536
 
 
+def normalize_timestamp(timestamp):
+    """
+    Format a timestamp (string or numeric) into a standardized
+    xxxxxxxxxx.xxxxx (10.5) format.
+
+    Note that timestamps using values greater than or equal to November 20th,
+    2286 at 17:46 UTC will use 11 digits to represent the number of
+    seconds.
+
+    :param timestamp: unix timestamp
+    :returns: normalized timestamp as a string
+    """
+    return "%016.05f" % (float(timestamp))
+
+
 def read_metadata(path_or_fd):
     """
     Helper function to read the pickled metadata from a File/Directory.
@@ -207,7 +220,6 @@ def validate_account(metadata):
 
 def validate_object(metadata):
     if not metadata:
-        logging.warn('validate_object: No metadata')
         return False
 
     if X_TIMESTAMP not in metadata.keys() or \
@@ -451,38 +463,6 @@ def create_account_metadata(acc_path):
     return rmd
 
 
-def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
-    """
-    Ensure that a pickle file gets written to disk.  The file is first written
-    to a tmp file location in the destination directory path, ensured it is
-    synced to disk, then moved to its final destination name.
-
-    This version takes advantage of Gluster's dot-prefix-dot-suffix naming
-    where the a file named ".thefile.name.9a7aasv" is hashed to the same
-    Gluster node as "thefile.name". This ensures the renaming of a temp file
-    once written does not move it to another Gluster node.
-
-    :param obj: python object to be pickled
-    :param dest: path of final destination file
-    :param tmp: path to tmp to use, defaults to None (ignored)
-    :param pickle_protocol: protocol to pickle the obj with, defaults to 0
-    """
-    dirname = os.path.dirname(dest)
-    basename = os.path.basename(dest)
-    tmpname = '.' + basename + '.' + \
-        md5(basename + str(random.random())).hexdigest()
-    tmppath = os.path.join(dirname, tmpname)
-    with open(tmppath, 'wb') as fo:
-        pickle.dump(obj, fo, pickle_protocol)
-        # TODO: This flush() method call turns into a flush() system call
-        # We'll need to wrap this as well, but we would do this by writing
-        #a context manager for our own open() method which returns an object
-        # in fo which makes the gluster API call.
-        fo.flush()
-        do_fsync(fo)
-    do_rename(tmppath, dest)
-
-
 # The following dir_xxx calls should definitely be replaced
 # with a Metadata class to encapsulate their implementation.
 # :FIXME: For now we have them as functions, but we should
@@ -557,11 +537,3 @@ def rmobjdir(dir_path):
         raise
     else:
         return True
-
-
-# Over-ride Swift's utils.write_pickle with ours
-#
-# FIXME: Is this even invoked anymore given we don't perform container or
-# account updates?
-import swift.common.utils
-swift.common.utils.write_pickle = write_pickle