path: root/gluster/swift/common/
diff options
authorPeter Portante <>2013-09-20 11:25:04 -0400
committerLuis Pabon <>2013-10-29 07:25:59 -0700
commit100d6b01bd40d8b01335e5ecd4a592df79e75b63 (patch)
treefb6875e07b37808b56aa2f0078040c9ef480a58e /gluster/swift/common/
parent70a4cef96c42fe30e62d23ae816a330d3d35df63 (diff)
Rebase to lastest OpenStack Swift DiskFile API
As of October 28th, 2013, we rebase to OpenStack Swift master (commit 4bfe674) to pick up the lastest officially supported DiskFile API changes. We use a snapshot of OpenStack Swift stored in the gluster-swift launchpad downloads area so that we can deliberately rebase at our own pace. With this refactoring, all the code for handling I/O is wholly contained in the swift tree for object operations. This will allow us to use a different fs_utils implementation in the future (for example, one based on a yet-to-be-implemented python bindings over libgfapi). This also means the "Fake_file" class has been removed entirely. Change-Id: I767983f88c59786e30b6c64da16d1cb6ab3c3e7f Signed-off-by: Peter Portante <> Reviewed-on: Reviewed-by: Luis Pabon <> Tested-by: Luis Pabon <>
Diffstat (limited to 'gluster/swift/common/')
1 files changed, 17 insertions, 45 deletions
diff --git a/gluster/swift/common/ b/gluster/swift/common/
index 5d4b6a4..6773836 100644
--- a/gluster/swift/common/
+++ b/gluster/swift/common/
@@ -17,15 +17,13 @@ import os
import stat
import errno
import xattr
-import random
import logging
from hashlib import md5
from eventlet import sleep
import cPickle as pickle
-from swift.common.utils import normalize_timestamp
from gluster.swift.common.exceptions import GlusterFileSystemIOError
-from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \
- do_stat, do_fstat, do_listdir, do_walk, do_rmdir
+from gluster.swift.common.fs_utils import os_path, do_stat, do_listdir, \
+ do_walk, do_rmdir, do_fstat
from gluster.swift.common import Glusterfs
X_CONTENT_TYPE = 'Content-Type'
@@ -56,6 +54,21 @@ PICKLE_PROTOCOL = 2
CHUNK_SIZE = 65536
+def normalize_timestamp(timestamp):
+ """
+ Format a timestamp (string or numeric) into a standardized
+ xxxxxxxxxx.xxxxx (10.5) format.
+ Note that timestamps using values greater than or equal to November 20th,
+ 2286 at 17:46 UTC will use 11 digits to represent the number of
+ seconds.
+ :param timestamp: unix timestamp
+ :returns: normalized timestamp as a string
+ """
+ return "%016.05f" % (float(timestamp))
def read_metadata(path_or_fd):
Helper function to read the pickled metadata from a File/Directory.
@@ -207,7 +220,6 @@ def validate_account(metadata):
def validate_object(metadata):
if not metadata:
- logging.warn('validate_object: No metadata')
return False
if X_TIMESTAMP not in metadata.keys() or \
@@ -451,38 +463,6 @@ def create_account_metadata(acc_path):
return rmd
-def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
- """
- Ensure that a pickle file gets written to disk. The file is first written
- to a tmp file location in the destination directory path, ensured it is
- synced to disk, then moved to its final destination name.
- This version takes advantage of Gluster's dot-prefix-dot-suffix naming
- where the a file named "" is hashed to the same
- Gluster node as "". This ensures the renaming of a temp file
- once written does not move it to another Gluster node.
- :param obj: python object to be pickled
- :param dest: path of final destination file
- :param tmp: path to tmp to use, defaults to None (ignored)
- :param pickle_protocol: protocol to pickle the obj with, defaults to 0
- """
- dirname = os.path.dirname(dest)
- basename = os.path.basename(dest)
- tmpname = '.' + basename + '.' + \
- md5(basename + str(random.random())).hexdigest()
- tmppath = os.path.join(dirname, tmpname)
- with open(tmppath, 'wb') as fo:
- pickle.dump(obj, fo, pickle_protocol)
- # TODO: This flush() method call turns into a flush() system call
- # We'll need to wrap this as well, but we would do this by writing
- #a context manager for our own open() method which returns an object
- # in fo which makes the gluster API call.
- fo.flush()
- do_fsync(fo)
- do_rename(tmppath, dest)
# The following dir_xxx calls should definitely be replaced
# with a Metadata class to encapsulate their implementation.
# :FIXME: For now we have them as functions, but we should
@@ -557,11 +537,3 @@ def rmobjdir(dir_path):
return True
-# Over-ride Swift's utils.write_pickle with ours
-# FIXME: Is this even invoked anymore given we don't perform container or
-# account updates?
-import swift.common.utils
-swift.common.utils.write_pickle = write_pickle