Rebase to stable/kilo

This change ports most of swiftonfile object server fixes and changes
into gluster-swift. Storage policy as a feature is not usable here
(it doesn't make sense).

The hacky way of creating zero byte tracker objects for object
expiration has not been ported to this release due to scalability
issues and the need to have a separate volume.

Change-Id: I17ba27dacea9ac000bdb8934700996e4d17f4251
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Reviewed-on: http://review.gluster.org/13269
Reviewed-by: Thiago da Silva <thiago@redhat.com>
Tested-by: Thiago da Silva <thiago@redhat.com>

1 files changed, 40 insertions, 1 deletions

diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index b6a5a09..e6f4bcc 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -17,6 +17,7 @@ import os
 import stat
 import json
 import errno
+import random
 import logging
 from hashlib import md5
 from eventlet import sleep
@@ -29,7 +30,7 @@ from swift.common.db import utf8encodekeys
 from gluster.swift.common.fs_utils import do_getctime, do_getmtime, do_stat, \
     do_listdir, do_walk, do_rmdir, do_log_rl, get_filename_from_fd, do_open, \
     do_isdir, do_getsize, do_getxattr, do_setxattr, do_removexattr, do_read, \
-    do_close, do_dup, do_lseek, do_fstat
+    do_close, do_dup, do_lseek, do_fstat, do_fsync, do_rename
 from gluster.swift.common import Glusterfs
 
 X_CONTENT_TYPE = 'Content-Type'
@@ -607,3 +608,41 @@ def rmobjdir(dir_path):
         raise
     else:
         return True
+
+
+def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
+    """
+    Ensure that a pickle file gets written to disk.  The file is first written
+    to a tmp file location in the destination directory path, ensured it is
+    synced to disk, then moved to its final destination name.
+
+    This version takes advantage of Gluster's dot-prefix-dot-suffix naming
+    where the a file named ".thefile.name.9a7aasv" is hashed to the same
+    Gluster node as "thefile.name". This ensures the renaming of a temp file
+    once written does not move it to another Gluster node.
+
+    :param obj: python object to be pickled
+    :param dest: path of final destination file
+    :param tmp: path to tmp to use, defaults to None (ignored)
+    :param pickle_protocol: protocol to pickle the obj with, defaults to 0
+    """
+    dirname = os.path.dirname(dest)
+    # Create destination directory
+    try:
+        os.makedirs(dirname)
+    except OSError as err:
+        if err.errno != errno.EEXIST:
+            raise
+    basename = os.path.basename(dest)
+    tmpname = '.' + basename + '.' + \
+        md5(basename + str(random.random())).hexdigest()
+    tmppath = os.path.join(dirname, tmpname)
+    with open(tmppath, 'wb') as fo:
+        pickle.dump(obj, fo, pickle_protocol)
+        # TODO: This flush() method call turns into a flush() system call
+        # We'll need to wrap this as well, but we would do this by writing
+        # a context manager for our own open() method which returns an object
+        # in fo which makes the gluster API call.
+        fo.flush()
+        do_fsync(fo)
+    do_rename(tmppath, dest)