diff options
| author | Peter Portante <peter.portante@redhat.com> | 2012-11-14 11:31:43 -0500 |
|---|---|---|
| committer | Vijay Bellur <vbellur@redhat.com> | 2012-12-11 04:53:27 -0800 |
| commit | 29c23f20a6213affb646c322b7219a0f7c5c3dfc (patch) | |
| tree | 86fa1c87d9d3d1b3baa18546973d41bbc4a4a101 /ufo/gluster | |
| parent | 5886396d969a317cfd46e3da82543b0868325acc (diff) | |
object-storage: use temp file optimization for pkl
Override OpenStack Swift's swift.common.utils.write_pickle with our own
implementation that uses the GlusterFS temporary file operation. A file name
'.<FILENAME>.<RANDOM>' will hash to the same GlusterFS node as a file named
'<FILENAME>', those avoiding a move of the file on a rename.
This is part of the work needed to address BZ 876660
(https://bugzilla.redhat.com/show_bug.cgi?id=876660).
Change-Id: I1cb9f97f289ab2ca76ec9221366df74de08268bb
BUG: 876660
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4224
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Mohammed Junaid <junaid@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Diffstat (limited to 'ufo/gluster')
| -rw-r--r-- | ufo/gluster/swift/common/utils.py | 31 | ||||
| -rw-r--r-- | ufo/gluster/swift/obj/server.py | 1 |
2 files changed, 32 insertions, 0 deletions
diff --git a/ufo/gluster/swift/common/utils.py b/ufo/gluster/swift/common/utils.py index 56376f8eed5..d35abe5825b 100644 --- a/ufo/gluster/swift/common/utils.py +++ b/ufo/gluster/swift/common/utils.py @@ -17,6 +17,7 @@ import logging import os import errno import xattr +import random from hashlib import md5 import cPickle as pickle from ConfigParser import ConfigParser, NoSectionError, NoOptionError @@ -455,3 +456,33 @@ def create_container_metadata(cont_path, memcache=None): def create_account_metadata(acc_path, memcache=None): metadata = get_account_metadata(acc_path, memcache) return restore_metadata(acc_path, metadata) + +def write_pickle(obj, dest, tmp=None, pickle_protocol=0): + """ + Ensure that a pickle file gets written to disk. The file is first written + to a tmp file location in the destination directory path, ensured it is + synced to disk, then moved to its final destination name. + + This version takes advantage of Gluster's dot-prefix-dot-suffix naming + where the a file named ".thefile.name.9a7aasv" is hashed to the same + Gluster node as "thefile.name". This ensures the renaming of a temp file + once written does not move it to another Gluster node. + + :param obj: python object to be pickled + :param dest: path of final destination file + :param tmp: path to tmp to use, defaults to None (ignored) + :param pickle_protocol: protocol to pickle the obj with, defaults to 0 + """ + dirname = os.path.dirname(dest) + basename = os.path.basename(dest) + tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest() + tmppath = os.path.join(dirname, tmpname) + with open(tmppath, 'wb') as fo: + pickle.dump(obj, fo, pickle_protocol) + fo.flush() + os.fsync(fo) + os.rename(tmppath, dest) + +# Over-ride Swift's utils.write_pickle with ours +import swift.common.utils +swift.common.utils.write_pickle = write_pickle diff --git a/ufo/gluster/swift/obj/server.py b/ufo/gluster/swift/obj/server.py index 43cdd8890d2..1c2b6cb1d7f 100644 --- a/ufo/gluster/swift/obj/server.py +++ b/ufo/gluster/swift/obj/server.py @@ -18,6 +18,7 @@ # Simply importing this monkey patches the constraint handling to fit our # needs import gluster.swift.common.constraints +import gluster.swift.common.utils from swift.obj import server from gluster.swift.common.DiskFile import Gluster_DiskFile |