diff options
16 files changed, 429 insertions, 495 deletions
diff --git a/swift/1.4.8/gluster-swift-plugin.spec b/swift/1.4.8/gluster-swift-plugin.spec index 181666a3608..50731bc844f 100644 --- a/swift/1.4.8/gluster-swift-plugin.spec +++ b/swift/1.4.8/gluster-swift-plugin.spec @@ -13,7 +13,7 @@ %define _confdir /etc/swift %define _swiftdir /usr/lib/python2.6/site-packages/swift %define _ufo_version 1.0 -%define _ufo_release 5 +%define _ufo_release 6 Summary : GlusterFS Unified File and Object Storage. Name : gluster-swift-plugin @@ -51,10 +51,10 @@ cp DiskFile.py %{buildroot}/%{_swiftdir}/plugins cp Glusterfs.py %{buildroot}/%{_swiftdir}/plugins cp __init__.py %{buildroot}/%{_swiftdir}/plugins cp utils.py %{buildroot}/%{_swiftdir}/plugins +cp fs_utils.py %{buildroot}/%{_swiftdir}/plugins cp middleware/__init__.py %{buildroot}/%{_swiftdir}/plugins/middleware cp middleware/gluster.py %{buildroot}/%{_swiftdir}/plugins/middleware -cp middleware/glusterfs.py %{buildroot}/%{_swiftdir}/plugins/middleware cp -r conf/* %{buildroot}/%{_confdir}/ diff --git a/swift/1.4.8/gluster-swift.spec b/swift/1.4.8/gluster-swift.spec index 640b944e8cb..47fafcb3203 100644 --- a/swift/1.4.8/gluster-swift.spec +++ b/swift/1.4.8/gluster-swift.spec @@ -4,7 +4,7 @@ Name: gluster-swift Version: 1.4.8 -Release: 3%{?dist} +Release: 6%{?dist} Summary: OpenStack Object Storage (swift) Group: Development/Languages diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py index e8024768d8b..e057dff956d 100644 --- a/swift/1.4.8/plugins/DiskDir.py +++ b/swift/1.4.8/plugins/DiskDir.py @@ -28,15 +28,13 @@ from swift.plugins.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP,\ X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ X_CONTAINER_COUNT, CONTAINER -from swift import plugins -def strip_obj_storage_path(path, string='/mnt/gluster-object'): - """ - strip /mnt/gluster-object - """ - return path.replace(string, '').strip('/') - DATADIR = 'containers' +# Create a dummy db_file in /etc/swift +_db_file = '/etc/swift/db_file.db' +if not os.path.exists(_db_file): + file(_db_file, 'w+') + def _read_metadata(dd): """ Filter read metadata so that it always returns a tuple that includes @@ -148,7 +146,7 @@ class DiskDir(DiskCommon): """ def __init__(self, path, device, partition, account, container, logger, - uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object=None): + uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path device = account if container: @@ -162,15 +160,12 @@ class DiskDir(DiskCommon): self.account = account self.device_path = os.path.join(path, device) if not check_mount(path, device): - check_valid_account(account, fs_object) + check_valid_account(account) self.logger = logger self.metadata = {} self.uid = int(uid) self.gid = int(gid) - # Create a dummy db_file in /etc/swift - self.db_file = '/etc/swift/db_file.db' - if not os.path.exists(self.db_file): - file(self.db_file, 'w+') + self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: @@ -319,9 +314,10 @@ class DiskDir(DiskCommon): for obj in objects: list_item = [] list_item.append(obj) - metadata = read_metadata(self.datadir + '/' + obj) + obj_path = os.path.join(self.datadir, obj) + metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): - metadata = create_object_metadata(self.datadir + '/' + obj) + metadata = create_object_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) @@ -420,12 +416,12 @@ class DiskDir(DiskCommon): class DiskAccount(DiskDir): - def __init__(self, root, account, fs_object = None): + def __init__(self, root, account): self.root = root self.account = account self.datadir = os.path.join(self.root, self.account) if not check_mount(root, account): - check_valid_account(account, fs_object) + check_valid_account(account) self.metadata = _read_metadata(self.datadir) if not self.metadata or not validate_account(self.metadata): self.metadata = create_account_metadata(self.datadir) @@ -472,9 +468,10 @@ class DiskAccount(DiskDir): list_item = [] metadata = None list_item.append(cont) - metadata = _read_metadata(self.datadir + '/' + cont) + cont_path = os.path.join(self.datadir, cont) + metadata = _read_metadata(cont_path) if not metadata or not validate_container(metadata): - metadata = create_container_metadata(self.datadir + '/' + cont) + metadata = create_container_metadata(cont_path) if metadata: list_item.append(metadata[X_OBJECTS_COUNT][0]) diff --git a/swift/1.4.8/plugins/DiskFile.py b/swift/1.4.8/plugins/DiskFile.py index 6f77eaaa57a..f4ca366b236 100644 --- a/swift/1.4.8/plugins/DiskFile.py +++ b/swift/1.4.8/plugins/DiskFile.py @@ -55,7 +55,7 @@ class Gluster_DiskFile(DiskFile): def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, - uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object = None): + uid=DEFAULT_UID, gid=DEFAULT_GID): self.disk_chunk_size = disk_chunk_size device = account #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc @@ -75,7 +75,7 @@ class Gluster_DiskFile(DiskFile): self.device_path = os.path.join(path, device) if not check_mount(path, device): - check_valid_account(account, fs_object) + check_valid_account(account) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') diff --git a/swift/1.4.8/plugins/Glusterfs.py b/swift/1.4.8/plugins/Glusterfs.py index 5e191e1bd8f..69902d85d48 100644 --- a/swift/1.4.8/plugins/Glusterfs.py +++ b/swift/1.4.8/plugins/Glusterfs.py @@ -17,115 +17,124 @@ import os, fcntl, time from ConfigParser import ConfigParser from swift.common.utils import TRUE_VALUES from hashlib import md5 -from swift.plugins.utils import mkdirs - -class Glusterfs(object): - def __init__(self): - self.name = 'glusterfs' - self.fs_conf = ConfigParser() - self.fs_conf.read(os.path.join('/etc/swift', 'fs.conf')) - self.mount_path = self.fs_conf.get('DEFAULT', 'mount_path', '/mnt/gluster-object') - self.auth_account = self.fs_conf.get('DEFAULT', 'auth_account', 'auth') - self.mount_ip = self.fs_conf.get('DEFAULT', 'mount_ip', 'localhost') - self.remote_cluster = self.fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES - self.object_only = self.fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES - - def busy_wait(self, mount_path): - # Iterate for definite number of time over a given - # interval for successful mount - for i in range(0, 5): - if os.path.ismount(os.path.join(mount_path)): - return True - time.sleep(2) - return False - - def mount(self, account): - mount_path = os.path.join(self.mount_path, account) - export = self.get_export_from_account_id(account) - - pid_dir = "/var/lib/glusterd/vols/%s/run/" %export - pid_file = os.path.join(pid_dir, 'swift.pid'); - - if not os.path.exists(pid_dir): - mkdirs(pid_dir) - - fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) - with os.fdopen(fd, 'r+b') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) - except: - ex = sys.exc_info()[1] - if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): - # This means that some other process is mounting the - # filesystem, so wait for the mount process to complete - return self.busy_wait(mount_path) - - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (self.mount_ip, export, \ - mount_path) - if os.system(mnt_cmd) or not self.busy_wait(mount_path): - raise Exception('Mount failed %s: %s' % (self.name, mnt_cmd)) - return False - return True - - def unmount(self, mount_path): - umnt_cmd = 'umount %s 2>> /dev/null' % mount_path - if os.system(umnt_cmd): - logging.error('Unable to unmount %s %s' % (mount_path, self.name)) - - def get_export_list_local(self): - export_list = [] - cmnd = 'gluster volume info' - - if os.system(cmnd + ' >> /dev/null'): - raise Exception('Getting volume failed %s', self.name) - return export_list - - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) +from swift.plugins.fs_utils import mkdirs - return export_list +# +# Read the fs.conf file once at startup (module load) +# +_fs_conf = ConfigParser() +MOUNT_PATH = '/mnt/gluster-object' +AUTH_ACCOUNT = 'auth' +MOUNT_IP = 'localhost' +REMOTE_CLUSTER = False +OBJECT_ONLY = False +if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): + try: + MOUNT_PATH = _fs_conf.get('DEFAULT', 'mount_path', '/mnt/gluster-object') + except (NoSectionError, NoOptionError): + pass + try: + AUTH_ACCOUNT = _fs_conf.get('DEFAULT', 'auth_account', 'auth') + except (NoSectionError, NoOptionError): + pass + try: + MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') + except (NoSectionError, NoOptionError): + pass + try: + REMOTE_CLUSTER = _fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + try: + OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass +NAME = 'glusterfs' + + +def strip_obj_storage_path(path, mp=MOUNT_PATH): + """ + strip the mount path off, also stripping the leading and trailing slashes + """ + return path.replace(mp, '').strip(os.path.sep) + +def _busy_wait(full_mount_path): + # Iterate for definite number of time over a given + # interval for successful mount + for i in range(0, 5): + if os.path.ismount(os.path.join(full_mount_path)): + return True + time.sleep(2) + return False + +def mount(account): + global mount_path, mount_ip + + full_mount_path = os.path.join(mount_path, account) + export = get_export_from_account_id(account) + + pid_dir = "/var/lib/glusterd/vols/%s/run/" % export + pid_file = os.path.join(pid_dir, 'swift.pid'); + + if not os.path.exists(pid_dir): + mkdirs(pid_dir) + + fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) + with os.fdopen(fd, 'r+b') as f: + try: + fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) + except: + ex = sys.exc_info()[1] + if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): + # This means that some other process is mounting the + # filesystem, so wait for the mount process to complete + return _busy_wait(full_mount_path) + + mnt_cmd = 'mount -t glusterfs %s:%s %s' % (mount_ip, export, \ + full_mount_path) + if os.system(mnt_cmd) or not _busy_wait(full_mount_path): + raise Exception('Mount failed %s: %s' % (NAME, mnt_cmd)) + return True + +def unmount(full_mount_path): + umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path + if os.system(umnt_cmd): + logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) + +def get_export_list(): + global mount_ip + + if remote_cluster: + cmnd = 'ssh %s gluster volume info' % mount_ip + else: + cmnd = 'gluster volume info' - def get_export_list_remote(self): - export_list = [] - cmnd = 'ssh %s gluster volume info' % self.mount_ip - - if os.system(cmnd + ' >> /dev/null'): + if os.system(cmnd + ' >> /dev/null'): + if remove_cluster: raise Exception('Getting volume info failed %s, make sure to have \ - passwordless ssh on %s', self.name, self.mount_ip) - return export_list - - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) - - return export_list - - def get_export_list(self): - if self.remote_cluster: - return self.get_export_list_remote() + passwordless ssh on %s', NAME, mount_ip) else: - return self.get_export_list_local() + raise Exception('Getting volume failed %s', NAME) + + export_list = [] + fp = os.popen(cmnd) + while True: + item = fp.readline() + if not item: + break + item = item.strip('\n').strip(' ') + if item.lower().startswith('volume name:'): + export_list.append(item.split(':')[1].strip(' ')) + + return export_list - def get_export_from_account_id(self, account): - if not account: - print 'account is none, returning' - raise AttributeError +def get_export_from_account_id(account): + if not account: + raise ValueError('No account given') - for export in self.get_export_list(): - if account == 'AUTH_' + export: - return export + for export in get_export_list(): + if account == 'AUTH_' + export: + return export - raise Exception('No export found %s %s' % (account, self.name)) - return None + raise Exception('No export found %s %s' % (account, NAME)) diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf b/swift/1.4.8/plugins/conf/account-server/1.conf-gluster index a82bc35eaa4..911e2837f50 100644 --- a/swift/1.4.8/plugins/conf/account-server/1.conf +++ b/swift/1.4.8/plugins/conf/account-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs account-server +pipeline = account-server [app:account-server] use = egg:swift#account -[filter:glusterfs] -use = egg:swift#glusterfs - [account-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf b/swift/1.4.8/plugins/conf/container-server/1.conf-gluster index e6d313c3872..eb5650afa74 100644 --- a/swift/1.4.8/plugins/conf/container-server/1.conf +++ b/swift/1.4.8/plugins/conf/container-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs container-server +pipeline = container-server [app:container-server] use = egg:swift#container -[filter:glusterfs] -use = egg:swift#glusterfs - [container-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/fs.conf b/swift/1.4.8/plugins/conf/fs.conf-gluster index b6ec5121f9f..b6ec5121f9f 100644 --- a/swift/1.4.8/plugins/conf/fs.conf +++ b/swift/1.4.8/plugins/conf/fs.conf-gluster diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf b/swift/1.4.8/plugins/conf/object-server/1.conf-gluster index 665aa94be8b..13dda69bae2 100644 --- a/swift/1.4.8/plugins/conf/object-server/1.conf +++ b/swift/1.4.8/plugins/conf/object-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs object-server +pipeline = object-server [app:object-server] use = egg:swift#object -[filter:glusterfs] -use = egg:swift#glusterfs - [object-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf b/swift/1.4.8/plugins/conf/proxy-server.conf-gluster index 9bec6425355..9bec6425355 100644 --- a/swift/1.4.8/plugins/conf/proxy-server.conf +++ b/swift/1.4.8/plugins/conf/proxy-server.conf-gluster diff --git a/swift/1.4.8/plugins/conf/swift.conf b/swift/1.4.8/plugins/conf/swift.conf-gluster index e506b6f54a8..e506b6f54a8 100644 --- a/swift/1.4.8/plugins/conf/swift.conf +++ b/swift/1.4.8/plugins/conf/swift.conf-gluster diff --git a/swift/1.4.8/plugins/fs_utils.py b/swift/1.4.8/plugins/fs_utils.py new file mode 100644 index 00000000000..76fcbbc2b9c --- /dev/null +++ b/swift/1.4.8/plugins/fs_utils.py @@ -0,0 +1,157 @@ +# Copyright (c) 2011 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno + +def do_mkdir(path): + try: + os.mkdir(path) + except Exception, err: + logging.exception("Mkdir failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_makedirs(path): + try: + os.makedirs(path) + except Exception, err: + logging.exception("Makedirs failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_listdir(path): + try: + buf = os.listdir(path) + except Exception, err: + logging.exception("Listdir failed on %s err: %s", path, str(err)) + raise + return buf + +def do_chown(path, uid, gid): + try: + os.chown(path, uid, gid) + except Exception, err: + logging.exception("Chown failed on %s err: %s", path, str(err)) + raise + return True + +def do_stat(path): + try: + #Check for fd. + if isinstance(path, int): + buf = os.fstat(path) + else: + buf = os.stat(path) + except Exception, err: + logging.exception("Stat failed on %s err: %s", path, str(err)) + raise + + return buf + +def do_open(path, mode): + try: + fd = open(path, mode) + except Exception, err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + return fd + +def do_close(fd): + #fd could be file or int type. + try: + if isinstance(fd, int): + os.close(fd) + else: + fd.close() + except Exception, err: + logging.exception("Close failed on %s err: %s", fd, str(err)) + raise + return True + +def do_unlink(path, log = True): + try: + os.unlink(path) + except Exception, err: + if log: + logging.exception("Unlink failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rmdir(path): + try: + os.rmdir(path) + except Exception, err: + logging.exception("Rmdir failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rename(old_path, new_path): + try: + os.rename(old_path, new_path) + except Exception, err: + logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ + str(err)) + raise + return True + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + try: + do_makedirs(path) + except OSError, err: + #TODO: check, isdir will fail if mounted and volume stopped. + #if err.errno != errno.EEXIST or not os.path.isdir(path) + if err.errno != errno.EEXIST: + raise + +def dir_empty(path): + """ + Return true if directory/container is empty. + :param path: Directory path. + :returns: True/False. + """ + if os.path.isdir(path): + try: + files = do_listdir(path) + except Exception, err: + logging.exception("listdir failed on %s err: %s", path, str(err)) + raise + if not files: + return True + else: + return False + else: + if not os.path.exists(path): + return True + +def rmdirs(path): + if os.path.isdir(path) and dir_empty(path): + do_rmdir(path) + else: + logging.error("rmdirs failed dir may not be empty or not valid dir") + return False + diff --git a/swift/1.4.8/plugins/middleware/glusterfs.py b/swift/1.4.8/plugins/middleware/glusterfs.py deleted file mode 100644 index 7472146406c..00000000000 --- a/swift/1.4.8/plugins/middleware/glusterfs.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Monkey patch constraints -import swift.plugins.constraints -from swift.plugins.Glusterfs import Glusterfs -from ConfigParser import ConfigParser - -fs_conf = ConfigParser() -if fs_conf.read('/etc/swift/fs.conf'): - try: - mount_path = fs_conf.get ('DEFAULT', 'mount_path') - except NoSectionError, NoOptionError: - # FIXME - How to log during module initialization - logger.exception(_('ERROR mount_path not present')) - mount_path = '' -else: - mount_path = '' - - -class Gluster(object): - """ - Update the environment with keys that reflect GlusterFS middleware enabled - """ - def __init__(self, app, conf): - self.app = app - self.conf = conf - - def __call__(self, env, start_response): - env['Gluster_enabled'] = True - env['fs_object'] = Glusterfs() - env['root'] = mount_path - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def gluster_filter(app): - return Gluster(app, conf) - return gluster_filter diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py index a1ac64db5c9..bc16ff6f738 100644 --- a/swift/1.4.8/plugins/utils.py +++ b/swift/1.4.8/plugins/utils.py @@ -18,10 +18,11 @@ import os import errno import xattr from hashlib import md5 -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from swift.obj.server import ASYNCDIR import cPickle as pickle from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.utils import normalize_timestamp, TRUE_VALUES +from swift.plugins.fs_utils import * +from swift.plugins import Glusterfs X_CONTENT_TYPE = 'Content-Type' X_CONTENT_LENGTH = 'Content-Length' @@ -35,15 +36,14 @@ X_CONTAINER_COUNT = 'X-Container-Count' X_OBJECT_TYPE = 'X-Object-Type' DIR_TYPE = 'application/directory' ACCOUNT = 'Account' -MOUNT_PATH = '/mnt/gluster-object' METADATA_KEY = 'user.swift.metadata' MAX_XATTR_SIZE = 65536 CONTAINER = 'container' DIR = 'dir' MARKER_DIR = 'marker_dir' TEMP_DIR = 'tmp' +ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR FILE = 'file' -DIR_TYPE = 'application/directory' FILE_TYPE = 'application/octet-stream' OBJECT = 'Object' OBJECT_TYPE = 'application/octet-stream' @@ -56,131 +56,6 @@ MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' -def mkdirs(path): - """ - Ensures the path is a directory or makes it if not. Errors if the path - exists but is a file or on permissions failure. - - :param path: path to create - """ - if not os.path.isdir(path): - try: - do_makedirs(path) - except OSError, err: - #TODO: check, isdir will fail if mounted and volume stopped. - #if err.errno != errno.EEXIST or not os.path.isdir(path) - if err.errno != errno.EEXIST: - raise - -def rmdirs(path): - if os.path.isdir(path) and dir_empty(path): - do_rmdir(path) - else: - logging.error("rmdirs failed dir may not be empty or not valid dir") - return False - -def strip_obj_storage_path(path, string='/mnt/gluster-object'): - """ - strip /mnt/gluster-object - """ - return path.replace(string, '').strip('/') - -def do_mkdir(path): - try: - os.mkdir(path) - except Exception, err: - logging.exception("Mkdir failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_makedirs(path): - try: - os.makedirs(path) - except Exception, err: - logging.exception("Makedirs failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_listdir(path): - try: - buf = os.listdir(path) - except Exception, err: - logging.exception("Listdir failed on %s err: %s", path, str(err)) - raise - return buf - -def do_chown(path, uid, gid): - try: - os.chown(path, uid, gid) - except Exception, err: - logging.exception("Chown failed on %s err: %s", path, str(err)) - raise - return True - -def do_stat(path): - try: - #Check for fd. - if isinstance(path, int): - buf = os.fstat(path) - else: - buf = os.stat(path) - except Exception, err: - logging.exception("Stat failed on %s err: %s", path, str(err)) - raise - - return buf - -def do_open(path, mode): - try: - fd = open(path, mode) - except Exception, err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - return fd - -def do_close(fd): - #fd could be file or int type. - try: - if isinstance(fd, int): - os.close(fd) - else: - fd.close() - except Exception, err: - logging.exception("Close failed on %s err: %s", fd, str(err)) - raise - return True - -def do_unlink(path, log = True): - try: - os.unlink(path) - except Exception, err: - if log: - logging.exception("Unlink failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rmdir(path): - try: - os.rmdir(path) - except Exception, err: - logging.exception("Rmdir failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rename(old_path, new_path): - try: - os.rename(old_path, new_path) - except Exception, err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - str(err)) - raise - return True - - def read_metadata(path): """ Helper function to read the pickled metadata from a File/Directory. @@ -262,26 +137,6 @@ def clean_metadata(path): raise key += 1 -def dir_empty(path): - """ - Return true if directory/container is empty. - :param path: Directory path. - :returns: True/False. - """ - if os.path.isdir(path): - try: - files = do_listdir(path) - except Exception, err: - logging.exception("listdir failed on %s err: %s", path, str(err)) - raise - if not files: - return True - else: - return False - else: - if not os.path.exists(path): - return True - def get_device_from_account(account): if account.startswith(RESELLER_PREFIX): device = account.replace(RESELLER_PREFIX, '', 1) @@ -302,27 +157,26 @@ def check_user_xattr(path): #Remove xattr may fail in case of concurrent remove. return True -def _check_valid_account(account, fs_object): - mount_path = getattr(fs_object, 'mount_path', MOUNT_PATH) +def _check_valid_account(account): + full_mount_path = os.path.join(Glusterfs.MOUNT_PATH, account) - if os.path.ismount(os.path.join(mount_path, account)): + if os.path.ismount(full_mount_path): return True - if not check_account_exists(fs_object.get_export_from_account_id(account), fs_object): + if not Glusterfs.check_account_exists(Glusterfs.get_export_from_account_id(account)): logging.error('Account not present %s', account) return False - if not os.path.isdir(os.path.join(mount_path, account)): - mkdirs(os.path.join(mount_path, account)) + if not os.path.isdir(full_mount_path): + mkdirs(full_mount_path) - if fs_object: - if not fs_object.mount(account): - return False + if not Glusterfs.mount(account): + return False return True -def check_valid_account(account, fs_object): - return _check_valid_account(account, fs_object) +def check_valid_account(account): + return _check_valid_account(account) def validate_container(metadata): if not metadata: @@ -401,7 +255,7 @@ def is_marker(metadata): def _update_list(path, const_path, src_list, reg_file=True, object_count=0, bytes_used=0, obj_list=[]): - obj_path = strip_obj_storage_path(path, const_path) + obj_path = Glusterfs.strip_obj_storage_path(path, const_path) for i in src_list: if obj_path: @@ -460,7 +314,7 @@ def get_container_details(cont_path, memcache=None): """ mkey = '' if memcache: - mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + strip_obj_storage_path(cont_path) + mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(cont_path) cd = memcache.get(mkey) if cd: if not cd.dir_list: @@ -517,7 +371,7 @@ def get_account_details(acc_path, memcache=None): acc_stats = None mkey = '' if memcache: - mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + strip_obj_storage_path(acc_path) + mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(acc_path) ad = memcache.get(mkey) if ad: # FIXME: Do we really need to stat the file? If we are object @@ -627,18 +481,15 @@ def create_account_metadata(acc_path, memcache=None): metadata = get_account_metadata(acc_path, memcache) return restore_metadata(acc_path, metadata) -def check_account_exists(account, fs_object): - if account not in get_account_list(fs_object): +def check_account_exists(account): + if account not in get_account_list(): logging.warn('Account %s does not exist' % account) return False else: return True -def get_account_list(fs_object): - account_list = [] - if fs_object: - account_list = fs_object.get_export_list() - return account_list +def get_account_list(): + return Glusterfs.get_export_list() def get_account_id(account): return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest() @@ -647,10 +498,10 @@ def get_account_id(account): __swift_conf = ConfigParser() __swift_conf.read(os.path.join('/etc/swift', 'swift.conf')) try: - _plugin_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES + _gluster_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES except NoOptionError, NoSectionError: - _plugin_enabled = False + _gluster_enabled = False del __swift_conf -def plugin_enabled(): - return _plugin_enabled +def Gluster_enabled(): + return _gluster_enabled diff --git a/swift/1.4.8/swift.diff b/swift/1.4.8/swift.diff index 0024d7b8a3a..522e651cb0d 100644 --- a/swift/1.4.8/swift.diff +++ b/swift/1.4.8/swift.diff @@ -1,5 +1,5 @@ diff --git a/setup.py b/setup.py -index d195d34..ab236ee 100644 +index d195d34..ef625ff 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ @@ -9,17 +9,16 @@ index d195d34..ab236ee 100644 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -@@ -94,6 +95,8 @@ setup( +@@ -94,6 +95,7 @@ setup( 'tempurl=swift.common.middleware.tempurl:filter_factory', 'formpost=swift.common.middleware.formpost:filter_factory', 'name_check=swift.common.middleware.name_check:filter_factory', + 'gluster=swift.plugins.middleware.gluster:filter_factory', -+ 'glusterfs=swift.plugins.middleware.glusterfs:filter_factory', ], }, ) diff --git a/swift/account/server.py b/swift/account/server.py -index 800b3c0..77f9879 100644 +index 800b3c0..ba13786 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -1,4 +1,5 @@ @@ -28,59 +27,54 @@ index 800b3c0..77f9879 100644 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -@@ -35,6 +36,9 @@ from swift.common.utils import get_logger, get_param, hash_path, \ +@@ -35,6 +36,10 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc -+from swift.plugins.utils import plugin_enabled -+if plugin_enabled(): ++from swift.plugins.utils import Gluster_enabled ++if Gluster_enabled(): + from swift.plugins.DiskDir import DiskAccount ++ from swift.plugins import Glusterfs DATADIR = 'accounts' -@@ -52,8 +56,12 @@ class AccountController(object): +@@ -45,15 +50,21 @@ class AccountController(object): + + def __init__(self, conf): + self.logger = get_logger(conf, log_route='account-server') +- self.root = conf.get('devices', '/srv/node') +- self.mount_check = conf.get('mount_check', 'true').lower() in \ +- ('true', 't', '1', 'on', 'yes', 'y') ++ if Gluster_enabled(): ++ self.root = Glusterfs.MOUNT_PATH ++ self.mount_check = False ++ else: ++ self.root = conf.get('devices', '/srv/node') ++ self.mount_check = conf.get('mount_check', 'true').lower() in \ ++ ('true', 't', '1', 'on', 'yes', 'y') + self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' -+ self.fs_object = None def _get_account_broker(self, drive, part, account): -+ if self.fs_object: -+ return DiskAccount(self.root, account, self.fs_object); -+ ++ if Gluster_enabled(): ++ return DiskAccount(self.root, account) hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') -@@ -153,6 +161,9 @@ class AccountController(object): +@@ -153,6 +164,9 @@ class AccountController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) -+ if self.fs_object and not self.fs_object.object_only: ++ if Gluster_enabled() and not Glusterfs.OBJECT_ONLY: + broker.list_containers_iter(None, None,None, + None, None) info = broker.get_info() headers = { 'X-Account-Container-Count': info['container_count'], -@@ -305,8 +316,17 @@ class AccountController(object): - broker.update_metadata(metadata) - return HTTPNoContent(request=req) - -+ def plugin(self, env): -+ if env.get('Gluster_enabled', False): -+ self.fs_object = env.get('fs_object') -+ self.root = env.get('root') -+ self.mount_check = False -+ else: -+ self.fs_object = None -+ - def __call__(self, env, start_response): - start_time = time.time() -+ self.plugin(env) - req = Request(env) - self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(req.path_info): diff --git a/swift/container/server.py b/swift/container/server.py -index 8a18cfd..952b8cd 100644 +index 8a18cfd..c4982f1 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -1,4 +1,5 @@ @@ -89,67 +83,57 @@ index 8a18cfd..952b8cd 100644 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -@@ -37,6 +38,9 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ +@@ -37,6 +38,11 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc -+from swift.plugins.utils import plugin_enabled -+if plugin_enabled(): ++from swift.plugins.utils import Gluster_enabled ++if Gluster_enabled(): + from swift.plugins.DiskDir import DiskDir ++ from swift.plugins import Glusterfs ++ DATADIR = 'containers' -@@ -62,6 +66,7 @@ class ContainerController(object): - ContainerBroker, self.mount_check, logger=self.logger) - self.auto_create_account_prefix = \ - conf.get('auto_create_account_prefix') or '.' -+ self.fs_object = None +@@ -50,9 +56,13 @@ class ContainerController(object): - def _get_container_broker(self, drive, part, account, container): - """ -@@ -73,6 +78,10 @@ class ContainerController(object): + def __init__(self, conf): + self.logger = get_logger(conf, log_route='container-server') +- self.root = conf.get('devices', '/srv/node/') +- self.mount_check = conf.get('mount_check', 'true').lower() in \ +- ('true', 't', '1', 'on', 'yes', 'y') ++ if Gluster_enabled(): ++ self.root = Glusterfs.MOUNT_PATH ++ self.mount_check = False ++ else: ++ self.root = conf.get('devices', '/srv/node/') ++ self.mount_check = conf.get('mount_check', 'true').lower() in \ ++ ('true', 't', '1', 'on', 'yes', 'y') + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.allowed_sync_hosts = [h.strip() +@@ -73,6 +83,9 @@ class ContainerController(object): :param container: container name :returns: ContainerBroker object """ -+ if self.fs_object: ++ if Gluster_enabled(): + return DiskDir(self.root, drive, part, account, -+ container, self.logger, -+ fs_object = self.fs_object) ++ container, self.logger) hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') -@@ -245,6 +254,9 @@ class ContainerController(object): +@@ -245,6 +258,9 @@ class ContainerController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) -+ if self.fs_object and not self.fs_object.object_only: ++ if Gluster_enabled() and not Glusterfs.OBJECT_ONLY: + broker.list_objects_iter(None, None, None, None, + None, None) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], -@@ -427,8 +439,19 @@ class ContainerController(object): - broker.update_metadata(metadata) - return HTTPNoContent(request=req) - -+ def plugin(self, env): -+ if env.get('Gluster_enabled', False): -+ self.fs_object = env.get('fs_object') -+ if not self.fs_object: -+ raise NoneTypeError -+ self.root = env.get('root') -+ self.mount_check = False -+ else: -+ self.fs_object = None -+ - def __call__(self, env, start_response): - start_time = time.time() -+ self.plugin(env) - req = Request(env) - self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(req.path_info): diff --git a/swift/obj/server.py b/swift/obj/server.py -index 9cca16b..82eaa40 100644 +index 9cca16b..7a671c2 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -1,4 +1,5 @@ @@ -158,46 +142,63 @@ index 9cca16b..82eaa40 100644 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -@@ -45,6 +46,10 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ +@@ -45,6 +46,11 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ DiskFileNotExist from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \ quarantine_renamer -+from swift.plugins.utils import plugin_enabled -+if plugin_enabled(): ++from swift.plugins.utils import Gluster_enabled ++if Gluster_enabled(): + from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \ + OBJECT, DIR_TYPE, FILE_TYPE ++ from swift.plugins import Glusterfs DATADIR = 'objects' -@@ -340,6 +345,9 @@ class DiskFile(object): +@@ -340,6 +346,10 @@ class DiskFile(object): raise DiskFileNotExist('Data File does not exist.') -+if plugin_enabled(): ++if Gluster_enabled(): + from swift.plugins.DiskFile import Gluster_DiskFile + ++ class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" -@@ -377,6 +385,17 @@ class ObjectController(object): - 'expiring_objects' +@@ -351,9 +361,13 @@ class ObjectController(object): + /etc/swift/object-server.conf-sample. + """ + self.logger = get_logger(conf, log_route='object-server') +- self.devices = conf.get('devices', '/srv/node/') +- self.mount_check = conf.get('mount_check', 'true').lower() in \ +- ('true', 't', '1', 'on', 'yes', 'y') ++ if Gluster_enabled(): ++ self.devices = Glusterfs.MOUNT_PATH ++ self.mount_check = False ++ else: ++ self.devices = conf.get('devices', '/srv/node/') ++ self.mount_check = conf.get('mount_check', 'true').lower() in \ ++ ('true', 't', '1', 'on', 'yes', 'y') + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) +@@ -378,6 +392,15 @@ class ObjectController(object): self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) -+ self.fs_object = None -+ + + def get_DiskFile_obj(self, path, device, partition, account, container, obj, + logger, keep_data_fp=False, disk_chunk_size=65536): -+ if self.fs_object: ++ if Gluster_enabled(): + return Gluster_DiskFile(path, device, partition, account, container, -+ obj, logger, keep_data_fp, -+ disk_chunk_size, fs_object = self.fs_object); ++ obj, logger, keep_data_fp, disk_chunk_size) + else: + return DiskFile(path, device, partition, account, container, + obj, logger, keep_data_fp, disk_chunk_size) - ++ def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): -@@ -493,7 +512,7 @@ class ObjectController(object): + """ +@@ -493,7 +516,7 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) @@ -206,7 +207,7 @@ index 9cca16b..82eaa40 100644 obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'X-Delete-At' in file.metadata and \ -@@ -548,7 +567,7 @@ class ObjectController(object): +@@ -548,7 +571,7 @@ class ObjectController(object): if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') @@ -215,7 +216,7 @@ index 9cca16b..82eaa40 100644 obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time -@@ -580,12 +599,26 @@ class ObjectController(object): +@@ -580,12 +603,28 @@ class ObjectController(object): if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) @@ -226,9 +227,7 @@ index 9cca16b..82eaa40 100644 - 'Content-Length': str(os.fstat(fd).st_size), - } + content_type = request.headers['content-type'] -+ if self.fs_object and not content_type: -+ content_type = FILE_TYPE -+ if not self.fs_object: ++ if not Gluster_enabled(): + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, @@ -236,7 +235,11 @@ index 9cca16b..82eaa40 100644 + 'Content-Length': str(os.fstat(fd).st_size), + } + else: -+ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE ++ if not content_type: ++ content_type = FILE_TYPE ++ x_object_type = FILE ++ else: ++ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, @@ -248,16 +251,16 @@ index 9cca16b..82eaa40 100644 metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) -@@ -612,7 +645,7 @@ class ObjectController(object): +@@ -612,7 +651,7 @@ class ObjectController(object): 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], 'x-trans-id': request.headers.get('x-trans-id', '-')}, - device) -+ (self.fs_object and account) or device) ++ (Gluster_enabled() and account) or device) resp = HTTPCreated(request=request, etag=etag) return resp -@@ -626,9 +659,9 @@ class ObjectController(object): +@@ -626,9 +665,9 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) @@ -270,7 +273,7 @@ index 9cca16b..82eaa40 100644 if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): if request.headers.get('if-match') == '*': -@@ -702,7 +735,7 @@ class ObjectController(object): +@@ -702,7 +741,7 @@ class ObjectController(object): return resp if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) @@ -279,7 +282,7 @@ index 9cca16b..82eaa40 100644 obj, self.logger, disk_chunk_size=self.disk_chunk_size) if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): -@@ -744,7 +777,7 @@ class ObjectController(object): +@@ -744,7 +783,7 @@ class ObjectController(object): if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) response_class = HTTPNoContent @@ -288,22 +291,3 @@ index 9cca16b..82eaa40 100644 obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'x-if-delete-at' in request.headers and \ int(request.headers['x-if-delete-at']) != \ -@@ -797,9 +830,18 @@ class ObjectController(object): - raise hashes - return Response(body=pickle.dumps(hashes)) - -+ def plugin(self, env): -+ if env.get('Gluster_enabled', False): -+ self.fs_object = env.get('fs_object') -+ self.devices = env.get('root') -+ self.mount_check = False -+ else: -+ self.fs_object = None -+ - def __call__(self, env, start_response): - """WSGI Application entry point for the Swift Object Server.""" - start_time = time.time() -+ self.plugin(env) - req = Request(env) - self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(req.path_info): diff --git a/swift/1.4.8/test/unit/plugins/test_utils.py b/swift/1.4.8/test/unit/plugins/test_utils.py index fb0cb1f2c8f..e46c0adbd0b 100644 --- a/swift/1.4.8/test/unit/plugins/test_utils.py +++ b/swift/1.4.8/test/unit/plugins/test_utils.py @@ -24,9 +24,9 @@ import tempfile import hashlib import tarfile import shutil -from swift.common.utils import normalize_timestamp from collections import defaultdict -from plugins import utils +from swift.common.utils import normalize_timestamp +from swift.plugins import utils, Glusterfs # # Somewhat hacky way of emulating the operation of xattr calls. They are made @@ -542,7 +542,7 @@ class TestUtils(unittest.TestCase): self.st_mtime = mtime return MockStat(100) cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path), cd) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat @@ -571,7 +571,7 @@ class TestUtils(unittest.TestCase): # Be sure we don't miss due to mtimes not matching self.fail("do_stat should not have been called") cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path + "u"), cd) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path + "u"), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat @@ -580,7 +580,7 @@ class TestUtils(unittest.TestCase): retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d finally: utils._get_container_details_from_fs = orig_gcdff @@ -599,7 +599,7 @@ class TestUtils(unittest.TestCase): # Be sure we don't miss due to mtimes not matching self.fail("do_stat should not have been called") cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path), cd) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat @@ -608,7 +608,7 @@ class TestUtils(unittest.TestCase): retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: @@ -631,7 +631,7 @@ class TestUtils(unittest.TestCase): self.st_mtime = mtime return MockStat(200) cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path), cd) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat @@ -640,7 +640,7 @@ class TestUtils(unittest.TestCase): retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: @@ -678,7 +678,7 @@ class TestUtils(unittest.TestCase): return MockStat(100) ad = mock_get_account_details_from_fs(the_path, None) ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path), ad) + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), ad) orig_gcdff = utils._get_account_details_from_fs orig_ds = utils.do_stat utils._get_account_details_from_fs = mock_get_account_details_from_fs @@ -707,7 +707,7 @@ class TestUtils(unittest.TestCase): return MockStat(100) ad = mock_get_account_details_from_fs(the_path, None) ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path + 'u'), ad) + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path + 'u'), ad) orig_gcdff = utils._get_account_details_from_fs orig_ds = utils.do_stat utils._get_account_details_from_fs = mock_get_account_details_from_fs @@ -737,7 +737,7 @@ class TestUtils(unittest.TestCase): ad = mock_get_account_details_from_fs(the_path, None) ad.container_list = ['x', 'y'] ad.mtime = 200 - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + utils.strip_obj_storage_path(the_path), ad) + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), ad) orig_gcdff = utils._get_account_details_from_fs orig_ds = utils.do_stat utils._get_account_details_from_fs = mock_get_account_details_from_fs |