diff options
Diffstat (limited to 'swift/1.4.8/plugins')
| -rw-r--r-- | swift/1.4.8/plugins/DiskDir.py | 35 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/DiskFile.py | 4 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/Glusterfs.py | 217 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/account-server/1.conf-gluster (renamed from swift/1.4.8/plugins/conf/account-server/1.conf) | 5 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/container-server/1.conf-gluster (renamed from swift/1.4.8/plugins/conf/container-server/1.conf) | 5 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/fs.conf-gluster (renamed from swift/1.4.8/plugins/conf/fs.conf) | 0 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/object-server/1.conf-gluster (renamed from swift/1.4.8/plugins/conf/object-server/1.conf) | 5 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/proxy-server.conf-gluster (renamed from swift/1.4.8/plugins/conf/proxy-server.conf) | 0 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/swift.conf-gluster (renamed from swift/1.4.8/plugins/conf/swift.conf) | 0 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/fs_utils.py | 157 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/middleware/glusterfs.py | 55 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/utils.py | 199 |
12 files changed, 316 insertions, 366 deletions
diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py index e8024768d8b..e057dff956d 100644 --- a/swift/1.4.8/plugins/DiskDir.py +++ b/swift/1.4.8/plugins/DiskDir.py @@ -28,15 +28,13 @@ from swift.plugins.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP,\ X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ X_CONTAINER_COUNT, CONTAINER -from swift import plugins -def strip_obj_storage_path(path, string='/mnt/gluster-object'): - """ - strip /mnt/gluster-object - """ - return path.replace(string, '').strip('/') - DATADIR = 'containers' +# Create a dummy db_file in /etc/swift +_db_file = '/etc/swift/db_file.db' +if not os.path.exists(_db_file): + file(_db_file, 'w+') + def _read_metadata(dd): """ Filter read metadata so that it always returns a tuple that includes @@ -148,7 +146,7 @@ class DiskDir(DiskCommon): """ def __init__(self, path, device, partition, account, container, logger, - uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object=None): + uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path device = account if container: @@ -162,15 +160,12 @@ class DiskDir(DiskCommon): self.account = account self.device_path = os.path.join(path, device) if not check_mount(path, device): - check_valid_account(account, fs_object) + check_valid_account(account) self.logger = logger self.metadata = {} self.uid = int(uid) self.gid = int(gid) - # Create a dummy db_file in /etc/swift - self.db_file = '/etc/swift/db_file.db' - if not os.path.exists(self.db_file): - file(self.db_file, 'w+') + self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: @@ -319,9 +314,10 @@ class DiskDir(DiskCommon): for obj in objects: list_item = [] list_item.append(obj) - metadata = read_metadata(self.datadir + '/' + obj) + obj_path = os.path.join(self.datadir, obj) + metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): - metadata = create_object_metadata(self.datadir + '/' + obj) + metadata = create_object_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) @@ -420,12 +416,12 @@ class DiskDir(DiskCommon): class DiskAccount(DiskDir): - def __init__(self, root, account, fs_object = None): + def __init__(self, root, account): self.root = root self.account = account self.datadir = os.path.join(self.root, self.account) if not check_mount(root, account): - check_valid_account(account, fs_object) + check_valid_account(account) self.metadata = _read_metadata(self.datadir) if not self.metadata or not validate_account(self.metadata): self.metadata = create_account_metadata(self.datadir) @@ -472,9 +468,10 @@ class DiskAccount(DiskDir): list_item = [] metadata = None list_item.append(cont) - metadata = _read_metadata(self.datadir + '/' + cont) + cont_path = os.path.join(self.datadir, cont) + metadata = _read_metadata(cont_path) if not metadata or not validate_container(metadata): - metadata = create_container_metadata(self.datadir + '/' + cont) + metadata = create_container_metadata(cont_path) if metadata: list_item.append(metadata[X_OBJECTS_COUNT][0]) diff --git a/swift/1.4.8/plugins/DiskFile.py b/swift/1.4.8/plugins/DiskFile.py index 6f77eaaa57a..f4ca366b236 100644 --- a/swift/1.4.8/plugins/DiskFile.py +++ b/swift/1.4.8/plugins/DiskFile.py @@ -55,7 +55,7 @@ class Gluster_DiskFile(DiskFile): def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, - uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object = None): + uid=DEFAULT_UID, gid=DEFAULT_GID): self.disk_chunk_size = disk_chunk_size device = account #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc @@ -75,7 +75,7 @@ class Gluster_DiskFile(DiskFile): self.device_path = os.path.join(path, device) if not check_mount(path, device): - check_valid_account(account, fs_object) + check_valid_account(account) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') diff --git a/swift/1.4.8/plugins/Glusterfs.py b/swift/1.4.8/plugins/Glusterfs.py index 5e191e1bd8f..69902d85d48 100644 --- a/swift/1.4.8/plugins/Glusterfs.py +++ b/swift/1.4.8/plugins/Glusterfs.py @@ -17,115 +17,124 @@ import os, fcntl, time from ConfigParser import ConfigParser from swift.common.utils import TRUE_VALUES from hashlib import md5 -from swift.plugins.utils import mkdirs - -class Glusterfs(object): - def __init__(self): - self.name = 'glusterfs' - self.fs_conf = ConfigParser() - self.fs_conf.read(os.path.join('/etc/swift', 'fs.conf')) - self.mount_path = self.fs_conf.get('DEFAULT', 'mount_path', '/mnt/gluster-object') - self.auth_account = self.fs_conf.get('DEFAULT', 'auth_account', 'auth') - self.mount_ip = self.fs_conf.get('DEFAULT', 'mount_ip', 'localhost') - self.remote_cluster = self.fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES - self.object_only = self.fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES - - def busy_wait(self, mount_path): - # Iterate for definite number of time over a given - # interval for successful mount - for i in range(0, 5): - if os.path.ismount(os.path.join(mount_path)): - return True - time.sleep(2) - return False - - def mount(self, account): - mount_path = os.path.join(self.mount_path, account) - export = self.get_export_from_account_id(account) - - pid_dir = "/var/lib/glusterd/vols/%s/run/" %export - pid_file = os.path.join(pid_dir, 'swift.pid'); - - if not os.path.exists(pid_dir): - mkdirs(pid_dir) - - fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) - with os.fdopen(fd, 'r+b') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) - except: - ex = sys.exc_info()[1] - if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): - # This means that some other process is mounting the - # filesystem, so wait for the mount process to complete - return self.busy_wait(mount_path) - - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (self.mount_ip, export, \ - mount_path) - if os.system(mnt_cmd) or not self.busy_wait(mount_path): - raise Exception('Mount failed %s: %s' % (self.name, mnt_cmd)) - return False - return True - - def unmount(self, mount_path): - umnt_cmd = 'umount %s 2>> /dev/null' % mount_path - if os.system(umnt_cmd): - logging.error('Unable to unmount %s %s' % (mount_path, self.name)) - - def get_export_list_local(self): - export_list = [] - cmnd = 'gluster volume info' - - if os.system(cmnd + ' >> /dev/null'): - raise Exception('Getting volume failed %s', self.name) - return export_list - - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) +from swift.plugins.fs_utils import mkdirs - return export_list +# +# Read the fs.conf file once at startup (module load) +# +_fs_conf = ConfigParser() +MOUNT_PATH = '/mnt/gluster-object' +AUTH_ACCOUNT = 'auth' +MOUNT_IP = 'localhost' +REMOTE_CLUSTER = False +OBJECT_ONLY = False +if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): + try: + MOUNT_PATH = _fs_conf.get('DEFAULT', 'mount_path', '/mnt/gluster-object') + except (NoSectionError, NoOptionError): + pass + try: + AUTH_ACCOUNT = _fs_conf.get('DEFAULT', 'auth_account', 'auth') + except (NoSectionError, NoOptionError): + pass + try: + MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') + except (NoSectionError, NoOptionError): + pass + try: + REMOTE_CLUSTER = _fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + try: + OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass +NAME = 'glusterfs' + + +def strip_obj_storage_path(path, mp=MOUNT_PATH): + """ + strip the mount path off, also stripping the leading and trailing slashes + """ + return path.replace(mp, '').strip(os.path.sep) + +def _busy_wait(full_mount_path): + # Iterate for definite number of time over a given + # interval for successful mount + for i in range(0, 5): + if os.path.ismount(os.path.join(full_mount_path)): + return True + time.sleep(2) + return False + +def mount(account): + global mount_path, mount_ip + + full_mount_path = os.path.join(mount_path, account) + export = get_export_from_account_id(account) + + pid_dir = "/var/lib/glusterd/vols/%s/run/" % export + pid_file = os.path.join(pid_dir, 'swift.pid'); + + if not os.path.exists(pid_dir): + mkdirs(pid_dir) + + fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) + with os.fdopen(fd, 'r+b') as f: + try: + fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) + except: + ex = sys.exc_info()[1] + if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): + # This means that some other process is mounting the + # filesystem, so wait for the mount process to complete + return _busy_wait(full_mount_path) + + mnt_cmd = 'mount -t glusterfs %s:%s %s' % (mount_ip, export, \ + full_mount_path) + if os.system(mnt_cmd) or not _busy_wait(full_mount_path): + raise Exception('Mount failed %s: %s' % (NAME, mnt_cmd)) + return True + +def unmount(full_mount_path): + umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path + if os.system(umnt_cmd): + logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) + +def get_export_list(): + global mount_ip + + if remote_cluster: + cmnd = 'ssh %s gluster volume info' % mount_ip + else: + cmnd = 'gluster volume info' - def get_export_list_remote(self): - export_list = [] - cmnd = 'ssh %s gluster volume info' % self.mount_ip - - if os.system(cmnd + ' >> /dev/null'): + if os.system(cmnd + ' >> /dev/null'): + if remove_cluster: raise Exception('Getting volume info failed %s, make sure to have \ - passwordless ssh on %s', self.name, self.mount_ip) - return export_list - - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) - - return export_list - - def get_export_list(self): - if self.remote_cluster: - return self.get_export_list_remote() + passwordless ssh on %s', NAME, mount_ip) else: - return self.get_export_list_local() + raise Exception('Getting volume failed %s', NAME) + + export_list = [] + fp = os.popen(cmnd) + while True: + item = fp.readline() + if not item: + break + item = item.strip('\n').strip(' ') + if item.lower().startswith('volume name:'): + export_list.append(item.split(':')[1].strip(' ')) + + return export_list - def get_export_from_account_id(self, account): - if not account: - print 'account is none, returning' - raise AttributeError +def get_export_from_account_id(account): + if not account: + raise ValueError('No account given') - for export in self.get_export_list(): - if account == 'AUTH_' + export: - return export + for export in get_export_list(): + if account == 'AUTH_' + export: + return export - raise Exception('No export found %s %s' % (account, self.name)) - return None + raise Exception('No export found %s %s' % (account, NAME)) diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf b/swift/1.4.8/plugins/conf/account-server/1.conf-gluster index a82bc35eaa4..911e2837f50 100644 --- a/swift/1.4.8/plugins/conf/account-server/1.conf +++ b/swift/1.4.8/plugins/conf/account-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs account-server +pipeline = account-server [app:account-server] use = egg:swift#account -[filter:glusterfs] -use = egg:swift#glusterfs - [account-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf b/swift/1.4.8/plugins/conf/container-server/1.conf-gluster index e6d313c3872..eb5650afa74 100644 --- a/swift/1.4.8/plugins/conf/container-server/1.conf +++ b/swift/1.4.8/plugins/conf/container-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs container-server +pipeline = container-server [app:container-server] use = egg:swift#container -[filter:glusterfs] -use = egg:swift#glusterfs - [container-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/fs.conf b/swift/1.4.8/plugins/conf/fs.conf-gluster index b6ec5121f9f..b6ec5121f9f 100644 --- a/swift/1.4.8/plugins/conf/fs.conf +++ b/swift/1.4.8/plugins/conf/fs.conf-gluster diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf b/swift/1.4.8/plugins/conf/object-server/1.conf-gluster index 665aa94be8b..13dda69bae2 100644 --- a/swift/1.4.8/plugins/conf/object-server/1.conf +++ b/swift/1.4.8/plugins/conf/object-server/1.conf-gluster @@ -6,14 +6,11 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = glusterfs object-server +pipeline = object-server [app:object-server] use = egg:swift#object -[filter:glusterfs] -use = egg:swift#glusterfs - [object-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf b/swift/1.4.8/plugins/conf/proxy-server.conf-gluster index 9bec6425355..9bec6425355 100644 --- a/swift/1.4.8/plugins/conf/proxy-server.conf +++ b/swift/1.4.8/plugins/conf/proxy-server.conf-gluster diff --git a/swift/1.4.8/plugins/conf/swift.conf b/swift/1.4.8/plugins/conf/swift.conf-gluster index e506b6f54a8..e506b6f54a8 100644 --- a/swift/1.4.8/plugins/conf/swift.conf +++ b/swift/1.4.8/plugins/conf/swift.conf-gluster diff --git a/swift/1.4.8/plugins/fs_utils.py b/swift/1.4.8/plugins/fs_utils.py new file mode 100644 index 00000000000..76fcbbc2b9c --- /dev/null +++ b/swift/1.4.8/plugins/fs_utils.py @@ -0,0 +1,157 @@ +# Copyright (c) 2011 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno + +def do_mkdir(path): + try: + os.mkdir(path) + except Exception, err: + logging.exception("Mkdir failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_makedirs(path): + try: + os.makedirs(path) + except Exception, err: + logging.exception("Makedirs failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_listdir(path): + try: + buf = os.listdir(path) + except Exception, err: + logging.exception("Listdir failed on %s err: %s", path, str(err)) + raise + return buf + +def do_chown(path, uid, gid): + try: + os.chown(path, uid, gid) + except Exception, err: + logging.exception("Chown failed on %s err: %s", path, str(err)) + raise + return True + +def do_stat(path): + try: + #Check for fd. + if isinstance(path, int): + buf = os.fstat(path) + else: + buf = os.stat(path) + except Exception, err: + logging.exception("Stat failed on %s err: %s", path, str(err)) + raise + + return buf + +def do_open(path, mode): + try: + fd = open(path, mode) + except Exception, err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + return fd + +def do_close(fd): + #fd could be file or int type. + try: + if isinstance(fd, int): + os.close(fd) + else: + fd.close() + except Exception, err: + logging.exception("Close failed on %s err: %s", fd, str(err)) + raise + return True + +def do_unlink(path, log = True): + try: + os.unlink(path) + except Exception, err: + if log: + logging.exception("Unlink failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rmdir(path): + try: + os.rmdir(path) + except Exception, err: + logging.exception("Rmdir failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rename(old_path, new_path): + try: + os.rename(old_path, new_path) + except Exception, err: + logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ + str(err)) + raise + return True + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + try: + do_makedirs(path) + except OSError, err: + #TODO: check, isdir will fail if mounted and volume stopped. + #if err.errno != errno.EEXIST or not os.path.isdir(path) + if err.errno != errno.EEXIST: + raise + +def dir_empty(path): + """ + Return true if directory/container is empty. + :param path: Directory path. + :returns: True/False. + """ + if os.path.isdir(path): + try: + files = do_listdir(path) + except Exception, err: + logging.exception("listdir failed on %s err: %s", path, str(err)) + raise + if not files: + return True + else: + return False + else: + if not os.path.exists(path): + return True + +def rmdirs(path): + if os.path.isdir(path) and dir_empty(path): + do_rmdir(path) + else: + logging.error("rmdirs failed dir may not be empty or not valid dir") + return False + diff --git a/swift/1.4.8/plugins/middleware/glusterfs.py b/swift/1.4.8/plugins/middleware/glusterfs.py deleted file mode 100644 index 7472146406c..00000000000 --- a/swift/1.4.8/plugins/middleware/glusterfs.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Monkey patch constraints -import swift.plugins.constraints -from swift.plugins.Glusterfs import Glusterfs -from ConfigParser import ConfigParser - -fs_conf = ConfigParser() -if fs_conf.read('/etc/swift/fs.conf'): - try: - mount_path = fs_conf.get ('DEFAULT', 'mount_path') - except NoSectionError, NoOptionError: - # FIXME - How to log during module initialization - logger.exception(_('ERROR mount_path not present')) - mount_path = '' -else: - mount_path = '' - - -class Gluster(object): - """ - Update the environment with keys that reflect GlusterFS middleware enabled - """ - def __init__(self, app, conf): - self.app = app - self.conf = conf - - def __call__(self, env, start_response): - env['Gluster_enabled'] = True - env['fs_object'] = Glusterfs() - env['root'] = mount_path - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def gluster_filter(app): - return Gluster(app, conf) - return gluster_filter diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py index a1ac64db5c9..bc16ff6f738 100644 --- a/swift/1.4.8/plugins/utils.py +++ b/swift/1.4.8/plugins/utils.py @@ -18,10 +18,11 @@ import os import errno import xattr from hashlib import md5 -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from swift.obj.server import ASYNCDIR import cPickle as pickle from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.utils import normalize_timestamp, TRUE_VALUES +from swift.plugins.fs_utils import * +from swift.plugins import Glusterfs X_CONTENT_TYPE = 'Content-Type' X_CONTENT_LENGTH = 'Content-Length' @@ -35,15 +36,14 @@ X_CONTAINER_COUNT = 'X-Container-Count' X_OBJECT_TYPE = 'X-Object-Type' DIR_TYPE = 'application/directory' ACCOUNT = 'Account' -MOUNT_PATH = '/mnt/gluster-object' METADATA_KEY = 'user.swift.metadata' MAX_XATTR_SIZE = 65536 CONTAINER = 'container' DIR = 'dir' MARKER_DIR = 'marker_dir' TEMP_DIR = 'tmp' +ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR FILE = 'file' -DIR_TYPE = 'application/directory' FILE_TYPE = 'application/octet-stream' OBJECT = 'Object' OBJECT_TYPE = 'application/octet-stream' @@ -56,131 +56,6 @@ MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' -def mkdirs(path): - """ - Ensures the path is a directory or makes it if not. Errors if the path - exists but is a file or on permissions failure. - - :param path: path to create - """ - if not os.path.isdir(path): - try: - do_makedirs(path) - except OSError, err: - #TODO: check, isdir will fail if mounted and volume stopped. - #if err.errno != errno.EEXIST or not os.path.isdir(path) - if err.errno != errno.EEXIST: - raise - -def rmdirs(path): - if os.path.isdir(path) and dir_empty(path): - do_rmdir(path) - else: - logging.error("rmdirs failed dir may not be empty or not valid dir") - return False - -def strip_obj_storage_path(path, string='/mnt/gluster-object'): - """ - strip /mnt/gluster-object - """ - return path.replace(string, '').strip('/') - -def do_mkdir(path): - try: - os.mkdir(path) - except Exception, err: - logging.exception("Mkdir failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_makedirs(path): - try: - os.makedirs(path) - except Exception, err: - logging.exception("Makedirs failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_listdir(path): - try: - buf = os.listdir(path) - except Exception, err: - logging.exception("Listdir failed on %s err: %s", path, str(err)) - raise - return buf - -def do_chown(path, uid, gid): - try: - os.chown(path, uid, gid) - except Exception, err: - logging.exception("Chown failed on %s err: %s", path, str(err)) - raise - return True - -def do_stat(path): - try: - #Check for fd. - if isinstance(path, int): - buf = os.fstat(path) - else: - buf = os.stat(path) - except Exception, err: - logging.exception("Stat failed on %s err: %s", path, str(err)) - raise - - return buf - -def do_open(path, mode): - try: - fd = open(path, mode) - except Exception, err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - return fd - -def do_close(fd): - #fd could be file or int type. - try: - if isinstance(fd, int): - os.close(fd) - else: - fd.close() - except Exception, err: - logging.exception("Close failed on %s err: %s", fd, str(err)) - raise - return True - -def do_unlink(path, log = True): - try: - os.unlink(path) - except Exception, err: - if log: - logging.exception("Unlink failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rmdir(path): - try: - os.rmdir(path) - except Exception, err: - logging.exception("Rmdir failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rename(old_path, new_path): - try: - os.rename(old_path, new_path) - except Exception, err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - str(err)) - raise - return True - - def read_metadata(path): """ Helper function to read the pickled metadata from a File/Directory. @@ -262,26 +137,6 @@ def clean_metadata(path): raise key += 1 -def dir_empty(path): - """ - Return true if directory/container is empty. - :param path: Directory path. - :returns: True/False. - """ - if os.path.isdir(path): - try: - files = do_listdir(path) - except Exception, err: - logging.exception("listdir failed on %s err: %s", path, str(err)) - raise - if not files: - return True - else: - return False - else: - if not os.path.exists(path): - return True - def get_device_from_account(account): if account.startswith(RESELLER_PREFIX): device = account.replace(RESELLER_PREFIX, '', 1) @@ -302,27 +157,26 @@ def check_user_xattr(path): #Remove xattr may fail in case of concurrent remove. return True -def _check_valid_account(account, fs_object): - mount_path = getattr(fs_object, 'mount_path', MOUNT_PATH) +def _check_valid_account(account): + full_mount_path = os.path.join(Glusterfs.MOUNT_PATH, account) - if os.path.ismount(os.path.join(mount_path, account)): + if os.path.ismount(full_mount_path): return True - if not check_account_exists(fs_object.get_export_from_account_id(account), fs_object): + if not Glusterfs.check_account_exists(Glusterfs.get_export_from_account_id(account)): logging.error('Account not present %s', account) return False - if not os.path.isdir(os.path.join(mount_path, account)): - mkdirs(os.path.join(mount_path, account)) + if not os.path.isdir(full_mount_path): + mkdirs(full_mount_path) - if fs_object: - if not fs_object.mount(account): - return False + if not Glusterfs.mount(account): + return False return True -def check_valid_account(account, fs_object): - return _check_valid_account(account, fs_object) +def check_valid_account(account): + return _check_valid_account(account) def validate_container(metadata): if not metadata: @@ -401,7 +255,7 @@ def is_marker(metadata): def _update_list(path, const_path, src_list, reg_file=True, object_count=0, bytes_used=0, obj_list=[]): - obj_path = strip_obj_storage_path(path, const_path) + obj_path = Glusterfs.strip_obj_storage_path(path, const_path) for i in src_list: if obj_path: @@ -460,7 +314,7 @@ def get_container_details(cont_path, memcache=None): """ mkey = '' if memcache: - mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + strip_obj_storage_path(cont_path) + mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(cont_path) cd = memcache.get(mkey) if cd: if not cd.dir_list: @@ -517,7 +371,7 @@ def get_account_details(acc_path, memcache=None): acc_stats = None mkey = '' if memcache: - mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + strip_obj_storage_path(acc_path) + mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(acc_path) ad = memcache.get(mkey) if ad: # FIXME: Do we really need to stat the file? If we are object @@ -627,18 +481,15 @@ def create_account_metadata(acc_path, memcache=None): metadata = get_account_metadata(acc_path, memcache) return restore_metadata(acc_path, metadata) -def check_account_exists(account, fs_object): - if account not in get_account_list(fs_object): +def check_account_exists(account): + if account not in get_account_list(): logging.warn('Account %s does not exist' % account) return False else: return True -def get_account_list(fs_object): - account_list = [] - if fs_object: - account_list = fs_object.get_export_list() - return account_list +def get_account_list(): + return Glusterfs.get_export_list() def get_account_id(account): return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest() @@ -647,10 +498,10 @@ def get_account_id(account): __swift_conf = ConfigParser() __swift_conf.read(os.path.join('/etc/swift', 'swift.conf')) try: - _plugin_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES + _gluster_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES except NoOptionError, NoSectionError: - _plugin_enabled = False + _gluster_enabled = False del __swift_conf -def plugin_enabled(): - return _plugin_enabled +def Gluster_enabled(): + return _gluster_enabled |