diff options
Diffstat (limited to 'swift/1.4.8/plugins/utils.py')
| -rw-r--r-- | swift/1.4.8/plugins/utils.py | 692 |
1 files changed, 692 insertions, 0 deletions
diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py new file mode 100644 index 00000000000..57c6180fdd1 --- /dev/null +++ b/swift/1.4.8/plugins/utils.py @@ -0,0 +1,692 @@ +# Copyright (c) 2011 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno +from hashlib import md5 +from swift.common.utils import normalize_timestamp +from xattr import setxattr, removexattr, getxattr, removexattr +import cPickle as pickle + +X_CONTENT_TYPE = 'Content-Type' +X_CONTENT_LENGTH = 'Content-Length' +X_TIMESTAMP = 'X-Timestamp' +X_PUT_TIMESTAMP = 'X-PUT-Timestamp' +X_TYPE = 'X-Type' +X_ETAG = 'ETag' +X_OBJECTS_COUNT = 'X-Object-Count' +X_BYTES_USED = 'X-Bytes-Used' +X_CONTAINER_COUNT = 'X-Container-Count' +X_OBJECT_TYPE = 'X-Object-Type' +DIR_TYPE = 'application/directory' +ACCOUNT = 'Account' +MOUNT_PATH = '/mnt/gluster-object' +METADATA_KEY = 'user.swift.metadata' +CONTAINER = 'container' +DIR = 'dir' +MARKER_DIR = 'marker_dir' +FILE = 'file' +DIR_TYPE = 'application/directory' +FILE_TYPE = 'application/octet-stream' +OBJECT = 'Object' +OBJECT_TYPE = 'application/octet-stream' +DEFAULT_UID = -1 +DEFAULT_GID = -1 +PICKLE_PROTOCOL = 2 +CHUNK_SIZE = 65536 + + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + try: + do_makedirs(path) + except OSError, err: + #TODO: check, isdir will fail if mounted and volume stopped. + #if err.errno != errno.EEXIST or not os.path.isdir(path) + if err.errno != errno.EEXIST: + raise + +def rmdirs(path): + if os.path.isdir(path) and dir_empty(path): + do_rmdir(path) + else: + logging.error("rmdirs failed dir may not be empty or not valid dir") + return False + +def strip_obj_storage_path(path, string='/mnt/gluster-object'): + """ + strip /mnt/gluster-object + """ + return path.replace(string, '').strip('/') + +def do_mkdir(path): + try: + os.mkdir(path) + except Exception, err: + logging.exception("Mkdir failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_makedirs(path): + try: + os.makedirs(path) + except Exception, err: + logging.exception("Makedirs failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + + +def do_listdir(path): + try: + buf = os.listdir(path) + except Exception, err: + logging.exception("Listdir failed on %s err: %s", path, str(err)) + raise + return buf + +def do_chown(path, uid, gid): + try: + os.chown(path, uid, gid) + except Exception, err: + logging.exception("Chown failed on %s err: %s", path, str(err)) + raise + return True + +def do_stat(path): + try: + #Check for fd. + if isinstance(path, int): + buf = os.fstat(path) + else: + buf = os.stat(path) + except Exception, err: + logging.exception("Stat failed on %s err: %s", path, str(err)) + raise + + return buf + +def do_open(path, mode): + try: + fd = open(path, mode) + except Exception, err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + return fd + +def do_close(fd): + #fd could be file or int type. + try: + if isinstance(fd, int): + os.close(fd) + else: + fd.close() + except Exception, err: + logging.exception("Close failed on %s err: %s", fd, str(err)) + raise + return True + +def do_unlink(path, log = True): + try: + os.unlink(path) + except Exception, err: + if log: + logging.exception("Unlink failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rmdir(path): + try: + os.rmdir(path) + except Exception, err: + logging.exception("Rmdir failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rename(old_path, new_path): + try: + os.rename(old_path, new_path) + except Exception, err: + logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ + str(err)) + raise + return True + +def do_setxattr(path, key, value): + fd = None + if not os.path.isdir(path): + fd = do_open(path, 'rb') + else: + fd = path + if fd or os.path.isdir(path): + try: + setxattr(fd, key, value) + except Exception, err: + logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err)) + raise + finally: + if fd and not os.path.isdir(path): + do_close(fd) + else: + logging.error("Open failed path %s", path) + return False + return True + + + +def do_getxattr(path, key, log = True): + fd = None + if not os.path.isdir(path): + fd = do_open(path, 'rb') + else: + fd = path + if fd or os.path.isdir(path): + try: + value = getxattr(fd, key) + except Exception, err: + if log: + logging.exception("getxattr failed on %s key %s err: %s", path, key, str(err)) + raise + finally: + if fd and not os.path.isdir(path): + do_close(fd) + else: + logging.error("Open failed path %s", path) + return False + return value + +def do_removexattr(path, key): + fd = None + if not os.path.isdir(path): + fd = do_open(path, 'rb') + else: + fd = path + if fd or os.path.isdir(path): + try: + removexattr(fd, key) + except Exception, err: + logging.exception("removexattr failed on %s key %s err: %s", path, key, str(err)) + raise + finally: + if fd and not os.path.isdir(path): + do_close(fd) + else: + logging.error("Open failed path %s", path) + return False + return True + +def read_metadata(path): + """ + Helper function to read the pickled metadata from a File/Directory . + + :param path: File/Directory to read metadata from. + + :returns: dictionary of metadata + """ + metadata = '' + key = 0 + while True: + try: + metadata += do_getxattr(path, '%s%s' % (METADATA_KEY, (key or '')), + log = False) + except Exception: + break + key += 1 + if metadata: + return pickle.loads(metadata) + else: + return {} + + +def write_metadata(path, metadata): + """ + Helper function to write pickled metadata for a File/Directory. + + :param path: File/Directory path to write the metadata + :param metadata: metadata to write + """ + metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) + key = 0 + while metastr: + do_setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:254]) + metastr = metastr[254:] + key += 1 + +def clean_metadata(path): + key = 0 + while True: + value = do_getxattr(path, '%s%s' % (METADATA_KEY, (key or ''))) + do_removexattr(path, '%s%s' % (METADATA_KEY, (key or ''))) + key += 1 + + +def dir_empty(path): + """ + Return true if directory/container is empty. + :param path: Directory path. + :returns: True/False. + """ + if os.path.isdir(path): + try: + files = do_listdir(path) + except Exception, err: + logging.exception("listdir failed on %s err: %s", path, str(err)) + raise + if not files: + return True + else: + return False + else: + if not os.path.exists(path): + return True + + +def get_device_from_account(account): + if account.startswith(RESELLER_PREFIX): + device = account.replace(RESELLER_PREFIX, '', 1) + return device + +def check_user_xattr(path): + if not os.path.exists(path): + return False + do_setxattr(path, 'user.test.key1', 'value1') + try: + removexattr(path, 'user.test.key1') + except Exception, err: + logging.exception("removexattr failed on %s err: %s", path, str(err)) + #Remove xattr may fail in case of concurrent remove. + return True + + +def _check_valid_account(account, fs_object): + mount_path = getattr(fs_object, 'mount_path', MOUNT_PATH) + + if not check_account_exists(fs_object.get_export_from_account_id(account), \ + fs_object): + logging.error('Account not present %s', account) + return False + + if not os.path.ismount(os.path.join(mount_path, account)): + if not os.path.isdir(os.path.join(mount_path, account)): + mkdirs(os.path.join(mount_path, account)) + + fs_object.unmount(os.path.join(mount_path, account)) + + if fs_object: + if not fs_object.mount(account): + return False + + if not check_user_xattr(os.path.join(mount_path, account)): + logging.error('Error: No support for user.xattr on backend %s' % account) + return False + + chmod_cmd = ['chmod 777 %s' % (mount_path), \ + 'chmod 777 %s/%s' % (mount_path, account)] + + for cmd in chmod_cmd: + if os.system(cmd): + logging.error('Chmod failed: %s' % (cmd)) + return False + + return True + +def check_valid_account(account, fs_object): + return _check_valid_account(account, fs_object) + +def validate_container(metadata): + if not metadata: + logging.error('No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys(): + #logging.error('Container error %s' % metadata) + return False + + if metadata[X_TYPE] == CONTAINER: + return True + + logging.error('Container error %s' % metadata) + return False + +def validate_account(metadata): + if not metadata: + logging.error('No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys() or \ + X_CONTAINER_COUNT not in metadata.keys(): + #logging.error('Account error %s' % metadata) + return False + + if metadata[X_TYPE] == ACCOUNT: + return True + + logging.error('Account error %s' % metadata) + return False + +def validate_object(metadata): + if not metadata: + logging.error('No metadata') + return False + + if X_TIMESTAMP not in metadata.keys() or \ + X_CONTENT_TYPE not in metadata.keys() or \ + X_ETAG not in metadata.keys() or \ + X_CONTENT_LENGTH not in metadata.keys() or \ + X_TYPE not in metadata.keys() or \ + X_OBJECT_TYPE not in metadata.keys(): + #logging.error('Object error %s' % metadata) + return False + + if metadata[X_TYPE] == OBJECT: + return True + + logging.error('Object error %s' % metadata) + return False + +def is_marker(metadata): + if not metadata: + logging.error('No metadata') + return False + + if X_OBJECT_TYPE not in metadata.keys(): + logging.error('X_OBJECT_TYPE missing %s' % metadata) + return False + + if metadata[X_OBJECT_TYPE] == MARKER_DIR: + return True + else: + return False + +def _update_list(path, const_path, src_list, reg_file=True, object_count=0, + bytes_used=0, obj_list=[]): + obj_path = strip_obj_storage_path(path, const_path) + + for i in src_list: + if obj_path: + obj_list.append(os.path.join(obj_path, i)) + else: + obj_list.append(i) + + object_count += 1 + + if reg_file: + bytes_used += os.path.getsize(path + '/' + i) + + return object_count, bytes_used + +def update_list(path, const_path, dirs=[], files=[], object_count=0, + bytes_used=0, obj_list=[]): + object_count, bytes_used = _update_list (path, const_path, files, True, + object_count, bytes_used, + obj_list) + object_count, bytes_used = _update_list (path, const_path, dirs, False, + object_count, bytes_used, + obj_list) + return object_count, bytes_used + +def get_container_details_from_fs(cont_path, const_path, + memcache=None): + """ + get container details by traversing the filesystem + """ + bytes_used = 0 + object_count = 0 + obj_list=[] + dir_list = [] + + if os.path.isdir(cont_path): + for (path, dirs, files) in os.walk(cont_path): + object_count, bytes_used = update_list(path, const_path, dirs, files, + object_count, bytes_used, + obj_list) + + dir_list.append(path + ':' + str(do_stat(path).st_mtime)) + + if memcache: + memcache.set(strip_obj_storage_path(cont_path), obj_list) + memcache.set(strip_obj_storage_path(cont_path) + '-dir_list', + ','.join(dir_list)) + memcache.set(strip_obj_storage_path(cont_path) + '-cont_meta', + [object_count, bytes_used]) + + return obj_list, object_count, bytes_used + +def get_container_details_from_memcache(cont_path, const_path, + memcache): + """ + get container details stored in memcache + """ + + bytes_used = 0 + object_count = 0 + obj_list=[] + + dir_contents = memcache.get(strip_obj_storage_path(cont_path) + '-dir_list') + if not dir_contents: + return get_container_details_from_fs(cont_path, const_path, + memcache=memcache) + + for i in dir_contents.split(','): + path, mtime = i.split(':') + if mtime != str(do_stat(path).st_mtime): + return get_container_details_from_fs(cont_path, const_path, + memcache=memcache) + + obj_list = memcache.get(strip_obj_storage_path(cont_path)) + + object_count, bytes_used = memcache.get(strip_obj_storage_path(cont_path) + '-cont_meta') + + return obj_list, object_count, bytes_used + +def get_container_details(cont_path, memcache=None): + """ + Return object_list, object_count and bytes_used. + """ + if memcache: + object_list, object_count, bytes_used = get_container_details_from_memcache(cont_path, cont_path, + memcache=memcache) + else: + object_list, object_count, bytes_used = get_container_details_from_fs(cont_path, cont_path) + + return object_list, object_count, bytes_used + +def get_account_details_from_fs(acc_path, memcache=None): + container_list = [] + container_count = 0 + + if os.path.isdir(acc_path): + for name in do_listdir(acc_path): + if not os.path.isdir(acc_path + '/' + name) or \ + name.lower() == 'tmp': + continue + container_count += 1 + container_list.append(name) + + if memcache: + memcache.set(strip_obj_storage_path(acc_path) + '_container_list', container_list) + memcache.set(strip_obj_storage_path(acc_path)+'_mtime', str(do_stat(acc_path).st_mtime)) + memcache.set(strip_obj_storage_path(acc_path)+'_container_count', container_count) + + return container_list, container_count + +def get_account_details_from_memcache(acc_path, memcache=None): + if memcache: + mtime = memcache.get(strip_obj_storage_path(acc_path)+'_mtime') + if not mtime or mtime != str(do_stat(acc_path).st_mtime): + return get_account_details_from_fs(acc_path, memcache) + container_list = memcache.get(strip_obj_storage_path(acc_path) + '_container_list') + container_count = memcache.get(strip_obj_storage_path(acc_path)+'_container_count') + return container_list, container_count + + +def get_account_details(acc_path, memcache=None): + """ + Return container_list and container_count. + """ + if memcache: + return get_account_details_from_memcache(acc_path, memcache) + else: + return get_account_details_from_fs(acc_path, memcache) + + + +def get_etag(path): + etag = None + if os.path.exists(path): + etag = md5() + if not os.path.isdir(path): + fp = open(path, 'rb') + if fp: + while True: + chunk = fp.read(CHUNK_SIZE) + if chunk: + etag.update(chunk) + else: + break + fp.close() + + etag = etag.hexdigest() + + return etag + + +def get_object_metadata(obj_path): + """ + Return metadata of object. + """ + metadata = {} + if os.path.exists(obj_path): + if not os.path.isdir(obj_path): + metadata = { + X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)), + X_CONTENT_TYPE: FILE_TYPE, + X_ETAG: get_etag(obj_path), + X_CONTENT_LENGTH: os.path.getsize(obj_path), + X_TYPE: OBJECT, + X_OBJECT_TYPE: FILE, + } + else: + metadata = { + X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)), + X_CONTENT_TYPE: DIR_TYPE, + X_ETAG: get_etag(obj_path), + X_CONTENT_LENGTH: 0, + X_TYPE: OBJECT, + X_OBJECT_TYPE: DIR, + } + + return metadata + +def get_container_metadata(cont_path, memcache=None): + objects = [] + object_count = 0 + bytes_used = 0 + objects, object_count, bytes_used = get_container_details(cont_path, + memcache=memcache) + metadata = {X_TYPE: CONTAINER, + X_TIMESTAMP: normalize_timestamp(os.path.getctime(cont_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)), + X_OBJECTS_COUNT: object_count, + X_BYTES_USED: bytes_used} + return metadata + +def get_account_metadata(acc_path, memcache=None): + containers = [] + container_count = 0 + containers, container_count = get_account_details(acc_path, memcache) + metadata = {X_TYPE: ACCOUNT, + X_TIMESTAMP: normalize_timestamp(os.path.getctime(acc_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(acc_path)), + X_OBJECTS_COUNT: 0, + X_BYTES_USED: 0, + X_CONTAINER_COUNT: container_count} + return metadata + +def restore_object(obj_path, metadata): + meta = read_metadata(obj_path) + if meta: + meta.update(metadata) + write_metadata(obj_path, meta) + else: + write_metadata(obj_path, metadata) + +def restore_container(cont_path, metadata): + meta = read_metadata(cont_path) + if meta: + meta.update(metadata) + write_metadata(cont_path, meta) + else: + write_metadata(cont_path, metadata) + +def restore_account(acc_path, metadata): + meta = read_metadata(acc_path) + if meta: + meta.update(metadata) + write_metadata(acc_path, meta) + else: + write_metadata(acc_path, metadata) + +def create_object_metadata(obj_path): + meta = get_object_metadata(obj_path) + restore_object(obj_path, meta) + return meta + +def create_container_metadata(cont_path, memcache=None): + meta = get_container_metadata(cont_path, memcache) + restore_container(cont_path, meta) + return meta + +def create_account_metadata(acc_path, memcache=None): + meta = get_account_metadata(acc_path, memcache) + restore_account(acc_path, meta) + return meta + + +def check_account_exists(account, fs_object): + if account not in get_account_list(fs_object): + logging.error('Account not exists %s' % account) + return False + else: + return True + +def get_account_list(fs_object): + account_list = [] + if fs_object: + account_list = fs_object.get_export_list() + return account_list + + +def get_account_id(account): + return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest() + |