diff --git a/setup.py b/setup.py index d195d34..ab236ee 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/python # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -94,6 +95,8 @@ setup( 'tempurl=swift.common.middleware.tempurl:filter_factory', 'formpost=swift.common.middleware.formpost:filter_factory', 'name_check=swift.common.middleware.name_check:filter_factory', + 'gluster=swift.plugins.middleware.gluster:filter_factory', + 'glusterfs=swift.plugins.middleware.glusterfs:filter_factory', ], }, ) diff --git a/swift/account/server.py b/swift/account/server.py index 800b3c0..77f9879 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,6 +36,9 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc +from swift.plugins.utils import plugin_enabled +if plugin_enabled(): + from swift.plugins.DiskDir import DiskAccount DATADIR = 'accounts' @@ -52,8 +56,12 @@ class AccountController(object): self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' + self.fs_object = None def _get_account_broker(self, drive, part, account): + if self.fs_object: + return DiskAccount(self.root, account, self.fs_object); + hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') @@ -153,6 +161,9 @@ class AccountController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) + if self.fs_object and not self.fs_object.object_only: + broker.list_containers_iter(None, None,None, + None, None) info = broker.get_info() headers = { 'X-Account-Container-Count': info['container_count'], @@ -305,8 +316,17 @@ class AccountController(object): broker.update_metadata(metadata) return HTTPNoContent(request=req) + def plugin(self, env): + if env.get('Gluster_enabled', False): + self.fs_object = env.get('fs_object') + self.root = env.get('root') + self.mount_check = False + else: + self.fs_object = None + def __call__(self, env, start_response): start_time = time.time() + self.plugin(env) req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): diff --git a/swift/container/server.py b/swift/container/server.py index 8a18cfd..952b8cd 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +38,9 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc +from swift.plugins.utils import plugin_enabled +if plugin_enabled(): + from swift.plugins.DiskDir import DiskDir DATADIR = 'containers' @@ -62,6 +66,7 @@ class ContainerController(object): ContainerBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' + self.fs_object = None def _get_container_broker(self, drive, part, account, container): """ @@ -73,6 +78,10 @@ class ContainerController(object): :param container: container name :returns: ContainerBroker object """ + if self.fs_object: + return DiskDir(self.root, drive, part, account, + container, self.logger, + fs_object = self.fs_object) hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') @@ -245,6 +254,9 @@ class ContainerController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) + if self.fs_object and not self.fs_object.object_only: + broker.list_objects_iter(None, None, None, None, + None, None) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], @@ -427,8 +439,19 @@ class ContainerController(object): broker.update_metadata(metadata) return HTTPNoContent(request=req) + def plugin(self, env): + if env.get('Gluster_enabled', False): + self.fs_object = env.get('fs_object') + if not self.fs_object: + raise NoneTypeError + self.root = env.get('root') + self.mount_check = False + else: + self.fs_object = None + def __call__(self, env, start_response): start_time = time.time() + self.plugin(env) req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): diff --git a/swift/obj/server.py b/swift/obj/server.py index 9cca16b..82eaa40 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,6 +46,10 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ DiskFileNotExist from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \ quarantine_renamer +from swift.plugins.utils import plugin_enabled +if plugin_enabled(): + from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \ + OBJECT, DIR_TYPE, FILE_TYPE DATADIR = 'objects' @@ -340,6 +345,9 @@ class DiskFile(object): raise DiskFileNotExist('Data File does not exist.') +if plugin_enabled(): + from swift.plugins.DiskFile import Gluster_DiskFile + class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" @@ -377,6 +385,17 @@ class ObjectController(object): 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) + self.fs_object = None + + def get_DiskFile_obj(self, path, device, partition, account, container, obj, + logger, keep_data_fp=False, disk_chunk_size=65536): + if self.fs_object: + return Gluster_DiskFile(path, device, partition, account, container, + obj, logger, keep_data_fp, + disk_chunk_size, fs_object = self.fs_object); + else: + return DiskFile(path, device, partition, account, container, + obj, logger, keep_data_fp, disk_chunk_size) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): @@ -493,7 +512,7 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'X-Delete-At' in file.metadata and \ @@ -548,7 +567,7 @@ class ObjectController(object): if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time @@ -580,12 +599,26 @@ class ObjectController(object): if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) - metadata = { - 'X-Timestamp': request.headers['x-timestamp'], - 'Content-Type': request.headers['content-type'], - 'ETag': etag, - 'Content-Length': str(os.fstat(fd).st_size), - } + content_type = request.headers['content-type'] + if self.fs_object and not content_type: + content_type = FILE_TYPE + if not self.fs_object: + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, + 'ETag': etag, + 'Content-Length': str(os.fstat(fd).st_size), + } + else: + x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, + 'ETag': etag, + 'Content-Length': str(os.fstat(fd).st_size), + X_TYPE: OBJECT, + X_OBJECT_TYPE: x_object_type, + } metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) @@ -612,7 +645,7 @@ class ObjectController(object): 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], 'x-trans-id': request.headers.get('x-trans-id', '-')}, - device) + (self.fs_object and account) or device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -626,9 +659,9 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, - obj, self.logger, keep_data_fp=True, - disk_chunk_size=self.disk_chunk_size) + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, + obj, self.logger, keep_data_fp=True, + disk_chunk_size=self.disk_chunk_size) if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): if request.headers.get('if-match') == '*': @@ -702,7 +735,7 @@ class ObjectController(object): return resp if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): @@ -744,7 +777,7 @@ class ObjectController(object): if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) response_class = HTTPNoContent - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'x-if-delete-at' in request.headers and \ int(request.headers['x-if-delete-at']) != \ @@ -797,9 +830,18 @@ class ObjectController(object): raise hashes return Response(body=pickle.dumps(hashes)) + def plugin(self, env): + if env.get('Gluster_enabled', False): + self.fs_object = env.get('fs_object') + self.devices = env.get('root') + self.mount_check = False + else: + self.fs_object = None + def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() + self.plugin(env) req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info):