diff --git a/setup.py b/setup.py index d195d34..ef625ff 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/python # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -94,6 +95,7 @@ setup( 'tempurl=swift.common.middleware.tempurl:filter_factory', 'formpost=swift.common.middleware.formpost:filter_factory', 'name_check=swift.common.middleware.name_check:filter_factory', + 'gluster=swift.plugins.middleware.gluster:filter_factory', ], }, ) diff --git a/swift/account/server.py b/swift/account/server.py index 800b3c0..ba13786 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,6 +36,10 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc +from swift.plugins.utils import Gluster_enabled +if Gluster_enabled(): + from swift.plugins.DiskDir import DiskAccount + from swift.plugins import Glusterfs DATADIR = 'accounts' @@ -45,15 +50,21 @@ class AccountController(object): def __init__(self, conf): self.logger = get_logger(conf, log_route='account-server') - self.root = conf.get('devices', '/srv/node') - self.mount_check = conf.get('mount_check', 'true').lower() in \ - ('true', 't', '1', 'on', 'yes', 'y') + if Gluster_enabled(): + self.root = Glusterfs.MOUNT_PATH + self.mount_check = False + else: + self.root = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' def _get_account_broker(self, drive, part, account): + if Gluster_enabled(): + return DiskAccount(self.root, account) hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') @@ -153,6 +164,9 @@ class AccountController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) + if Gluster_enabled() and not Glusterfs.OBJECT_ONLY: + broker.list_containers_iter(None, None,None, + None, None) info = broker.get_info() headers = { 'X-Account-Container-Count': info['container_count'], diff --git a/swift/container/server.py b/swift/container/server.py index 8a18cfd..c4982f1 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +38,11 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc +from swift.plugins.utils import Gluster_enabled +if Gluster_enabled(): + from swift.plugins.DiskDir import DiskDir + from swift.plugins import Glusterfs + DATADIR = 'containers' @@ -50,9 +56,13 @@ class ContainerController(object): def __init__(self, conf): self.logger = get_logger(conf, log_route='container-server') - self.root = conf.get('devices', '/srv/node/') - self.mount_check = conf.get('mount_check', 'true').lower() in \ - ('true', 't', '1', 'on', 'yes', 'y') + if Gluster_enabled(): + self.root = Glusterfs.MOUNT_PATH + self.mount_check = False + else: + self.root = conf.get('devices', '/srv/node/') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.allowed_sync_hosts = [h.strip() @@ -73,6 +83,9 @@ class ContainerController(object): :param container: container name :returns: ContainerBroker object """ + if Gluster_enabled(): + return DiskDir(self.root, drive, part, account, + container, self.logger) hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') @@ -245,6 +258,9 @@ class ContainerController(object): broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) + if Gluster_enabled() and not Glusterfs.OBJECT_ONLY: + broker.list_objects_iter(None, None, None, None, + None, None) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], diff --git a/swift/obj/server.py b/swift/obj/server.py index 9cca16b..7a671c2 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -1,4 +1,5 @@ # Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,6 +46,11 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ DiskFileNotExist from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \ quarantine_renamer +from swift.plugins.utils import Gluster_enabled +if Gluster_enabled(): + from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \ + OBJECT, DIR_TYPE, FILE_TYPE + from swift.plugins import Glusterfs DATADIR = 'objects' @@ -340,6 +346,10 @@ class DiskFile(object): raise DiskFileNotExist('Data File does not exist.') +if Gluster_enabled(): + from swift.plugins.DiskFile import Gluster_DiskFile + + class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" @@ -351,9 +361,13 @@ class ObjectController(object): /etc/swift/object-server.conf-sample. """ self.logger = get_logger(conf, log_route='object-server') - self.devices = conf.get('devices', '/srv/node/') - self.mount_check = conf.get('mount_check', 'true').lower() in \ - ('true', 't', '1', 'on', 'yes', 'y') + if Gluster_enabled(): + self.devices = Glusterfs.MOUNT_PATH + self.mount_check = False + else: + self.devices = conf.get('devices', '/srv/node/') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) @@ -378,6 +392,15 @@ class ObjectController(object): self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) + def get_DiskFile_obj(self, path, device, partition, account, container, obj, + logger, keep_data_fp=False, disk_chunk_size=65536): + if Gluster_enabled(): + return Gluster_DiskFile(path, device, partition, account, container, + obj, logger, keep_data_fp, disk_chunk_size) + else: + return DiskFile(path, device, partition, account, container, + obj, logger, keep_data_fp, disk_chunk_size) + def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ @@ -493,7 +516,7 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'X-Delete-At' in file.metadata and \ @@ -548,7 +571,7 @@ class ObjectController(object): if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time @@ -580,12 +603,28 @@ class ObjectController(object): if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) - metadata = { - 'X-Timestamp': request.headers['x-timestamp'], - 'Content-Type': request.headers['content-type'], - 'ETag': etag, - 'Content-Length': str(os.fstat(fd).st_size), - } + content_type = request.headers['content-type'] + if not Gluster_enabled(): + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, + 'ETag': etag, + 'Content-Length': str(os.fstat(fd).st_size), + } + else: + if not content_type: + content_type = FILE_TYPE + x_object_type = FILE + else: + x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': content_type, + 'ETag': etag, + 'Content-Length': str(os.fstat(fd).st_size), + X_TYPE: OBJECT, + X_OBJECT_TYPE: x_object_type, + } metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) @@ -612,7 +651,7 @@ class ObjectController(object): 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], 'x-trans-id': request.headers.get('x-trans-id', '-')}, - device) + (Gluster_enabled() and account) or device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -626,9 +665,9 @@ class ObjectController(object): content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, - obj, self.logger, keep_data_fp=True, - disk_chunk_size=self.disk_chunk_size) + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, + obj, self.logger, keep_data_fp=True, + disk_chunk_size=self.disk_chunk_size) if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): if request.headers.get('if-match') == '*': @@ -702,7 +741,7 @@ class ObjectController(object): return resp if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if file.is_deleted() or ('X-Delete-At' in file.metadata and int(file.metadata['X-Delete-At']) <= time.time()): @@ -744,7 +783,7 @@ class ObjectController(object): if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) response_class = HTTPNoContent - file = DiskFile(self.devices, device, partition, account, container, + file = self.get_DiskFile_obj(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) if 'x-if-delete-at' in request.headers and \ int(request.headers['x-if-delete-at']) != \