From 2bd696e392e420a2521dcca0b8613122d8169025 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Fri, 18 Mar 2016 18:09:46 +0530 Subject: Enhance object expiration This change re-introduces object expiration feature with some internal differences from earlier implementation such as: * Creation of zero-byte tracker object is performed directly on the mount point. Earlier HTTP request was sent to object server for the same. This incurred overhead of metadata creation for these zero-byte files which isn't necessarry as all required information is encoded in the path itself. * Crawling of zero-byte tracker objects is done by the object expirer daemon itself and not container server. * Deletion of tracker object is performed by the object expiration daemon directly on mount point. Deletion of actual data object is not carried out by object expiration daemon directly. The object expirer sends a DELETE request to object server which deletes the actual object. This behaviour is not changed. There is no change in behaviour in comparison with older implementation. This is asserted by re-enabling existing functional tests without any changes. Change-Id: I01dc77cc4db3be3147d54e3aa2a19ed182498900 Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/13913 Reviewed-by: Thiago da Silva Tested-by: Thiago da Silva --- gluster/swift/obj/expirer.py | 160 +++++++++++++++++++++++++++++++++++++++++++ gluster/swift/obj/server.py | 78 ++++++++++++++++++--- 2 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 gluster/swift/obj/expirer.py (limited to 'gluster/swift/obj') diff --git a/gluster/swift/obj/expirer.py b/gluster/swift/obj/expirer.py new file mode 100644 index 0000000..564a2c9 --- /dev/null +++ b/gluster/swift/obj/expirer.py @@ -0,0 +1,160 @@ +# Copyright (c) 2016 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This import will monkey-patch Ring and other classes. +# Do not remove. +import gluster.swift.common.constraints # noqa + +import errno +import os + +from gluster.swift.common.utils import rmobjdir + +from swift.obj.expirer import ObjectExpirer as SwiftObjectExpirer +from swift.common.http import HTTP_NOT_FOUND +from swift.common.internal_client import InternalClient, UnexpectedResponse +from swift.common.utils import ThreadPool + +EXCLUDE_DIRS = ('.trashcan', '.glusterfs') + + +class GlusterSwiftInternalClient(InternalClient): + + def __init__(self, conf_path, user_agent, request_tries, + allow_modify_pipeline=False, devices=None): + super(GlusterSwiftInternalClient, self).__init__( + conf_path, user_agent, request_tries, allow_modify_pipeline) + self.devices = devices + + def get_account_info(self, account): + # Supposed to return container count and object count in gsexpiring + # account. This is used by object expirer only for logging. + return (0, 0) + + def delete_container(self, account, container, acceptable_statuses=None): + container_path = os.path.join(self.devices, account, container) + try: + os.rmdir(container_path) + except OSError as err: + if err.errno != errno.ENOENT: + raise + + def iter_containers(self, account): + account_path = os.path.join(self.devices, account) + for container in os.listdir(account_path): + if container in EXCLUDE_DIRS: + continue + container_path = os.path.join(account_path, container) + if os.path.isdir(container_path): + yield {'name': container.encode('utf8')} + + def iter_objects(self, account, container): + container_path = os.path.join(self.devices, account, container) + # TODO: Use a slightly better implementation of os.walk() + for (root, dirs, files) in os.walk(container_path): + for f in files: + obj_path = os.path.join(root, f) + obj = obj_path[(len(container_path) + 1):] + yield {'name': obj.encode('utf8')} + + +class ObjectExpirer(SwiftObjectExpirer): + + def __init__(self, conf, logger=None, swift=None): + + conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf' + self.devices = conf.get('devices', '/mnt/gluster-object') + # Do not retry DELETEs on getting 404. Hence default is set to 1. + request_tries = int(conf.get('request_tries') or 1) + # Use our extended version of InternalClient + swift = GlusterSwiftInternalClient( + conf_path, 'Gluster Swift Object Expirer', request_tries, + devices=self.devices) + # Let the parent class initialize self.swift + super(ObjectExpirer, self).__init__(conf, logger=logger, swift=swift) + + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() + if not self.reseller_prefix.endswith('_'): + self.reseller_prefix = self.reseller_prefix + '_' + + # nthread=0 is intentional. This ensures that no green pool is + # used. Call to force_run_in_thread() will ensure that the method + # passed as arg is run in a real external thread using eventlet.tpool + # which has a threadpool of 20 threads (default) + self.threadpool = ThreadPool(nthreads=0) + + def _delete_tracker_object(self, container, obj): + container_path = os.path.join(self.devices, + self.expiring_objects_account, + container) + tracker_object_path = os.path.join(container_path, obj) + + try: + os.unlink(tracker_object_path) + except OSError as err: + if err.errno in (errno.ENOENT, errno.ESTALE): + # Ignore removal from another entity. + return + elif err.errno == errno.EISDIR: + # Handle race: Was a file during crawl, but now it's a + # directory. There are no 'directory marker' objects in + # gsexpiring volume. + return + else: + raise + + # This part of code is very similar to DiskFile._unlinkold() + dirname = os.path.dirname(tracker_object_path) + while dirname and dirname != container_path: + if not rmobjdir(dirname, marker_dir_check=False): + # If a directory with objects has been found, we can stop + # garbage collection + break + else: + # Traverse upwards till the root of container + dirname = os.path.dirname(dirname) + + def pop_queue(self, container, obj): + """ + In Swift, this method removes tracker object entry directly from + container database. In gluster-swift, this method deletes tracker + object directly from filesystem. + """ + self.threadpool.force_run_in_thread(self._delete_tracker_object, + container, obj) + + def delete_actual_object(self, actual_obj, timestamp): + """ + Swift's expirer will re-attempt expiring if the source object is not + available (404 or ANY other error) up to self.reclaim_age seconds + before it gives up and deletes the entry in the queue. + + Don't do this in gluster-swift. GlusterFS isn't eventually consistent + and has no concept of hand-off nodes. If actual data object doesn't + exist (404), remove tracker object from the queue (filesystem). + + However if DELETE fails due a reason other than 404, do not remove + tracker object yet, follow Swift's behaviour of waiting till + self.reclaim_age seconds. + + This method is just a wrapper around parent class's method. All this + wrapper does is ignore 404 failures. + """ + try: + super(ObjectExpirer, self).delete_actual_object( + actual_obj, timestamp) + except UnexpectedResponse as err: + if err.resp.status_int != HTTP_NOT_FOUND: + raise diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py index 1d8d418..3e27cc3 100644 --- a/gluster/swift/obj/server.py +++ b/gluster/swift/obj/server.py @@ -14,16 +14,19 @@ # limitations under the License. """ Object Server for Gluster for Swift """ +import errno +import os from swift.common.swob import HTTPConflict, HTTPNotImplemented -from swift.common.utils import public, timing_stats, replication -from gluster.swift.common.exceptions import AlreadyExistsAsFile, \ - AlreadyExistsAsDir +from swift.common.utils import public, timing_stats, replication, mkdirs from swift.common.request_helpers import split_and_validate_path - from swift.obj import server from gluster.swift.obj.diskfile import DiskFileManager +from gluster.swift.common.fs_utils import do_ismount +from gluster.swift.common.ring import Ring +from gluster.swift.common.exceptions import AlreadyExistsAsFile, \ + AlreadyExistsAsDir class GlusterSwiftDiskFileRouter(object): @@ -55,6 +58,9 @@ class ObjectController(server.ObjectController): """ # Replaces Swift's DiskFileRouter object reference with ours. self._diskfile_router = GlusterSwiftDiskFileRouter(conf, self.logger) + self.devices = conf.get('devices', '/mnt/gluster-object') + self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.object_ring = self.get_object_ring() def container_update(self, *args, **kwargs): """ @@ -65,15 +71,65 @@ class ObjectController(server.ObjectController): """ return - def delete_at_update(self, *args, **kwargs): - """ - Update the expiring objects container when objects are updated. + def get_object_ring(self): + return Ring(self.swift_dir, ring_name='object') - For Gluster, this is a no-op as there are no container DB entries - to be created that tracks objects to be expired. Objects to be - expired will be determined by crawling the filesystem directly. + def _create_expiring_tracker_object(self, object_path): + try: + + # Check if gsexpiring volume is present in ring + if not any(d.get('device', None) == self.expiring_objects_account + for d in self.object_ring.devs): + raise Exception("%s volume not in ring" % + self.expiring_objects_account) + + # Check if gsexpiring is mounted. + expiring_objects_account_path = \ + os.path.join(self.devices, self.expiring_objects_account) + mount_check = self._diskfile_router['junk'].mount_check + if mount_check and not do_ismount(expiring_objects_account_path): + raise Exception("Path %s doesn't exist or is not a mount " + "point" % expiring_objects_account_path) + + # Create object directory + object_dir = os.path.dirname(object_path) + try: + mkdirs(object_dir) + except OSError as err: + mkdirs(object_dir) # handle race + + # Create zero-byte file + try: + os.mknod(object_path) + except OSError as err: + if err.errno != errno.EEXIST: + raise + except Exception as e: + self.logger.error("Creation of tracker object %s failed: %s" % + (object_path, str(e))) + + def async_update(self, op, account, container, obj, host, partition, + contdevice, headers_out, objdevice, policy): """ - return + In Openstack Swift, this method is called by: + * container_update (a no-op in gluster-swift) + * delete_at_update (to PUT objects into .expiring_objects account) + + The Swift's version of async_update only sends the request to + container-server to PUT the object. The container-server calls + container_update method which makes an entry for the object in it's + database. No actual object is created on disk. + + But in gluster-swift container_update is a no-op, so we'll + have to PUT an actual object. We override async_update to create a + container first and then the corresponding "tracker object" which + tracks expired objects scheduled for deletion. + """ + object_path = os.path.join(self.devices, account, container, obj) + + threadpool = self._diskfile_router[policy].threadpools[objdevice] + threadpool.run_in_thread(self._create_expiring_tracker_object, + object_path) @public @timing_stats() -- cgit