From 5a04cede1f5bb44d6c64b186335146dd4e70a6ea Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 20 Apr 2016 15:10:43 +0530 Subject: Make swift's expirer compatible with gluster-swift Swift's object expirer in kilo series was incompatible with gluster-swift. This change does the following: * Optimizes crawl in account and container server for listing requests for containers and tracker objects in gsexpiring volume. * Enables container server to delete tracker objects from gsexpiring volume. Swift's expirer sends request directly to container server to remove tracker object entry. * delete_tracker_object() is now a common utility function that is invoked from container server and gluster-swift's object expirer. * Run functional test to be run against both swift's object expirer and gluster-swift's object expirer Change-Id: Ib5b7f7f08fe7dda574f6dd80be2f38bdfaee32bc Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/14038 Reviewed-by: Thiago da Silva Tested-by: Thiago da Silva --- etc/object-expirer.conf-gluster | 8 + gluster/swift/common/DiskDir.py | 97 +++++- gluster/swift/common/utils.py | 57 ++++ gluster/swift/obj/expirer.py | 40 +-- .../test_object_expirer.py | 337 --------------------- .../test_object_expirer_gluster_swift.py | 337 +++++++++++++++++++++ .../test_object_expirer_swift.py | 335 ++++++++++++++++++++ test/unit/common/test_diskdir.py | 70 ++++- 8 files changed, 896 insertions(+), 385 deletions(-) delete mode 100644 test/object_expirer_functional/test_object_expirer.py create mode 100644 test/object_expirer_functional/test_object_expirer_gluster_swift.py create mode 100644 test/object_expirer_functional/test_object_expirer_swift.py diff --git a/etc/object-expirer.conf-gluster b/etc/object-expirer.conf-gluster index 32be5a1..8be8626 100644 --- a/etc/object-expirer.conf-gluster +++ b/etc/object-expirer.conf-gluster @@ -14,6 +14,14 @@ log_level = INFO auto_create_account_prefix = gs expiring_objects_account_name = expiring +# The expirer will re-attempt expiring if the source object is not available +# up to reclaim_age seconds before it gives up and deletes the entry in the +# queue. In gluster-swift, you'd almost always want to set this to zero. +reclaim_age = 0 + +# Do not retry DELETEs on getting 404. Hence default is set to 1. +request_tries = 1 + # The swift-object-expirer daemon will run every 'interval' number of seconds # interval = 300 diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py index d314a1f..204ae1d 100644 --- a/gluster/swift/common/DiskDir.py +++ b/gluster/swift/common/DiskDir.py @@ -17,19 +17,22 @@ import os import errno from gluster.swift.common.fs_utils import dir_empty, mkdirs, do_chown, \ - do_exists, do_touch + do_exists, do_touch, do_stat from gluster.swift.common.utils import validate_account, validate_container, \ get_container_details, get_account_details, create_container_metadata, \ create_account_metadata, DEFAULT_GID, get_container_metadata, \ get_account_metadata, DEFAULT_UID, validate_object, \ create_object_metadata, read_metadata, write_metadata, X_CONTENT_TYPE, \ X_CONTENT_LENGTH, X_TIMESTAMP, X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, \ - X_BYTES_USED, X_CONTAINER_COUNT, DIR_TYPE, rmobjdir, dir_is_object + X_BYTES_USED, X_CONTAINER_COUNT, DIR_TYPE, rmobjdir, dir_is_object, \ + list_objects_gsexpiring_container, normalize_timestamp from gluster.swift.common import Glusterfs from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ GlusterFileSystemIOError +from gluster.swift.obj.expirer import delete_tracker_object from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE from swift.common.swob import HTTPBadRequest +from swift.common.utils import ThreadPool DATADIR = 'containers' @@ -176,6 +179,12 @@ class DiskCommon(object): self.datadir = os.path.join(root, drive) self._dir_exists = None + # nthread=0 is intentional. This ensures that no green pool is + # used. Call to force_run_in_thread() will ensure that the method + # passed as arg is run in a real external thread using eventlet.tpool + # which has a threadpool of 20 threads (default) + self.threadpool = ThreadPool(nthreads=0) + def _dir_exists_read_metadata(self): self._dir_exists = do_exists(self.datadir) if self._dir_exists: @@ -342,6 +351,24 @@ class DiskDir(DiskCommon): self.container = container self.datadir = os.path.join(self.datadir, self.container) + if self.account == 'gsexpiring': + # Do not bother crawling the entire container tree just to update + # object count and bytes used. Return immediately before metadata + # validation and creation happens. + info = do_stat(self.datadir) + if not info: + # Container no longer exists. + return + semi_fake_md = { + 'X-Object-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0), + 'X-Type': ('container', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + self.metadata = semi_fake_md + return + if not self._dir_exists_read_metadata(): return @@ -387,7 +414,10 @@ class DiskDir(DiskCommon): container_list = [] - objects = self._update_object_count() + if self.account == 'gsexpiring': + objects = list_objects_gsexpiring_container(self.datadir) + else: + objects = self._update_object_count() if objects: objects.sort() else: @@ -419,12 +449,23 @@ class DiskDir(DiskCommon): objects = filter_delimiter(objects, delimiter, prefix, marker, path) - if out_content_type == 'text/plain': + if out_content_type == 'text/plain' or \ + self.account == 'gsexpiring': + # When out_content_type == 'text/plain': + # # The client is only asking for a plain list of objects and NOT # asking for any extended information about objects such as # bytes used or etag. + # + # When self.account == 'gsexpiring': + # + # This is a JSON request sent by the object expirer to list + # tracker objects in a container in gsexpiring volume. + # When out_content_type is 'application/json', the caller + # expects each record entry to have the following ordered + # fields: (name, timestamp, size, content_type, etag) for obj in objects: - container_list.append((obj, 0, 0, 0, 0)) + container_list.append((obj, '0', 0, 'text/plain', '')) if len(container_list) >= limit: break return container_list @@ -498,7 +539,8 @@ class DiskDir(DiskCommon): reported_put_timestamp, reported_delete_timestamp, reported_object_count, and reported_bytes_used. """ - if self._dir_exists and Glusterfs._container_update_object_count: + if self._dir_exists and Glusterfs._container_update_object_count and \ + self.account != 'gsexpiring': self._update_object_count() data = {'account': self.account, 'container': self.container, @@ -560,9 +602,15 @@ class DiskDir(DiskCommon): write_metadata(self.datadir, self.metadata) def delete_object(self, name, timestamp, obj_policy_index): - # NOOP - should never be called since object file removal occurs - # within a directory implicitly. - return + if self.account == 'gsexpiring': + # The request originated from object expirer. This should + # delete tracker object. + self.threadpool.force_run_in_thread(delete_tracker_object, + self.datadir, name) + else: + # NOOP - should never be called since object file removal occurs + # within a directory implicitly. + return def delete_db(self, timestamp): """ @@ -626,6 +674,22 @@ class DiskAccount(DiskCommon): super(DiskAccount, self).__init__(root, drive, account, logger, **kwargs) + if self.account == 'gsexpiring': + # Do not bother updating object count, container count and bytes + # used. Return immediately before metadata validation and + # creation happens. + info = do_stat(self.datadir) + semi_fake_md = { + 'X-Object-Count': (0, 0), + 'X-Container-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + self.metadata = semi_fake_md + return + # Since accounts should always exist (given an account maps to a # gluster volume directly, and the mount has already been checked at # the beginning of the REST API handling), just assert that that @@ -750,10 +814,20 @@ class DiskAccount(DiskCommon): containers = filter_delimiter(containers, delimiter, prefix, marker) - if response_content_type == 'text/plain': + if response_content_type == 'text/plain' or \ + self.account == 'gsexpiring': + # When response_content_type == 'text/plain': + # # The client is only asking for a plain list of containers and NOT # asking for any extended information about container such as # bytes used or object count. + # + # When self.account == 'gsexpiring': + # This is a JSON request sent by the object expirer to list + # containers in gsexpiring volume. When out_content_type is + # 'application/json', the caller expects each record entry to have + # the following ordered fields: + # (name, object_count, bytes_used, is_subdir) for container in containers: # When response_content_type == 'text/plain', Swift will only # consume the name of the container (first element of tuple). @@ -796,7 +870,8 @@ class DiskAccount(DiskCommon): delete_timestamp, container_count, object_count, bytes_used, hash, id """ - if Glusterfs._account_update_container_count: + if Glusterfs._account_update_container_count and \ + self.account != 'gsexpiring': self._update_container_count() data = {'account': self.account, 'created_at': '1', diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index 1bbc56c..26e8c1b 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -370,6 +370,63 @@ def get_container_details(cont_path): return obj_list, object_count, bytes_used +def list_objects_gsexpiring_container(container_path): + """ + This method does a simple walk, unlike get_container_details which + walks the filesystem tree and does a getxattr() on every directory + to check if it's a directory marker object and stat() on every file + to get it's size. These are not required for gsexpiring volume as + it can never have directory marker objects in it and all files are + zero-byte in size. + """ + obj_list = [] + + for (root, dirs, files) in os.walk(container_path): + for f in files: + obj_path = os.path.join(root, f) + obj = obj_path[(len(container_path) + 1):] + obj_list.append(obj) + # Yield the co-routine cooperatively + sleep() + + return obj_list + + +def delete_tracker_object(container_path, obj): + """ + Delete zero-byte tracker object from gsexpiring volume. + Called by: + - gluster.swift.obj.expirer.ObjectExpirer.pop_queue() + - gluster.swift.common.DiskDir.DiskDir.delete_object() + """ + tracker_object_path = os.path.join(container_path, obj) + + try: + os.unlink(tracker_object_path) + except OSError as err: + if err.errno in (errno.ENOENT, errno.ESTALE): + # Ignore removal from another entity. + return + elif err.errno == errno.EISDIR: + # Handle race: Was a file during crawl, but now it's a + # directory. There are no 'directory marker' objects in + # gsexpiring volume. + return + else: + raise + + # This part of code is very similar to DiskFile._unlinkold() + dirname = os.path.dirname(tracker_object_path) + while dirname and dirname != container_path: + if not rmobjdir(dirname, marker_dir_check=False): + # If a directory with objects has been found, we can stop + # garbage collection + break + else: + # Traverse upwards till the root of container + dirname = os.path.dirname(dirname) + + def get_account_details(acc_path): """ Return container_list and container_count. diff --git a/gluster/swift/obj/expirer.py b/gluster/swift/obj/expirer.py index 564a2c9..38f870e 100644 --- a/gluster/swift/obj/expirer.py +++ b/gluster/swift/obj/expirer.py @@ -20,7 +20,7 @@ import gluster.swift.common.constraints # noqa import errno import os -from gluster.swift.common.utils import rmobjdir +from gluster.swift.common.utils import delete_tracker_object from swift.obj.expirer import ObjectExpirer as SwiftObjectExpirer from swift.common.http import HTTP_NOT_FOUND @@ -95,45 +95,17 @@ class ObjectExpirer(SwiftObjectExpirer): # which has a threadpool of 20 threads (default) self.threadpool = ThreadPool(nthreads=0) - def _delete_tracker_object(self, container, obj): - container_path = os.path.join(self.devices, - self.expiring_objects_account, - container) - tracker_object_path = os.path.join(container_path, obj) - - try: - os.unlink(tracker_object_path) - except OSError as err: - if err.errno in (errno.ENOENT, errno.ESTALE): - # Ignore removal from another entity. - return - elif err.errno == errno.EISDIR: - # Handle race: Was a file during crawl, but now it's a - # directory. There are no 'directory marker' objects in - # gsexpiring volume. - return - else: - raise - - # This part of code is very similar to DiskFile._unlinkold() - dirname = os.path.dirname(tracker_object_path) - while dirname and dirname != container_path: - if not rmobjdir(dirname, marker_dir_check=False): - # If a directory with objects has been found, we can stop - # garbage collection - break - else: - # Traverse upwards till the root of container - dirname = os.path.dirname(dirname) - def pop_queue(self, container, obj): """ In Swift, this method removes tracker object entry directly from container database. In gluster-swift, this method deletes tracker object directly from filesystem. """ - self.threadpool.force_run_in_thread(self._delete_tracker_object, - container, obj) + container_path = os.path.join(self.devices, + self.expiring_objects_account, + container) + self.threadpool.force_run_in_thread(delete_tracker_object, + container_path, obj) def delete_actual_object(self, actual_obj, timestamp): """ diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer.py deleted file mode 100644 index 279994f..0000000 --- a/test/object_expirer_functional/test_object_expirer.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time -import logging - -from gluster.swift.obj.expirer import ObjectExpirer, GlusterSwiftInternalClient - -from swift.common.utils import readconf - -from test import get_config -from test.functional.tests import Base, Utils -from test.functional.swift_test_client import Account, Connection, \ - ResponseError - -config = get_config('func_test') - - -class TestObjectExpirerEnv: - @classmethod - def setUp(cls): - cls.conn = Connection(config) - cls.conn.authenticate() - cls.account = Account(cls.conn, - config.get('account', - config['username'])) - cls.account.delete_containers() - cls.container = cls.account.container(Utils.create_name()) - if not cls.container.create(): - raise ResponseError(cls.conn.response) - cls.file_size = 8 - cls.root_dir = os.path.join('/mnt/gluster-object', - cls.account.conn.storage_url.split('/')[2].split('_')[1]) - devices = config.get('devices', '/mnt/gluster-object') - cls.client = GlusterSwiftInternalClient('/etc/swift/object-expirer.conf', - 'Test Object Expirer', 1, - devices=devices) - conf = readconf('/etc/swift/object-expirer.conf', 'object-expirer') - cls.expirer = ObjectExpirer(conf) - - -class TestObjectExpirer(Base): - env = TestObjectExpirerEnv - set_up = False - - def test_object_expiry_X_Delete_At_PUT(self): - obj = self.env.container.file(Utils.create_name()) - x_delete_at = str(int(time.time()) + 2) - obj.write_random(self.env.file_size, - hdrs={'X-Delete-At': x_delete_at}) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Ensure X-Delete-At is saved as object metadata. - self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - def test_object_expiry_X_Delete_After_PUT(self): - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size, - hdrs={'X-Delete-After': 2}) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Ensure X-Delete-At is saved as object metadata. - self.assertTrue(str(obj.info()['x_delete_at'])) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - def test_object_expiry_X_Delete_At_POST(self): - - # Create normal object - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size) - obj.read() - self.assert_status(200) - - # Send POST on that object and set it to be expired. - x_delete_at = str(int(time.time()) + 2) - obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, - cfg={'x_delete_at': x_delete_at}) - - # Ensure X-Delete-At is saved as object metadata. - self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - - def test_object_expiry_X_Delete_After_POST(self): - - # Create normal object - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size) - obj.read() - self.assert_status(200) - - # Send POST on that object and set it to be expired. - obj.sync_metadata(metadata={'X-Delete-After': 2}, - cfg={'x_delete_after': 2}) - - # Ensure X-Delete-At is saved as object metadata. - self.assertTrue(str(obj.info()['x_delete_at'])) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - - def test_object_expiry_err(self): - obj = self.env.container.file(Utils.create_name()) - - # X-Delete-At is invalid or is in the past - for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): - self.assertRaises(ResponseError, - obj.write_random, - self.env.file_size, - hdrs={'X-Delete-At': i}) - self.assert_status(400) - - # X-Delete-After is invalid. - for i in (-2, 'abc', 3.7): - self.assertRaises(ResponseError, - obj.write_random, - self.env.file_size, - hdrs={'X-Delete-After': i}) - self.assert_status(400) - diff --git a/test/object_expirer_functional/test_object_expirer_gluster_swift.py b/test/object_expirer_functional/test_object_expirer_gluster_swift.py new file mode 100644 index 0000000..279994f --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer_gluster_swift.py @@ -0,0 +1,337 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import logging + +from gluster.swift.obj.expirer import ObjectExpirer, GlusterSwiftInternalClient + +from swift.common.utils import readconf + +from test import get_config +from test.functional.tests import Base, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + +config = get_config('func_test') + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + devices = config.get('devices', '/mnt/gluster-object') + cls.client = GlusterSwiftInternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1, + devices=devices) + conf = readconf('/etc/swift/object-expirer.conf', 'object-expirer') + cls.expirer = ObjectExpirer(conf) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/object_expirer_functional/test_object_expirer_swift.py b/test/object_expirer_functional/test_object_expirer_swift.py new file mode 100644 index 0000000..c59cc67 --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer_swift.py @@ -0,0 +1,335 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import logging + +from swift.common.manager import Manager +from swift.common.internal_client import InternalClient + + +from test import get_config +from test.functional.tests import Base, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + +config = get_config('func_test') + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + devices = config.get('devices', '/mnt/gluster-object') + cls.client = InternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1) + cls.expirer = Manager(['object-expirer']) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index 623164c..964bc2f 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -402,6 +402,37 @@ class TestContainerBroker(unittest.TestCase): fp.write("file path: %s\n" % fullname) return fullname + def test_gsexpiring_fake_md(self): + # Create account + account_path = os.path.join(self.path, "gsexpiring") + os.makedirs(account_path) + + # Create container + cpath = os.path.join(account_path, "container") + os.mkdir(cpath) + + # Create 10 objects in container. These should not reflect in + # X-Object-Count. + for o in xrange(10): + os.mkdir(os.path.join(cpath, str(o))) + + orig_stat = os.stat(cpath) + expected_metadata = { + 'X-Object-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0), + 'X-Type': ('container', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + + # Create DiskDir instance + disk_dir = dd.DiskDir(self.path, 'gsexpiring', account='gsexpiring', + container='container', logger=FakeLogger()) + self.assertEqual(expected_metadata, disk_dir.metadata) + info = disk_dir.get_info() + self.assertEqual(info['object_count'], 0) + self.assertEqual(info['bytes_used'], 0) + def test_creation(self): # Test swift.common.db.ContainerBroker.__init__ broker = self._get_broker(account='a', container='c') @@ -814,10 +845,10 @@ class TestContainerBroker(unittest.TestCase): out_content_type="text/plain") self.assertEquals(len(listing), 100) for (name, ts, clen, ctype, etag) in listing: - self.assertEqual(ts, 0) + self.assertEqual(ts, '0') self.assertEqual(clen, 0) - self.assertEqual(ctype, 0) - self.assertEqual(etag, 0) + self.assertEqual(ctype, 'text/plain') + self.assertEqual(etag, '') # Check that limit is still honored. listing = broker.list_objects_iter(25, '', None, None, '', @@ -1343,6 +1374,39 @@ class TestDiskAccount(unittest.TestCase): _destroyxattr() shutil.rmtree(self.td) + def test_gsexpiring_fake_md(self): + # Create account + account_path = os.path.join(self.td, "gsexpiring") + os.makedirs(account_path) + + # Create 10 containers - these should not reflect in + # X-Container-Count. Also, create 10 objects in each + # container. These should not reflect in X-Object-Count. + for c in xrange(10): + cpath = os.path.join(account_path, str(c)) + os.mkdir(cpath) + for o in xrange(10): + os.mkdir(os.path.join(cpath, str(o))) + + orig_stat = os.stat(account_path) + expected_metadata = { + 'X-Object-Count': (0, 0), + 'X-Container-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + + # Create DiskAccount instance + da = dd.DiskAccount(self.td, 'gsexpiring', 'gsexpiring', + self.fake_logger) + self.assertEqual(expected_metadata, da.metadata) + info = da.get_info() + self.assertEqual(info['container_count'], 0) + self.assertEqual(info['object_count'], 0) + self.assertEqual(info['bytes_used'], 0) + def test_constructor_no_metadata(self): da = dd.DiskAccount(self.td, self.fake_drives[0], self.fake_accounts[0], self.fake_logger) -- cgit