summaryrefslogtreecommitdiffstats
path: root/test
diff options
context:
space:
mode:
authorPrashanth Pai <ppai@redhat.com>2016-04-20 15:10:43 +0530
committerThiago da Silva <thiago@redhat.com>2016-04-20 12:08:03 -0700
commit5a04cede1f5bb44d6c64b186335146dd4e70a6ea (patch)
tree34babe4bcc295562ca3b740c98bc2299669c12f3 /test
parent2bd696e392e420a2521dcca0b8613122d8169025 (diff)
Make swift's expirer compatible with gluster-swift
Swift's object expirer in kilo series was incompatible with gluster-swift. This change does the following: * Optimizes crawl in account and container server for listing requests for containers and tracker objects in gsexpiring volume. * Enables container server to delete tracker objects from gsexpiring volume. Swift's expirer sends request directly to container server to remove tracker object entry. * delete_tracker_object() is now a common utility function that is invoked from container server and gluster-swift's object expirer. * Run functional test to be run against both swift's object expirer and gluster-swift's object expirer Change-Id: Ib5b7f7f08fe7dda574f6dd80be2f38bdfaee32bc Signed-off-by: Prashanth Pai <ppai@redhat.com> Reviewed-on: http://review.gluster.org/14038 Reviewed-by: Thiago da Silva <thiago@redhat.com> Tested-by: Thiago da Silva <thiago@redhat.com>
Diffstat (limited to 'test')
-rw-r--r--test/object_expirer_functional/test_object_expirer_gluster_swift.py (renamed from test/object_expirer_functional/test_object_expirer.py)0
-rw-r--r--test/object_expirer_functional/test_object_expirer_swift.py335
-rw-r--r--test/unit/common/test_diskdir.py70
3 files changed, 402 insertions, 3 deletions
diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer_gluster_swift.py
index 279994f..279994f 100644
--- a/test/object_expirer_functional/test_object_expirer.py
+++ b/test/object_expirer_functional/test_object_expirer_gluster_swift.py
diff --git a/test/object_expirer_functional/test_object_expirer_swift.py b/test/object_expirer_functional/test_object_expirer_swift.py
new file mode 100644
index 0000000..c59cc67
--- /dev/null
+++ b/test/object_expirer_functional/test_object_expirer_swift.py
@@ -0,0 +1,335 @@
+# Copyright (c) 2014 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import logging
+
+from swift.common.manager import Manager
+from swift.common.internal_client import InternalClient
+
+
+from test import get_config
+from test.functional.tests import Base, Utils
+from test.functional.swift_test_client import Account, Connection, \
+ ResponseError
+
+config = get_config('func_test')
+
+
+class TestObjectExpirerEnv:
+ @classmethod
+ def setUp(cls):
+ cls.conn = Connection(config)
+ cls.conn.authenticate()
+ cls.account = Account(cls.conn,
+ config.get('account',
+ config['username']))
+ cls.account.delete_containers()
+ cls.container = cls.account.container(Utils.create_name())
+ if not cls.container.create():
+ raise ResponseError(cls.conn.response)
+ cls.file_size = 8
+ cls.root_dir = os.path.join('/mnt/gluster-object',
+ cls.account.conn.storage_url.split('/')[2].split('_')[1])
+ devices = config.get('devices', '/mnt/gluster-object')
+ cls.client = InternalClient('/etc/swift/object-expirer.conf',
+ 'Test Object Expirer', 1)
+ cls.expirer = Manager(['object-expirer'])
+
+
+class TestObjectExpirer(Base):
+ env = TestObjectExpirerEnv
+ set_up = False
+
+ def test_object_expiry_X_Delete_At_PUT(self):
+ obj = self.env.container.file(Utils.create_name())
+ x_delete_at = str(int(time.time()) + 2)
+ obj.write_random(self.env.file_size,
+ hdrs={'X-Delete-At': x_delete_at})
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+ def test_object_expiry_X_Delete_After_PUT(self):
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size,
+ hdrs={'X-Delete-After': 2})
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertTrue(str(obj.info()['x_delete_at']))
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+ def test_object_expiry_X_Delete_At_POST(self):
+
+ # Create normal object
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size)
+ obj.read()
+ self.assert_status(200)
+
+ # Send POST on that object and set it to be expired.
+ x_delete_at = str(int(time.time()) + 2)
+ obj.sync_metadata(metadata={'X-Delete-At': x_delete_at},
+ cfg={'x_delete_at': x_delete_at})
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+
+ def test_object_expiry_X_Delete_After_POST(self):
+
+ # Create normal object
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size)
+ obj.read()
+ self.assert_status(200)
+
+ # Send POST on that object and set it to be expired.
+ obj.sync_metadata(metadata={'X-Delete-After': 2},
+ cfg={'x_delete_after': 2})
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertTrue(str(obj.info()['x_delete_at']))
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+
+ def test_object_expiry_err(self):
+ obj = self.env.container.file(Utils.create_name())
+
+ # X-Delete-At is invalid or is in the past
+ for i in (-2, 'abc', str(int(time.time()) - 2), 5.8):
+ self.assertRaises(ResponseError,
+ obj.write_random,
+ self.env.file_size,
+ hdrs={'X-Delete-At': i})
+ self.assert_status(400)
+
+ # X-Delete-After is invalid.
+ for i in (-2, 'abc', 3.7):
+ self.assertRaises(ResponseError,
+ obj.write_random,
+ self.env.file_size,
+ hdrs={'X-Delete-After': i})
+ self.assert_status(400)
+
diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py
index 623164c..964bc2f 100644
--- a/test/unit/common/test_diskdir.py
+++ b/test/unit/common/test_diskdir.py
@@ -402,6 +402,37 @@ class TestContainerBroker(unittest.TestCase):
fp.write("file path: %s\n" % fullname)
return fullname
+ def test_gsexpiring_fake_md(self):
+ # Create account
+ account_path = os.path.join(self.path, "gsexpiring")
+ os.makedirs(account_path)
+
+ # Create container
+ cpath = os.path.join(account_path, "container")
+ os.mkdir(cpath)
+
+ # Create 10 objects in container. These should not reflect in
+ # X-Object-Count.
+ for o in xrange(10):
+ os.mkdir(os.path.join(cpath, str(o)))
+
+ orig_stat = os.stat(cpath)
+ expected_metadata = {
+ 'X-Object-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0),
+ 'X-Type': ('container', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+
+ # Create DiskDir instance
+ disk_dir = dd.DiskDir(self.path, 'gsexpiring', account='gsexpiring',
+ container='container', logger=FakeLogger())
+ self.assertEqual(expected_metadata, disk_dir.metadata)
+ info = disk_dir.get_info()
+ self.assertEqual(info['object_count'], 0)
+ self.assertEqual(info['bytes_used'], 0)
+
def test_creation(self):
# Test swift.common.db.ContainerBroker.__init__
broker = self._get_broker(account='a', container='c')
@@ -814,10 +845,10 @@ class TestContainerBroker(unittest.TestCase):
out_content_type="text/plain")
self.assertEquals(len(listing), 100)
for (name, ts, clen, ctype, etag) in listing:
- self.assertEqual(ts, 0)
+ self.assertEqual(ts, '0')
self.assertEqual(clen, 0)
- self.assertEqual(ctype, 0)
- self.assertEqual(etag, 0)
+ self.assertEqual(ctype, 'text/plain')
+ self.assertEqual(etag, '')
# Check that limit is still honored.
listing = broker.list_objects_iter(25, '', None, None, '',
@@ -1343,6 +1374,39 @@ class TestDiskAccount(unittest.TestCase):
_destroyxattr()
shutil.rmtree(self.td)
+ def test_gsexpiring_fake_md(self):
+ # Create account
+ account_path = os.path.join(self.td, "gsexpiring")
+ os.makedirs(account_path)
+
+ # Create 10 containers - these should not reflect in
+ # X-Container-Count. Also, create 10 objects in each
+ # container. These should not reflect in X-Object-Count.
+ for c in xrange(10):
+ cpath = os.path.join(account_path, str(c))
+ os.mkdir(cpath)
+ for o in xrange(10):
+ os.mkdir(os.path.join(cpath, str(o)))
+
+ orig_stat = os.stat(account_path)
+ expected_metadata = {
+ 'X-Object-Count': (0, 0),
+ 'X-Container-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0),
+ 'X-Type': ('Account', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+
+ # Create DiskAccount instance
+ da = dd.DiskAccount(self.td, 'gsexpiring', 'gsexpiring',
+ self.fake_logger)
+ self.assertEqual(expected_metadata, da.metadata)
+ info = da.get_info()
+ self.assertEqual(info['container_count'], 0)
+ self.assertEqual(info['object_count'], 0)
+ self.assertEqual(info['bytes_used'], 0)
+
def test_constructor_no_metadata(self):
da = dd.DiskAccount(self.td, self.fake_drives[0],
self.fake_accounts[0], self.fake_logger)