diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/unit/common/test_diskdir.py | 715 | ||||
| -rw-r--r-- | test/unit/common/test_utils.py | 47 |
2 files changed, 735 insertions, 27 deletions
diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index 1058367..733cc17 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -22,6 +22,8 @@ import cPickle as pickle import unittest import shutil import tarfile +import hashlib +from time import time from nose import SkipTest from swift.common.utils import normalize_timestamp from gluster.swift.common import utils @@ -29,12 +31,23 @@ import gluster.swift.common.Glusterfs from test_utils import _initxattr, _destroyxattr, _setxattr, _getxattr from test.unit import FakeLogger -gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' -try: - os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) -except OSError as e: - if e.errno != errno.EEXIST: - raise +def setup(): + global _saved_RUN_DIR, _saved_do_getsize + _saved_do_getsize = gluster.swift.common.Glusterfs._do_getsize + gluster.swift.common.Glusterfs._do_getsize = True + _saved_RUN_DIR = gluster.swift.common.Glusterfs.RUN_DIR + gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' + try: + os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +def teardown(): + shutil.rmtree(gluster.swift.common.Glusterfs.RUN_DIR) + gluster.swift.common.Glusterfs.RUN_DIR = _saved_RUN_DIR + gluster.swift.common.Glusterfs._do_getsize = _saved_do_getsize + import gluster.swift.common.DiskDir as dd @@ -243,6 +256,7 @@ class TestDiskDirModuleFunctions(unittest.TestCase): l = list(dd.filter_limit([1,2,3], 4)) assert l == [1,2,3] + class TestDiskCommon(unittest.TestCase): """ Tests for gluster.swift.common.DiskDir.DiskCommon """ @@ -393,6 +407,678 @@ class TestDiskDir(unittest.TestCase): self.fail("Implement me") +class TestContainerBroker(unittest.TestCase): + """ + Tests for DiskDir.DiskDir class (duck-typed + swift.common.db.ContainerBroker). + """ + + def __init__(self, *args, **kwargs): + super(TestContainerBroker, self).__init__(*args, **kwargs) + self.initial_ts = normalize_timestamp('1') + + def setUp(self): + _initxattr() + self.path = tempfile.mkdtemp() + self.drive = 'drv' + self.container = None + + def tearDown(self): + self.container = None + _destroyxattr() + shutil.rmtree(self.path) + + def _get_broker(self, account=None, container=None): + assert account is not None + assert container is not None + self.container = os.path.join(self.path, self.drive, container) + return dd.DiskDir(self.path, self.drive, account=account, + container=container, logger=FakeLogger()) + + def _create_file(self, p): + fullname = os.path.join(self.container, p) + dirs = os.path.dirname(fullname) + try: + os.makedirs(dirs) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(fullname, 'w') as fp: + fp.write("file path: %s\n" % fullname) + return fullname + + def test_creation(self): + # Test swift.common.db.ContainerBroker.__init__ + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_existing(self): + # Test swift.common.db.ContainerBroker.__init__ + os.makedirs(os.path.join(self.path, self.drive, 'c')) + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_existing_bad_metadata(self): + # Test swift.common.db.ContainerBroker.__init__ + container = os.path.join(self.path, self.drive, 'c') + os.makedirs(container) + utils.write_metadata(container, dict(a=1, b=2)) + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_empty(self): + # Test swift.common.db.ContainerBroker.empty + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + obj = self._create_file('o.txt') + self.assert_(not broker.empty()) + os.unlink(obj) + self.assert_(broker.empty()) + + def test_put_object(self): + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + # put_object() should be a NOOP + self.assert_(broker.empty()) + + def test_delete_object(self): + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + obj = self._create_file('o.txt') + self.assert_(not broker.empty()) + broker.delete_object('o', normalize_timestamp(time())) + # delete_object() should be a NOOP + self.assert_(not broker.empty()) + os.unlink(obj) + self.assert_(broker.empty()) + + def test_get_info(self): + # Test swift.common.db.ContainerBroker.get_info + broker = self._get_broker(account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + obj1 = os.path.join(self.container, 'o1') + with open(obj1, 'w') as fp: + fp.write("%s\n" % ('x' * 122)) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + + obj2 = os.path.join(self.container, 'o2') + with open(obj2, 'w') as fp: + fp.write("%s\n" % ('x' * 122)) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + + with open(obj2, 'w') as fp: + fp.write("%s\n" % ('x' * 999)) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + + os.unlink(obj1) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + + os.unlink(obj2) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + def test_set_x_syncs(self): + broker = self._get_broker(account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + broker.set_x_container_sync_points(1, 2) + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], 1) + self.assertEquals(info['x_container_sync_point2'], 2) + + def test_list_objects_iter(self): + # Test swift.common.db.ContainerBroker.list_objects_iter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + for obj1 in xrange(4): + for obj2 in xrange(125): + self._create_file('%d.d/%04d' % (obj1, obj2)) + for obj in xrange(125): + self._create_file('2.d/0051.d/%04d' % obj) + for obj in xrange(125): + self._create_file('3.d/%04d.d/0049' % obj) + + listing = broker.list_objects_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0099') + + listing = broker.list_objects_iter(100, '', '0.d/0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0049') + + listing = broker.list_objects_iter(100, '0.d/0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0.d/0100') + self.assertEquals(listing[-1][0], '1.d/0074') + + listing = broker.list_objects_iter(55, '1.d/0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1.d/0075') + self.assertEquals(listing[-1][0], '2.d/0004') + + listing = broker.list_objects_iter(10, '', None, '0.d/01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0100') + self.assertEquals(listing[-1][0], '0.d/0109') + + listing = broker.list_objects_iter(10, '', None, '0.d/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0009') + + listing = broker.list_objects_iter(10, '', None, None, '', '0.d') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0009') + + listing = broker.list_objects_iter(10, '', None, '', '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2', None, None, '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2.d/', None, None, '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2.d/0050', None, '2.d/', '/') + self.assertEquals(len(listing), 9) + self.assertEquals(listing[0][0], '2.d/0051') + self.assertEquals(listing[1][0], '2.d/0052') + self.assertEquals(listing[-1][0], '2.d/0059') + + listing = broker.list_objects_iter(10, '3.d/0045', None, '3.d/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['3.d/0046', '3.d/0047', + '3.d/0048', '3.d/0049', + '3.d/0050']) + + # FIXME + #broker.put_object('3/0049/', normalize_timestamp(time()), 0, + # 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + #listing = broker.list_objects_iter(10, '3/0048', None, None, None) + #self.assertEquals(len(listing), 10) + #self.assertEquals([row[0] for row in listing], + # ['3.d/0048.d/0049', '3.d/0049', '3.d/0049.d/', + # '3.d/0049.d/0049', '3.d/0050', '3.d/0050.d/0049', '3.d/0051', '3.d/0051.d/0049', + # '3.d/0052', '3.d/0052.d/0049']) + + listing = broker.list_objects_iter(10, '3.d/0048', None, '3.d/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['3.d/0049', '3.d/0050', + '3.d/0051', '3.d/0052', '3.d/0053']) + + listing = broker.list_objects_iter(10, None, None, '3.d/0049.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], + ['3.d/0049.d/0049']) + + # FIXME + #listing = broker.list_objects_iter(10, None, None, None, None, + # '3.d/0049') + #self.assertEquals(len(listing), 1) + #self.assertEquals([row[0] for row in listing], ['3.d/0049.d/0049']) + + listing = broker.list_objects_iter(2, None, None, '3.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3.d/0000']) + + # FIXME + #listing = broker.list_objects_iter(2, None, None, None, None, '3') + #self.assertEquals(len(listing), 2) + #self.assertEquals([row[0] for row in listing], ['3.d/0000', '3.d/0001']) + + def test_list_objects_iter_prefix_delim(self): + # Test swift.common.db.ContainerBroker.list_objects_iter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + os.mkdir(os.path.join(self.container, 'pets')) + os.mkdir(os.path.join(self.container, 'pets', 'dogs')) + obj1 = os.path.join(self.container, 'pets', 'dogs', '1') + with open(obj1, 'w') as fp: + fp.write("one\n") + obj2 = os.path.join(self.container, 'pets', 'dogs', '2') + with open(obj2, 'w') as fp: + fp.write("two\n") + os.mkdir(os.path.join(self.container, 'pets', 'fish')) + obja = os.path.join(self.container, 'pets', 'fish', 'a') + with open(obja, 'w') as fp: + fp.write("A\n") + objb = os.path.join(self.container, 'pets', 'fish', 'b') + with open(objb, 'w') as fp: + fp.write("B\n") + objf = os.path.join(self.container, 'pets', 'fish_info.txt') + with open(objf, 'w') as fp: + fp.write("one fish\n") + objs = os.path.join(self.container, 'snakes') + with open(objs, 'w') as fp: + fp.write("slither\n") + + listing = broker.list_objects_iter(100, None, None, 'pets/f', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, 'pets/fish', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, 'pets/fish/', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish/a', 'pets/fish/b']) + + def test_double_check_trailing_delimiter(self): + # Test swift.common.db.ContainerBroker.list_objects_iter for a + # container that has an odd file with a trailing delimiter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + self._create_file('a') + self._create_file('a.d/a') + self._create_file('a.d/a.d/a') + self._create_file('a.d/a.d/b') + self._create_file('a.d/b') + self._create_file('b') + self._create_file('b.d/a') + self._create_file('b.d/b') + self._create_file('c') + self._create_file('a.d/0') + self._create_file('0') + self._create_file('00') + self._create_file('0.d/0') + self._create_file('0.d/00') + self._create_file('0.d/1') + self._create_file('0.d/1.d/0') + self._create_file('1') + self._create_file('1.d/0') + + listing = broker.list_objects_iter(25, None, None, None, None) + self.assertEquals(len(listing), 18) + self.assertEquals([row[0] for row in listing], + ['0', '0.d/0', '0.d/00', '0.d/1', '0.d/1.d/0', '00', + '1', '1.d/0', 'a', 'a.d/0', 'a.d/a', 'a.d/a.d/a', + 'a.d/a.d/b', 'a.d/b', 'b', 'b.d/a', 'b.d/b', 'c']) + listing = broker.list_objects_iter(25, None, None, '', '/') + self.assertEquals(len(listing), 6) + self.assertEquals([row[0] for row in listing], + ['0', '00', '1', 'a', 'b', 'c']) + listing = broker.list_objects_iter(25, None, None, 'a.d/', '/') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a.d/0', 'a.d/a', 'a.d/b']) + listing = broker.list_objects_iter(25, None, None, '0.d/', '/') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['0.d/0', '0.d/00', '0.d/1']) + listing = broker.list_objects_iter(25, None, None, '0.d/1.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['0.d/1.d/0']) + listing = broker.list_objects_iter(25, None, None, 'b.d/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b.d/a', 'b.d/b']) + + def test_metadata(self): + # Initializes a good broker for us + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + # Add our first item + first_timestamp = normalize_timestamp(1) + first_value = '1' + broker.update_metadata({'First': [first_value, first_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + # Add our second item + second_timestamp = normalize_timestamp(2) + second_value = '2' + broker.update_metadata({'Second': [second_value, second_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + # Update our first item + first_timestamp = normalize_timestamp(3) + first_value = '1b' + broker.update_metadata({'First': [first_value, first_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + # Delete our second item (by setting to empty string) + second_timestamp = normalize_timestamp(4) + second_value = '' + broker.update_metadata({'Second': [second_value, second_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + + def test_delete_db(self): + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + broker.delete_db(normalize_timestamp(time())) + self.assertTrue(broker.is_deleted()) + + +class TestAccountBroker(unittest.TestCase): + """ + Tests for DiskDir.DiskAccount class (duck-typed + swift.common.db.AccountBroker). + """ + + def __init__(self, *args, **kwargs): + super(TestAccountBroker, self).__init__(*args, **kwargs) + self.initial_ts = normalize_timestamp('1') + + def setUp(self): + _initxattr() + self.path = tempfile.mkdtemp() + self.drive = 'drv' + self.drive_fullpath = os.path.join(self.path, self.drive) + os.mkdir(self.drive_fullpath) + self.account = None + + def tearDown(self): + self.account = None + _destroyxattr() + shutil.rmtree(self.path) + + def _get_broker(self, account=None): + assert account is not None + self.account = account + return dd.DiskAccount(self.path, self.drive, account=account, + logger=FakeLogger()) + + def _create_container(self, name): + cont = os.path.join(self.drive_fullpath, name) + try: + os.mkdir(cont) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return cont + + def test_creation(self): + # Test swift.common.db.AccountBroker.__init__ + broker = self._get_broker(account='a') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_bad_metadata(self): + # Test swift.common.db.AccountBroker.__init__ + utils.write_metadata(self.drive_fullpath, dict(a=1, b=2)) + broker = self._get_broker(account='a') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_empty(self): + # Test swift.common.db.AccountBroker.empty + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + c1 = self._create_container('c1') + self.assert_(not broker.empty()) + os.rmdir(c1) + self.assert_(broker.empty()) + + def test_put_container(self): + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) + # put_container() should be a NOOP + self.assert_(broker.empty()) + + def test_put_container_for_deletes(self): + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + c1 = self._create_container('c1') + self.assert_(not broker.empty()) + broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) + # put_container() should be a NOOP + self.assert_(not broker.empty()) + os.rmdir(c1) + self.assert_(broker.empty()) + + def test_get_info(self): + # Test swift.common.db.AccountBroker.get_info + broker = self._get_broker(account='test1') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + c1 = self._create_container('c1') + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + c2 = self._create_container('c2') + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + c2 = self._create_container('c2') + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + os.rmdir(c1) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + os.rmdir(c2) + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + def test_list_containers_iter(self): + # Test swift.common.db.AccountBroker.list_containers_iter + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + for cont1 in xrange(4): + for cont2 in xrange(125): + self._create_container('%d-%04d' % (cont1, cont2)) + for cont in xrange(125): + self._create_container('2-0051-%04d' % cont) + for cont in xrange(125): + self._create_container('3-%04d-0049' % cont) + + listing = broker.list_containers_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0099') + + listing = broker.list_containers_iter(100, '', '0-0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0049') + + listing = broker.list_containers_iter(100, '0-0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '1-0074') + + listing = broker.list_containers_iter(55, '1-0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1-0075') + self.assertEquals(listing[-1][0], '2-0004') + + listing = broker.list_containers_iter(10, '', None, '0-01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-01', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0009') + + listing = broker.list_containers_iter(10, '', None, '', '-') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0', '1', '2', '3']) + + listing = broker.list_containers_iter(10, '2-', None, None, '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3']) + + listing = broker.list_containers_iter(10, '', None, '2', '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['2']) + + listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2-0051') + self.assertEquals(listing[1][0], '2-0052') + self.assertEquals(listing[-1][0], '2-0060') + + listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0046', '3-0047', '3-0048', '3-0049', '3-0050', + '3-0051', '3-0052', '3-0053', '3-0054', '3-0055']) + + self._create_container('3-0049-') + listing = broker.list_containers_iter(10, '3-0048', None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', + '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', + '3-0052', '3-0052-0049']) + + listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0049', '3-0050', '3-0051', '3-0052', '3-0053', + '3-0054', '3-0055', '3-0056', '3-0057', '3-0058']) + + listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], + ['3-0049-', '3-0049-0049']) + + def test_double_check_trailing_delimiter(self): + # Test swift.common.db.AccountBroker.list_containers_iter for an + # account that has an odd file with a trailing delimiter + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + self._create_container('a') + self._create_container('a-') + self._create_container('a-a') + self._create_container('a-a-a') + self._create_container('a-a-b') + self._create_container('a-b') + self._create_container('b') + self._create_container('b-a') + self._create_container('b-b') + self._create_container('c') + listing = broker.list_containers_iter(15, None, None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', + 'b-a', 'b-b', 'c']) + listing = broker.list_containers_iter(15, None, None, '', '-') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a', 'b', 'c']) + listing = broker.list_containers_iter(15, None, None, 'a-', '-') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a-', 'a-a', 'a-b']) + listing = broker.list_containers_iter(15, None, None, 'b-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b-a', 'b-b']) + + def test_delete_db(self): + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + broker.delete_db(normalize_timestamp(time())) + # Deleting the "db" should be a NOOP + self.assertFalse(broker.is_deleted()) + + class TestDiskAccount(unittest.TestCase): """ Tests for gluster.swift.common.DiskDir.DiskAccount """ @@ -524,23 +1210,6 @@ class TestDiskAccount(unittest.TestCase): assert data['hash'] == '' assert data['id'] == '' - def test_get_container_timestamp(self): - tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") - orig_cwd = os.getcwd() - datadir = os.path.join(self.td, self.fake_drives[0]) - os.chdir(datadir) - try: - tf.extractall() - finally: - os.chdir(orig_cwd) - md = dd.create_container_metadata(os.path.join(datadir, 'c2')) - assert 'X-PUT-Timestamp' in md, repr(md) - da = dd.DiskAccount(self.td, self.fake_drives[0], - self.fake_accounts[0], self.fake_logger) - raise SkipTest - cts = da.get_container_timestamp('c2') - assert md['X-PUT-Timestamp'][0] == cts, repr(cts) - def test_update_put_timestamp_not_updated(self): da = dd.DiskAccount(self.td, self.fake_drives[0], self.fake_accounts[0], self.fake_logger) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0f6dba3..06f1a46 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -38,9 +38,11 @@ _xattr_set_err = {} _xattr_get_err = {} _xattr_rem_err = {} + def _xkey(path, key): return "%s:%s" % (path, key) + def _setxattr(path, key, value, *args, **kwargs): _xattr_op_cnt['set'] += 1 xkey = _xkey(path, key) @@ -51,6 +53,7 @@ def _setxattr(path, key, value, *args, **kwargs): global _xattrs _xattrs[xkey] = value + def _getxattr(path, key, *args, **kwargs): _xattr_op_cnt['get'] += 1 xkey = _xkey(path, key) @@ -67,6 +70,7 @@ def _getxattr(path, key, *args, **kwargs): raise e return ret_val + def _removexattr(path, key, *args, **kwargs): _xattr_op_cnt['remove'] += 1 xkey = _xkey(path, key) @@ -82,6 +86,7 @@ def _removexattr(path, key, *args, **kwargs): e.errno = errno.ENODATA raise e + def _initxattr(): global _xattrs _xattrs = {} @@ -102,6 +107,7 @@ def _initxattr(): xattr.getxattr = _getxattr xattr.removexattr = _removexattr + def _destroyxattr(): # Restore the current methods just in case global _xattr_set; xattr.setxattr = _xattr_set @@ -789,6 +795,40 @@ class TestUtils(unittest.TestCase): cd = utils._get_container_details_from_fs(td) assert cd.bytes_used == 0, repr(cd.bytes_used) + # Should not include the directories + assert cd.object_count == 5, repr(cd.object_count) + assert set(cd.obj_list) == set(['file1', 'file3', 'file2', + 'dir1/file1', 'dir1/file2' + ]), repr(cd.obj_list) + + full_dir1 = os.path.join(td, 'dir1') + full_dir2 = os.path.join(td, 'dir2') + full_dir3 = os.path.join(td, 'dir3') + exp_dir_dict = { td: os.path.getmtime(td), + full_dir1: os.path.getmtime(full_dir1), + full_dir2: os.path.getmtime(full_dir2), + full_dir3: os.path.getmtime(full_dir3), + } + for d,m in cd.dir_list: + assert d in exp_dir_dict + assert exp_dir_dict[d] == m + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_container_details_from_fs_ufo(self): + orig_cwd = os.getcwd() + __obj_only = Glusterfs.OBJECT_ONLY + td = tempfile.mkdtemp() + try: + tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") + os.chdir(td) + tf.extractall() + + Glusterfs.OBJECT_ONLY = False + + cd = utils._get_container_details_from_fs(td) + assert cd.bytes_used == 0, repr(cd.bytes_used) assert cd.object_count == 8, repr(cd.object_count) assert set(cd.obj_list) == set(['file1', 'file3', 'file2', 'dir3', 'dir1', 'dir2', @@ -809,24 +849,23 @@ class TestUtils(unittest.TestCase): finally: os.chdir(orig_cwd) shutil.rmtree(td) - + Glusterfs.OBJECT_ONLY = __obj_only def test_get_container_details_from_fs_do_getsize_true(self): orig_cwd = os.getcwd() + __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() - __do_getsize = Glusterfs._do_getsize Glusterfs._do_getsize = True cd = utils._get_container_details_from_fs(td) assert cd.bytes_used == 30, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) + assert cd.object_count == 5, repr(cd.object_count) assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', 'dir1/file1', 'dir1/file2' ]), repr(cd.obj_list) |