From 9d4e67e741f13b4b93620fbb972886e1dc975fee Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 15 Jul 2013 16:52:46 -0400 Subject: Updates to support Havana interim version 1.9.1. The code changes are basically: * Apply refactoring in the DiskFile class to use the new DiskWriter abstraction * Move and rename our diskfile module to match upstream * ThreadPools allow us to remove the tpool usage around fsync * Update the Ring subclass to support the get_part() method * Update to use the 1.9.1 proxy server unit tests * Move the DebugLogger class to test.unit * Rebuild the Rings to use the new layout * Remove backup ring builder files * Update spec files to 1.9.1, and tox to use swift 1.9.1 * Updated version to 1.9.0-0 Change-Id: Ica12cac8b351627d67500723f1dbd8a54d45f7c8 Signed-off-by: Peter Portante Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/5331 --- test/unit/common/data/README.rings | 2 +- test/unit/common/data/account.builder | Bin 537 -> 566 bytes test/unit/common/data/account.ring.gz | Bin 183 -> 183 bytes .../common/data/backups/1365124498.account.builder | Bin 537 -> 0 bytes .../data/backups/1365124498.container.builder | Bin 537 -> 0 bytes .../common/data/backups/1365124498.object.builder | Bin 228 -> 0 bytes .../common/data/backups/1365124499.object.builder | Bin 537 -> 0 bytes test/unit/common/data/container.builder | Bin 537 -> 566 bytes test/unit/common/data/container.ring.gz | Bin 185 -> 184 bytes test/unit/common/data/object.builder | Bin 537 -> 566 bytes test/unit/common/data/object.ring.gz | Bin 182 -> 182 bytes test/unit/common/test_diskdir.py | 12 +- test/unit/common/test_diskfile.py | 985 --------------------- test/unit/common/test_ring.py | 4 + 14 files changed, 11 insertions(+), 992 deletions(-) delete mode 100644 test/unit/common/data/backups/1365124498.account.builder delete mode 100644 test/unit/common/data/backups/1365124498.container.builder delete mode 100644 test/unit/common/data/backups/1365124498.object.builder delete mode 100644 test/unit/common/data/backups/1365124499.object.builder delete mode 100644 test/unit/common/test_diskfile.py (limited to 'test/unit/common') diff --git a/test/unit/common/data/README.rings b/test/unit/common/data/README.rings index 6457501..4ff802e 100644 --- a/test/unit/common/data/README.rings +++ b/test/unit/common/data/README.rings @@ -1,3 +1,3 @@ The unit tests expect certain ring data built using the following command: - ../../../../bin/gluster-swift-gen-builders test iops \ No newline at end of file + ../../../../bin/gluster-swift-gen-builders test iops diff --git a/test/unit/common/data/account.builder b/test/unit/common/data/account.builder index 090ba4b..c3c0a33 100644 Binary files a/test/unit/common/data/account.builder and b/test/unit/common/data/account.builder differ diff --git a/test/unit/common/data/account.ring.gz b/test/unit/common/data/account.ring.gz index 6d4c854..dc34c31 100644 Binary files a/test/unit/common/data/account.ring.gz and b/test/unit/common/data/account.ring.gz differ diff --git a/test/unit/common/data/backups/1365124498.account.builder b/test/unit/common/data/backups/1365124498.account.builder deleted file mode 100644 index 090ba4b..0000000 Binary files a/test/unit/common/data/backups/1365124498.account.builder and /dev/null differ diff --git a/test/unit/common/data/backups/1365124498.container.builder b/test/unit/common/data/backups/1365124498.container.builder deleted file mode 100644 index 733d27d..0000000 Binary files a/test/unit/common/data/backups/1365124498.container.builder and /dev/null differ diff --git a/test/unit/common/data/backups/1365124498.object.builder b/test/unit/common/data/backups/1365124498.object.builder deleted file mode 100644 index ff877ec..0000000 Binary files a/test/unit/common/data/backups/1365124498.object.builder and /dev/null differ diff --git a/test/unit/common/data/backups/1365124499.object.builder b/test/unit/common/data/backups/1365124499.object.builder deleted file mode 100644 index 8b8cd6c..0000000 Binary files a/test/unit/common/data/backups/1365124499.object.builder and /dev/null differ diff --git a/test/unit/common/data/container.builder b/test/unit/common/data/container.builder index 733d27d..22b9b80 100644 Binary files a/test/unit/common/data/container.builder and b/test/unit/common/data/container.builder differ diff --git a/test/unit/common/data/container.ring.gz b/test/unit/common/data/container.ring.gz index 592b84b..269a1eb 100644 Binary files a/test/unit/common/data/container.ring.gz and b/test/unit/common/data/container.ring.gz differ diff --git a/test/unit/common/data/object.builder b/test/unit/common/data/object.builder index 8b8cd6c..b5bdda9 100644 Binary files a/test/unit/common/data/object.builder and b/test/unit/common/data/object.builder differ diff --git a/test/unit/common/data/object.ring.gz b/test/unit/common/data/object.ring.gz index d2f7192..1c8199a 100644 Binary files a/test/unit/common/data/object.ring.gz and b/test/unit/common/data/object.ring.gz differ diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index bbdb168..f32c3ad 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -272,8 +272,8 @@ class TestDiskCommon(unittest.TestCase): self.fake_accounts[0], self.fake_logger) assert dc.metadata == {} assert dc.db_file == dd._db_file - assert dc.pending_timeout == 0 - assert dc.stale_reads_ok == False + assert dc.pending_timeout == 10 + assert dc.stale_reads_ok is False assert dc.root == self.td assert dc.logger == self.fake_logger assert dc.account == self.fake_accounts[0] @@ -290,8 +290,8 @@ class TestDiskCommon(unittest.TestCase): dc._dir_exists_read_metadata() assert dc.metadata == fake_md, repr(dc.metadata) assert dc.db_file == dd._db_file - assert dc.pending_timeout == 0 - assert dc.stale_reads_ok == False + assert dc.pending_timeout == 10 + assert dc.stale_reads_ok is False assert dc.root == self.td assert dc.logger == self.fake_logger assert dc.account == self.fake_accounts[0] @@ -303,8 +303,8 @@ class TestDiskCommon(unittest.TestCase): dc._dir_exists_read_metadata() assert dc.metadata == {} assert dc.db_file == dd._db_file - assert dc.pending_timeout == 0 - assert dc.stale_reads_ok == False + assert dc.pending_timeout == 10 + assert dc.stale_reads_ok is False assert dc.root == self.td assert dc.logger == self.fake_logger assert dc.account == "dne0" diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py deleted file mode 100644 index 410f113..0000000 --- a/test/unit/common/test_diskfile.py +++ /dev/null @@ -1,985 +0,0 @@ -# Copyright (c) 2012-2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for gluster.swift.common.DiskFile """ - -import os -import stat -import errno -import unittest -import tempfile -import shutil -import mock -from mock import patch -from hashlib import md5 - -import gluster.swift.common.utils -import gluster.swift.common.DiskFile -from swift.common.utils import normalize_timestamp -from gluster.swift.common.DiskFile import Gluster_DiskFile -from swift.common.exceptions import DiskFileNotExist, DiskFileError -from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \ - X_OBJECT_TYPE, DIR_OBJECT -from test_utils import _initxattr, _destroyxattr -from test.unit import FakeLogger - -from gluster.swift.common.exceptions import * - -_metadata = {} - -def _mock_read_metadata(filename): - if filename in _metadata: - md = _metadata[filename] - else: - md = {} - return md - -def _mock_write_metadata(filename, metadata): - _metadata[filename] = metadata - -def _mock_clear_metadata(): - _metadata = {} - - -class MockException(Exception): - pass - - -def _mock_rmobjdir(p): - raise MockException("gluster.swift.common.DiskFile.rmobjdir() called") - -def _mock_do_fsync(fd): - return - -class MockRenamerCalled(Exception): - pass - - -def _mock_renamer(a, b): - raise MockRenamerCalled() - - -class TestDiskFile(unittest.TestCase): - """ Tests for gluster.swift.common.DiskFile """ - - def setUp(self): - self.lg = FakeLogger() - _initxattr() - _mock_clear_metadata() - self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata - self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata - gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata - gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata - self._saved_ut_wm = gluster.swift.common.utils.write_metadata - self._saved_ut_rm = gluster.swift.common.utils.read_metadata - gluster.swift.common.utils.write_metadata = _mock_write_metadata - gluster.swift.common.utils.read_metadata = _mock_read_metadata - self._saved_do_fsync = gluster.swift.common.DiskFile.do_fsync - gluster.swift.common.DiskFile.do_fsync = _mock_do_fsync - - def tearDown(self): - self.lg = None - _destroyxattr() - gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm - gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm - gluster.swift.common.utils.write_metadata = self._saved_ut_wm - gluster.swift.common.utils.read_metadata = self._saved_ut_rm - gluster.swift.common.DiskFile.do_fsync = self._saved_do_fsync - - def test_constructor_no_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == "/tmp/foo/vol0/bar" - assert gdf.device_path == "/tmp/foo/vol0" - assert gdf._container_path == "/tmp/foo/vol0/bar" - assert gdf.disk_chunk_size == 65536 - assert gdf.iter_hook == None - assert gdf.logger == self.lg - assert gdf.uid == DEFAULT_UID - assert gdf.gid == DEFAULT_GID - assert gdf.metadata == {} - assert gdf.meta_file == None - assert gdf.data_file == None - assert gdf.fp == None - assert gdf.iter_etag == None - assert not gdf.started_at_0 - assert not gdf.read_to_eof - assert gdf.quarantined_dir == None - assert not gdf.keep_cache - assert not gdf._is_dir - - def test_constructor_leadtrail_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "b/a" - assert gdf.name == "bar/b/a" - assert gdf.datadir == "/tmp/foo/vol0/bar/b/a" - assert gdf.device_path == "/tmp/foo/vol0" - - def test_constructor_no_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - stats = os.stat(the_file) - ts = normalize_timestamp(stats.st_ctime) - etag = md5() - etag.update("1234") - etag = etag.hexdigest() - exp_md = { - 'Content-Length': 4, - 'ETag': etag, - 'X-Timestamp': ts, - 'Content-Type': 'application/octet-stream'} - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'file', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_invalid_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - inv_md = { - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = inv_md - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata != inv_md - finally: - shutil.rmtree(td) - - def test_constructor_isdir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'dir', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_dir] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_keep_data_fp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg, keep_data_fp=True) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is not None - finally: - shutil.rmtree(td) - - def test_constructor_chunk_size(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, disk_chunk_size=8192) - assert gdf.disk_chunk_size == 8192 - - def test_constructor_iter_hook(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, iter_hook='hook') - assert gdf.iter_hook == 'hook' - - def test_close(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - # Should be a no-op, as by default is_dir is False, but fp is None - gdf.close() - - gdf._is_dir = True - gdf.fp = "123" - # Should still be a no-op as is_dir is True (marker directory) - self.assertRaises(AssertionError, gdf.close) - assert gdf.fp == "123" - - gdf._is_dir = False - saved_dc = gluster.swift.common.DiskFile.do_close - self.called = False - def our_do_close(fp): - self.called = True - gluster.swift.common.DiskFile.do_close = our_do_close - try: - gdf.close() - assert self.called - assert gdf.fp is None - finally: - gluster.swift.common.DiskFile.do_close = saved_dc - - def test_is_deleted(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.is_deleted() - gdf.data_file = "/tmp/foo/bar" - assert not gdf.is_deleted() - - def test_create_dir_object_no_md(self): - td = tempfile.mkdtemp() - the_cont = os.path.join(td, "vol0", "bar") - the_dir = "dir" - try: - os.makedirs(the_cont) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - os.path.join(the_dir, "z"), self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - gdf._create_dir_object(the_dir) - full_dir_path = os.path.join(the_cont, the_dir) - assert os.path.isdir(full_dir_path) - assert full_dir_path not in _metadata - finally: - shutil.rmtree(td) - - def test_create_dir_object_with_md(self): - td = tempfile.mkdtemp() - the_cont = os.path.join(td, "vol0", "bar") - the_dir = "dir" - try: - os.makedirs(the_cont) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - os.path.join(the_dir, "z"), self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - dir_md = {'Content-Type': 'application/directory', - X_OBJECT_TYPE: DIR_OBJECT} - gdf._create_dir_object(the_dir, dir_md) - full_dir_path = os.path.join(the_cont, the_dir) - assert os.path.isdir(full_dir_path) - assert full_dir_path in _metadata - finally: - shutil.rmtree(td) - - def test_create_dir_object_exists(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_path) - with open(the_dir, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - def _mock_do_chown(p, u, g): - assert u == DEFAULT_UID - assert g == DEFAULT_GID - dc = gluster.swift.common.DiskFile.do_chown - gluster.swift.common.DiskFile.do_chown = _mock_do_chown - self.assertRaises(DiskFileError, - gdf._create_dir_object, - the_dir) - gluster.swift.common.DiskFile.do_chown = dc - self.assertFalse(os.path.isdir(the_dir)) - self.assertFalse(the_dir in _metadata) - finally: - shutil.rmtree(td) - - def test_create_dir_object_do_stat_failure(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_path) - with open(the_dir, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - def _mock_do_chown(p, u, g): - assert u == DEFAULT_UID - assert g == DEFAULT_GID - dc = gluster.swift.common.DiskFile.do_chown - gluster.swift.common.DiskFile.do_chown = _mock_do_chown - self.assertRaises(DiskFileError, - gdf._create_dir_object, - the_dir) - gluster.swift.common.DiskFile.do_chown = dc - self.assertFalse(os.path.isdir(the_dir)) - self.assertFalse(the_dir in _metadata) - finally: - shutil.rmtree(td) - - def test_put_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "z") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - md = { 'Content-Type': 'application/octet-stream', 'a': 'b' } - gdf.put_metadata(md.copy()) - assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md) - assert _metadata[the_dir] == md - finally: - shutil.rmtree(td) - - def test_put_w_tombstone(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - - gdf.put_metadata({'x': '1'}, tombstone=True) - assert gdf.metadata == {} - - def test_put_w_meta_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_file_no_content_type(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir_create(self): - td = tempfile.mkdtemp() - the_cont = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_cont, "dir") - try: - os.makedirs(the_cont) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - assert gdf.metadata == {} - newmd = { - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/directory'} - gdf.put(None, newmd, extension='.dir') - assert gdf.data_file == the_dir - for key,val in newmd.items(): - assert gdf.metadata[key] == val - assert _metadata[the_dir][key] == val - assert gdf.metadata[X_OBJECT_TYPE] == DIR_OBJECT - assert _metadata[the_dir][X_OBJECT_TYPE] == DIR_OBJECT - finally: - shutil.rmtree(td) - - def test_put_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - origmd = gdf.metadata.copy() - origfmd = _metadata[the_dir] - newmd = gdf.metadata.copy() - # FIXME: This is a hack to get to the code-path; it is not clear - # how this can happen normally. - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - try: - gdf.put(None, newmd, extension='.data') - except DiskFileError: - pass - else: - self.fail("Expected to encounter" - " 'already-exists-as-dir' exception") - assert gdf.metadata == origmd - assert _metadata[the_dir] == origfmd - finally: - shutil.rmtree(td) - - def test_put(self): - td = tempfile.mkdtemp() - the_cont = os.path.join(td, "vol0", "bar") - try: - os.makedirs(the_cont) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == the_cont - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - - def test_put_ENOSPC(self): - td = tempfile.mkdtemp() - the_cont = os.path.join(td, "vol0", "bar") - try: - os.makedirs(the_cont) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == the_cont - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - def mock_open(*args, **kwargs): - raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC)) - - with mock.patch("os.open", mock_open): - try: - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - except DiskFileNoSpace: - pass - finally: - shutil.rmtree(td) - - def test_put_obj_path(self): - the_obj_path = os.path.join("b", "a") - the_file = os.path.join(the_obj_path, "z") - td = tempfile.mkdtemp() - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - the_file, self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == the_obj_path - assert gdf.name == os.path.join("bar", "b", "a") - assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a") - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - def test_unlinkold_no_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir - gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir - try: - gdf.unlinkold(None) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir - - def test_unlinkold_same_timestamp(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.metadata['X-Timestamp'] = 1 - _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir - gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir - try: - gdf.unlinkold(1) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir - - def test_unlinkold_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_not_found(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - # Handle the case the file is not in the directory listing. - os.unlink(the_file) - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_unlink_error(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - - def _mock_os_unlink_eacces_err(f): - raise OSError(errno.EACCES, os.strerror(errno.EACCES)) - - stats = os.stat(the_path) - try: - os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) - - # Handle the case os_unlink() raises an OSError - with patch("os.unlink", _mock_os_unlink_eacces_err): - try: - gdf.unlinkold(normalize_timestamp(later)) - except OSError as e: - assert e.errno == errno.EACCES - else: - self.fail("Excepted an OSError when unlinking file") - finally: - os.chmod(the_path, stats.st_mode) - - assert os.path.isdir(gdf.datadir) - assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.metadata['Content-Length'] - gdf.metadata['Content-Length'] = 3 - assert 4 == gdf.get_data_file_size() - assert 4 == gdf.metadata['Content-Length'] - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dne(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - - def test_get_data_file_size_dne_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - gdf.data_file = gdf.data_file + ".dne" - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - finally: - shutil.rmtree(td) - - def test_get_data_file_size_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - stats = os.stat(the_path) - try: - os.chmod(the_path, 0) - - def _mock_getsize_eaccess_err(f): - raise OSError(errno.EACCES, os.strerror(errno.EACCES)) - - with patch("os.path.getsize", _mock_getsize_eaccess_err): - try: - s = gdf.get_data_file_size() - except OSError as err: - assert err.errno == errno.EACCES - else: - self.fail("Expected OSError exception") - finally: - os.chmod(the_path, stats.st_mode) - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert 0 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_filter_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.filter_metadata() - assert gdf.metadata == {} - - gdf.metadata[X_TYPE] = 'a' - gdf.metadata[X_OBJECT_TYPE] = 'b' - gdf.metadata['foobar'] = 'c' - gdf.filter_metadata() - assert X_TYPE not in gdf.metadata - assert X_OBJECT_TYPE not in gdf.metadata - assert 'foobar' in gdf.metadata - - def test_mkstemp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - # At the end of previous with block a close on fd is called. - # Calling os.close on the same fd will raise an OSError - # exception and we must catch it. - try: - os.close(fd) - except OSError as err: - pass - else: - self.fail("Exception expected") - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_close(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - # Closing the fd prematurely should not raise any exceptions. - os.close(fd) - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_unlink(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - os.unlink(saved_tmppath) - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) diff --git a/test/unit/common/test_ring.py b/test/unit/common/test_ring.py index 32dd7bb..de32c7b 100644 --- a/test/unit/common/test_ring.py +++ b/test/unit/common/test_ring.py @@ -51,6 +51,10 @@ class TestRing(unittest.TestCase): for node in self.ring.get_more_nodes(0): assert node['device'] == 'volume_not_in_ring' + def test_second_device_part(self): + part = self.ring.get_part('iops') + assert part == 0 + def test_second_device_with_reseller_prefix(self): part, node = self.ring.get_nodes('AUTH_iops') assert node[0]['device'] == 'iops' -- cgit