From 5cef798f8dcdee0d0512e47b67ac67d5f8d6c14c Mon Sep 17 00:00:00 2001 From: Luis Pabon Date: Fri, 21 Jun 2013 16:41:50 -0400 Subject: OpenStack Swift Functional Tests for G4S This commit has the following changes: * G4S no longer accepts URLs that end in /. A HTTP code of 400 is returned when a / at the end of the object is detected. * Directories can be created as objects setting the content-type to application/directory and content-length to 0. * Functional tests have been adjusted to work with G4S constraints Change-Id: I31038a59699a8e3eeaba902db322218c6400093e Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/5246 Reviewed-by: Peter Portante Tested-by: Peter Portante --- test/unit/common/test_diskfile.py | 32 ++--- test/unit/common/test_utils.py | 57 +++++++- test/unit/proxy/test_server.py | 278 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 349 insertions(+), 18 deletions(-) (limited to 'test/unit') diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py index b8878e8..857ba9d 100644 --- a/test/unit/common/test_diskfile.py +++ b/test/unit/common/test_diskfile.py @@ -23,7 +23,7 @@ import tempfile import shutil from hashlib import md5 from swift.common.utils import normalize_timestamp -from swift.common.exceptions import DiskFileNotExist +from swift.common.exceptions import DiskFileNotExist, DiskFileError import gluster.swift.common.DiskFile import gluster.swift.common.utils from gluster.swift.common.DiskFile import Gluster_DiskFile, \ @@ -54,8 +54,8 @@ class MockException(Exception): pass -def _mock_rmdirs(p): - raise MockException("gluster.swift.common.DiskFile.rmdirs() called") +def _mock_rmobjdir(p): + raise MockException("gluster.swift.common.DiskFile.rmobjdir() called") def _mock_do_fsync(fd): return @@ -348,12 +348,12 @@ class TestDiskFile(unittest.TestCase): assert g == DEFAULT_GID dc = gluster.swift.common.DiskFile.do_chown gluster.swift.common.DiskFile.do_chown = _mock_do_chown - try: - gdf._create_dir_object(the_dir) - finally: - gluster.swift.common.DiskFile.do_chown = dc - assert os.path.isdir(the_dir) - assert the_dir in _metadata + self.assertRaises(DiskFileError, + gdf._create_dir_object, + the_dir) + gluster.swift.common.DiskFile.do_chown = dc + self.assertFalse(os.path.isdir(the_dir)) + self.assertFalse(the_dir in _metadata) finally: shutil.rmtree(td) @@ -571,14 +571,14 @@ class TestDiskFile(unittest.TestCase): gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", "z", self.lg) assert gdf.metadata == {} - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs + _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir + gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir try: gdf.unlinkold(None) except MockException as exp: self.fail(str(exp)) finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs + gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir def test_unlinkold_same_timestamp(self): assert not os.path.exists("/tmp/foo") @@ -586,14 +586,14 @@ class TestDiskFile(unittest.TestCase): "z", self.lg) assert gdf.metadata == {} gdf.metadata['X-Timestamp'] = 1 - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs + _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir + gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir try: gdf.unlinkold(1) except MockException as exp: self.fail(str(exp)) finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs + gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir def test_unlinkold_file(self): td = tempfile.mkdtemp() @@ -717,7 +717,7 @@ class TestDiskFile(unittest.TestCase): os.chmod(gdf.datadir, stats.st_mode) os.rmdir = __os_rmdir assert os.path.isdir(gdf.datadir) - assert os.path.isdir(gdf.data_file) + self.assertTrue(gdf.data_file is None) finally: shutil.rmtree(td) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 566c70e..20984b1 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -367,7 +367,7 @@ class TestUtils(unittest.TestCase): for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) @@ -413,7 +413,7 @@ class TestUtils(unittest.TestCase): for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) @@ -802,3 +802,56 @@ class TestUtils(unittest.TestCase): utils.X_OBJECT_TYPE: 'na' } ret = utils.validate_object(md) assert ret + +class TestUtilsDirObjects(unittest.TestCase): + def setUp(self): + _initxattr() + self.dirs = ['dir1', + 'dir1/dir2', + 'dir1/dir2/dir3' ] + self.files = ['file1', + 'file2', + 'dir1/dir2/file3'] + self.tempdir = tempfile.mkdtemp() + self.rootdir = os.path.join(self.tempdir, 'a') + for d in self.dirs: + os.makedirs(os.path.join(self.rootdir, d)) + for f in self.files: + open(os.path.join(self.rootdir, f), 'w').close() + + def tearDown(self): + _destroyxattr() + shutil.rmtree(self.tempdir) + + def _set_dir_object(self, obj): + metadata = utils.read_metadata(os.path.join(self.rootdir, obj)) + metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT + utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]), + metadata) + + def _clear_dir_object(self, obj): + metadata = utils.read_metadata(os.path.join(self.rootdir, obj)) + metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT + utils.write_metadata(os.path.join(self.rootdir, obj), + metadata) + + def test_rmobjdir_removing_files(self): + self.assertFalse(utils.rmobjdir(self.rootdir)) + + # Remove the files + for f in self.files: + os.unlink(os.path.join(self.rootdir, f)) + + self.assertTrue(utils.rmobjdir(self.rootdir)) + + def test_rmobjdir_removing_dirs(self): + self.assertFalse(utils.rmobjdir(self.rootdir)) + + # Remove the files + for f in self.files: + os.unlink(os.path.join(self.rootdir, f)) + + self._set_dir_object(self.dirs[0]) + self.assertFalse(utils.rmobjdir(self.rootdir)) + self._clear_dir_object(self.dirs[0]) + self.assertTrue(utils.rmobjdir(self.rootdir)) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 57e3111..3bb2c02 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -742,6 +742,46 @@ class TestObjectController(unittest.TestCase): res = method(req) self.assertEquals(res.status_int, expected) + def test_illegal_object_name(self): + prolis = _test_sockets[0] + prosrv = _test_servers[0] + + # Create a container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/illegal_name HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a file obj + fakedata = 'a' * 1024 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/illegal_name/file/ HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octect-stream\r\n' + '\r\n%s' % (str(len(fakedata)), fakedata)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 400' + self.assertEquals(headers[:len(exp)], exp) + + # Delete continer + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/illegal_name HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + def test_GET_newest_large_file(self): calls = [0] @@ -4547,6 +4587,244 @@ class TestContainerController(unittest.TestCase): exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) + def test_dir_object_not_lost(self): + prolis = _test_sockets[0] + prosrv = _test_servers[0] + + # Create a container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/dir_obj_test HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a dir obj A + dir_list = ['a', 'a/b', 'a/b/c'] + + for dir_obj in dir_list: + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/dir_obj_test/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n' + 'Content-type: application/directory\r\n\r\n' % dir_obj) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Check we see all the objects we created + req = Request.blank('/v1/a/dir_obj_test', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + for dir_obj in dir_list: + self.assertTrue(dir_obj in obj_list) + + # Now let's create a file obj + fakedata = 'a' * 1024 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/dir_obj_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octect-stream\r\n' + '\r\n%s' % (str(len(fakedata)), fakedata)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Now check we get all dir objs and the file obj + req = Request.blank('/v1/a/dir_obj_test', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + for dir_obj in dir_list: + self.assertTrue(dir_obj in obj_list) + self.assertTrue('a/b/c/file1' in obj_list) + + # Delete dir objects, file should still be available + for dir_obj in dir_list: + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/dir_obj_test/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + '\r\n' % dir_obj) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + + # Now check file is still available + req = Request.blank('/v1/a/dir_obj_test', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + for dir_obj in dir_list: + self.assertFalse(dir_obj in obj_list) + self.assertTrue('a/b/c/file1' in obj_list) + + # Delete file + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/dir_obj_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + + # Delete continer + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/dir_obj_test HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + + def test_container_lists_dir_and_file_objects(self): + prolis = _test_sockets[0] + prosrv = _test_servers[0] + + # Create a container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a file obj + fakedata = 'a' * 1024 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/list_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octect-stream\r\n' + '\r\n%s' % (str(len(fakedata)), fakedata)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a second file obj + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/list_test/a/b/c/file2 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octect-stream\r\n' + '\r\n%s' % (str(len(fakedata)), fakedata)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a third file obj + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/list_test/file3 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octect-stream\r\n' + '\r\n%s' % (str(len(fakedata)), fakedata)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Create a dir obj + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/list_test/a/b/c/dir1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n' + 'Content-type: application/directory\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + + # Path tests + req = Request.blank('/v1/a/list_test?path=', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + self.assertFalse('a/b/c/file1' in obj_list) + self.assertFalse('a/b/c/file2' in obj_list) + self.assertFalse('a/b/c/dir1' in obj_list) + self.assertTrue('file3' in obj_list) + + req = Request.blank('/v1/a/list_test?path=a/b/c', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + self.assertTrue('a/b/c/file1' in obj_list) + self.assertTrue('a/b/c/file2' in obj_list) + self.assertTrue('a/b/c/dir1' in obj_list) + self.assertFalse('file3' in obj_list) + + # Try to delete, but expect failure since the + # container is not empty + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 409' + self.assertEquals(headers[:len(exp)], exp) + + # Get object list + req = Request.blank('/v1/a/list_test', + environ={'REQUEST_METHOD': 'GET'}) + res = req.get_response(prosrv) + obj_list = res.body.split('\n') + self.assertTrue('a/b/c/file1' in obj_list) + self.assertTrue('a/b/c/file2' in obj_list) + self.assertTrue('a/b/c/dir1' in obj_list) + self.assertTrue('file3' in obj_list) + self.assertEqual(res.headers['x-container-object-count'], '4') + + # Now let's delete the objects + for obj in obj_list: + if not obj: + continue + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/list_test/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + '\r\n' % obj) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + + # Delete continer which has stale directies a/b/c + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('DELETE /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + def test_response_get_accept_ranges_header(self): with save_globals(): set_http_connect(200, 200, body='{}') -- cgit