OpenStack Swift Functional Tests for G4S

This commit has the following changes:
  * G4S no longer accepts URLs that end in /.  A HTTP code
of 400 is returned when a / at the end of the object
is detected.

  * Directories can be created as objects setting the
content-type to application/directory and content-length
to 0.

  * Functional tests have been adjusted to work with
G4S constraints

Change-Id: I31038a59699a8e3eeaba902db322218c6400093e
Signed-off-by: Luis Pabon <lpabon@redhat.com>
Reviewed-on: http://review.gluster.org/5246
Reviewed-by: Peter Portante <pportant@redhat.com>
Tested-by: Peter Portante <pportant@redhat.com>

3 files changed, 349 insertions, 18 deletions

diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py
index b8878e8..857ba9d 100644
--- a/test/unit/common/test_diskfile.py
+++ b/test/unit/common/test_diskfile.py
@@ -23,7 +23,7 @@ import tempfile
 import shutil
 from hashlib import md5
 from swift.common.utils import normalize_timestamp
-from swift.common.exceptions import DiskFileNotExist
+from swift.common.exceptions import DiskFileNotExist, DiskFileError
 import gluster.swift.common.DiskFile
 import gluster.swift.common.utils
 from gluster.swift.common.DiskFile import Gluster_DiskFile, \
@@ -54,8 +54,8 @@ class MockException(Exception):
     pass
 
 
-def _mock_rmdirs(p):
-    raise MockException("gluster.swift.common.DiskFile.rmdirs() called")
+def _mock_rmobjdir(p):
+    raise MockException("gluster.swift.common.DiskFile.rmobjdir() called")
 
 def _mock_do_fsync(fd):
     return
@@ -348,12 +348,12 @@ class TestDiskFile(unittest.TestCase):
                 assert g == DEFAULT_GID
             dc = gluster.swift.common.DiskFile.do_chown
             gluster.swift.common.DiskFile.do_chown = _mock_do_chown
-            try:
-                gdf._create_dir_object(the_dir)
-            finally:
-                gluster.swift.common.DiskFile.do_chown = dc
-            assert os.path.isdir(the_dir)
-            assert the_dir in _metadata
+            self.assertRaises(DiskFileError,
+                    gdf._create_dir_object,
+                    the_dir)
+            gluster.swift.common.DiskFile.do_chown = dc
+            self.assertFalse(os.path.isdir(the_dir))
+            self.assertFalse(the_dir in _metadata)
         finally:
             shutil.rmtree(td)
 
@@ -571,14 +571,14 @@ class TestDiskFile(unittest.TestCase):
         gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
                                "z", self.lg)
         assert gdf.metadata == {}
-        _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
-        gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
+        _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir
+        gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir
         try:
             gdf.unlinkold(None)
         except MockException as exp:
             self.fail(str(exp))
         finally:
-            gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
+            gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir
 
     def test_unlinkold_same_timestamp(self):
         assert not os.path.exists("/tmp/foo")
@@ -586,14 +586,14 @@ class TestDiskFile(unittest.TestCase):
                                "z", self.lg)
         assert gdf.metadata == {}
         gdf.metadata['X-Timestamp'] = 1
-        _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
-        gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
+        _saved_rmobjdir = gluster.swift.common.DiskFile.rmobjdir
+        gluster.swift.common.DiskFile.rmobjdir = _mock_rmobjdir
         try:
             gdf.unlinkold(1)
         except MockException as exp:
             self.fail(str(exp))
         finally:
-            gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
+            gluster.swift.common.DiskFile.rmobjdir = _saved_rmobjdir
 
     def test_unlinkold_file(self):
         td = tempfile.mkdtemp()
@@ -717,7 +717,7 @@ class TestDiskFile(unittest.TestCase):
                 os.chmod(gdf.datadir, stats.st_mode)
                 os.rmdir = __os_rmdir
             assert os.path.isdir(gdf.datadir)
-            assert os.path.isdir(gdf.data_file)
+            self.assertTrue(gdf.data_file is None)
         finally:
             shutil.rmtree(td)
 
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 566c70e..20984b1 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -367,7 +367,7 @@ class TestUtils(unittest.TestCase):
             for key in self.obj_keys:
                 assert key in md, "Expected key %s in %r" % (key, md)
             assert md[utils.X_TYPE] == utils.OBJECT
-            assert md[utils.X_OBJECT_TYPE] == utils.DIR
+            assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
             assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
             assert md[utils.X_CONTENT_LENGTH] == 0
             assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td))
@@ -413,7 +413,7 @@ class TestUtils(unittest.TestCase):
             for key in self.obj_keys:
                 assert key in md, "Expected key %s in %r" % (key, md)
             assert md[utils.X_TYPE] == utils.OBJECT
-            assert md[utils.X_OBJECT_TYPE] == utils.DIR
+            assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
             assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
             assert md[utils.X_CONTENT_LENGTH] == 0
             assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td))
@@ -802,3 +802,56 @@ class TestUtils(unittest.TestCase):
                utils.X_OBJECT_TYPE: 'na' }
         ret = utils.validate_object(md)
         assert ret
+
+class TestUtilsDirObjects(unittest.TestCase):
+    def setUp(self):
+        _initxattr()
+        self.dirs = ['dir1',
+                'dir1/dir2',
+                'dir1/dir2/dir3' ]
+        self.files = ['file1',
+                'file2',
+                'dir1/dir2/file3']
+        self.tempdir = tempfile.mkdtemp()
+        self.rootdir = os.path.join(self.tempdir, 'a')
+        for d in self.dirs:
+            os.makedirs(os.path.join(self.rootdir, d))
+        for f in self.files:
+            open(os.path.join(self.rootdir, f), 'w').close()
+
+    def tearDown(self):
+        _destroyxattr()
+        shutil.rmtree(self.tempdir)
+
+    def _set_dir_object(self, obj):
+        metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
+        metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
+        utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
+                metadata)
+
+    def _clear_dir_object(self, obj):
+        metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
+        metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
+        utils.write_metadata(os.path.join(self.rootdir, obj),
+                metadata)
+
+    def test_rmobjdir_removing_files(self):
+        self.assertFalse(utils.rmobjdir(self.rootdir))
+
+        # Remove the files
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        self.assertTrue(utils.rmobjdir(self.rootdir))
+
+    def test_rmobjdir_removing_dirs(self):
+        self.assertFalse(utils.rmobjdir(self.rootdir))
+
+        # Remove the files
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        self._set_dir_object(self.dirs[0])
+        self.assertFalse(utils.rmobjdir(self.rootdir))
+        self._clear_dir_object(self.dirs[0])
+        self.assertTrue(utils.rmobjdir(self.rootdir))
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 57e3111..3bb2c02 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -742,6 +742,46 @@ class TestObjectController(unittest.TestCase):
             res = method(req)
             self.assertEquals(res.status_int, expected)
 
+    def test_illegal_object_name(self):
+        prolis = _test_sockets[0]
+        prosrv = _test_servers[0]
+
+        # Create a container
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/illegal_name HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a file obj
+        fakedata = 'a' * 1024
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/illegal_name/file/ HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octect-stream\r\n'
+                 '\r\n%s' % (str(len(fakedata)), fakedata))
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 400'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Delete continer
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('DELETE /v1/a/illegal_name HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 204'
+        self.assertEquals(headers[:len(exp)], exp)
+
     def test_GET_newest_large_file(self):
         calls = [0]
 
@@ -4547,6 +4587,244 @@ class TestContainerController(unittest.TestCase):
         exp = 'HTTP/1.1 404'
         self.assertEquals(headers[:len(exp)], exp)
 
+    def test_dir_object_not_lost(self):
+        prolis = _test_sockets[0]
+        prosrv = _test_servers[0]
+
+        # Create a container
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/dir_obj_test HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a dir obj A
+        dir_list = ['a', 'a/b', 'a/b/c']
+
+        for dir_obj in dir_list:
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+            fd = sock.makefile()
+            fd.write('PUT /v1/a/dir_obj_test/%s HTTP/1.1\r\nHost: localhost\r\n'
+                     'Connection: close\r\nX-Storage-Token: t\r\n'
+                     'Content-Length: 0\r\n'
+                     'Content-type: application/directory\r\n\r\n' % dir_obj)
+            fd.flush()
+            headers = readuntil2crlfs(fd)
+            exp = 'HTTP/1.1 201'
+            self.assertEquals(headers[:len(exp)], exp)
+
+        # Check we see all the objects we created
+        req = Request.blank('/v1/a/dir_obj_test',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        for dir_obj in dir_list:
+            self.assertTrue(dir_obj in obj_list)
+
+        # Now let's create a file obj
+        fakedata = 'a' * 1024
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/dir_obj_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octect-stream\r\n'
+                 '\r\n%s' % (str(len(fakedata)), fakedata))
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Now check we get all dir objs and the file obj
+        req = Request.blank('/v1/a/dir_obj_test',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        for dir_obj in dir_list:
+            self.assertTrue(dir_obj in obj_list)
+        self.assertTrue('a/b/c/file1' in obj_list)
+
+        # Delete dir objects, file should still be available
+        for dir_obj in dir_list:
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+            fd = sock.makefile()
+            fd.write('DELETE /v1/a/dir_obj_test/%s HTTP/1.1\r\nHost: localhost\r\n'
+                     'Connection: close\r\nX-Storage-Token: t\r\n'
+                     '\r\n' % dir_obj)
+            fd.flush()
+            headers = readuntil2crlfs(fd)
+            exp = 'HTTP/1.1 204'
+            self.assertEquals(headers[:len(exp)], exp)
+
+        # Now check file is still available
+        req = Request.blank('/v1/a/dir_obj_test',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        for dir_obj in dir_list:
+            self.assertFalse(dir_obj in obj_list)
+        self.assertTrue('a/b/c/file1' in obj_list)
+
+        # Delete file
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('DELETE /v1/a/dir_obj_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 '\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 204'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Delete continer
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('DELETE /v1/a/dir_obj_test HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 204'
+        self.assertEquals(headers[:len(exp)], exp)
+
+    def test_container_lists_dir_and_file_objects(self):
+        prolis = _test_sockets[0]
+        prosrv = _test_servers[0]
+
+        # Create a container
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a file obj
+        fakedata = 'a' * 1024
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/list_test/a/b/c/file1 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octect-stream\r\n'
+                 '\r\n%s' % (str(len(fakedata)), fakedata))
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a second file obj
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/list_test/a/b/c/file2 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octect-stream\r\n'
+                 '\r\n%s' % (str(len(fakedata)), fakedata))
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a third file obj
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/list_test/file3 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octect-stream\r\n'
+                 '\r\n%s' % (str(len(fakedata)), fakedata))
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Create a dir obj
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('PUT /v1/a/list_test/a/b/c/dir1 HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n'
+                 'Content-type: application/directory\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Path tests
+        req = Request.blank('/v1/a/list_test?path=',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        self.assertFalse('a/b/c/file1' in obj_list)
+        self.assertFalse('a/b/c/file2' in obj_list)
+        self.assertFalse('a/b/c/dir1' in obj_list)
+        self.assertTrue('file3' in obj_list)
+
+        req = Request.blank('/v1/a/list_test?path=a/b/c',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        self.assertTrue('a/b/c/file1' in obj_list)
+        self.assertTrue('a/b/c/file2' in obj_list)
+        self.assertTrue('a/b/c/dir1' in obj_list)
+        self.assertFalse('file3' in obj_list)
+
+        # Try to delete, but expect failure since the
+        # container is not empty
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('DELETE /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 409'
+        self.assertEquals(headers[:len(exp)], exp)
+
+        # Get object list
+        req = Request.blank('/v1/a/list_test',
+                            environ={'REQUEST_METHOD': 'GET'})
+        res = req.get_response(prosrv)
+        obj_list = res.body.split('\n')
+        self.assertTrue('a/b/c/file1' in obj_list)
+        self.assertTrue('a/b/c/file2' in obj_list)
+        self.assertTrue('a/b/c/dir1' in obj_list)
+        self.assertTrue('file3' in obj_list)
+        self.assertEqual(res.headers['x-container-object-count'], '4')
+
+        # Now let's delete the objects
+        for obj in obj_list:
+            if not obj:
+                continue
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+            fd = sock.makefile()
+            fd.write('DELETE /v1/a/list_test/%s HTTP/1.1\r\nHost: localhost\r\n'
+                     'Connection: close\r\nX-Storage-Token: t\r\n'
+                     '\r\n' % obj)
+            fd.flush()
+            headers = readuntil2crlfs(fd)
+            exp = 'HTTP/1.1 204'
+            self.assertEquals(headers[:len(exp)], exp)
+
+        # Delete continer which has stale directies a/b/c
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        fd = sock.makefile()
+        fd.write('DELETE /v1/a/list_test HTTP/1.1\r\nHost: localhost\r\n'
+                 'Connection: close\r\nX-Storage-Token: t\r\n'
+                 'Content-Length: 0\r\n\r\n')
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 204'
+        self.assertEquals(headers[:len(exp)], exp)
+
     def test_response_get_accept_ranges_header(self):
         with save_globals():
             set_http_connect(200, 200, body='{}')