From bc9bdcf25b1062958563ddb8bab0dd1243f6004c Mon Sep 17 00:00:00 2001 From: Luis Pabon Date: Thu, 5 Dec 2013 16:24:28 -0500 Subject: Rebase to OpenStack Swift v1.10.0.172.g9fe7748 * Updated Proxy unit test * Updated Functional tests * Updated Tox to point to the new swift snapshot available on http://launchpad.net/gluster-swift Change-Id: Ia91593c6a28d5a3fe70715ddc60546931ae71635 Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/6445 --- test/unit/proxy/test_server.py | 222 +++++++++++++++++++++++++++++++++-------- 1 file changed, 182 insertions(+), 40 deletions(-) (limited to 'test/unit/proxy') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 4537ab6..4c75562 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -50,7 +50,7 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH from swift.common import utils -from swift.common.utils import mkdirs, normalize_timestamp +from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers.base import get_container_memcache_key, \ @@ -66,7 +66,7 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) STATIC_TIME = time.time() _request_instances = weakref.WeakKeyDictionary() _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ - _testdir = None + _testdir = _orig_SysLogHandler = None def request_init(self, *args, **kwargs): @@ -78,7 +78,9 @@ def request_init(self, *args, **kwargs): def do_setup(the_object_server): utils.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ - _orig_container_listing_limit, _test_coros + _orig_container_listing_limit, _test_coros, _orig_SysLogHandler + _orig_SysLogHandler = utils.SysLogHandler + utils.SysLogHandler = mock.MagicMock() Request._orig_init = Request.__init__ Request.__init__ = request_init monkey_patch_mimetools() @@ -201,6 +203,7 @@ def teardown(): _orig_container_listing_limit rmtree(os.path.dirname(_testdir)) Request.__init__ = Request._orig_init + utils.SysLogHandler = _orig_SysLogHandler def sortHeaderNames(headerNames): @@ -656,6 +659,34 @@ class TestProxyServer(unittest.TestCase): {'region': 2, 'zone': 1, 'ip': '127.0.0.1'}] self.assertEquals(exp_sorted, app_sorted) + def test_info_defaults(self): + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + object_ring=FakeRing()) + + self.assertTrue(app.expose_info) + self.assertTrue(isinstance(app.disallowed_sections, list)) + self.assertEqual(0, len(app.disallowed_sections)) + self.assertTrue(app.admin_key is None) + + def test_get_info_controller(self): + path = '/info' + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + object_ring=FakeRing()) + + controller, path_parts = app.get_controller(path) + + self.assertTrue('version' in path_parts) + self.assertTrue(path_parts['version'] is None) + self.assertTrue('disallowed_sections' in path_parts) + self.assertTrue('expose_info' in path_parts) + self.assertTrue('admin_key' in path_parts) + + self.assertEqual(controller.__name__, 'InfoController') + class TestObjectController(unittest.TestCase): @@ -825,7 +856,7 @@ class TestObjectController(unittest.TestCase): controller = \ proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') - controller.error_limit( + self.app.error_limit( self.app.object_ring.get_part_nodes(1)[0], 'test') set_http_connect(200, 200, # account, container 201, 201, 201, # 3 working backends @@ -2282,6 +2313,73 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(got_exc) + def test_node_read_timeout_retry(self): + with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.object_ring.get_nodes('account') + for dev in self.app.object_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + self.app.update_request(req) + + self.app.node_timeout = 0.1 + set_http_connect(200, 200, 200, slow=[3]) + resp = req.get_response(self.app) + got_exc = False + try: + resp.body + except ChunkReadTimeout: + got_exc = True + self.assert_(got_exc) + + set_http_connect(200, 200, 200, body='lalala', slow=[2]) + resp = req.get_response(self.app) + got_exc = False + try: + self.assertEquals(resp.body, 'lalala') + except ChunkReadTimeout: + got_exc = True + self.assert_(not got_exc) + + set_http_connect(200, 200, 200, body='lalala', slow=[2], + etags=['a', 'a', 'a']) + resp = req.get_response(self.app) + got_exc = False + try: + self.assertEquals(resp.body, 'lalala') + except ChunkReadTimeout: + got_exc = True + self.assert_(not got_exc) + + set_http_connect(200, 200, 200, body='lalala', slow=[2], + etags=['a', 'b', 'a']) + resp = req.get_response(self.app) + got_exc = False + try: + self.assertEquals(resp.body, 'lalala') + except ChunkReadTimeout: + got_exc = True + self.assert_(not got_exc) + + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + set_http_connect(200, 200, 200, body='lalala', slow=[2], + etags=['a', 'b', 'b']) + resp = req.get_response(self.app) + got_exc = False + try: + resp.body + except ChunkReadTimeout: + got_exc = True + self.assert_(got_exc) + def test_node_write_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') @@ -2320,44 +2418,35 @@ class TestObjectController(unittest.TestCase): with save_globals(): try: self.app.object_ring.max_more_nodes = 2 - controller = proxy_server.ObjectController(self.app, 'account', - 'container', - 'object') partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] - for node in controller.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(self.app.object_ring, + partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.app.object_ring.max_more_nodes = 20 self.app.request_node_count = lambda r: 20 - controller = proxy_server.ObjectController(self.app, 'account', - 'container', - 'object') partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] - for node in controller.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(self.app.object_ring, + partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 9) self.app.log_handoffs = True self.app.logger = FakeLogger() self.app.object_ring.max_more_nodes = 2 - controller = proxy_server.ObjectController(self.app, 'account', - 'container', - 'object') partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] - for node in controller.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(self.app.object_ring, + partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.assertEquals( @@ -2368,15 +2457,12 @@ class TestObjectController(unittest.TestCase): self.app.log_handoffs = False self.app.logger = FakeLogger() self.app.object_ring.max_more_nodes = 2 - controller = proxy_server.ObjectController(self.app, 'account', - 'container', - 'object') partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] - for node in controller.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(self.app.object_ring, + partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.assertEquals(self.app.logger.log_dict['warning'], []) @@ -2385,21 +2471,19 @@ class TestObjectController(unittest.TestCase): def test_iter_nodes_calls_sort_nodes(self): with mock.patch.object(self.app, 'sort_nodes') as sort_nodes: - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - for node in controller.iter_nodes(self.app.object_ring, 0): + for node in self.app.iter_nodes(self.app.object_ring, 0): pass sort_nodes.assert_called_once_with( self.app.object_ring.get_part_nodes(0)) def test_iter_nodes_skips_error_limited(self): with mock.patch.object(self.app, 'sort_nodes', lambda n: n): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - first_nodes = list(controller.iter_nodes(self.app.object_ring, 0)) - second_nodes = list(controller.iter_nodes(self.app.object_ring, 0)) + first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) + second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) self.assertTrue(first_nodes[0] in second_nodes) - controller.error_limit(first_nodes[0], 'test') - second_nodes = list(controller.iter_nodes(self.app.object_ring, 0)) + self.app.error_limit(first_nodes[0], 'test') + second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) self.assertTrue(first_nodes[0] not in second_nodes) def test_iter_nodes_gives_extra_if_error_limited_inline(self): @@ -2408,33 +2492,31 @@ class TestObjectController(unittest.TestCase): mock.patch.object(self.app, 'request_node_count', lambda r: 6), mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - first_nodes = list(controller.iter_nodes(self.app.object_ring, 0)) + first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) second_nodes = [] - for node in controller.iter_nodes(self.app.object_ring, 0): + for node in self.app.iter_nodes(self.app.object_ring, 0): if not second_nodes: - controller.error_limit(node, 'test') + self.app.error_limit(node, 'test') second_nodes.append(node) self.assertEquals(len(first_nodes), 6) self.assertEquals(len(second_nodes), 7) def test_iter_nodes_with_custom_node_iter(self): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') node_list = [dict(id=n) for n in xrange(10)] with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 3)): - got_nodes = list(controller.iter_nodes(self.app.object_ring, 0, - node_iter=iter(node_list))) + got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, + node_iter=iter(node_list))) self.assertEqual(node_list[:3], got_nodes) with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 1000000)): - got_nodes = list(controller.iter_nodes(self.app.object_ring, 0, - node_iter=iter(node_list))) + got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, + node_iter=iter(node_list))) self.assertEqual(node_list, got_nodes) def test_best_response_sets_headers(self): @@ -4609,6 +4691,7 @@ class TestObjectController(unittest.TestCase): fd.read(1) fd.close() sock.close() + sleep(0) # let eventlet do it's thing # Make sure the GC is run again for pythons without reference counting for i in xrange(4): gc.collect() @@ -6743,6 +6826,65 @@ class TestSegmentedIterable(unittest.TestCase): self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34') +class TestProxyObjectPerformance(unittest.TestCase): + + def setUp(self): + # This is just a simple test that can be used to verify and debug the + # various data paths between the proxy server and the object + # server. Used as a play ground to debug buffer sizes for sockets. + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is transmitting in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + # Small, fast for testing + obj_len = 2 * 64 * 1024 + # Use 1 GB or more for measurements + #obj_len = 2 * 512 * 1024 * 1024 + self.path = '/v1/a/c/o.large' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n' % (self.path, str(obj_len))) + fd.write('a' * obj_len) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + self.obj_len = obj_len + + def test_GET_debug_large_file(self): + for i in range(0, 10): + start = time.time() + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is reading in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + fd.write('GET %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n' % self.path) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + total = 0 + while True: + buf = fd.read(100000) + if not buf: + break + total += len(buf) + self.assertEqual(total, self.obj_len) + + end = time.time() + print "Run %02d took %07.03f" % (i, end - start) + + if __name__ == '__main__': setup() try: -- cgit