summaryrefslogtreecommitdiffstats
path: root/test/unit/proxy/test_server.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/unit/proxy/test_server.py')
-rw-r--r--test/unit/proxy/test_server.py222
1 files changed, 182 insertions, 40 deletions
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 4537ab6..4c75562 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -50,7 +50,7 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
from swift.common import utils
-from swift.common.utils import mkdirs, normalize_timestamp
+from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers.base import get_container_memcache_key, \
@@ -66,7 +66,7 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
STATIC_TIME = time.time()
_request_instances = weakref.WeakKeyDictionary()
_test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \
- _testdir = None
+ _testdir = _orig_SysLogHandler = None
def request_init(self, *args, **kwargs):
@@ -78,7 +78,9 @@ def request_init(self, *args, **kwargs):
def do_setup(the_object_server):
utils.HASH_PATH_SUFFIX = 'endcap'
global _testdir, _test_servers, _test_sockets, \
- _orig_container_listing_limit, _test_coros
+ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler
+ _orig_SysLogHandler = utils.SysLogHandler
+ utils.SysLogHandler = mock.MagicMock()
Request._orig_init = Request.__init__
Request.__init__ = request_init
monkey_patch_mimetools()
@@ -201,6 +203,7 @@ def teardown():
_orig_container_listing_limit
rmtree(os.path.dirname(_testdir))
Request.__init__ = Request._orig_init
+ utils.SysLogHandler = _orig_SysLogHandler
def sortHeaderNames(headerNames):
@@ -656,6 +659,34 @@ class TestProxyServer(unittest.TestCase):
{'region': 2, 'zone': 1, 'ip': '127.0.0.1'}]
self.assertEquals(exp_sorted, app_sorted)
+ def test_info_defaults(self):
+ app = proxy_server.Application({}, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing(),
+ object_ring=FakeRing())
+
+ self.assertTrue(app.expose_info)
+ self.assertTrue(isinstance(app.disallowed_sections, list))
+ self.assertEqual(0, len(app.disallowed_sections))
+ self.assertTrue(app.admin_key is None)
+
+ def test_get_info_controller(self):
+ path = '/info'
+ app = proxy_server.Application({}, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing(),
+ object_ring=FakeRing())
+
+ controller, path_parts = app.get_controller(path)
+
+ self.assertTrue('version' in path_parts)
+ self.assertTrue(path_parts['version'] is None)
+ self.assertTrue('disallowed_sections' in path_parts)
+ self.assertTrue('expose_info' in path_parts)
+ self.assertTrue('admin_key' in path_parts)
+
+ self.assertEqual(controller.__name__, 'InfoController')
+
class TestObjectController(unittest.TestCase):
@@ -825,7 +856,7 @@ class TestObjectController(unittest.TestCase):
controller = \
proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg')
- controller.error_limit(
+ self.app.error_limit(
self.app.object_ring.get_part_nodes(1)[0], 'test')
set_http_connect(200, 200, # account, container
201, 201, 201, # 3 working backends
@@ -2282,6 +2313,73 @@ class TestObjectController(unittest.TestCase):
got_exc = True
self.assert_(got_exc)
+ def test_node_read_timeout_retry(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs.values():
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs.values():
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.object_ring.get_nodes('account')
+ for dev in self.app.object_ring.devs.values():
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ self.app.update_request(req)
+
+ self.app.node_timeout = 0.1
+ set_http_connect(200, 200, 200, slow=[3])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assert_(got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala', slow=[2])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEquals(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assert_(not got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala', slow=[2],
+ etags=['a', 'a', 'a'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEquals(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assert_(not got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala', slow=[2],
+ etags=['a', 'b', 'a'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEquals(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assert_(not got_exc)
+
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ set_http_connect(200, 200, 200, body='lalala', slow=[2],
+ etags=['a', 'b', 'b'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assert_(got_exc)
+
def test_node_write_timeout(self):
with save_globals():
self.app.account_ring.get_nodes('account')
@@ -2320,44 +2418,35 @@ class TestObjectController(unittest.TestCase):
with save_globals():
try:
self.app.object_ring.max_more_nodes = 2
- controller = proxy_server.ObjectController(self.app, 'account',
- 'container',
- 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container',
'object')
collected_nodes = []
- for node in controller.iter_nodes(self.app.object_ring,
- partition):
+ for node in self.app.iter_nodes(self.app.object_ring,
+ partition):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 5)
self.app.object_ring.max_more_nodes = 20
self.app.request_node_count = lambda r: 20
- controller = proxy_server.ObjectController(self.app, 'account',
- 'container',
- 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container',
'object')
collected_nodes = []
- for node in controller.iter_nodes(self.app.object_ring,
- partition):
+ for node in self.app.iter_nodes(self.app.object_ring,
+ partition):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 9)
self.app.log_handoffs = True
self.app.logger = FakeLogger()
self.app.object_ring.max_more_nodes = 2
- controller = proxy_server.ObjectController(self.app, 'account',
- 'container',
- 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container',
'object')
collected_nodes = []
- for node in controller.iter_nodes(self.app.object_ring,
- partition):
+ for node in self.app.iter_nodes(self.app.object_ring,
+ partition):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 5)
self.assertEquals(
@@ -2368,15 +2457,12 @@ class TestObjectController(unittest.TestCase):
self.app.log_handoffs = False
self.app.logger = FakeLogger()
self.app.object_ring.max_more_nodes = 2
- controller = proxy_server.ObjectController(self.app, 'account',
- 'container',
- 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container',
'object')
collected_nodes = []
- for node in controller.iter_nodes(self.app.object_ring,
- partition):
+ for node in self.app.iter_nodes(self.app.object_ring,
+ partition):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 5)
self.assertEquals(self.app.logger.log_dict['warning'], [])
@@ -2385,21 +2471,19 @@ class TestObjectController(unittest.TestCase):
def test_iter_nodes_calls_sort_nodes(self):
with mock.patch.object(self.app, 'sort_nodes') as sort_nodes:
- controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
- for node in controller.iter_nodes(self.app.object_ring, 0):
+ for node in self.app.iter_nodes(self.app.object_ring, 0):
pass
sort_nodes.assert_called_once_with(
self.app.object_ring.get_part_nodes(0))
def test_iter_nodes_skips_error_limited(self):
with mock.patch.object(self.app, 'sort_nodes', lambda n: n):
- controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
- first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
- second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+ first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
+ second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
self.assertTrue(first_nodes[0] in second_nodes)
- controller.error_limit(first_nodes[0], 'test')
- second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+ self.app.error_limit(first_nodes[0], 'test')
+ second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
self.assertTrue(first_nodes[0] not in second_nodes)
def test_iter_nodes_gives_extra_if_error_limited_inline(self):
@@ -2408,33 +2492,31 @@ class TestObjectController(unittest.TestCase):
mock.patch.object(self.app, 'request_node_count',
lambda r: 6),
mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)):
- controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
- first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+ first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
second_nodes = []
- for node in controller.iter_nodes(self.app.object_ring, 0):
+ for node in self.app.iter_nodes(self.app.object_ring, 0):
if not second_nodes:
- controller.error_limit(node, 'test')
+ self.app.error_limit(node, 'test')
second_nodes.append(node)
self.assertEquals(len(first_nodes), 6)
self.assertEquals(len(second_nodes), 7)
def test_iter_nodes_with_custom_node_iter(self):
- controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
node_list = [dict(id=n) for n in xrange(10)]
with nested(
mock.patch.object(self.app, 'sort_nodes', lambda n: n),
mock.patch.object(self.app, 'request_node_count',
lambda r: 3)):
- got_nodes = list(controller.iter_nodes(self.app.object_ring, 0,
- node_iter=iter(node_list)))
+ got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0,
+ node_iter=iter(node_list)))
self.assertEqual(node_list[:3], got_nodes)
with nested(
mock.patch.object(self.app, 'sort_nodes', lambda n: n),
mock.patch.object(self.app, 'request_node_count',
lambda r: 1000000)):
- got_nodes = list(controller.iter_nodes(self.app.object_ring, 0,
- node_iter=iter(node_list)))
+ got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0,
+ node_iter=iter(node_list)))
self.assertEqual(node_list, got_nodes)
def test_best_response_sets_headers(self):
@@ -4609,6 +4691,7 @@ class TestObjectController(unittest.TestCase):
fd.read(1)
fd.close()
sock.close()
+ sleep(0) # let eventlet do it's thing
# Make sure the GC is run again for pythons without reference counting
for i in xrange(4):
gc.collect()
@@ -6743,6 +6826,65 @@ class TestSegmentedIterable(unittest.TestCase):
self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
+class TestProxyObjectPerformance(unittest.TestCase):
+
+ def setUp(self):
+ # This is just a simple test that can be used to verify and debug the
+ # various data paths between the proxy server and the object
+ # server. Used as a play ground to debug buffer sizes for sockets.
+ prolis = _test_sockets[0]
+ sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+ # Client is transmitting in 2 MB chunks
+ fd = sock.makefile('wb', 2 * 1024 * 1024)
+ # Small, fast for testing
+ obj_len = 2 * 64 * 1024
+ # Use 1 GB or more for measurements
+ #obj_len = 2 * 512 * 1024 * 1024
+ self.path = '/v1/a/c/o.large'
+ fd.write('PUT %s HTTP/1.1\r\n'
+ 'Host: localhost\r\n'
+ 'Connection: close\r\n'
+ 'X-Storage-Token: t\r\n'
+ 'Content-Length: %s\r\n'
+ 'Content-Type: application/octet-stream\r\n'
+ '\r\n' % (self.path, str(obj_len)))
+ fd.write('a' * obj_len)
+ fd.flush()
+ headers = readuntil2crlfs(fd)
+ exp = 'HTTP/1.1 201'
+ self.assertEqual(headers[:len(exp)], exp)
+ self.obj_len = obj_len
+
+ def test_GET_debug_large_file(self):
+ for i in range(0, 10):
+ start = time.time()
+
+ prolis = _test_sockets[0]
+ sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+ # Client is reading in 2 MB chunks
+ fd = sock.makefile('wb', 2 * 1024 * 1024)
+ fd.write('GET %s HTTP/1.1\r\n'
+ 'Host: localhost\r\n'
+ 'Connection: close\r\n'
+ 'X-Storage-Token: t\r\n'
+ '\r\n' % self.path)
+ fd.flush()
+ headers = readuntil2crlfs(fd)
+ exp = 'HTTP/1.1 200'
+ self.assertEqual(headers[:len(exp)], exp)
+
+ total = 0
+ while True:
+ buf = fd.read(100000)
+ if not buf:
+ break
+ total += len(buf)
+ self.assertEqual(total, self.obj_len)
+
+ end = time.time()
+ print "Run %02d took %07.03f" % (i, end - start)
+
+
if __name__ == '__main__':
setup()
try: