Rebase to OpenStack Swift v1.10.0.172.g9fe7748v1.10.2-0v1.10.2

* Updated Proxy unit test
* Updated Functional tests
* Updated Tox to point to the new swift snapshot available
  on http://launchpad.net/gluster-swift

Change-Id: Ia91593c6a28d5a3fe70715ddc60546931ae71635
Signed-off-by: Luis Pabon <lpabon@redhat.com>
Reviewed-on: http://review.gluster.org/6445

2 files changed, 220 insertions, 52 deletions

diff --git a/test/unit/__init__.py b/test/unit/__init__.py
index 76e09c1..fa46715 100644
--- a/test/unit/__init__.py
+++ b/test/unit/__init__.py
@@ -35,13 +35,6 @@ import logging.handlers
 from httplib import HTTPException
 
 
-class DebugLogger(object):
-    """A simple stdout logger for eventlet wsgi."""
-
-    def write(self, *args):
-        print args
-
-
 class FakeRing(object):
 
     def __init__(self, replicas=3, max_more_nodes=0):
@@ -254,6 +247,7 @@ class FakeLogger(logging.Logger):
         self.level = logging.NOTSET
         if 'facility' in kwargs:
             self.facility = kwargs['facility']
+        self.statsd_client = None
 
     def _clear(self):
         self.log_dict = defaultdict(list)
@@ -276,10 +270,12 @@ class FakeLogger(logging.Logger):
     error = _store_and_log_in('error')
     info = _store_and_log_in('info')
     warning = _store_and_log_in('warning')
+    warn = _store_and_log_in('warning')
     debug = _store_and_log_in('debug')
 
     def exception(self, *args, **kwargs):
-        self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
+        self.log_dict['exception'].append((args, kwargs,
+                                           str(sys.exc_info()[1])))
         print 'FakeLogger Exception: %s' % self.log_dict
 
     # mock out the StatsD logging methods:
@@ -323,7 +319,7 @@ class FakeLogger(logging.Logger):
     def emit(self, record):
         pass
 
-    def handle(self, record):
+    def _handle(self, record):
         try:
             line = record.getMessage()
         except TypeError:
@@ -332,6 +328,9 @@ class FakeLogger(logging.Logger):
             raise
         self.lines_dict[record.levelno].append(line)
 
+    def handle(self, record):
+        self._handle(record)
+
     def flush(self):
         pass
 
@@ -339,6 +338,26 @@ class FakeLogger(logging.Logger):
         pass
 
 
+class DebugLogger(FakeLogger):
+    """A simple stdout logging version of FakeLogger"""
+
+    def __init__(self, *args, **kwargs):
+        FakeLogger.__init__(self, *args, **kwargs)
+        self.formatter = logging.Formatter("%(server)s: %(message)s")
+
+    def handle(self, record):
+        self._handle(record)
+        print self.formatter.format(record)
+
+    def write(self, *args):
+        print args
+
+
+def debug_logger(name='test'):
+    """get a named adapted debug logger"""
+    return LogAdapter(DebugLogger(), name)
+
+
 original_syslog_handler = logging.handlers.SysLogHandler
 
 
@@ -446,6 +465,8 @@ def fake_http_connect(*code_iter, **kwargs):
             self.body = body
             self.headers = headers or {}
             self.timestamp = timestamp
+            if kwargs.get('slow') and isinstance(kwargs['slow'], list):
+                kwargs['slow'][0] -= 1
 
         def getresponse(self):
             if kwargs.get('raise_exc'):
@@ -489,13 +510,18 @@ def fake_http_connect(*code_iter, **kwargs):
                     headers['x-container-timestamp'] = '1'
             except StopIteration:
                 pass
-            if 'slow' in kwargs:
+            if self.am_slow():
                 headers['content-length'] = '4'
             headers.update(self.headers)
             return headers.items()
 
+        def am_slow(self):
+            if kwargs.get('slow') and isinstance(kwargs['slow'], list):
+                return kwargs['slow'][0] >= 0
+            return bool(kwargs.get('slow'))
+
         def read(self, amt=None):
-            if 'slow' in kwargs:
+            if self.am_slow():
                 if self.sent < 4:
                     self.sent += 1
                     sleep(0.1)
@@ -505,7 +531,7 @@ def fake_http_connect(*code_iter, **kwargs):
             return rv
 
         def send(self, amt=None):
-            if 'slow' in kwargs:
+            if self.am_slow():
                 if self.received < 4:
                     self.received += 1
                     sleep(0.1)
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 4537ab6..4c75562 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -50,7 +50,7 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
     MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
     MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
 from swift.common import utils
-from swift.common.utils import mkdirs, normalize_timestamp
+from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
 from swift.common.wsgi import monkey_patch_mimetools
 from swift.proxy.controllers.obj import SegmentedIterable
 from swift.proxy.controllers.base import get_container_memcache_key, \
@@ -66,7 +66,7 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
 STATIC_TIME = time.time()
 _request_instances = weakref.WeakKeyDictionary()
 _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \
-    _testdir = None
+    _testdir = _orig_SysLogHandler = None
 
 
 def request_init(self, *args, **kwargs):
@@ -78,7 +78,9 @@ def request_init(self, *args, **kwargs):
 def do_setup(the_object_server):
     utils.HASH_PATH_SUFFIX = 'endcap'
     global _testdir, _test_servers, _test_sockets, \
-        _orig_container_listing_limit, _test_coros
+        _orig_container_listing_limit, _test_coros, _orig_SysLogHandler
+    _orig_SysLogHandler = utils.SysLogHandler
+    utils.SysLogHandler = mock.MagicMock()
     Request._orig_init = Request.__init__
     Request.__init__ = request_init
     monkey_patch_mimetools()
@@ -201,6 +203,7 @@ def teardown():
         _orig_container_listing_limit
     rmtree(os.path.dirname(_testdir))
     Request.__init__ = Request._orig_init
+    utils.SysLogHandler = _orig_SysLogHandler
 
 
 def sortHeaderNames(headerNames):
@@ -656,6 +659,34 @@ class TestProxyServer(unittest.TestCase):
                           {'region': 2, 'zone': 1, 'ip': '127.0.0.1'}]
             self.assertEquals(exp_sorted, app_sorted)
 
+    def test_info_defaults(self):
+        app = proxy_server.Application({}, FakeMemcache(),
+                                       account_ring=FakeRing(),
+                                       container_ring=FakeRing(),
+                                       object_ring=FakeRing())
+
+        self.assertTrue(app.expose_info)
+        self.assertTrue(isinstance(app.disallowed_sections, list))
+        self.assertEqual(0, len(app.disallowed_sections))
+        self.assertTrue(app.admin_key is None)
+
+    def test_get_info_controller(self):
+        path = '/info'
+        app = proxy_server.Application({}, FakeMemcache(),
+                                       account_ring=FakeRing(),
+                                       container_ring=FakeRing(),
+                                       object_ring=FakeRing())
+
+        controller, path_parts = app.get_controller(path)
+
+        self.assertTrue('version' in path_parts)
+        self.assertTrue(path_parts['version'] is None)
+        self.assertTrue('disallowed_sections' in path_parts)
+        self.assertTrue('expose_info' in path_parts)
+        self.assertTrue('admin_key' in path_parts)
+
+        self.assertEqual(controller.__name__, 'InfoController')
+
 
 class TestObjectController(unittest.TestCase):
 
@@ -825,7 +856,7 @@ class TestObjectController(unittest.TestCase):
 
             controller = \
                 proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg')
-            controller.error_limit(
+            self.app.error_limit(
                 self.app.object_ring.get_part_nodes(1)[0], 'test')
             set_http_connect(200, 200,        # account, container
                              201, 201, 201,   # 3 working backends
@@ -2282,6 +2313,73 @@ class TestObjectController(unittest.TestCase):
                 got_exc = True
             self.assert_(got_exc)
 
+    def test_node_read_timeout_retry(self):
+        with save_globals():
+            self.app.account_ring.get_nodes('account')
+            for dev in self.app.account_ring.devs.values():
+                dev['ip'] = '127.0.0.1'
+                dev['port'] = 1
+            self.app.container_ring.get_nodes('account')
+            for dev in self.app.container_ring.devs.values():
+                dev['ip'] = '127.0.0.1'
+                dev['port'] = 1
+            self.app.object_ring.get_nodes('account')
+            for dev in self.app.object_ring.devs.values():
+                dev['ip'] = '127.0.0.1'
+                dev['port'] = 1
+            req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+            self.app.update_request(req)
+
+            self.app.node_timeout = 0.1
+            set_http_connect(200, 200, 200, slow=[3])
+            resp = req.get_response(self.app)
+            got_exc = False
+            try:
+                resp.body
+            except ChunkReadTimeout:
+                got_exc = True
+            self.assert_(got_exc)
+
+            set_http_connect(200, 200, 200, body='lalala', slow=[2])
+            resp = req.get_response(self.app)
+            got_exc = False
+            try:
+                self.assertEquals(resp.body, 'lalala')
+            except ChunkReadTimeout:
+                got_exc = True
+            self.assert_(not got_exc)
+
+            set_http_connect(200, 200, 200, body='lalala', slow=[2],
+                             etags=['a', 'a', 'a'])
+            resp = req.get_response(self.app)
+            got_exc = False
+            try:
+                self.assertEquals(resp.body, 'lalala')
+            except ChunkReadTimeout:
+                got_exc = True
+            self.assert_(not got_exc)
+
+            set_http_connect(200, 200, 200, body='lalala', slow=[2],
+                             etags=['a', 'b', 'a'])
+            resp = req.get_response(self.app)
+            got_exc = False
+            try:
+                self.assertEquals(resp.body, 'lalala')
+            except ChunkReadTimeout:
+                got_exc = True
+            self.assert_(not got_exc)
+
+            req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+            set_http_connect(200, 200, 200, body='lalala', slow=[2],
+                             etags=['a', 'b', 'b'])
+            resp = req.get_response(self.app)
+            got_exc = False
+            try:
+                resp.body
+            except ChunkReadTimeout:
+                got_exc = True
+            self.assert_(got_exc)
+
     def test_node_write_timeout(self):
         with save_globals():
             self.app.account_ring.get_nodes('account')
@@ -2320,44 +2418,35 @@ class TestObjectController(unittest.TestCase):
         with save_globals():
             try:
                 self.app.object_ring.max_more_nodes = 2
-                controller = proxy_server.ObjectController(self.app, 'account',
-                                                           'container',
-                                                           'object')
                 partition, nodes = self.app.object_ring.get_nodes('account',
                                                                   'container',
                                                                   'object')
                 collected_nodes = []
-                for node in controller.iter_nodes(self.app.object_ring,
-                                                  partition):
+                for node in self.app.iter_nodes(self.app.object_ring,
+                                                partition):
                     collected_nodes.append(node)
                 self.assertEquals(len(collected_nodes), 5)
 
                 self.app.object_ring.max_more_nodes = 20
                 self.app.request_node_count = lambda r: 20
-                controller = proxy_server.ObjectController(self.app, 'account',
-                                                           'container',
-                                                           'object')
                 partition, nodes = self.app.object_ring.get_nodes('account',
                                                                   'container',
                                                                   'object')
                 collected_nodes = []
-                for node in controller.iter_nodes(self.app.object_ring,
-                                                  partition):
+                for node in self.app.iter_nodes(self.app.object_ring,
+                                                partition):
                     collected_nodes.append(node)
                 self.assertEquals(len(collected_nodes), 9)
 
                 self.app.log_handoffs = True
                 self.app.logger = FakeLogger()
                 self.app.object_ring.max_more_nodes = 2
-                controller = proxy_server.ObjectController(self.app, 'account',
-                                                           'container',
-                                                           'object')
                 partition, nodes = self.app.object_ring.get_nodes('account',
                                                                   'container',
                                                                   'object')
                 collected_nodes = []
-                for node in controller.iter_nodes(self.app.object_ring,
-                                                  partition):
+                for node in self.app.iter_nodes(self.app.object_ring,
+                                                partition):
                     collected_nodes.append(node)
                 self.assertEquals(len(collected_nodes), 5)
                 self.assertEquals(
@@ -2368,15 +2457,12 @@ class TestObjectController(unittest.TestCase):
                 self.app.log_handoffs = False
                 self.app.logger = FakeLogger()
                 self.app.object_ring.max_more_nodes = 2
-                controller = proxy_server.ObjectController(self.app, 'account',
-                                                           'container',
-                                                           'object')
                 partition, nodes = self.app.object_ring.get_nodes('account',
                                                                   'container',
                                                                   'object')
                 collected_nodes = []
-                for node in controller.iter_nodes(self.app.object_ring,
-                                                  partition):
+                for node in self.app.iter_nodes(self.app.object_ring,
+                                                partition):
                     collected_nodes.append(node)
                 self.assertEquals(len(collected_nodes), 5)
                 self.assertEquals(self.app.logger.log_dict['warning'], [])
@@ -2385,21 +2471,19 @@ class TestObjectController(unittest.TestCase):
 
     def test_iter_nodes_calls_sort_nodes(self):
         with mock.patch.object(self.app, 'sort_nodes') as sort_nodes:
-            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
-            for node in controller.iter_nodes(self.app.object_ring, 0):
+            for node in self.app.iter_nodes(self.app.object_ring, 0):
                 pass
             sort_nodes.assert_called_once_with(
                 self.app.object_ring.get_part_nodes(0))
 
     def test_iter_nodes_skips_error_limited(self):
         with mock.patch.object(self.app, 'sort_nodes', lambda n: n):
-            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
-            first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
-            second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+            first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
+            second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
             self.assertTrue(first_nodes[0] in second_nodes)
 
-            controller.error_limit(first_nodes[0], 'test')
-            second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+            self.app.error_limit(first_nodes[0], 'test')
+            second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
             self.assertTrue(first_nodes[0] not in second_nodes)
 
     def test_iter_nodes_gives_extra_if_error_limited_inline(self):
@@ -2408,33 +2492,31 @@ class TestObjectController(unittest.TestCase):
                 mock.patch.object(self.app, 'request_node_count',
                                   lambda r: 6),
                 mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)):
-            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
-            first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
+            first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0))
             second_nodes = []
-            for node in controller.iter_nodes(self.app.object_ring, 0):
+            for node in self.app.iter_nodes(self.app.object_ring, 0):
                 if not second_nodes:
-                    controller.error_limit(node, 'test')
+                    self.app.error_limit(node, 'test')
                 second_nodes.append(node)
             self.assertEquals(len(first_nodes), 6)
             self.assertEquals(len(second_nodes), 7)
 
     def test_iter_nodes_with_custom_node_iter(self):
-        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
         node_list = [dict(id=n) for n in xrange(10)]
         with nested(
                 mock.patch.object(self.app, 'sort_nodes', lambda n: n),
                 mock.patch.object(self.app, 'request_node_count',
                                   lambda r: 3)):
-            got_nodes = list(controller.iter_nodes(self.app.object_ring, 0,
-                                                   node_iter=iter(node_list)))
+            got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0,
+                                                 node_iter=iter(node_list)))
         self.assertEqual(node_list[:3], got_nodes)
 
         with nested(
                 mock.patch.object(self.app, 'sort_nodes', lambda n: n),
                 mock.patch.object(self.app, 'request_node_count',
                                   lambda r: 1000000)):
-            got_nodes = list(controller.iter_nodes(self.app.object_ring, 0,
-                                                   node_iter=iter(node_list)))
+            got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0,
+                                                 node_iter=iter(node_list)))
         self.assertEqual(node_list, got_nodes)
 
     def test_best_response_sets_headers(self):
@@ -4609,6 +4691,7 @@ class TestObjectController(unittest.TestCase):
         fd.read(1)
         fd.close()
         sock.close()
+        sleep(0)  # let eventlet do it's thing
         # Make sure the GC is run again for pythons without reference counting
         for i in xrange(4):
             gc.collect()
@@ -6743,6 +6826,65 @@ class TestSegmentedIterable(unittest.TestCase):
         self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
 
 
+class TestProxyObjectPerformance(unittest.TestCase):
+
+    def setUp(self):
+        # This is just a simple test that can be used to verify and debug the
+        # various data paths between the proxy server and the object
+        # server. Used as a play ground to debug buffer sizes for sockets.
+        prolis = _test_sockets[0]
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        # Client is transmitting in 2 MB chunks
+        fd = sock.makefile('wb', 2 * 1024 * 1024)
+        # Small, fast for testing
+        obj_len = 2 * 64 * 1024
+        # Use 1 GB or more for measurements
+        #obj_len = 2 * 512 * 1024 * 1024
+        self.path = '/v1/a/c/o.large'
+        fd.write('PUT %s HTTP/1.1\r\n'
+                 'Host: localhost\r\n'
+                 'Connection: close\r\n'
+                 'X-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octet-stream\r\n'
+                 '\r\n' % (self.path, str(obj_len)))
+        fd.write('a' * obj_len)
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEqual(headers[:len(exp)], exp)
+        self.obj_len = obj_len
+
+    def test_GET_debug_large_file(self):
+        for i in range(0, 10):
+            start = time.time()
+
+            prolis = _test_sockets[0]
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+            # Client is reading in 2 MB chunks
+            fd = sock.makefile('wb', 2 * 1024 * 1024)
+            fd.write('GET %s HTTP/1.1\r\n'
+                     'Host: localhost\r\n'
+                     'Connection: close\r\n'
+                     'X-Storage-Token: t\r\n'
+                     '\r\n' % self.path)
+            fd.flush()
+            headers = readuntil2crlfs(fd)
+            exp = 'HTTP/1.1 200'
+            self.assertEqual(headers[:len(exp)], exp)
+
+            total = 0
+            while True:
+                buf = fd.read(100000)
+                if not buf:
+                    break
+                total += len(buf)
+            self.assertEqual(total, self.obj_len)
+
+            end = time.time()
+            print "Run %02d took %07.03f" % (i, end - start)
+
+
 if __name__ == '__main__':
     setup()
     try: