From 2ccd2c4d969cdd6e7feedd21ac2e5cb8498ff37d Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 12 Mar 2014 16:54:30 +0530 Subject: Sync with OpenStack Swift v1.13.0 Also, bumped version of gluster-swift to v1.13.0 Change-Id: I797dc704c9523540cba847b1e8ff3da97b79630c Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/7229 Reviewed-by: Chetan Risbud Reviewed-by: Luis Pabon Tested-by: Luis Pabon --- doc/markdown/openstack_swift_sync.md | 9 +- gluster/swift/__init__.py | 2 +- test/functional/swift_test_client.py | 23 +- test/functional/swift_testing.py | 33 +- test/functional/test_account.py | 148 ++++- test/functional/tests.py | 12 +- test/unit/proxy/controllers/test_account.py | 89 ++- test/unit/proxy/controllers/test_obj.py | 13 +- test/unit/proxy/test_server.py | 840 +++------------------------- tools/requirements.txt | 1 - tools/test-requires | 6 +- tox.ini | 2 +- 12 files changed, 368 insertions(+), 810 deletions(-) diff --git a/doc/markdown/openstack_swift_sync.md b/doc/markdown/openstack_swift_sync.md index f3feac8..ba2fadb 100644 --- a/doc/markdown/openstack_swift_sync.md +++ b/doc/markdown/openstack_swift_sync.md @@ -14,7 +14,8 @@ $ python setup.py sdist $ ls dist ``` -* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net +* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net. +* Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad. ## Setup Tox Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link. @@ -22,7 +23,11 @@ Now that the swift source is availabe on launchpad.net, copy its link location a ## Update tests This part is a little more complicated and now we need to *merge* the latest tests with ours. -I suggest using a tool called `meld` to make this work easier. +[meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from: + +* Latest swift (say v1.13) +* Previous swift (say v1.12) +* gluster-swift (v1.12) Files that need to be merged: diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py index 4fe17ba..17f2fcf 100644 --- a/gluster/swift/__init__.py +++ b/gluster/swift/__init__.py @@ -45,6 +45,6 @@ class PkgInfo(object): ### ### Change the Package version here ### -_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False) +_pkginfo = PkgInfo('1.13.0', '0', 'gluster_swift', False) __version__ = _pkginfo.pretty_version __canonical_version__ = _pkginfo.canonical_version diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 47e023e..ecb2a6d 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -40,7 +40,7 @@ class RequestError(Exception): class ResponseError(Exception): - def __init__(self, response, method, path): + def __init__(self, response, method=None, path=None): self.status = response.status self.reason = response.reason self.method = method @@ -310,10 +310,11 @@ class Base: def __str__(self): return self.name - def header_fields(self, fields): + def header_fields(self, required_fields, optional_fields=()): headers = dict(self.conn.response.getheaders()) ret = {} - for field in fields: + + for field in required_fields: if field[1] not in headers: raise ValueError("%s was not found in response header" % (field[1])) @@ -322,6 +323,15 @@ class Base: ret[field[0]] = int(headers[field[1]]) except ValueError: ret[field[0]] = headers[field[1]] + + for field in optional_fields: + if field[1] not in headers: + continue + try: + ret[field[0]] = int(headers[field[1]]) + except ValueError: + ret[field[0]] = headers[field[1]] + return ret @@ -480,10 +490,11 @@ class Container(Base): parms=parms, cfg=cfg) if self.conn.response.status == 204: - fields = [['bytes_used', 'x-container-bytes-used'], - ['object_count', 'x-container-object-count']] + required_fields = [['bytes_used', 'x-container-bytes-used'], + ['object_count', 'x-container-object-count']] + optional_fields = [['versions', 'x-versions-location']] - return self.header_fields(fields) + return self.header_fields(required_fields, optional_fields) raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py index b5642b3..f05cb48 100644 --- a/test/functional/swift_testing.py +++ b/test/functional/swift_testing.py @@ -18,6 +18,7 @@ import os import socket import sys from time import sleep +from urlparse import urlparse from test import get_config @@ -119,18 +120,23 @@ conn = [None, None, None] def retry(func, *args, **kwargs): """ - You can use the kwargs to override the 'retries' (default: 5) and - 'use_account' (default: 1). + You can use the kwargs to override: + 'retries' (default: 5) + 'use_account' (default: 1) - which user's token to pass + 'url_account' (default: matches 'use_account') - which user's storage URL + 'resource' (default: url[url_account] - URL to connect to; retry() + will interpolate the variable :storage_url: if present """ global url, token, parsed, conn retries = kwargs.get('retries', 5) - use_account = 1 - if 'use_account' in kwargs: - use_account = kwargs['use_account'] - del kwargs['use_account'] - use_account -= 1 - attempts = 0 - backoff = 1 + attempts, backoff = 0, 1 + + # use account #1 by default; turn user's 1-indexed account into 0-indexed + use_account = kwargs.pop('use_account', 1) - 1 + + # access our own account by default + url_account = kwargs.pop('url_account', use_account + 1) - 1 + while attempts <= retries: attempts += 1 try: @@ -146,8 +152,13 @@ def retry(func, *args, **kwargs): if not parsed[use_account] or not conn[use_account]: parsed[use_account], conn[use_account] = \ http_connection(url[use_account]) - return func(url[use_account], token[use_account], - parsed[use_account], conn[use_account], + + # default resource is the account url[url_account] + resource = kwargs.pop('resource', '%(storage_url)s') + template_vars = {'storage_url': url[url_account]} + parsed_result = urlparse(resource % template_vars) + return func(url[url_account], token[use_account], + parsed_result, conn[url_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: diff --git a/test/functional/test_account.py b/test/functional/test_account.py index b2f743f..d456090 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -16,12 +16,16 @@ # limitations under the License. import unittest +import json from nose import SkipTest from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH - +from swift.common.middleware.acl import format_acl +from test.functional.swift_test_client import Connection +from test import get_config from swift_testing import check_response, retry, skip, web_front_end +import swift_testing class TestAccount(unittest.TestCase): @@ -66,6 +70,148 @@ class TestAccount(unittest.TestCase): self.assert_(resp.status in (200, 204), resp.status) self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + def test_tempauth_account_acls(self): + if skip: + raise SkipTest + + # Determine whether this cluster has account ACLs; if not, skip test + conn = Connection(get_config('func_test')) + conn.authenticate() + status = conn.make_request( + 'GET', '/info', cfg={'verbatim_path': True}) + if status // 100 != 2: + # Can't tell if account ACLs are enabled; skip tests proactively. + raise SkipTest + else: + cluster_info = json.loads(conn.response.read()) + if not cluster_info.get('tempauth', {}).get('account_acls'): + raise SkipTest + if 'keystoneauth' in cluster_info: + # Unfortunate hack -- tempauth (with account ACLs) is expected + # to play nice with Keystone (without account ACLs), but Zuul + # functest framework doesn't give us an easy way to get a + # tempauth user. + raise SkipTest + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('PUT', parsed.path, '', new_headers) + return check_response(conn) + + def delete(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('DELETE', parsed.path, '', new_headers) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + try: + # User1 can POST to their own account (and reset the ACLs) + resp = retry(post, headers={'X-Account-Access-Control': '{}'}, + use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can GET their own empty account + resp = retry(get, use_account=1) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User2 can't GET User1's account + resp = retry(get, use_account=2, url_account=1) + resp.read() + self.assertEqual(resp.status, 403) + + # User1 is swift_owner of their own account, so they can POST an + # ACL -- let's do this and make User2 (test_user[1]) an admin + acl_user = swift_testing.swift_test_user[1] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # User1 can see the new header + resp = retry(get, use_account=1) + resp.read() + self.assertEqual(resp.status // 100, 2) + data_from_headers = resp.getheader('x-account-access-control') + expected = json.dumps(acl, separators=(',', ':')) + self.assertEqual(data_from_headers, expected) + + # Now User2 should be able to GET the account and see the ACL + resp = retry(head, use_account=2, url_account=1) + resp.read() + data_from_headers = resp.getheader('x-account-access-control') + self.assertEqual(data_from_headers, expected) + + # Revoke User2's admin access, grant User2 read-write access + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # User2 can still GET the account, but not see the ACL + # (since it's privileged data) + resp = retry(head, use_account=2, url_account=1) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('x-account-access-control'), None) + + # User2 can PUT and DELETE a container + resp = retry(put, use_account=2, url_account=1, + resource='%(storage_url)s/mycontainer', headers={}) + resp.read() + self.assertEqual(resp.status, 201) + resp = retry(delete, use_account=2, url_account=1, + resource='%(storage_url)s/mycontainer', headers={}) + resp.read() + self.assertEqual(resp.status, 204) + + # Revoke User2's read-write access, grant User2 read-only access + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # User2 can still GET the account, but not see the ACL + # (since it's privileged data) + resp = retry(head, use_account=2, url_account=1) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('x-account-access-control'), None) + + # User2 can't PUT a container + resp = retry(put, use_account=2, url_account=1, + resource='%(storage_url)s/mycontainer', headers={}) + resp.read() + self.assertEqual(resp.status, 403) + + finally: + # Make sure to clean up even if tests fail -- User2 should not + # have access to User1's account in other functional tests! + resp = retry(post, headers={'X-Account-Access-Control': '{}'}, + use_account=1) + resp.read() + def test_unicode_metadata(self): if skip: raise SkipTest diff --git a/test/functional/tests.py b/test/functional/tests.py index 9469cff..0d9a9ef 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -149,7 +149,7 @@ def timeout(seconds, method, *args, **kwargs): return False -class Utils: +class Utils(object): @classmethod def create_ascii_name(cls, length=None): return uuid.uuid4().hex @@ -201,7 +201,7 @@ class Base2(object): Utils.create_name = Utils.create_ascii_name -class TestAccountEnv: +class TestAccountEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) @@ -388,7 +388,7 @@ class TestAccountUTF8(Base2, TestAccount): set_up = False -class TestAccountNoContainersEnv: +class TestAccountNoContainersEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) @@ -417,7 +417,7 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): set_up = False -class TestContainerEnv: +class TestContainerEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) @@ -708,7 +708,7 @@ class TestContainerUTF8(Base2, TestContainer): set_up = False -class TestContainerPathsEnv: +class TestContainerPathsEnv(object): @classmethod def setUp(cls): raise SkipTest('Objects ending in / are not supported') @@ -888,7 +888,7 @@ class TestContainerPaths(Base): ['dir1/subdir with spaces/file B']) -class TestFileEnv: +class TestFileEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index eefd57d..47f76dc 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -16,12 +16,14 @@ import mock import unittest -from swift.common.swob import Request +from swift.common.swob import Request, Response +from swift.common.middleware.acl import format_acl from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_account_info from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.common.request_helpers import get_sys_meta_prefix +import swift.proxy.controllers.base class TestAccountController(unittest.TestCase): @@ -152,6 +154,91 @@ class TestAccountController(unittest.TestCase): self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + def _make_user_and_sys_acl_headers_data(self): + acl = { + 'admin': ['AUTH_alice', 'AUTH_bob'], + 'read-write': ['AUTH_carol'], + 'read-only': [], + } + user_prefix = 'x-account-' # external, user-facing + user_headers = {(user_prefix + 'access-control'): format_acl( + version=2, acl_dict=acl)} + sys_prefix = get_sys_meta_prefix('account') # internal, system-facing + sys_headers = {(sys_prefix + 'core-access-control'): format_acl( + version=2, acl_dict=acl)} + return user_headers, sys_headers + + def test_account_acl_headers_translated_for_GET_HEAD(self): + # Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers + # from the account server will remap those headers to X-Account-Acl-* + + hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() + controller = proxy_server.AccountController(self.app, 'acct') + + for verb in ('GET', 'HEAD'): + req = Request.blank('/v1/acct', environ={'swift_owner': True}) + controller.GETorHEAD_base = lambda *_: Response( + headers=hdrs_int, environ={ + 'PATH_INFO': '/acct', + 'REQUEST_METHOD': verb, + }) + method = getattr(controller, verb) + resp = method(req) + for header, value in hdrs_ext.items(): + if value: + self.assertEqual(resp.headers.get(header), value) + else: + # blank ACLs should result in no header + self.assert_(header not in resp.headers) + + def test_add_acls_impossible_cases(self): + # For test coverage: verify that defensive coding does defend, in cases + # that shouldn't arise naturally + + # add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST + resp = Response() + controller = proxy_server.AccountController(self.app, 'a') + resp.environ['PATH_INFO'] = '/a' + resp.environ['REQUEST_METHOD'] = 'OPTIONS' + controller.add_acls_from_sys_metadata(resp) + self.assertEqual(1, len(resp.headers)) # we always get Content-Type + self.assertEqual(2, len(resp.environ)) + + def test_memcache_key_impossible_cases(self): + # For test coverage: verify that defensive coding does defend, in cases + # that shouldn't arise naturally + self.assertRaises( + ValueError, + lambda: swift.proxy.controllers.base.get_container_memcache_key( + '/a', None)) + + def test_stripping_swift_admin_headers(self): + # Verify that a GET/HEAD which receives privileged headers from the + # account server will strip those headers for non-swift_owners + + hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() + headers = { + 'x-account-meta-harmless': 'hi mom', + 'x-account-meta-temp-url-key': 's3kr1t', + } + controller = proxy_server.AccountController(self.app, 'acct') + + for verb in ('GET', 'HEAD'): + for env in ({'swift_owner': True}, {'swift_owner': False}): + req = Request.blank('/v1/acct', environ=env) + controller.GETorHEAD_base = lambda *_: Response( + headers=headers, environ={ + 'PATH_INFO': '/acct', + 'REQUEST_METHOD': verb, + }) + method = getattr(controller, verb) + resp = method(req) + self.assertEqual(resp.headers.get('x-account-meta-harmless'), + 'hi mom') + privileged_header_present = ( + 'x-account-meta-temp-url-key' in resp.headers) + self.assertEqual(privileged_header_present, env['swift_owner']) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index cae62b0..aada616 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -21,6 +21,7 @@ import mock import swift from swift.proxy import server as proxy_server +from swift.common.swob import HTTPException from test.unit import FakeRing, FakeMemcache, fake_http_connect @@ -44,7 +45,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase): self.app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9)) - self.app.request_node_count = lambda ring: 10000000 + self.app.request_node_count = lambda replicas: 10000000 self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes def test_iter_nodes_local_first_noops_when_no_affinity(self): @@ -107,14 +108,20 @@ class TestObjController(unittest.TestCase): # and now test that we add the header to log_info req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['x-copy-from'] = 'somewhere' - controller.PUT(req) + try: + controller.PUT(req) + except HTTPException: + pass self.assertEquals( req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) # and then check that we don't do that for originating POSTs req = swift.common.swob.Request.blank('/v1/a/c/o') req.method = 'POST' req.headers['x-copy-from'] = 'elsewhere' - controller.PUT(req) + try: + controller.PUT(req) + except HTTPException: + pass self.assertEquals(req.environ.get('swift.log_info'), None) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index b68be9f..4086a32 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -46,7 +46,8 @@ from gluster.swift.container import server as container_server from gluster.swift.obj import server as object_server from swift.common import ring from swift.common.middleware import proxy_logging -from swift.common.exceptions import ChunkReadTimeout, SegmentError +from swift.common.middleware.acl import parse_acl, format_acl +from swift.common.exceptions import ChunkReadTimeout from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ @@ -54,38 +55,28 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \ from swift.common import utils from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools -from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers -from swift.common.swob import Request, Response, HTTPNotFound, \ - HTTPUnauthorized +from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.swob import Request, Response, HTTPUnauthorized # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) STATIC_TIME = time.time() -_request_instances = weakref.WeakKeyDictionary() _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ _testdir = _orig_SysLogHandler = None -def request_init(self, *args, **kwargs): - self._orig_init(*args, **kwargs) - - _request_instances[self] = None - - def do_setup(the_object_server): utils.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_SysLogHandler = utils.SysLogHandler utils.SysLogHandler = mock.MagicMock() - Request._orig_init = Request.__init__ - Request.__init__ = request_init monkey_patch_mimetools() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of @@ -99,8 +90,6 @@ def do_setup(the_object_server): mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) mkdirs(os.path.join(_testdir, 'a')) mkdirs(os.path.join(_testdir, 'a', 'tmp')) - _orig_container_listing_limit = \ - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', @@ -211,10 +200,7 @@ def setup(): def teardown(): for server in _test_coros: server.kill() - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \ - _orig_container_listing_limit rmtree(os.path.dirname(_testdir)) - Request.__init__ = Request._orig_init utils.SysLogHandler = _orig_SysLogHandler @@ -1716,7 +1702,7 @@ class TestObjectController(unittest.TestCase): except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) - self.app.node_timeout = 0.1 + self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=True) resp = req.get_response(self.app) got_exc = False @@ -1731,7 +1717,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - self.app.node_timeout = 0.1 + self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=[3]) resp = req.get_response(self.app) got_exc = False @@ -3316,402 +3302,6 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) self.assertEquals(headers[:len(exp)], exp) - def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self): - raise SkipTest("Not until we support pure object requests") - # Create a container for our segmented/manifest object testing - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ - _test_sockets - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Create the object segments - segment_etags = [] - for segment in xrange(5): - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - segment_etags.append(md5('1234 ').hexdigest()) - - # Create the nonzero size manifest file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\nabcd ') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - - # Create the object manifest file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n' - 'X-Object-Manifest: segmented_nonzero/name/\r\n' - 'Foo: barbaz\r\nContent-Type: text/jibberish\r\n' - '\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 202' - self.assertEquals(headers[:len(exp)], exp) - - # Ensure retrieving the manifest file gets the whole object - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: ' - 't\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) - self.assert_('Content-Type: text/jibberish' in headers) - self.assert_('Foo: barbaz' in headers) - expected_etag = md5(''.join(segment_etags)).hexdigest() - self.assert_('Etag: "%s"' % expected_etag in headers) - body = fd.read() - self.assertEquals(body, '1234 1234 1234 1234 1234 ') - - # Get lobjects with Range smaller than manifest file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' - 'Range: bytes=0-4\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 206' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) - self.assert_('Content-Type: text/jibberish' in headers) - self.assert_('Foo: barbaz' in headers) - expected_etag = md5(''.join(segment_etags)).hexdigest() - body = fd.read() - self.assertEquals(body, '1234 ') - - # Get lobjects with Range bigger than manifest file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' - 'Range: bytes=11-15\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 206' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) - self.assert_('Content-Type: text/jibberish' in headers) - self.assert_('Foo: barbaz' in headers) - expected_etag = md5(''.join(segment_etags)).hexdigest() - body = fd.read() - self.assertEquals(body, '234 1') - - def test_chunked_put_lobjects(self): - raise SkipTest("Not until we support pure object requests") - # Create a container for our segmented/manifest object testing - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Create the object segments - segment_etags = [] - for segment in xrange(5): - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s ' - 'HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 5\r\n' - '\r\n' - '1234 ' % str(segment)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - segment_etags.append(md5('1234 ').hexdigest()) - # Create the object manifest file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'X-Object-Manifest: segmented%20object/object%20name/\r\n' - 'Content-Type: text/jibberish\r\n' - 'Foo: barbaz\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Check retrieving the listing the manifest would retrieve - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ ' - 'HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - body = fd.read() - self.assertEquals( - body, - 'object name/0\n' - 'object name/1\n' - 'object name/2\n' - 'object name/3\n' - 'object name/4\n') - # Ensure retrieving the manifest file gets the whole object - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in - headers) - self.assert_('Content-Type: text/jibberish' in headers) - self.assert_('Foo: barbaz' in headers) - expected_etag = md5(''.join(segment_etags)).hexdigest() - self.assert_('Etag: "%s"' % expected_etag in headers) - body = fd.read() - self.assertEquals(body, '1234 1234 1234 1234 1234 ') - # Do it again but exceeding the container listing limit - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2 - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - - fd = sock.makefile() - fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in - headers) - self.assert_('Content-Type: text/jibberish' in headers) - body = fd.read() - # A bit fragile of a test; as it makes the assumption that all - # will be sent in a single chunk. - self.assertEquals( - body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n') - # Make a copy of the manifested object, which should - # error since the number of segments exceeds - # CONTAINER_LISTING_LIMIT. - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - 'X-Copy-From: segmented%20object/object%20name\r\n' - 'Content-Length: 0\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 413' - self.assertEquals(headers[:len(exp)], exp) - body = fd.read() - # After adjusting the CONTAINER_LISTING_LIMIT, make a copy of - # the manifested object which should consolidate the segments. - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000 - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - 'X-Copy-From: segmented%20object/object%20name\r\n' - 'Content-Length: 0\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - body = fd.read() - # Retrieve and validate the copy. - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('x-object-manifest:' not in headers.lower()) - self.assert_('Content-Length: 25\r' in headers) - body = fd.read() - self.assertEquals(body, '1234 1234 1234 1234 1234 ') - # Create an object manifest file pointing to nothing - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'X-Object-Manifest: segmented%20object/empty/\r\n' - 'Content-Type: text/jibberish\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Ensure retrieving the manifest file gives a zero-byte file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers) - self.assert_('Content-Type: text/jibberish' in headers) - body = fd.read() - self.assertEquals(body, '') - # Check copy content type - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'Content-Type: text/jibberish\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'X-Copy-From: c/obj\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Ensure getting the copied file gets original content-type - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('Content-Type: text/jibberish' in headers) - # Check set content type - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'Content-Type: foo/bar\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Ensure getting the copied file gets original content-type - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('Content-Type: foo/bar' in - headers.split('\r\n'), repr(headers.split('\r\n'))) - # Check set content type with charset - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: 0\r\n' - 'Content-Type: foo/bar; charset=UTF-8\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) - # Ensure getting the copied file gets original content-type - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - self.assert_('Content-Type: foo/bar; charset=UTF-8' in - headers.split('\r\n'), repr(headers.split('\r\n'))) - def test_mismatched_etags(self): with save_globals(): # no etag supplied, object servers return success w/ diff values @@ -4085,53 +3675,63 @@ class TestObjectController(unittest.TestCase): self.assertTrue('X-Delete-At in past' in resp.body) def test_leak_1(self): - prolis = _test_sockets[0] - prosrv = _test_servers[0] - obj_len = prosrv.client_chunk_size * 2 - # PUT test file - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - 'Content-Length: %s\r\n' - 'Content-Type: application/octet-stream\r\n' - '\r\n%s' % (obj_len, 'a' * obj_len)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - # Remember Request instance count, make sure the GC is run for pythons - # without reference counting. - for i in xrange(4): - sleep(0) # let eventlet do its thing - gc.collect() - else: - sleep(0) - before_request_instances = len(_request_instances) - # GET test file, but disconnect early - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Auth-Token: t\r\n' - '\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - fd.read(1) - fd.close() - sock.close() - # Make sure the GC is run again for pythons without reference counting - for i in xrange(4): - sleep(0) # let eventlet do its thing - gc.collect() - else: - sleep(0) - self.assertEquals(before_request_instances, len(_request_instances)) + _request_instances = weakref.WeakKeyDictionary() + _orig_init = Request.__init__ + + def request_init(self, *args, **kwargs): + _orig_init(self, *args, **kwargs) + _request_instances[self] = None + + with mock.patch.object(Request, "__init__", request_init): + prolis = _test_sockets[0] + prosrv = _test_servers[0] + obj_len = prosrv.client_chunk_size * 2 + # PUT test file + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Auth-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (obj_len, 'a' * obj_len)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + # Remember Request instance count, make sure the GC is run for + # pythons without reference counting. + for i in xrange(4): + sleep(0) # let eventlet do its thing + gc.collect() + else: + sleep(0) + before_request_instances = len(_request_instances) + # GET test file, but disconnect early + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Auth-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + fd.read(1) + fd.close() + sock.close() + # Make sure the GC is run again for pythons without reference + # counting + for i in xrange(4): + sleep(0) # let eventlet do its thing + gc.collect() + else: + sleep(0) + self.assertEquals( + before_request_instances, len(_request_instances)) def test_OPTIONS(self): with save_globals(): @@ -4926,9 +4526,10 @@ class TestContainerController(unittest.TestCase): controller = \ proxy_server.ContainerController(self.app, 'a', 'c') set_http_connect(200, 201, 201, 201, give_connect=test_connect) - req = Request.blank('/v1/a/c', - environ={'REQUEST_METHOD': method}, - headers={test_header: test_value}) + req = Request.blank( + '/v1/a/c', + environ={'REQUEST_METHOD': method, 'swift_owner': True}, + headers={test_header: test_value}) self.app.update_request(req) getattr(controller, method)(req) self.assertEquals(test_errors, []) @@ -6006,313 +5607,6 @@ class Stub(object): pass -class TestSegmentedIterable(unittest.TestCase): - - def setUp(self): - self.controller = FakeObjectController() - - def test_load_next_segment_unexpected_error(self): - # Iterator value isn't a dict - self.assertRaises(Exception, - SegmentedIterable(self.controller, None, - [None])._load_next_segment) - self.assert_(self.controller.exception_args[0].startswith( - 'ERROR: While processing manifest')) - - def test_load_next_segment_with_no_segments(self): - self.assertRaises(StopIteration, - SegmentedIterable(self.controller, 'lc', - [])._load_next_segment) - - def test_load_next_segment_with_one_segment(self): - segit = SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}]) - segit._load_next_segment() - self.assertEquals( - self.controller.GETorHEAD_base_args[0][4], '/a/lc/o1') - data = ''.join(segit.segment_iter) - self.assertEquals(data, '1') - - def test_load_next_segment_with_two_segments(self): - segit = SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}, {'name': 'o2'}]) - segit._load_next_segment() - self.assertEquals( - self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o1') - data = ''.join(segit.segment_iter) - self.assertEquals(data, '1') - segit._load_next_segment() - self.assertEquals( - self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') - data = ''.join(segit.segment_iter) - self.assertEquals(data, '22') - - def test_load_next_segment_rate_limiting(self): - sleep_calls = [] - - def _stub_sleep(sleepy_time): - sleep_calls.append(sleepy_time) - orig_sleep = swift.proxy.controllers.obj.sleep - try: - swift.proxy.controllers.obj.sleep = _stub_sleep - segit = SegmentedIterable( - self.controller, 'lc', [ - {'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'}, - {'name': 'o4'}, {'name': 'o5'}]) - - # rate_limit_after_segment == 3, so the first 3 segments should - # invoke no sleeping. - for _ in xrange(3): - segit._load_next_segment() - self.assertEquals([], sleep_calls) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o3') - - # Loading of next (4th) segment starts rate-limiting. - segit._load_next_segment() - self.assertAlmostEqual(0.5, sleep_calls[0], places=2) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o4') - - sleep_calls = [] - segit._load_next_segment() - self.assertAlmostEqual(0.5, sleep_calls[0], places=2) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o5') - finally: - swift.proxy.controllers.obj.sleep = orig_sleep - - def test_load_next_segment_range_req_rate_limiting(self): - sleep_calls = [] - - def _stub_sleep(sleepy_time): - sleep_calls.append(sleepy_time) - orig_sleep = swift.proxy.controllers.obj.sleep - try: - swift.proxy.controllers.obj.sleep = _stub_sleep - segit = SegmentedIterable( - self.controller, 'lc', [ - {'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5}, - {'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'}, - {'name': 'o5'}, {'name': 'o6'}]) - - # this tests for a range request which skips over the whole first - # segment, after that 3 segments will be read in because the - # rate_limit_after_segment == 3, then sleeping starts - segit_iter = segit.app_iter_range(10, None) - segit_iter.next() - for _ in xrange(2): - # this is set to 2 instead of 3 because o2 was loaded after - # o0 and o1 were skipped. - segit._load_next_segment() - self.assertEquals([], sleep_calls) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o4') - - # Loading of next (5th) segment starts rate-limiting. - segit._load_next_segment() - self.assertAlmostEqual(0.5, sleep_calls[0], places=2) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o5') - - sleep_calls = [] - segit._load_next_segment() - self.assertAlmostEqual(0.5, sleep_calls[0], places=2) - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], - '/a/lc/o6') - finally: - swift.proxy.controllers.obj.sleep = orig_sleep - - def test_load_next_segment_with_two_segments_skip_first(self): - segit = SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}, {'name': 'o2'}]) - segit.ratelimit_index = 0 - segit.listing.next() - segit._load_next_segment() - self.assertEquals( - self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') - data = ''.join(segit.segment_iter) - self.assertEquals(data, '22') - - def test_load_next_segment_with_seek(self): - segit = SegmentedIterable(self.controller, 'lc', - [{'name': 'o1', 'bytes': 1}, - {'name': 'o2', 'bytes': 2}]) - segit.ratelimit_index = 0 - segit.listing.next() - segit.seek = 1 - segit._load_next_segment() - self.assertEquals( - self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') - self.assertEquals( - str(self.controller.GETorHEAD_base_args[-1][0].range), - 'bytes=1-') - data = ''.join(segit.segment_iter) - self.assertEquals(data, '2') - - def test_fetching_only_what_you_need(self): - segit = SegmentedIterable(self.controller, 'lc', - [{'name': 'o7', 'bytes': 7}, - {'name': 'o8', 'bytes': 8}, - {'name': 'o9', 'bytes': 9}]) - - body = ''.join(segit.app_iter_range(10, 20)) - self.assertEqual('8888899999', body) - - GoH_args = self.controller.GETorHEAD_base_args - self.assertEquals(2, len(GoH_args)) - - # Either one is fine, as they both indicate "from byte 3 to (the last) - # byte 8". - self.assert_(str(GoH_args[0][0].range) in ['bytes=3-', 'bytes=3-8']) - - # This one must ask only for the bytes it needs; otherwise we waste - # bandwidth pulling bytes from the object server and then throwing - # them out - self.assertEquals(str(GoH_args[1][0].range), 'bytes=0-4') - - def test_load_next_segment_with_get_error(self): - - def local_GETorHEAD_base(*args): - return HTTPNotFound() - - self.controller.GETorHEAD_base = local_GETorHEAD_base - self.assertRaises(Exception, - SegmentedIterable(self.controller, 'lc', - [{'name': 'o1'}])._load_next_segment) - self.assert_(self.controller.exception_args[0].startswith( - 'ERROR: While processing manifest')) - self.assertEquals(str(self.controller.exception_info[1]), - 'Could not load object segment /a/lc/o1: 404') - - def test_iter_unexpected_error(self): - # Iterator value isn't a dict - self.assertRaises(Exception, ''.join, - SegmentedIterable(self.controller, None, [None])) - self.assert_(self.controller.exception_args[0].startswith( - 'ERROR: While processing manifest')) - - def test_iter_with_no_segments(self): - segit = SegmentedIterable(self.controller, 'lc', []) - self.assertEquals(''.join(segit), '') - - def test_iter_with_one_segment(self): - segit = SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}]) - segit.response = Stub() - self.assertEquals(''.join(segit), '1') - - def test_iter_with_two_segments(self): - segit = SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}, {'name': 'o2'}]) - segit.response = Stub() - self.assertEquals(''.join(segit), '122') - - def test_iter_with_get_error(self): - - def local_GETorHEAD_base(*args): - return HTTPNotFound() - - self.controller.GETorHEAD_base = local_GETorHEAD_base - self.assertRaises(Exception, ''.join, - SegmentedIterable(self.controller, 'lc', [{'name': - 'o1'}])) - self.assert_(self.controller.exception_args[0].startswith( - 'ERROR: While processing manifest')) - self.assertEquals(str(self.controller.exception_info[1]), - 'Could not load object segment /a/lc/o1: 404') - - def test_app_iter_range_unexpected_error(self): - # Iterator value isn't a dict - self.assertRaises(Exception, - SegmentedIterable(self.controller, None, - [None]).app_iter_range(None, - None).next) - self.assert_(self.controller.exception_args[0].startswith( - 'ERROR: While processing manifest')) - - def test_app_iter_range_with_no_segments(self): - self.assertEquals(''.join(SegmentedIterable( - self.controller, 'lc', []).app_iter_range(None, None)), '') - self.assertEquals(''.join(SegmentedIterable( - self.controller, 'lc', []).app_iter_range(3, None)), '') - self.assertEquals(''.join(SegmentedIterable( - self.controller, 'lc', []).app_iter_range(3, 5)), '') - self.assertEquals(''.join(SegmentedIterable( - self.controller, 'lc', []).app_iter_range(None, 5)), '') - - def test_app_iter_range_with_one_segment(self): - listing = [{'name': 'o1', 'bytes': 1}] - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, None)), '1') - - segit = SegmentedIterable(self.controller, 'lc', listing) - self.assertEquals(''.join(segit.app_iter_range(3, None)), '') - - segit = SegmentedIterable(self.controller, 'lc', listing) - self.assertEquals(''.join(segit.app_iter_range(3, 5)), '') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1') - - def test_app_iter_range_with_two_segments(self): - listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}] - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, None)), '122') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(1, None)), '22') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12') - - def test_app_iter_range_with_many_segments(self): - listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}, - {'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4}, - {'name': 'o5', 'bytes': 5}] - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, None)), - '122333444455555') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(3, None)), - '333444455555') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334') - - segit = SegmentedIterable(self.controller, 'lc', listing) - segit.response = Stub() - self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34') - - class TestProxyObjectPerformance(unittest.TestCase): def setUp(self): diff --git a/tools/requirements.txt b/tools/requirements.txt index 1139866..bbac51a 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -5,4 +5,3 @@ netifaces>=0.5 pastedeploy>=1.3.3 simplejson>=2.0.9 xattr>=0.4 -python-swiftclient diff --git a/tools/test-requires b/tools/test-requires index d2c027f..63d499e 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,7 +1,4 @@ -# Install bounded pep8/pyflakes first, then let flake8 install -pep8==1.4.5 -pyflakes==0.7.2 -flake8==2.0 +# Hacking already pins down pep8, pyflakes and flake8 hacking>=0.5.6,<0.6 coverage nose @@ -10,5 +7,6 @@ openstack.nose_plugin nosehtmloutput sphinx>=1.1.2,<1.2 mock>=0.8.0 +python-swiftclient python-keystoneclient prettytable diff --git a/tox.ini b/tox.ini index 81621a0..ebcb5a1 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir} NOSE_OPENSTACK_SHOW_ELAPSED=1 NOSE_OPENSTACK_STDOUT=1 deps = - https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz + https://launchpad.net/gluster-swift/icehouse/1.13.0/+download/swift-1.13.0.tar.gz --download-cache={homedir}/.pipcache -r{toxinidir}/tools/test-requires -r{toxinidir}/tools/requirements.txt -- cgit