diff options
| author | Prashanth Pai <ppai@redhat.com> | 2014-03-12 16:54:30 +0530 | 
|---|---|---|
| committer | Luis Pabon <lpabon@redhat.com> | 2014-03-13 05:09:19 -0700 | 
| commit | 2ccd2c4d969cdd6e7feedd21ac2e5cb8498ff37d (patch) | |
| tree | 1886732cf56e26ab836f9fd67c853eff2005fbeb | |
| parent | 1a6b55714fddf7a1b526a31ee3f27589f01e98e5 (diff) | |
Sync with OpenStack Swift v1.13.0
Also, bumped version of gluster-swift to v1.13.0
Change-Id: I797dc704c9523540cba847b1e8ff3da97b79630c
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Reviewed-on: http://review.gluster.org/7229
Reviewed-by: Chetan Risbud <crisbud@redhat.com>
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>
| -rw-r--r-- | doc/markdown/openstack_swift_sync.md | 9 | ||||
| -rw-r--r-- | gluster/swift/__init__.py | 2 | ||||
| -rw-r--r-- | test/functional/swift_test_client.py | 23 | ||||
| -rw-r--r-- | test/functional/swift_testing.py | 33 | ||||
| -rwxr-xr-x | test/functional/test_account.py | 148 | ||||
| -rw-r--r-- | test/functional/tests.py | 12 | ||||
| -rw-r--r-- | test/unit/proxy/controllers/test_account.py | 89 | ||||
| -rwxr-xr-x | test/unit/proxy/controllers/test_obj.py | 13 | ||||
| -rw-r--r-- | test/unit/proxy/test_server.py | 840 | ||||
| -rw-r--r-- | tools/requirements.txt | 1 | ||||
| -rw-r--r-- | tools/test-requires | 6 | ||||
| -rw-r--r-- | tox.ini | 2 | 
12 files changed, 368 insertions, 810 deletions
diff --git a/doc/markdown/openstack_swift_sync.md b/doc/markdown/openstack_swift_sync.md index f3feac8..ba2fadb 100644 --- a/doc/markdown/openstack_swift_sync.md +++ b/doc/markdown/openstack_swift_sync.md @@ -14,7 +14,8 @@ $ python setup.py sdist  $ ls dist  ``` -* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net +* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net. +* Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad.  ## Setup Tox  Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link. @@ -22,7 +23,11 @@ Now that the swift source is availabe on launchpad.net, copy its link location a  ## Update tests  This part is a little more complicated and now we need to *merge* the latest tests with ours. -I suggest using a tool called `meld` to make this work easier. +[meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from: + +* Latest swift (say v1.13) +* Previous swift (say v1.12) +* gluster-swift (v1.12)  Files that need to be merged: diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py index 4fe17ba..17f2fcf 100644 --- a/gluster/swift/__init__.py +++ b/gluster/swift/__init__.py @@ -45,6 +45,6 @@ class PkgInfo(object):  ###  ### Change the Package version here  ### -_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False) +_pkginfo = PkgInfo('1.13.0', '0', 'gluster_swift', False)  __version__ = _pkginfo.pretty_version  __canonical_version__ = _pkginfo.canonical_version diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 47e023e..ecb2a6d 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -40,7 +40,7 @@ class RequestError(Exception):  class ResponseError(Exception): -    def __init__(self, response, method, path): +    def __init__(self, response, method=None, path=None):          self.status = response.status          self.reason = response.reason          self.method = method @@ -310,10 +310,11 @@ class Base:      def __str__(self):          return self.name -    def header_fields(self, fields): +    def header_fields(self, required_fields, optional_fields=()):          headers = dict(self.conn.response.getheaders())          ret = {} -        for field in fields: + +        for field in required_fields:              if field[1] not in headers:                  raise ValueError("%s was not found in response header" %                                   (field[1])) @@ -322,6 +323,15 @@ class Base:                  ret[field[0]] = int(headers[field[1]])              except ValueError:                  ret[field[0]] = headers[field[1]] + +        for field in optional_fields: +            if field[1] not in headers: +                continue +            try: +                ret[field[0]] = int(headers[field[1]]) +            except ValueError: +                ret[field[0]] = headers[field[1]] +          return ret @@ -480,10 +490,11 @@ class Container(Base):                                 parms=parms, cfg=cfg)          if self.conn.response.status == 204: -            fields = [['bytes_used', 'x-container-bytes-used'], -                      ['object_count', 'x-container-object-count']] +            required_fields = [['bytes_used', 'x-container-bytes-used'], +                               ['object_count', 'x-container-object-count']] +            optional_fields = [['versions', 'x-versions-location']] -            return self.header_fields(fields) +            return self.header_fields(required_fields, optional_fields)          raise ResponseError(self.conn.response, 'HEAD',                              self.conn.make_path(self.path)) diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py index b5642b3..f05cb48 100644 --- a/test/functional/swift_testing.py +++ b/test/functional/swift_testing.py @@ -18,6 +18,7 @@ import os  import socket  import sys  from time import sleep +from urlparse import urlparse  from test import get_config @@ -119,18 +120,23 @@ conn = [None, None, None]  def retry(func, *args, **kwargs):      """ -    You can use the kwargs to override the 'retries' (default: 5) and -    'use_account' (default: 1). +    You can use the kwargs to override: +      'retries' (default: 5) +      'use_account' (default: 1) - which user's token to pass +      'url_account' (default: matches 'use_account') - which user's storage URL +      'resource' (default: url[url_account] - URL to connect to; retry() +          will interpolate the variable :storage_url: if present      """      global url, token, parsed, conn      retries = kwargs.get('retries', 5) -    use_account = 1 -    if 'use_account' in kwargs: -        use_account = kwargs['use_account'] -        del kwargs['use_account'] -    use_account -= 1 -    attempts = 0 -    backoff = 1 +    attempts, backoff = 0, 1 + +    # use account #1 by default; turn user's 1-indexed account into 0-indexed +    use_account = kwargs.pop('use_account', 1) - 1 + +    # access our own account by default +    url_account = kwargs.pop('url_account', use_account + 1) - 1 +      while attempts <= retries:          attempts += 1          try: @@ -146,8 +152,13 @@ def retry(func, *args, **kwargs):              if not parsed[use_account] or not conn[use_account]:                  parsed[use_account], conn[use_account] = \                      http_connection(url[use_account]) -            return func(url[use_account], token[use_account], -                        parsed[use_account], conn[use_account], + +            # default resource is the account url[url_account] +            resource = kwargs.pop('resource', '%(storage_url)s') +            template_vars = {'storage_url': url[url_account]} +            parsed_result = urlparse(resource % template_vars) +            return func(url[url_account], token[use_account], +                        parsed_result, conn[url_account],                          *args, **kwargs)          except (socket.error, HTTPException):              if attempts > retries: diff --git a/test/functional/test_account.py b/test/functional/test_account.py index b2f743f..d456090 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -16,12 +16,16 @@  # limitations under the License.  import unittest +import json  from nose import SkipTest  from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \      MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH - +from swift.common.middleware.acl import format_acl +from test.functional.swift_test_client import Connection +from test import get_config  from swift_testing import check_response, retry, skip, web_front_end +import swift_testing  class TestAccount(unittest.TestCase): @@ -66,6 +70,148 @@ class TestAccount(unittest.TestCase):          self.assert_(resp.status in (200, 204), resp.status)          self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') +    def test_tempauth_account_acls(self): +        if skip: +            raise SkipTest + +        # Determine whether this cluster has account ACLs; if not, skip test +        conn = Connection(get_config('func_test')) +        conn.authenticate() +        status = conn.make_request( +            'GET', '/info', cfg={'verbatim_path': True}) +        if status // 100 != 2: +            # Can't tell if account ACLs are enabled; skip tests proactively. +            raise SkipTest +        else: +            cluster_info = json.loads(conn.response.read()) +            if not cluster_info.get('tempauth', {}).get('account_acls'): +                raise SkipTest +            if 'keystoneauth' in cluster_info: +                # Unfortunate hack -- tempauth (with account ACLs) is expected +                # to play nice with Keystone (without account ACLs), but Zuul +                # functest framework doesn't give us an easy way to get a +                # tempauth user. +                raise SkipTest + +        def post(url, token, parsed, conn, headers): +            new_headers = dict({'X-Auth-Token': token}, **headers) +            conn.request('POST', parsed.path, '', new_headers) +            return check_response(conn) + +        def put(url, token, parsed, conn, headers): +            new_headers = dict({'X-Auth-Token': token}, **headers) +            conn.request('PUT', parsed.path, '', new_headers) +            return check_response(conn) + +        def delete(url, token, parsed, conn, headers): +            new_headers = dict({'X-Auth-Token': token}, **headers) +            conn.request('DELETE', parsed.path, '', new_headers) +            return check_response(conn) + +        def head(url, token, parsed, conn): +            conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) +            return check_response(conn) + +        def get(url, token, parsed, conn): +            conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) +            return check_response(conn) + +        try: +            # User1 can POST to their own account (and reset the ACLs) +            resp = retry(post, headers={'X-Account-Access-Control': '{}'}, +                         use_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) +            self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + +            # User1 can GET their own empty account +            resp = retry(get, use_account=1) +            resp.read() +            self.assertEqual(resp.status // 100, 2) +            self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + +            # User2 can't GET User1's account +            resp = retry(get, use_account=2, url_account=1) +            resp.read() +            self.assertEqual(resp.status, 403) + +            # User1 is swift_owner of their own account, so they can POST an +            # ACL -- let's do this and make User2 (test_user[1]) an admin +            acl_user = swift_testing.swift_test_user[1] +            acl = {'admin': [acl_user]} +            headers = {'x-account-access-control': format_acl( +                version=2, acl_dict=acl)} +            resp = retry(post, headers=headers, use_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) + +            # User1 can see the new header +            resp = retry(get, use_account=1) +            resp.read() +            self.assertEqual(resp.status // 100, 2) +            data_from_headers = resp.getheader('x-account-access-control') +            expected = json.dumps(acl, separators=(',', ':')) +            self.assertEqual(data_from_headers, expected) + +            # Now User2 should be able to GET the account and see the ACL +            resp = retry(head, use_account=2, url_account=1) +            resp.read() +            data_from_headers = resp.getheader('x-account-access-control') +            self.assertEqual(data_from_headers, expected) + +            # Revoke User2's admin access, grant User2 read-write access +            acl = {'read-write': [acl_user]} +            headers = {'x-account-access-control': format_acl( +                version=2, acl_dict=acl)} +            resp = retry(post, headers=headers, use_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) + +            # User2 can still GET the account, but not see the ACL +            # (since it's privileged data) +            resp = retry(head, use_account=2, url_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) +            self.assertEqual(resp.getheader('x-account-access-control'), None) + +            # User2 can PUT and DELETE a container +            resp = retry(put, use_account=2, url_account=1, +                         resource='%(storage_url)s/mycontainer', headers={}) +            resp.read() +            self.assertEqual(resp.status, 201) +            resp = retry(delete, use_account=2, url_account=1, +                         resource='%(storage_url)s/mycontainer', headers={}) +            resp.read() +            self.assertEqual(resp.status, 204) + +            # Revoke User2's read-write access, grant User2 read-only access +            acl = {'read-only': [acl_user]} +            headers = {'x-account-access-control': format_acl( +                version=2, acl_dict=acl)} +            resp = retry(post, headers=headers, use_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) + +            # User2 can still GET the account, but not see the ACL +            # (since it's privileged data) +            resp = retry(head, use_account=2, url_account=1) +            resp.read() +            self.assertEqual(resp.status, 204) +            self.assertEqual(resp.getheader('x-account-access-control'), None) + +            # User2 can't PUT a container +            resp = retry(put, use_account=2, url_account=1, +                         resource='%(storage_url)s/mycontainer', headers={}) +            resp.read() +            self.assertEqual(resp.status, 403) + +        finally: +            # Make sure to clean up even if tests fail -- User2 should not +            # have access to User1's account in other functional tests! +            resp = retry(post, headers={'X-Account-Access-Control': '{}'}, +                         use_account=1) +            resp.read() +      def test_unicode_metadata(self):          if skip:              raise SkipTest diff --git a/test/functional/tests.py b/test/functional/tests.py index 9469cff..0d9a9ef 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -149,7 +149,7 @@ def timeout(seconds, method, *args, **kwargs):      return False -class Utils: +class Utils(object):      @classmethod      def create_ascii_name(cls, length=None):          return uuid.uuid4().hex @@ -201,7 +201,7 @@ class Base2(object):          Utils.create_name = Utils.create_ascii_name -class TestAccountEnv: +class TestAccountEnv(object):      @classmethod      def setUp(cls):          cls.conn = Connection(config) @@ -388,7 +388,7 @@ class TestAccountUTF8(Base2, TestAccount):      set_up = False -class TestAccountNoContainersEnv: +class TestAccountNoContainersEnv(object):      @classmethod      def setUp(cls):          cls.conn = Connection(config) @@ -417,7 +417,7 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):      set_up = False -class TestContainerEnv: +class TestContainerEnv(object):      @classmethod      def setUp(cls):          cls.conn = Connection(config) @@ -708,7 +708,7 @@ class TestContainerUTF8(Base2, TestContainer):      set_up = False -class TestContainerPathsEnv: +class TestContainerPathsEnv(object):      @classmethod      def setUp(cls):          raise SkipTest('Objects ending in / are not supported') @@ -888,7 +888,7 @@ class TestContainerPaths(Base):                         ['dir1/subdir with spaces/file B']) -class TestFileEnv: +class TestFileEnv(object):      @classmethod      def setUp(cls):          cls.conn = Connection(config) diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index eefd57d..47f76dc 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -16,12 +16,14 @@  import mock  import unittest -from swift.common.swob import Request +from swift.common.swob import Request, Response +from swift.common.middleware.acl import format_acl  from swift.proxy import server as proxy_server  from swift.proxy.controllers.base import headers_to_account_info  from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN  from test.unit import fake_http_connect, FakeRing, FakeMemcache  from swift.common.request_helpers import get_sys_meta_prefix +import swift.proxy.controllers.base  class TestAccountController(unittest.TestCase): @@ -152,6 +154,91 @@ class TestAccountController(unittest.TestCase):          self.assertEqual(context['headers'][user_meta_key], 'bar')          self.assertNotEqual(context['headers']['x-timestamp'], '1.0') +    def _make_user_and_sys_acl_headers_data(self): +        acl = { +            'admin': ['AUTH_alice', 'AUTH_bob'], +            'read-write': ['AUTH_carol'], +            'read-only': [], +        } +        user_prefix = 'x-account-'  # external, user-facing +        user_headers = {(user_prefix + 'access-control'): format_acl( +            version=2, acl_dict=acl)} +        sys_prefix = get_sys_meta_prefix('account')   # internal, system-facing +        sys_headers = {(sys_prefix + 'core-access-control'): format_acl( +            version=2, acl_dict=acl)} +        return user_headers, sys_headers + +    def test_account_acl_headers_translated_for_GET_HEAD(self): +        # Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers +        # from the account server will remap those headers to X-Account-Acl-* + +        hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() +        controller = proxy_server.AccountController(self.app, 'acct') + +        for verb in ('GET', 'HEAD'): +            req = Request.blank('/v1/acct', environ={'swift_owner': True}) +            controller.GETorHEAD_base = lambda *_: Response( +                headers=hdrs_int, environ={ +                    'PATH_INFO': '/acct', +                    'REQUEST_METHOD': verb, +                }) +            method = getattr(controller, verb) +            resp = method(req) +            for header, value in hdrs_ext.items(): +                if value: +                    self.assertEqual(resp.headers.get(header), value) +                else: +                    # blank ACLs should result in no header +                    self.assert_(header not in resp.headers) + +    def test_add_acls_impossible_cases(self): +        # For test coverage: verify that defensive coding does defend, in cases +        # that shouldn't arise naturally + +        # add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST +        resp = Response() +        controller = proxy_server.AccountController(self.app, 'a') +        resp.environ['PATH_INFO'] = '/a' +        resp.environ['REQUEST_METHOD'] = 'OPTIONS' +        controller.add_acls_from_sys_metadata(resp) +        self.assertEqual(1, len(resp.headers))  # we always get Content-Type +        self.assertEqual(2, len(resp.environ)) + +    def test_memcache_key_impossible_cases(self): +        # For test coverage: verify that defensive coding does defend, in cases +        # that shouldn't arise naturally +        self.assertRaises( +            ValueError, +            lambda: swift.proxy.controllers.base.get_container_memcache_key( +                '/a', None)) + +    def test_stripping_swift_admin_headers(self): +        # Verify that a GET/HEAD which receives privileged headers from the +        # account server will strip those headers for non-swift_owners + +        hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() +        headers = { +            'x-account-meta-harmless': 'hi mom', +            'x-account-meta-temp-url-key': 's3kr1t', +        } +        controller = proxy_server.AccountController(self.app, 'acct') + +        for verb in ('GET', 'HEAD'): +            for env in ({'swift_owner': True}, {'swift_owner': False}): +                req = Request.blank('/v1/acct', environ=env) +                controller.GETorHEAD_base = lambda *_: Response( +                    headers=headers, environ={ +                        'PATH_INFO': '/acct', +                        'REQUEST_METHOD': verb, +                    }) +                method = getattr(controller, verb) +                resp = method(req) +                self.assertEqual(resp.headers.get('x-account-meta-harmless'), +                                 'hi mom') +                privileged_header_present = ( +                    'x-account-meta-temp-url-key' in resp.headers) +                self.assertEqual(privileged_header_present, env['swift_owner']) +  if __name__ == '__main__':      unittest.main() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index cae62b0..aada616 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -21,6 +21,7 @@ import mock  import swift  from swift.proxy import server as proxy_server +from swift.common.swob import HTTPException  from test.unit import FakeRing, FakeMemcache, fake_http_connect @@ -44,7 +45,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):          self.app = proxy_server.Application(              None, FakeMemcache(), account_ring=FakeRing(),              container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9)) -        self.app.request_node_count = lambda ring: 10000000 +        self.app.request_node_count = lambda replicas: 10000000          self.app.sort_nodes = lambda l: l  # stop shuffling the primary nodes      def test_iter_nodes_local_first_noops_when_no_affinity(self): @@ -107,14 +108,20 @@ class TestObjController(unittest.TestCase):              # and now test that we add the header to log_info              req = swift.common.swob.Request.blank('/v1/a/c/o')              req.headers['x-copy-from'] = 'somewhere' -            controller.PUT(req) +            try: +                controller.PUT(req) +            except HTTPException: +                pass              self.assertEquals(                  req.environ.get('swift.log_info'), ['x-copy-from:somewhere'])              # and then check that we don't do that for originating POSTs              req = swift.common.swob.Request.blank('/v1/a/c/o')              req.method = 'POST'              req.headers['x-copy-from'] = 'elsewhere' -            controller.PUT(req) +            try: +                controller.PUT(req) +            except HTTPException: +                pass              self.assertEquals(req.environ.get('swift.log_info'), None) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index b68be9f..4086a32 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -46,7 +46,8 @@ from gluster.swift.container import server as container_server  from gluster.swift.obj import server as object_server  from swift.common import ring  from swift.common.middleware import proxy_logging -from swift.common.exceptions import ChunkReadTimeout, SegmentError +from swift.common.middleware.acl import parse_acl, format_acl +from swift.common.exceptions import ChunkReadTimeout  from swift.common.constraints import MAX_META_NAME_LENGTH, \      MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \      MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ @@ -54,38 +55,28 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \  from swift.common import utils  from swift.common.utils import mkdirs, normalize_timestamp, NullLogger  from swift.common.wsgi import monkey_patch_mimetools -from swift.proxy.controllers.obj import SegmentedIterable  from swift.proxy.controllers import base as proxy_base  from swift.proxy.controllers.base import get_container_memcache_key, \      get_account_memcache_key, cors_validation  import swift.proxy.controllers -from swift.common.swob import Request, Response, HTTPNotFound, \ -    HTTPUnauthorized +from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.swob import Request, Response, HTTPUnauthorized  # mocks  logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))  STATIC_TIME = time.time() -_request_instances = weakref.WeakKeyDictionary()  _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \      _testdir = _orig_SysLogHandler = None -def request_init(self, *args, **kwargs): -    self._orig_init(*args, **kwargs) - -    _request_instances[self] = None - -  def do_setup(the_object_server):      utils.HASH_PATH_SUFFIX = 'endcap'      global _testdir, _test_servers, _test_sockets, \          _orig_container_listing_limit, _test_coros, _orig_SysLogHandler      _orig_SysLogHandler = utils.SysLogHandler      utils.SysLogHandler = mock.MagicMock() -    Request._orig_init = Request.__init__ -    Request.__init__ = request_init      monkey_patch_mimetools()      # Since we're starting up a lot here, we're going to test more than      # just chunked puts; we're also going to test parts of @@ -99,8 +90,6 @@ def do_setup(the_object_server):      mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))      mkdirs(os.path.join(_testdir, 'a'))      mkdirs(os.path.join(_testdir, 'a', 'tmp')) -    _orig_container_listing_limit = \ -        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT      conf = {'devices': _testdir, 'swift_dir': _testdir,              'mount_check': 'false', 'allowed_headers':              'content-encoding, x-object-manifest, content-disposition, foo', @@ -211,10 +200,7 @@ def setup():  def teardown():      for server in _test_coros:          server.kill() -    swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \ -        _orig_container_listing_limit      rmtree(os.path.dirname(_testdir)) -    Request.__init__ = Request._orig_init      utils.SysLogHandler = _orig_SysLogHandler @@ -1716,7 +1702,7 @@ class TestObjectController(unittest.TestCase):              except ChunkReadTimeout:                  got_exc = True              self.assert_(not got_exc) -            self.app.node_timeout = 0.1 +            self.app.recoverable_node_timeout = 0.1              set_http_connect(200, 200, 200, slow=True)              resp = req.get_response(self.app)              got_exc = False @@ -1731,7 +1717,7 @@ class TestObjectController(unittest.TestCase):              req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})              self.app.update_request(req) -            self.app.node_timeout = 0.1 +            self.app.recoverable_node_timeout = 0.1              set_http_connect(200, 200, 200, slow=[3])              resp = req.get_response(self.app)              got_exc = False @@ -3316,402 +3302,6 @@ class TestObjectController(unittest.TestCase):          headers = readuntil2crlfs(fd)          self.assertEquals(headers[:len(exp)], exp) -    def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self): -        raise SkipTest("Not until we support pure object requests") -        # Create a container for our segmented/manifest object testing -        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ -            _test_sockets -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n' -                 'Connection: close\r\nX-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Create the object segments -        segment_etags = [] -        for segment in xrange(5): -            sock = connect_tcp(('localhost', prolis.getsockname()[1])) -            fd = sock.makefile() -            fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: ' -                     'localhost\r\nConnection: close\r\nX-Storage-Token: ' -                     't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment)) -            fd.flush() -            headers = readuntil2crlfs(fd) -            exp = 'HTTP/1.1 201' -            self.assertEquals(headers[:len(exp)], exp) -            segment_etags.append(md5('1234 ').hexdigest()) - -        # Create the nonzero size manifest file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' -                 'localhost\r\nConnection: close\r\nX-Storage-Token: ' -                 't\r\nContent-Length: 5\r\n\r\nabcd ') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) - -        # Create the object manifest file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' -                 'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n' -                 'X-Object-Manifest: segmented_nonzero/name/\r\n' -                 'Foo: barbaz\r\nContent-Type: text/jibberish\r\n' -                 '\r\n\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 202' -        self.assertEquals(headers[:len(exp)], exp) - -        # Ensure retrieving the manifest file gets the whole object -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' -                 'localhost\r\nConnection: close\r\nX-Auth-Token: ' -                 't\r\n\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        self.assert_('Foo: barbaz' in headers) -        expected_etag = md5(''.join(segment_etags)).hexdigest() -        self.assert_('Etag: "%s"' % expected_etag in headers) -        body = fd.read() -        self.assertEquals(body, '1234 1234 1234 1234 1234 ') - -        # Get lobjects with Range smaller than manifest file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' -                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' -                 'Range: bytes=0-4\r\n\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 206' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        self.assert_('Foo: barbaz' in headers) -        expected_etag = md5(''.join(segment_etags)).hexdigest() -        body = fd.read() -        self.assertEquals(body, '1234 ') - -        # Get lobjects with Range bigger than manifest file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: ' -                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' -                 'Range: bytes=11-15\r\n\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 206' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        self.assert_('Foo: barbaz' in headers) -        expected_etag = md5(''.join(segment_etags)).hexdigest() -        body = fd.read() -        self.assertEquals(body, '234 1') - -    def test_chunked_put_lobjects(self): -        raise SkipTest("Not until we support pure object requests") -        # Create a container for our segmented/manifest object testing -        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, -         obj2lis) = _test_sockets -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Create the object segments -        segment_etags = [] -        for segment in xrange(5): -            sock = connect_tcp(('localhost', prolis.getsockname()[1])) -            fd = sock.makefile() -            fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s ' -                     'HTTP/1.1\r\n' -                     'Host: localhost\r\n' -                     'Connection: close\r\n' -                     'X-Storage-Token: t\r\n' -                     'Content-Length: 5\r\n' -                     '\r\n' -                     '1234 ' % str(segment)) -            fd.flush() -            headers = readuntil2crlfs(fd) -            exp = 'HTTP/1.1 201' -            self.assertEquals(headers[:len(exp)], exp) -            segment_etags.append(md5('1234 ').hexdigest()) -        # Create the object manifest file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'X-Object-Manifest: segmented%20object/object%20name/\r\n' -                 'Content-Type: text/jibberish\r\n' -                 'Foo: barbaz\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Check retrieving the listing the manifest would retrieve -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ ' -                 'HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        body = fd.read() -        self.assertEquals( -            body, -            'object name/0\n' -            'object name/1\n' -            'object name/2\n' -            'object name/3\n' -            'object name/4\n') -        # Ensure retrieving the manifest file gets the whole object -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in -                     headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        self.assert_('Foo: barbaz' in headers) -        expected_etag = md5(''.join(segment_etags)).hexdigest() -        self.assert_('Etag: "%s"' % expected_etag in headers) -        body = fd.read() -        self.assertEquals(body, '1234 1234 1234 1234 1234 ') -        # Do it again but exceeding the container listing limit -        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2 -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) - -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in -                     headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        body = fd.read() -        # A bit fragile of a test; as it makes the assumption that all -        # will be sent in a single chunk. -        self.assertEquals( -            body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n') -        # Make a copy of the manifested object, which should -        # error since the number of segments exceeds -        # CONTAINER_LISTING_LIMIT. -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 'X-Copy-From: segmented%20object/object%20name\r\n' -                 'Content-Length: 0\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 413' -        self.assertEquals(headers[:len(exp)], exp) -        body = fd.read() -        # After adjusting the CONTAINER_LISTING_LIMIT, make a copy of -        # the manifested object which should consolidate the segments. -        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000 -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 'X-Copy-From: segmented%20object/object%20name\r\n' -                 'Content-Length: 0\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        body = fd.read() -        # Retrieve and validate the copy. -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('x-object-manifest:' not in headers.lower()) -        self.assert_('Content-Length: 25\r' in headers) -        body = fd.read() -        self.assertEquals(body, '1234 1234 1234 1234 1234 ') -        # Create an object manifest file pointing to nothing -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'X-Object-Manifest: segmented%20object/empty/\r\n' -                 'Content-Type: text/jibberish\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Ensure retrieving the manifest file gives a zero-byte file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers) -        self.assert_('Content-Type: text/jibberish' in headers) -        body = fd.read() -        self.assertEquals(body, '') -        # Check copy content type -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'Content-Type: text/jibberish\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'X-Copy-From: c/obj\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Ensure getting the copied file gets original content-type -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('Content-Type: text/jibberish' in headers) -        # Check set content type -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'Content-Type: foo/bar\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Ensure getting the copied file gets original content-type -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('Content-Type: foo/bar' in -                     headers.split('\r\n'), repr(headers.split('\r\n'))) -        # Check set content type with charset -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Storage-Token: t\r\n' -                 'Content-Length: 0\r\n' -                 'Content-Type: foo/bar; charset=UTF-8\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEquals(headers[:len(exp)], exp) -        # Ensure getting the copied file gets original content-type -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEquals(headers[:len(exp)], exp) -        self.assert_('Content-Type: foo/bar; charset=UTF-8' in -                     headers.split('\r\n'), repr(headers.split('\r\n'))) -      def test_mismatched_etags(self):          with save_globals():              # no etag supplied, object servers return success w/ diff values @@ -4085,53 +3675,63 @@ class TestObjectController(unittest.TestCase):              self.assertTrue('X-Delete-At in past' in resp.body)      def test_leak_1(self): -        prolis = _test_sockets[0] -        prosrv = _test_servers[0] -        obj_len = prosrv.client_chunk_size * 2 -        # PUT test file -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 'Content-Length: %s\r\n' -                 'Content-Type: application/octet-stream\r\n' -                 '\r\n%s' % (obj_len, 'a' * obj_len)) -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 201' -        self.assertEqual(headers[:len(exp)], exp) -        # Remember Request instance count, make sure the GC is run for pythons -        # without reference counting. -        for i in xrange(4): -            sleep(0)  # let eventlet do its thing -            gc.collect() -        else: -            sleep(0) -        before_request_instances = len(_request_instances) -        # GET test file, but disconnect early -        sock = connect_tcp(('localhost', prolis.getsockname()[1])) -        fd = sock.makefile() -        fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' -                 'Host: localhost\r\n' -                 'Connection: close\r\n' -                 'X-Auth-Token: t\r\n' -                 '\r\n') -        fd.flush() -        headers = readuntil2crlfs(fd) -        exp = 'HTTP/1.1 200' -        self.assertEqual(headers[:len(exp)], exp) -        fd.read(1) -        fd.close() -        sock.close() -        # Make sure the GC is run again for pythons without reference counting -        for i in xrange(4): -            sleep(0)  # let eventlet do its thing -            gc.collect() -        else: -            sleep(0) -        self.assertEquals(before_request_instances, len(_request_instances)) +        _request_instances = weakref.WeakKeyDictionary() +        _orig_init = Request.__init__ + +        def request_init(self, *args, **kwargs): +            _orig_init(self, *args, **kwargs) +            _request_instances[self] = None + +        with mock.patch.object(Request, "__init__", request_init): +            prolis = _test_sockets[0] +            prosrv = _test_servers[0] +            obj_len = prosrv.client_chunk_size * 2 +            # PUT test file +            sock = connect_tcp(('localhost', prolis.getsockname()[1])) +            fd = sock.makefile() +            fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n' +                     'Host: localhost\r\n' +                     'Connection: close\r\n' +                     'X-Auth-Token: t\r\n' +                     'Content-Length: %s\r\n' +                     'Content-Type: application/octet-stream\r\n' +                     '\r\n%s' % (obj_len, 'a' * obj_len)) +            fd.flush() +            headers = readuntil2crlfs(fd) +            exp = 'HTTP/1.1 201' +            self.assertEqual(headers[:len(exp)], exp) +            # Remember Request instance count, make sure the GC is run for +            # pythons without reference counting. +            for i in xrange(4): +                sleep(0)  # let eventlet do its thing +                gc.collect() +            else: +                sleep(0) +            before_request_instances = len(_request_instances) +            # GET test file, but disconnect early +            sock = connect_tcp(('localhost', prolis.getsockname()[1])) +            fd = sock.makefile() +            fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' +                     'Host: localhost\r\n' +                     'Connection: close\r\n' +                     'X-Auth-Token: t\r\n' +                     '\r\n') +            fd.flush() +            headers = readuntil2crlfs(fd) +            exp = 'HTTP/1.1 200' +            self.assertEqual(headers[:len(exp)], exp) +            fd.read(1) +            fd.close() +            sock.close() +            # Make sure the GC is run again for pythons without reference +            # counting +            for i in xrange(4): +                sleep(0)  # let eventlet do its thing +                gc.collect() +            else: +                sleep(0) +            self.assertEquals( +                before_request_instances, len(_request_instances))      def test_OPTIONS(self):          with save_globals(): @@ -4926,9 +4526,10 @@ class TestContainerController(unittest.TestCase):                  controller = \                      proxy_server.ContainerController(self.app, 'a', 'c')                  set_http_connect(200, 201, 201, 201, give_connect=test_connect) -                req = Request.blank('/v1/a/c', -                                    environ={'REQUEST_METHOD': method}, -                                    headers={test_header: test_value}) +                req = Request.blank( +                    '/v1/a/c', +                    environ={'REQUEST_METHOD': method, 'swift_owner': True}, +                    headers={test_header: test_value})                  self.app.update_request(req)                  getattr(controller, method)(req)                  self.assertEquals(test_errors, []) @@ -6006,313 +5607,6 @@ class Stub(object):      pass -class TestSegmentedIterable(unittest.TestCase): - -    def setUp(self): -        self.controller = FakeObjectController() - -    def test_load_next_segment_unexpected_error(self): -        # Iterator value isn't a dict -        self.assertRaises(Exception, -                          SegmentedIterable(self.controller, None, -                                            [None])._load_next_segment) -        self.assert_(self.controller.exception_args[0].startswith( -                     'ERROR: While processing manifest')) - -    def test_load_next_segment_with_no_segments(self): -        self.assertRaises(StopIteration, -                          SegmentedIterable(self.controller, 'lc', -                                            [])._load_next_segment) - -    def test_load_next_segment_with_one_segment(self): -        segit = SegmentedIterable(self.controller, 'lc', [{'name': -                                  'o1'}]) -        segit._load_next_segment() -        self.assertEquals( -            self.controller.GETorHEAD_base_args[0][4], '/a/lc/o1') -        data = ''.join(segit.segment_iter) -        self.assertEquals(data, '1') - -    def test_load_next_segment_with_two_segments(self): -        segit = SegmentedIterable(self.controller, 'lc', [{'name': -                                  'o1'}, {'name': 'o2'}]) -        segit._load_next_segment() -        self.assertEquals( -            self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o1') -        data = ''.join(segit.segment_iter) -        self.assertEquals(data, '1') -        segit._load_next_segment() -        self.assertEquals( -            self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') -        data = ''.join(segit.segment_iter) -        self.assertEquals(data, '22') - -    def test_load_next_segment_rate_limiting(self): -        sleep_calls = [] - -        def _stub_sleep(sleepy_time): -            sleep_calls.append(sleepy_time) -        orig_sleep = swift.proxy.controllers.obj.sleep -        try: -            swift.proxy.controllers.obj.sleep = _stub_sleep -            segit = SegmentedIterable( -                self.controller, 'lc', [ -                    {'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'}, -                    {'name': 'o4'}, {'name': 'o5'}]) - -            # rate_limit_after_segment == 3, so the first 3 segments should -            # invoke no sleeping. -            for _ in xrange(3): -                segit._load_next_segment() -            self.assertEquals([], sleep_calls) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o3') - -            # Loading of next (4th) segment starts rate-limiting. -            segit._load_next_segment() -            self.assertAlmostEqual(0.5, sleep_calls[0], places=2) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o4') - -            sleep_calls = [] -            segit._load_next_segment() -            self.assertAlmostEqual(0.5, sleep_calls[0], places=2) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o5') -        finally: -            swift.proxy.controllers.obj.sleep = orig_sleep - -    def test_load_next_segment_range_req_rate_limiting(self): -        sleep_calls = [] - -        def _stub_sleep(sleepy_time): -            sleep_calls.append(sleepy_time) -        orig_sleep = swift.proxy.controllers.obj.sleep -        try: -            swift.proxy.controllers.obj.sleep = _stub_sleep -            segit = SegmentedIterable( -                self.controller, 'lc', [ -                    {'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5}, -                    {'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'}, -                    {'name': 'o5'}, {'name': 'o6'}]) - -            # this tests for a range request which skips over the whole first -            # segment, after that 3 segments will be read in because the -            # rate_limit_after_segment == 3, then sleeping starts -            segit_iter = segit.app_iter_range(10, None) -            segit_iter.next() -            for _ in xrange(2): -                # this is set to 2 instead of 3 because o2 was loaded after -                # o0 and o1 were skipped. -                segit._load_next_segment() -            self.assertEquals([], sleep_calls) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o4') - -            # Loading of next (5th) segment starts rate-limiting. -            segit._load_next_segment() -            self.assertAlmostEqual(0.5, sleep_calls[0], places=2) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o5') - -            sleep_calls = [] -            segit._load_next_segment() -            self.assertAlmostEqual(0.5, sleep_calls[0], places=2) -            self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], -                              '/a/lc/o6') -        finally: -            swift.proxy.controllers.obj.sleep = orig_sleep - -    def test_load_next_segment_with_two_segments_skip_first(self): -        segit = SegmentedIterable(self.controller, 'lc', [{'name': -                                  'o1'}, {'name': 'o2'}]) -        segit.ratelimit_index = 0 -        segit.listing.next() -        segit._load_next_segment() -        self.assertEquals( -            self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') -        data = ''.join(segit.segment_iter) -        self.assertEquals(data, '22') - -    def test_load_next_segment_with_seek(self): -        segit = SegmentedIterable(self.controller, 'lc', -                                  [{'name': 'o1', 'bytes': 1}, -                                   {'name': 'o2', 'bytes': 2}]) -        segit.ratelimit_index = 0 -        segit.listing.next() -        segit.seek = 1 -        segit._load_next_segment() -        self.assertEquals( -            self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') -        self.assertEquals( -            str(self.controller.GETorHEAD_base_args[-1][0].range), -            'bytes=1-') -        data = ''.join(segit.segment_iter) -        self.assertEquals(data, '2') - -    def test_fetching_only_what_you_need(self): -        segit = SegmentedIterable(self.controller, 'lc', -                                  [{'name': 'o7', 'bytes': 7}, -                                   {'name': 'o8', 'bytes': 8}, -                                   {'name': 'o9', 'bytes': 9}]) - -        body = ''.join(segit.app_iter_range(10, 20)) -        self.assertEqual('8888899999', body) - -        GoH_args = self.controller.GETorHEAD_base_args -        self.assertEquals(2, len(GoH_args)) - -        # Either one is fine, as they both indicate "from byte 3 to (the last) -        # byte 8". -        self.assert_(str(GoH_args[0][0].range) in ['bytes=3-', 'bytes=3-8']) - -        # This one must ask only for the bytes it needs; otherwise we waste -        # bandwidth pulling bytes from the object server and then throwing -        # them out -        self.assertEquals(str(GoH_args[1][0].range), 'bytes=0-4') - -    def test_load_next_segment_with_get_error(self): - -        def local_GETorHEAD_base(*args): -            return HTTPNotFound() - -        self.controller.GETorHEAD_base = local_GETorHEAD_base -        self.assertRaises(Exception, -                          SegmentedIterable(self.controller, 'lc', -                          [{'name': 'o1'}])._load_next_segment) -        self.assert_(self.controller.exception_args[0].startswith( -                     'ERROR: While processing manifest')) -        self.assertEquals(str(self.controller.exception_info[1]), -                          'Could not load object segment /a/lc/o1: 404') - -    def test_iter_unexpected_error(self): -        # Iterator value isn't a dict -        self.assertRaises(Exception, ''.join, -                          SegmentedIterable(self.controller, None, [None])) -        self.assert_(self.controller.exception_args[0].startswith( -            'ERROR: While processing manifest')) - -    def test_iter_with_no_segments(self): -        segit = SegmentedIterable(self.controller, 'lc', []) -        self.assertEquals(''.join(segit), '') - -    def test_iter_with_one_segment(self): -        segit = SegmentedIterable(self.controller, 'lc', [{'name': -                                  'o1'}]) -        segit.response = Stub() -        self.assertEquals(''.join(segit), '1') - -    def test_iter_with_two_segments(self): -        segit = SegmentedIterable(self.controller, 'lc', [{'name': -                                  'o1'}, {'name': 'o2'}]) -        segit.response = Stub() -        self.assertEquals(''.join(segit), '122') - -    def test_iter_with_get_error(self): - -        def local_GETorHEAD_base(*args): -            return HTTPNotFound() - -        self.controller.GETorHEAD_base = local_GETorHEAD_base -        self.assertRaises(Exception, ''.join, -                          SegmentedIterable(self.controller, 'lc', [{'name': -                                                                    'o1'}])) -        self.assert_(self.controller.exception_args[0].startswith( -                     'ERROR: While processing manifest')) -        self.assertEquals(str(self.controller.exception_info[1]), -                          'Could not load object segment /a/lc/o1: 404') - -    def test_app_iter_range_unexpected_error(self): -        # Iterator value isn't a dict -        self.assertRaises(Exception, -                          SegmentedIterable(self.controller, None, -                                            [None]).app_iter_range(None, -                                                                   None).next) -        self.assert_(self.controller.exception_args[0].startswith( -            'ERROR: While processing manifest')) - -    def test_app_iter_range_with_no_segments(self): -        self.assertEquals(''.join(SegmentedIterable( -            self.controller, 'lc', []).app_iter_range(None, None)), '') -        self.assertEquals(''.join(SegmentedIterable( -            self.controller, 'lc', []).app_iter_range(3, None)), '') -        self.assertEquals(''.join(SegmentedIterable( -            self.controller, 'lc', []).app_iter_range(3, 5)), '') -        self.assertEquals(''.join(SegmentedIterable( -            self.controller, 'lc', []).app_iter_range(None, 5)), '') - -    def test_app_iter_range_with_one_segment(self): -        listing = [{'name': 'o1', 'bytes': 1}] - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, None)), '1') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        self.assertEquals(''.join(segit.app_iter_range(3, None)), '') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        self.assertEquals(''.join(segit.app_iter_range(3, 5)), '') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1') - -    def test_app_iter_range_with_two_segments(self): -        listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}] - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, None)), '122') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(1, None)), '22') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12') - -    def test_app_iter_range_with_many_segments(self): -        listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}, -                   {'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4}, -                   {'name': 'o5', 'bytes': 5}] - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, None)), -                          '122333444455555') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(3, None)), -                          '333444455555') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334') - -        segit = SegmentedIterable(self.controller, 'lc', listing) -        segit.response = Stub() -        self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34') - -  class TestProxyObjectPerformance(unittest.TestCase):      def setUp(self): diff --git a/tools/requirements.txt b/tools/requirements.txt index 1139866..bbac51a 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -5,4 +5,3 @@ netifaces>=0.5  pastedeploy>=1.3.3  simplejson>=2.0.9  xattr>=0.4 -python-swiftclient diff --git a/tools/test-requires b/tools/test-requires index d2c027f..63d499e 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,7 +1,4 @@ -# Install bounded pep8/pyflakes first, then let flake8 install -pep8==1.4.5 -pyflakes==0.7.2 -flake8==2.0 +# Hacking already pins down pep8, pyflakes and flake8  hacking>=0.5.6,<0.6  coverage  nose @@ -10,5 +7,6 @@ openstack.nose_plugin  nosehtmloutput  sphinx>=1.1.2,<1.2  mock>=0.8.0 +python-swiftclient  python-keystoneclient  prettytable @@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir}           NOSE_OPENSTACK_SHOW_ELAPSED=1           NOSE_OPENSTACK_STDOUT=1  deps = -  https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz +  https://launchpad.net/gluster-swift/icehouse/1.13.0/+download/swift-1.13.0.tar.gz    --download-cache={homedir}/.pipcache    -r{toxinidir}/tools/test-requires    -r{toxinidir}/tools/requirements.txt  | 
