From d5ebf36f7e9268b68b2e8d5b737f8252c5296e1a Mon Sep 17 00:00:00 2001 From: Luis Pabon Date: Tue, 11 Feb 2014 11:43:09 -0500 Subject: Sync with OpenStack Swift v1.12.0 * Bumped the version of gluster-swift to v1.12.0. * Added document on how to do a sync Change-Id: I676e110759b0bce3007a2a38f3b384b4ca9d710d Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/6977 Reviewed-by: Prashanth Pai Reviewed-by: Chetan Risbud Tested-by: Chetan Risbud --- doc/markdown/openstack_swift_sync.md | 62 ++++ gluster/swift/__init__.py | 2 +- test/functional/swift_testing.py | 175 +++++++++ test/functional/test_account.py | 203 +++++++++++ test/functional/test_container.py | 687 +++++++++++++++++++++++++++++++++++ test/functional/test_object.py | 624 +++++++++++++++++++++++++++++++ test/functional/tests.py | 22 +- test/unit/proxy/test_server.py | 679 ++-------------------------------- tox.ini | 2 +- 9 files changed, 1801 insertions(+), 655 deletions(-) create mode 100644 doc/markdown/openstack_swift_sync.md create mode 100644 test/functional/swift_testing.py create mode 100755 test/functional/test_account.py create mode 100755 test/functional/test_container.py create mode 100755 test/functional/test_object.py diff --git a/doc/markdown/openstack_swift_sync.md b/doc/markdown/openstack_swift_sync.md new file mode 100644 index 0000000..f3feac8 --- /dev/null +++ b/doc/markdown/openstack_swift_sync.md @@ -0,0 +1,62 @@ +# Syncing Gluster-Swift with Swift + +## Create a release +Create a release in launchpad.net so that we can place the latest swift source for download. We'll place the source here, and it will allow tox in gluster-swift to download the latest code. + +## Upload swift release + +* Clone the git swift repo +* Go to the release tag or just use the latest +* Type the following to package the swift code: + +``` +$ python setup.py sdist +$ ls dist +``` + +* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net + +## Setup Tox +Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link. + +## Update tests +This part is a little more complicated and now we need to *merge* the latest tests with ours. + +I suggest using a tool called `meld` to make this work easier. + +Files that need to be merged: + +* Update unit tests + +``` +$ export SWIFTDIR=../swift +$ meld $SWIFTDIR/tox.ini tox.ini +$ meld $SWIFTDIR/test-requirements.txt tools/test-requires +$ meld $SWIFTDIR/requirements.txt tools/requirements.txt +$ meld $SWIFTDIR/test/unit/proxy/test_servers.py test/unit/proxy/test_server.py +$ cp $SWIFTDIR/test/unit/proxy/controllers/*.py test/unit/proxy/controllers +$ meld $SWIFTDIR/test/unit/__init__.py test/unit/__init__.py +``` + +* Update all the functional tests +First check if there are any new files in the swift functional test directory. If there are, copy them over. + +* Remember to `git add` any new files + +* Now merge the existing ones: + +``` +for i in $SWIFTDIR/test/functional/*.py ; do + meld $i test/functional/`basename $i` +done +``` + +## Update the version +If needed, update the version now in `gluster/swift/__init__.py`. + +## Upload the patch +Upload the patch to Gerrit. + +## Update the release in launchpad.net +Upload the gluster-swift*.tar.gz built by Jenkins to launchpad.net once the fix has been commited to the main branch. + diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py index 9bfdc31..4fe17ba 100644 --- a/gluster/swift/__init__.py +++ b/gluster/swift/__init__.py @@ -45,6 +45,6 @@ class PkgInfo(object): ### ### Change the Package version here ### -_pkginfo = PkgInfo('1.11.0', '0', 'gluster_swift', False) +_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False) __version__ = _pkginfo.pretty_version __canonical_version__ = _pkginfo.canonical_version diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py new file mode 100644 index 0000000..b5642b3 --- /dev/null +++ b/test/functional/swift_testing.py @@ -0,0 +1,175 @@ +# Copyright (c) 2010-2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from httplib import HTTPException +import os +import socket +import sys +from time import sleep + +from test import get_config + +from swiftclient import get_auth, http_connection + +conf = get_config('func_test') +web_front_end = conf.get('web_front_end', 'integral') +normalized_urls = conf.get('normalized_urls', False) + +# If no conf was read, we will fall back to old school env vars +swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] +swift_test_tenant = ['', '', ''] +swift_test_perm = ['', '', ''] + +if conf: + swift_test_auth_version = str(conf.get('auth_version', '1')) + + swift_test_auth = 'http' + if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): + swift_test_auth = 'https' + if 'auth_prefix' not in conf: + conf['auth_prefix'] = '/' + try: + suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf + swift_test_auth += suffix + except KeyError: + pass # skip + + if swift_test_auth_version == "1": + swift_test_auth += 'v1.0' + + if 'account' in conf: + swift_test_user[0] = '%(account)s:%(username)s' % conf + else: + swift_test_user[0] = '%(username)s' % conf + swift_test_key[0] = conf['password'] + try: + swift_test_user[1] = '%s%s' % ( + '%s:' % conf['account2'] if 'account2' in conf else '', + conf['username2']) + swift_test_key[1] = conf['password2'] + except KeyError as err: + pass # old conf, no second account tests can be run + try: + swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account' + in conf else '', conf['username3']) + swift_test_key[2] = conf['password3'] + except KeyError as err: + pass # old conf, no third account tests can be run + + for _ in range(3): + swift_test_perm[_] = swift_test_user[_] + + else: + swift_test_user[0] = conf['username'] + swift_test_tenant[0] = conf['account'] + swift_test_key[0] = conf['password'] + swift_test_user[1] = conf['username2'] + swift_test_tenant[1] = conf['account2'] + swift_test_key[1] = conf['password2'] + swift_test_user[2] = conf['username3'] + swift_test_tenant[2] = conf['account'] + swift_test_key[2] = conf['password3'] + + for _ in range(3): + swift_test_perm[_] = swift_test_tenant[_] + ':' \ + + swift_test_user[_] + +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) +if skip: + print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' + +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) +if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) +if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + + +class AuthError(Exception): + pass + + +class InternalServerError(Exception): + pass + + +url = [None, None, None] +token = [None, None, None] +parsed = [None, None, None] +conn = [None, None, None] + + +def retry(func, *args, **kwargs): + """ + You can use the kwargs to override the 'retries' (default: 5) and + 'use_account' (default: 1). + """ + global url, token, parsed, conn + retries = kwargs.get('retries', 5) + use_account = 1 + if 'use_account' in kwargs: + use_account = kwargs['use_account'] + del kwargs['use_account'] + use_account -= 1 + attempts = 0 + backoff = 1 + while attempts <= retries: + attempts += 1 + try: + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = \ + get_auth(swift_test_auth, swift_test_user[use_account], + swift_test_key[use_account], + snet=False, + tenant_name=swift_test_tenant[use_account], + auth_version=swift_test_auth_version, + os_options={}) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + http_connection(url[use_account]) + return func(url[use_account], token[use_account], + parsed[use_account], conn[use_account], + *args, **kwargs) + except (socket.error, HTTPException): + if attempts > retries: + raise + parsed[use_account] = conn[use_account] = None + except AuthError: + url[use_account] = token[use_account] = None + continue + except InternalServerError: + pass + if attempts <= retries: + sleep(backoff) + backoff *= 2 + raise Exception('No result after %s retries.' % retries) + + +def check_response(conn): + resp = conn.getresponse() + if resp.status == 401: + resp.read() + raise AuthError() + elif resp.status // 100 == 5: + resp.read() + raise InternalServerError() + return resp diff --git a/test/functional/test_account.py b/test/functional/test_account.py new file mode 100755 index 0000000..b2f743f --- /dev/null +++ b/test/functional/test_account.py @@ -0,0 +1,203 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, web_front_end + + +class TestAccount(unittest.TestCase): + + def test_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, 'X-Account-Meta-Test': value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(post, '') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + + def test_unicode_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + uni_key = u'X-Account-Meta-uni\u0E12' + uni_value = u'uni\u0E12' + if (web_front_end == 'integral'): + resp = retry(post, uni_key, '1') + resp.read() + self.assertTrue(resp.status in (201, 204)) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + resp = retry(post, 'X-Account-Meta-uni', uni_value) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('X-Account-Meta-uni'), + uni_value.encode('utf-8')) + if (web_front_end == 'integral'): + resp = retry(post, uni_key, uni_value) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) + + def test_multi_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(post, 'X-Account-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + resp = retry(post, 'X-Account-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + self.assertEquals(resp.getheader('x-account-meta-two'), '2') + + def test_bad_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + resp = retry(post, + {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry( + post, + {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry(post, + {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry( + post, + {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Account-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functional/test_container.py b/test/functional/test_container.py new file mode 100755 index 0000000..15f7fc1 --- /dev/null +++ b/test/functional/test_container.py @@ -0,0 +1,687 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, skip2, skip3, \ + swift_test_perm, web_front_end + + +class TestContainer(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.name = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + + while True: + resp = retry(get) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_multi_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(post, 'X-Container-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + resp = retry(post, 'X-Container-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + self.assertEquals(resp.getheader('x-container-meta-two'), '2') + + def test_unicode_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + uni_key = u'X-Container-Meta-uni\u0E12' + uni_value = u'uni\u0E12' + if (web_front_end == 'integral'): + resp = retry(post, uni_key, '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + resp = retry(post, 'X-Container-Meta-uni', uni_value) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('X-Container-Meta-uni'), + uni_value.encode('utf-8')) + if (web_front_end == 'integral'): + resp = retry(post, uni_key, uni_value) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) + + def test_PUT_metadata(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn, name, value): + conn.request('PUT', parsed.path + '/' + name, '', + {'X-Auth-Token': token, + 'X-Container-Meta-Test': value}) + return check_response(conn) + + def head(url, token, parsed, conn, name): + conn.request('HEAD', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + name = uuid4().hex + resp = retry(put, name, 'Value') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + name = uuid4().hex + resp = retry(put, name, '') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + def test_POST_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Meta-Test': value}) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + + def test_PUT_bad_metadata(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn, name, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('PUT', parsed.path + '/' + name, '', headers) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + name = uuid4().hex + resp = retry( + put, name, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry( + put, name, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + resp = retry( + put, name, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry( + put, name, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + def test_POST_bad_metadata(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) + + resp = retry( + post, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry( + post, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry( + post, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry( + post, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + def test_public_container(self): + if skip: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name) + return check_response(conn) + + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception as err: + self.assert_(str(err).startswith('No result after '), err) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 204) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception as err: + self.assert_(str(err).startswith('No result after '), err) + + def test_cross_account_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get1) + resp.read() + + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + # Make the container accessible by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[1], + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now use the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can't access the container with the second account again + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + def test_cross_account_public_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get1) + resp.read() + + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + # Make the container completely public + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + + # But we shouldn't be able to write with the second account + def put2(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + # Now make the container also writeable by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the second account + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 201) + + def test_nonadmin_user(self): + if skip or skip3: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get1) + resp.read() + + # Ensure we can't access the container with the third account + def get3(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Make the container accessible by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + + # But we shouldn't be able to write with the third account + def put3(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Now make the container also writeable by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the third account + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + def test_long_name_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + container_name = 'X' * 2048 + conn.request('PUT', '%s/%s' % (parsed.path, container_name), + 'there', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 400) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/abc%%00def' % parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functional/test_object.py b/test/functional/test_object.py new file mode 100755 index 0000000..dad8635 --- /dev/null +++ b/test/functional/test_object.py @@ -0,0 +1,624 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift_testing import check_response, retry, skip, skip3, \ + swift_test_perm, web_front_end + + +class TestObject(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + self.obj = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, self.obj), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '%s/%s/%s' % (parsed.path, self.container, obj), + '', {'X-Auth-Token': token}) + return check_response(conn) + + # get list of objects in container + def list(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(list) + object_listing = resp.read() + self.assertEquals(resp.status, 200) + + # iterate over object listing and delete all objects + for obj in object_listing.splitlines(): + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + + # delete the container + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_copy_object(self): + if skip: + raise SkipTest + + source = '%s/%s' % (self.container, self.obj) + dest = '%s/%s' % (self.container, 'test_copy') + + # get contents of source + def get_source(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, source), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_source) + source_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(source_contents, 'test') + + # copy source to dest with X-Copy-From + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': source}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + def get_dest(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, dest), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + # verify dest does not exist + resp = retry(get_dest) + resp.read() + self.assertEquals(resp.status, 404) + + # copy source to dest with COPY + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination': dest}) + return check_response(conn) + resp = retry(copy) + resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_public_object(self): + if skip: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', + '%s/%s/%s' % (parsed.path, self.container, self.obj)) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception as err: + self.assert_(str(err).startswith('No result after ')) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 200) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception as err: + self.assert_(str(err).startswith('No result after ')) + + def test_private_object(self): + if skip or skip3: + raise SkipTest + + # Ensure we can't access the object with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, self.obj), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # create a shared container writable by account3 + shared_container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % ( + parsed.path, shared_container), '', + {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2], + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can not copy from private container + def copy(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, shared_container, 'private_object'), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': '%s/%s' % (self.container, self.obj)}) + return check_response(conn) + resp = retry(copy, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # verify third account can write "obj1" to shared container + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, shared_container, 'obj1'), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can copy "obj1" to shared container + def copy2(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % ( + parsed.path, shared_container, 'obj1'), '', + {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, 'obj1')}) + return check_response(conn) + resp = retry(copy2, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account STILL can not copy from private container + def copy3(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % ( + parsed.path, self.container, self.obj), '', + {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, + 'private_object')}) + return check_response(conn) + resp = retry(copy3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # clean up "obj1" + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/%s' % ( + parsed.path, shared_container, 'obj1'), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + # clean up shared_container + def delete(url, token, parsed, conn): + conn.request('DELETE', + parsed.path + '/' + shared_container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_manifest(self): + if skip: + raise SkipTest + # Data for the object segments + segments1 = ['one', 'two', 'three', 'four', 'five'] + segments2 = ['six', 'seven', 'eight'] + segments3 = ['nine', 'ten', 'eleven'] + + # Upload the first set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments1/%s' % ( + parsed.path, self.container, str(objnum)), segments1[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % ( + parsed.path, self.container), '', { + 'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments1/' % self.container, + 'Content-Type': 'text/jibberish', 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should get all the segments as the body) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + self.assertEquals(resp.getheader('content-type'), 'text/jibberish') + + # Get with a range at the start of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', { + 'X-Auth-Token': token, 'Range': 'bytes=3-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1[1:])) + self.assertEquals(resp.status, 206) + + # Get with a range in the middle of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', { + 'X-Auth-Token': token, 'Range': 'bytes=5-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:]) + self.assertEquals(resp.status, 206) + + # Get with a full start and stop range + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', { + 'X-Auth-Token': token, 'Range': 'bytes=5-10'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:11]) + self.assertEquals(resp.status, 206) + + # Upload the second set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments2/%s' % ( + parsed.path, self.container, str(objnum)), segments2[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should still be the first segments of course) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % ( + parsed.path, self.container), '', { + 'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments2/' % self.container, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should be the second set of segments now) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + # Create another container for the third set of segments + acontainer = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + acontainer, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the third set of segments in the other container + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments3/%s' % ( + parsed.path, acontainer, str(objnum)), segments3[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % ( + parsed.path, self.container), '', + {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments3/' % acontainer, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest to ensure it's the third set of segments + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + # (because the segments are in a protected container even if the + # manifest itself is not). + + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, acontainer), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % ( + parsed.path, self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + # Delete the manifest + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/manifest' % ( + parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the third set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments3/%s' % ( + parsed.path, acontainer, str(objnum)), '', + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the second set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments2/%s' % ( + parsed.path, self.container, str(objnum)), '', + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the first set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments1/%s' % ( + parsed.path, self.container, str(objnum)), '', + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the extra container + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, acontainer), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_delete_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/hi' % (parsed.path, self.container), + 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_delete_if_delete_at_bad(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', + '%s/%s/hi-delete-bad' % (parsed.path, self.container), + 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-If-Delete-At': 'bad'}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 400) + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/abc%%00def' % ( + parsed.path, + self.container), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functional/tests.py b/test/functional/tests.py index e4b778b..9469cff 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -2001,16 +2001,34 @@ class TestSlo(Base): def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") - file_item.copy(self.env.container.name, "copied-abcde", + file_item.copy(self.env.container.name, "copied-abcde-manifest-only", parms={'multipart-manifest': 'get'}) - copied = self.env.container.file("copied-abcde") + copied = self.env.container.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) try: json.loads(copied_contents) except ValueError: self.fail("COPY didn't copy the manifest (invalid json on GET)") + def test_slo_get_the_manifest(self): + manifest = self.env.container.file("manifest-abcde") + got_body = manifest.read(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + manifest.content_type) + try: + json.loads(got_body) + except ValueError: + self.fail("GET with multipart-manifest=get got invalid json") + + def test_slo_head_the_manifest(self): + manifest = self.env.container.file("manifest-abcde") + got_info = manifest.info(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + got_info['content_type']) + class TestSloUTF8(Base2, TestSlo): set_up = False diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index f669b9d..b68be9f 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -55,6 +55,7 @@ from swift.common import utils from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable +from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers @@ -1140,657 +1141,6 @@ class TestObjectController(unittest.TestCase): controller.DELETE(req) self.assertEquals(test_errors, []) - def test_GET_manifest_no_segments(self): - for hdict in [{"X-Object-Manifest": "segments/seg"}, - {"X-Static-Large-Object": "True"}]: - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps([]), # GET manifest - simplejson.dumps([])) # GET empty listing - - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET manifest - 200, # GET empty listing - headers=hdict, - body_iter=response_bodies) - - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, '') - - def test_GET_manifest_limited_listing(self): - listing1 = [{"hash": "454dfc73af632012ce3e6217dc464241", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "seg01", - "content_type": "application/octet-stream"}, - {"hash": "474bab96c67528d42d5c0c52b35228eb", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "seg02", - "content_type": "application/octet-stream"}] - - listing2 = [{"hash": "116baa5508693d1d1ca36abdd9f9478b", - "last_modified": "2012-11-08T04:05:37.849510", - "bytes": 2, - "name": "seg03", - "content_type": "application/octet-stream"}, - {"hash": "7bd6aaa1ef6013353f0420459574ac9d", - "last_modified": "2012-11-08T04:05:37.855180", - "bytes": 2, - "name": "seg04", - "content_type": "application/octet-stream" - }] - - listing3 = [{"hash": "6605f80e3cefaa24e9823544df4edbd6", - "last_modified": "2012-11-08T04:05:37.853710", - "bytes": 2, - "name": u'\N{SNOWMAN}seg05', - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - '', # GET manifest - simplejson.dumps(listing1), # GET listing1 - 'Aa', # GET seg01 - 'Bb', # GET seg02 - simplejson.dumps(listing2), # GET listing2 - 'Cc', # GET seg03 - 'Dd', # GET seg04 - simplejson.dumps(listing3), # GET listing3 - 'Ee', # GET seg05 - simplejson.dumps([])) # GET final empty listing - with save_globals(): - try: - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2 - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET manifest - 200, # GET listing1 - 200, # GET seg01 - 200, # GET seg02 - 200, # GET listing2 - 200, # GET seg03 - 200, # GET seg04 - 200, # GET listing3 - 200, # GET seg05 - 200, # GET final empty listing - headers={"X-Object-Manifest": "segments/seg"}, - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, 'AaBbCcDdEe') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/segments', - {'format': 'json', 'prefix': 'seg'}], - ['GET', '/a/segments/seg01', {}], - ['GET', '/a/segments/seg02', {}], - ['GET', '/a/segments', - {'format': 'json', 'prefix': 'seg', 'marker': 'seg02'}], - ['GET', '/a/segments/seg03', {}], - ['GET', '/a/segments/seg04', {}], - ['GET', '/a/segments', - {'format': 'json', 'prefix': 'seg', 'marker': 'seg04'}], - ['GET', '/a/segments/\xe2\x98\x83seg05', {}], - ['GET', '/a/segments', - {'format': 'json', 'prefix': 'seg', - 'marker': '\xe2\x98\x83seg05'}]]) - - finally: - # other tests in this file get very unhappy if this - # isn't set back, which leads to time-wasting - # debugging of other tests. - swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \ - _orig_container_listing_limit - - def test_GET_manifest_slo(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps(listing), # GET manifest - 'Aa', # GET seg01 - 'Bb') # GET seg02 - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - 200, # GET seg01 - 200, # GET seg02 - headers=[{}, {}, {"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, {}, {}], - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, 'AaBb') - self.assertEqual(resp.content_length, 4) - self.assertEqual(resp.content_type, 'text/html') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/seg02', {}]]) - - def test_GET_slo_multipart_manifest(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}] - json_listing = simplejson.dumps(listing) - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - json_listing) # GET manifest - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - headers={"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest?multipart-manifest=get') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, json_listing) - self.assertEqual(resp.content_type, 'application/json') - self.assertEqual(resp.charset.lower(), 'utf-8') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {'multipart-manifest': 'get'}]]) - - def test_GET_slo_multipart_manifest_from_copy(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}] - json_listing = simplejson.dumps(listing) - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - json_listing) # GET manifest - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - headers={"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest?multipart-manifest=get', - headers={'x-copy-from': '/a/c/manifest'}) - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, json_listing) - self.assertEqual(resp.content_type, 'text/html') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {'multipart-manifest': 'get'}]]) - - def test_GET_bad_etag_manifest_slo(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "invalidhash", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps(listing), # GET manifest - 'Aa', # GET seg01 - 'Bb') # GET seg02 - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - 200, # GET seg01 - 200, # GET seg02 - headers=[{}, {}, {"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, {}, {}], - body_iter=response_bodies, - give_connect=capture_requested_paths) - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.content_length, 4) # content incomplete - self.assertEqual(resp.content_type, 'text/html') - self.assertRaises(SegmentError, lambda: resp.body) - # dropped connection, exception is caught by eventlet as it is - # iterating over response - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/seg02', {}]]) - - def test_GET_nested_slo(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "8681fb3ada2715c8754706ee5f23d4f8", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 4, - "name": u"/d2/sub_manifest \u2661", "sub_slo": True, - "content_type": "application/octet-stream"}, - {"hash": "419af6d362a14b7a789ba1c7e772bbae", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg04", - "content_type": "application/octet-stream"}] - - sub_listing = [{"hash": "d526f1c8ef6c1e4e980e2b8471352d23", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg02", - "content_type": "application/octet-stream"}, - {"hash": "e4c8f1de1c0855c7c2be33196d3c3537", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg03", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps(listing), # GET manifest - simplejson.dumps(sub_listing), # GET sub_manifest - 'Aa', # GET seg01 - 'Bb', # GET seg02 - 'Cc', # GET seg03 - 'Dd') # GET seg04 - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - slob_headers = {"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'} - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - 200, # GET sub listing1 - 200, # GET seg01 - 200, # GET seg02 - 200, # GET seg03 - 200, # GET seg04 - headers=[{}, {}, slob_headers, slob_headers, {}, {}, {}, {}], - body_iter=response_bodies, - give_connect=capture_requested_paths) - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.content_length, 8) - self.assertEqual(resp.content_type, 'text/html') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}]]) - # iterating over body will retrieve manifest and sub manifest's - # objects - self.assertEqual(resp.body, 'AaBbCcDd') - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}], - ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d1/seg02', {}], - ['GET', '/a/d2/seg03', {}], - ['GET', '/a/d1/seg04', {}]]) - - def test_GET_nested_manifest_slo_with_range(self): - """ - Original whole slo is Aa1234Bb where 1234 is a sub-manifests. I'm - pulling out 34Bb - """ - listing = [{"hash": "98568d540134639be4655198a36614a4", # Aa - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "7b4b0ffa275d404bdc2fc6384916714f", # SubManifest1 - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 4, "sub_slo": True, - "name": "/d2/subManifest01", - "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", # Bb - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg02", - "content_type": "application/octet-stream"}] - - sublisting = [{"hash": "c20ad4d76fe97759aa27a0c99bff6710", # 12 - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d2/subSeg01", - "content_type": "application/octet-stream"}, - {"hash": "e369853df766fa44e1ed0ff613f563bd", # 34 - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d2/subSeg02", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps(listing)[1:1], # GET incomplete manifest - simplejson.dumps(listing), # GET complete manifest - simplejson.dumps(sublisting), # GET complete submanifest - '34', # GET subseg02 - 'Bb') # GET seg02 - etag_iter = ['', '', '', '', '', - 'e369853df766fa44e1ed0ff613f563bd', # subSeg02 - 'd526f1c8ef6c1e4e980e2b8471352d23'] # seg02 - headers = [{}, {}, - {'X-Static-Large-Object': 'True', - 'content-type': 'text/html; swift_bytes=4'}, - {'X-Static-Large-Object': 'True', - 'content-type': 'text/html; swift_bytes=4'}, - {'X-Static-Large-Object': 'True', - 'content-type': 'text/html; swift_bytes=4'}, - {}, {}] - self.assertTrue(len(response_bodies) == len(etag_iter) == len(headers)) - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 206, # GET incomplete listing - 200, # GET complete listing - 200, # GET complete sublisting - 200, # GET subSeg02 - 200, # GET seg02 - headers=headers, - etags=etag_iter, - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest') - req.range = 'bytes=4-7' - resp = controller.GET(req) - got_called = [False, ] - - def fake_start_response(*args, **kwargs): - got_called[0] = True - self.assertTrue(args[0].startswith('206')) - - app_iter = resp(req.environ, fake_start_response) - resp_body = ''.join(app_iter) # read in entire resp - self.assertEqual(resp.status_int, 206) - self.assertEqual(resp_body, '34Bb') - self.assertTrue(got_called[0]) - self.assertEqual(resp.content_length, 4) - self.assertEqual(resp.content_type, 'text/html') - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], # for incomplete manifest - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d2/subManifest01', {}], - ['GET', '/a/d2/subSeg02', {}], - ['GET', '/a/d1/seg02', {}]]) - - def test_GET_bad_404_manifest_slo(self): - listing = [{"hash": "98568d540134639be4655198a36614a4", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}, - {"hash": "invalidhash", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg03", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - simplejson.dumps(listing), # GET manifest - 'Aa', # GET seg01 - '') # GET seg02 - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # GET listing1 - 200, # GET seg01 - 404, # GET seg02 - headers=[{}, {}, {"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, {}, {}], - body_iter=response_bodies, - give_connect=capture_requested_paths) - req = Request.blank('/v1/a/c/manifest') - resp = controller.GET(req) - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.content_length, 6) # content incomplete - self.assertEqual(resp.content_type, 'text/html') - self.assertRaises(SegmentError, lambda: resp.body) - # dropped connection, exception is caught by eventlet as it is - # iterating over response - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}], - ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/seg02', {}], - ['GET', '/a/d2/seg02', {}], - ['GET', '/a/d2/seg02', {}]]) # 2nd segment not found - - def test_HEAD_manifest_slo(self): - listing = [{"hash": "454dfc73af632012ce3e6217dc464241", - "last_modified": "2012-11-08T04:05:37.866820", - "bytes": 2, - "name": "/d1/seg01", - "content_type": "application/octet-stream"}, - {"hash": "474bab96c67528d42d5c0c52b35228eb", - "last_modified": "2012-11-08T04:05:37.846710", - "bytes": 2, - "name": "/d2/seg02", - "content_type": "application/octet-stream"}] - - response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c - '', # HEAD manifest - simplejson.dumps(listing)) # GET manifest - with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'manifest') - - requested = [] - - def capture_requested_paths(ipaddr, port, device, partition, - method, path, headers=None, - query_string=None): - qs_dict = dict(urlparse.parse_qsl(query_string or '')) - requested.append([method, path, qs_dict]) - - set_http_connect( - 200, # HEAD /a - 200, # HEAD /a/c - 200, # HEAD listing1 - 200, # GET listing1 - headers={"X-Static-Large-Object": "True"}, - body_iter=response_bodies, - give_connect=capture_requested_paths) - - req = Request.blank('/v1/a/c/manifest', - environ={'REQUEST_METHOD': 'HEAD'}) - resp = controller.HEAD(req) - self.assertEqual(resp.status_int, 200) - - self.assertEqual( - requested, - [['HEAD', '/a', {}], - ['HEAD', '/a/c', {}], - ['HEAD', '/a/c/manifest', {}], - ['GET', '/a/c/manifest', {}]]) - def test_PUT_auto_content_type(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', @@ -5776,6 +5126,33 @@ class TestContainerController(unittest.TestCase): controller.HEAD(req) self.assert_(called[0]) + def test_OPTIONS_get_info_drops_origin(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'a', 'c') + + count = [0] + + def my_get_info(app, env, account, container=None, + ret_not_found=False, swift_source=None): + if count[0] > 11: + return {} + count[0] += 1 + if not container: + return {'some': 'stuff'} + return proxy_base.was_get_info( + app, env, account, container, ret_not_found, swift_source) + + proxy_base.was_get_info = proxy_base.get_info + with mock.patch.object(proxy_base, 'get_info', my_get_info): + proxy_base.get_info = my_get_info + req = Request.blank( + '/v1/a/c', + {'REQUEST_METHOD': 'OPTIONS'}, + headers={'Origin': 'http://foo.com', + 'Access-Control-Request-Method': 'GET'}) + controller.OPTIONS(req) + self.assertTrue(count[0] < 11) + def test_OPTIONS(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') diff --git a/tox.ini b/tox.ini index 07f0587..81621a0 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir} NOSE_OPENSTACK_SHOW_ELAPSED=1 NOSE_OPENSTACK_STDOUT=1 deps = - https://launchpad.net/gluster-swift/icehouse/1.11.0/+download/swift-1.11.0.71.gf310006.tar.gz + https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz --download-cache={homedir}/.pipcache -r{toxinidir}/tools/test-requires -r{toxinidir}/tools/requirements.txt -- cgit