summaryrefslogtreecommitdiffstats
path: root/test/functional
diff options
context:
space:
mode:
authorPrashanth Pai <ppai@redhat.com>2015-11-02 11:55:17 +0530
committerThiago da Silva <thiago@redhat.com>2016-03-07 10:38:49 -0800
commitea4750a366123f78411d90082733642376dc6afc (patch)
tree5124b5a407791afcd2dd1cfef00a3959cbb26033 /test/functional
parentc5d76cdd2e2e99d4ac65b645b17cf8a43e4ccab4 (diff)
Rebase to stable/kilo
This change ports most of swiftonfile object server fixes and changes into gluster-swift. Storage policy as a feature is not usable here (it doesn't make sense). The hacky way of creating zero byte tracker objects for object expiration has not been ported to this release due to scalability issues and the need to have a separate volume. Change-Id: I17ba27dacea9ac000bdb8934700996e4d17f4251 Signed-off-by: Prashanth Pai <ppai@redhat.com> Reviewed-on: http://review.gluster.org/13269 Reviewed-by: Thiago da Silva <thiago@redhat.com> Tested-by: Thiago da Silva <thiago@redhat.com>
Diffstat (limited to 'test/functional')
-rw-r--r--test/functional/__init__.py959
-rw-r--r--test/functional/gluster_swift_tests.py3
-rw-r--r--test/functional/swift_test_client.py246
-rwxr-xr-xtest/functional/test_account.py123
-rwxr-xr-xtest/functional/test_container.py549
-rwxr-xr-xtest/functional/test_object.py353
-rw-r--r--test/functional/tests.py1331
7 files changed, 3182 insertions, 382 deletions
diff --git a/test/functional/__init__.py b/test/functional/__init__.py
index e69de29..580de56 100644
--- a/test/functional/__init__.py
+++ b/test/functional/__init__.py
@@ -0,0 +1,959 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import httplib
+import mock
+import os
+import sys
+import pickle
+import socket
+import locale
+import eventlet
+import eventlet.debug
+import functools
+import random
+from ConfigParser import ConfigParser, NoSectionError
+from time import time, sleep
+from httplib import HTTPException
+from urlparse import urlparse
+from nose import SkipTest
+from contextlib import closing
+from gzip import GzipFile
+from shutil import rmtree
+from tempfile import mkdtemp
+from swift.common.middleware.memcache import MemcacheMiddleware
+from swift.common.storage_policy import parse_storage_policies, PolicyError
+
+from test import get_config
+from test.functional.swift_test_client import Account, Connection, \
+ ResponseError
+# This has the side effect of mocking out the xattr module so that unit tests
+# (and in this case, when in-process functional tests are called for) can run
+# on file systems that don't support extended attributes.
+from test.unit import debug_logger, FakeMemcache
+
+from swift.common import constraints, utils, ring, storage_policy
+from swift.common.ring import Ring
+from swift.common.wsgi import monkey_patch_mimetools, loadapp
+from swift.common.utils import config_true_value
+from swift.account import server as account_server
+from swift.container import server as container_server
+from swift.obj import server as object_server, mem_server as mem_object_server
+import swift.proxy.controllers.obj
+
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+DEBUG = True
+
+# In order to get the proper blocking behavior of sockets without using
+# threads, where we can set an arbitrary timeout for some piece of code under
+# test, we use eventlet with the standard socket library patched. We have to
+# perform this setup at module import time, since all the socket module
+# bindings in the swiftclient code will have been made by the time nose
+# invokes the package or class setup methods.
+eventlet.hubs.use_hub(utils.get_hub())
+eventlet.patcher.monkey_patch(all=False, socket=True)
+eventlet.debug.hub_exceptions(False)
+
+from swiftclient import get_auth, http_connection
+
+has_insecure = False
+try:
+ from swiftclient import __version__ as client_version
+ # Prevent a ValueError in StrictVersion with '2.0.3.68.ga99c2ff'
+ client_version = '.'.join(client_version.split('.')[:3])
+except ImportError:
+ # Pre-PBR we had version, not __version__. Anyhow...
+ client_version = '1.2'
+from distutils.version import StrictVersion
+if StrictVersion(client_version) >= StrictVersion('2.0'):
+ has_insecure = True
+
+
+config = {}
+web_front_end = None
+normalized_urls = None
+
+# If no config was read, we will fall back to old school env vars
+swift_test_auth_version = None
+swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
+swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, '', '']
+swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, '', '']
+swift_test_tenant = ['', '', '', '', '']
+swift_test_perm = ['', '', '', '', '']
+swift_test_domain = ['', '', '', '', '']
+swift_test_user_id = ['', '', '', '', '']
+swift_test_tenant_id = ['', '', '', '', '']
+
+skip, skip2, skip3, skip_service_tokens = False, False, False, False
+
+orig_collate = ''
+insecure = False
+
+orig_hash_path_suff_pref = ('', '')
+orig_swift_conf_name = None
+
+in_process = False
+_testdir = _test_servers = _test_coros = None
+
+
+class FakeMemcacheMiddleware(MemcacheMiddleware):
+ """
+ Caching middleware that fakes out caching in swift if memcached
+ does not appear to be running.
+ """
+
+ def __init__(self, app, conf):
+ super(FakeMemcacheMiddleware, self).__init__(app, conf)
+ self.memcache = FakeMemcache()
+
+
+class InProcessException(BaseException):
+ pass
+
+
+def _info(msg):
+ print >> sys.stderr, msg
+
+
+def _debug(msg):
+ if DEBUG:
+ _info('DEBUG: ' + msg)
+
+
+def _in_process_setup_swift_conf(swift_conf_src, testdir):
+ # override swift.conf contents for in-process functional test runs
+ conf = ConfigParser()
+ conf.read(swift_conf_src)
+ try:
+ section = 'swift-hash'
+ conf.set(section, 'swift_hash_path_suffix', 'inprocfunctests')
+ conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests')
+ section = 'swift-constraints'
+ max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2
+ conf.set(section, 'max_file_size', max_file_size)
+ except NoSectionError:
+ msg = 'Conf file %s is missing section %s' % (swift_conf_src, section)
+ raise InProcessException(msg)
+
+ test_conf_file = os.path.join(testdir, 'swift.conf')
+ with open(test_conf_file, 'w') as fp:
+ conf.write(fp)
+
+ return test_conf_file
+
+
+def _in_process_find_conf_file(conf_src_dir, conf_file_name, use_sample=True):
+ """
+ Look for a file first in conf_src_dir, if it exists, otherwise optionally
+ look in the source tree sample 'etc' dir.
+
+ :param conf_src_dir: Directory in which to search first for conf file. May
+ be None
+ :param conf_file_name: Name of conf file
+ :param use_sample: If True and the conf_file_name is not found, then return
+ any sample conf file found in the source tree sample
+ 'etc' dir by appending '-sample' to conf_file_name
+ :returns: Path to conf file
+ :raises InProcessException: If no conf file is found
+ """
+ dflt_src_dir = os.path.normpath(os.path.join(os.path.abspath(__file__),
+ os.pardir, os.pardir, os.pardir,
+ 'etc'))
+ conf_src_dir = dflt_src_dir if conf_src_dir is None else conf_src_dir
+ conf_file_path = os.path.join(conf_src_dir, conf_file_name)
+ if os.path.exists(conf_file_path):
+ return conf_file_path
+
+ if use_sample:
+ # fall back to using the corresponding sample conf file
+ conf_file_name += '-sample'
+ conf_file_path = os.path.join(dflt_src_dir, conf_file_name)
+ if os.path.exists(conf_file_path):
+ return conf_file_path
+
+ msg = 'Failed to find config file %s' % conf_file_name
+ raise InProcessException(msg)
+
+
+def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
+ """
+ If SWIFT_TEST_POLICY is set:
+ - look in swift.conf file for specified policy
+ - move this to be policy-0 but preserving its options
+ - copy its ring file to test dir, changing its devices to suit
+ in process testing, and renaming it to suit policy-0
+ Otherwise, create a default ring file.
+ """
+ conf = ConfigParser()
+ conf.read(swift_conf)
+ sp_prefix = 'storage-policy:'
+
+ try:
+ # policy index 0 will be created if no policy exists in conf
+ policies = parse_storage_policies(conf)
+ except PolicyError as e:
+ raise InProcessException(e)
+
+ # clear all policies from test swift.conf before adding test policy back
+ for policy in policies:
+ conf.remove_section(sp_prefix + str(policy.idx))
+
+ policy_specified = os.environ.get('SWIFT_TEST_POLICY')
+ if policy_specified:
+ policy_to_test = policies.get_by_name(policy_specified)
+ if policy_to_test is None:
+ raise InProcessException('Failed to find policy name "%s"'
+ % policy_specified)
+ _info('Using specified policy %s' % policy_to_test.name)
+ else:
+ policy_to_test = policies.default
+ _info('Defaulting to policy %s' % policy_to_test.name)
+
+ # make policy_to_test be policy index 0 and default for the test config
+ sp_zero_section = sp_prefix + '0'
+ conf.add_section(sp_zero_section)
+ for (k, v) in policy_to_test.get_info(config=True).items():
+ conf.set(sp_zero_section, k, v)
+ conf.set(sp_zero_section, 'default', True)
+
+ with open(swift_conf, 'w') as fp:
+ conf.write(fp)
+
+ # look for a source ring file
+ ring_file_src = ring_file_test = 'object.ring.gz'
+ if policy_to_test.idx:
+ ring_file_src = 'object-%s.ring.gz' % policy_to_test.idx
+ try:
+ ring_file_src = _in_process_find_conf_file(conf_src_dir, ring_file_src,
+ use_sample=False)
+ except InProcessException as e:
+ if policy_specified:
+ raise InProcessException('Failed to find ring file %s'
+ % ring_file_src)
+ ring_file_src = None
+
+ ring_file_test = os.path.join(testdir, ring_file_test)
+ if ring_file_src:
+ # copy source ring file to a policy-0 test ring file, re-homing servers
+ _info('Using source ring file %s' % ring_file_src)
+ ring_data = ring.RingData.load(ring_file_src)
+ obj_sockets = []
+ for dev in ring_data.devs:
+ device = 'sd%c1' % chr(len(obj_sockets) + ord('a'))
+ utils.mkdirs(os.path.join(_testdir, 'sda1'))
+ utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
+ obj_socket = eventlet.listen(('localhost', 0))
+ obj_sockets.append(obj_socket)
+ dev['port'] = obj_socket.getsockname()[1]
+ dev['ip'] = '127.0.0.1'
+ dev['device'] = device
+ dev['replication_port'] = dev['port']
+ dev['replication_ip'] = dev['ip']
+ ring_data.save(ring_file_test)
+ else:
+ # make default test ring, 2 replicas, 4 partitions, 2 devices
+ _info('No source object ring file, creating 2rep/4part/2dev ring')
+ obj_sockets = [eventlet.listen(('localhost', 0)) for _ in (0, 1)]
+ ring_data = ring.RingData(
+ [[0, 1, 0, 1], [1, 0, 1, 0]],
+ [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
+ 'port': obj_sockets[0].getsockname()[1]},
+ {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
+ 'port': obj_sockets[1].getsockname()[1]}],
+ 30)
+ with closing(GzipFile(ring_file_test, 'wb')) as f:
+ pickle.dump(ring_data, f)
+
+ for dev in ring_data.devs:
+ _debug('Ring file dev: %s' % dev)
+
+ return obj_sockets
+
+
+def in_process_setup(the_object_server=object_server):
+ _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
+ _info('Using object_server class: %s' % the_object_server.__name__)
+ conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR')
+
+ if conf_src_dir is not None:
+ if not os.path.isdir(conf_src_dir):
+ msg = 'Config source %s is not a dir' % conf_src_dir
+ raise InProcessException(msg)
+ _info('Using config source dir: %s' % conf_src_dir)
+
+ # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then
+ # prefer config files from there, otherwise read config from source tree
+ # sample files. A mixture of files from the two sources is allowed.
+ proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf')
+ _info('Using proxy config from %s' % proxy_conf)
+ swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
+ _info('Using swift config from %s' % swift_conf_src)
+
+ monkey_patch_mimetools()
+
+ global _testdir
+ _testdir = os.path.join(mkdtemp(), 'tmp_functional')
+ utils.mkdirs(_testdir)
+ rmtree(_testdir)
+ utils.mkdirs(os.path.join(_testdir, 'sda1'))
+ utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
+ utils.mkdirs(os.path.join(_testdir, 'sdb1'))
+ utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
+
+ swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
+ obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)
+
+ global orig_swift_conf_name
+ orig_swift_conf_name = utils.SWIFT_CONF_FILE
+ utils.SWIFT_CONF_FILE = swift_conf
+ constraints.reload_constraints()
+ storage_policy.SWIFT_CONF_FILE = swift_conf
+ storage_policy.reload_storage_policies()
+ global config
+ if constraints.SWIFT_CONSTRAINTS_LOADED:
+ # Use the swift constraints that are loaded for the test framework
+ # configuration
+ _c = dict((k, str(v))
+ for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
+ config.update(_c)
+ else:
+ # In-process swift constraints were not loaded, somethings wrong
+ raise SkipTest
+ global orig_hash_path_suff_pref
+ orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
+ utils.validate_hash_conf()
+
+ # We create the proxy server listening socket to get its port number so
+ # that we can add it as the "auth_port" value for the functional test
+ # clients.
+ prolis = eventlet.listen(('localhost', 0))
+
+ # The following set of configuration values is used both for the
+ # functional test frame work and for the various proxy, account, container
+ # and object servers.
+ config.update({
+ # Values needed by the various in-process swift servers
+ 'devices': _testdir,
+ 'swift_dir': _testdir,
+ 'mount_check': 'false',
+ 'client_timeout': '4',
+ 'allow_account_management': 'true',
+ 'account_autocreate': 'true',
+ 'allow_versions': 'True',
+ # Below are values used by the functional test framework, as well as
+ # by the various in-process swift servers
+ 'auth_host': '127.0.0.1',
+ 'auth_port': str(prolis.getsockname()[1]),
+ 'auth_ssl': 'no',
+ 'auth_prefix': '/auth/',
+ # Primary functional test account (needs admin access to the
+ # account)
+ 'account': 'test',
+ 'username': 'tester',
+ 'password': 'testing',
+ # User on a second account (needs admin access to the account)
+ 'account2': 'test2',
+ 'username2': 'tester2',
+ 'password2': 'testing2',
+ # User on same account as first, but without admin access
+ 'username3': 'tester3',
+ 'password3': 'testing3',
+ # Service user and prefix (emulates glance, cinder, etc. user)
+ 'account5': 'test5',
+ 'username5': 'tester5',
+ 'password5': 'testing5',
+ 'service_prefix': 'SERVICE',
+ # For tempauth middleware. Update reseller_prefix
+ 'reseller_prefix': 'AUTH, SERVICE',
+ 'SERVICE_require_group': 'service'
+ })
+
+ acc1lis = eventlet.listen(('localhost', 0))
+ acc2lis = eventlet.listen(('localhost', 0))
+ con1lis = eventlet.listen(('localhost', 0))
+ con2lis = eventlet.listen(('localhost', 0))
+
+ account_ring_path = os.path.join(_testdir, 'account.ring.gz')
+ with closing(GzipFile(account_ring_path, 'wb')) as f:
+ pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
+ [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
+ 'port': acc1lis.getsockname()[1]},
+ {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
+ 'port': acc2lis.getsockname()[1]}], 30),
+ f)
+ container_ring_path = os.path.join(_testdir, 'container.ring.gz')
+ with closing(GzipFile(container_ring_path, 'wb')) as f:
+ pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
+ [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
+ 'port': con1lis.getsockname()[1]},
+ {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
+ 'port': con2lis.getsockname()[1]}], 30),
+ f)
+
+ eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
+ # Turn off logging requests by the underlying WSGI software.
+ eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
+ logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
+ # Redirect logging other messages by the underlying WSGI software.
+ eventlet.wsgi.HttpProtocol.log_message = \
+ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
+ # Default to only 4 seconds for in-process functional test runs
+ eventlet.wsgi.WRITE_TIMEOUT = 4
+
+ acc1srv = account_server.AccountController(
+ config, logger=debug_logger('acct1'))
+ acc2srv = account_server.AccountController(
+ config, logger=debug_logger('acct2'))
+ con1srv = container_server.ContainerController(
+ config, logger=debug_logger('cont1'))
+ con2srv = container_server.ContainerController(
+ config, logger=debug_logger('cont2'))
+
+ objsrvs = [
+ (obj_sockets[index],
+ the_object_server.ObjectController(
+ config, logger=debug_logger('obj%d' % (index + 1))))
+ for index in range(len(obj_sockets))
+ ]
+
+ logger = debug_logger('proxy')
+
+ def get_logger(name, *args, **kwargs):
+ return logger
+
+ with mock.patch('swift.common.utils.get_logger', get_logger):
+ with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
+ FakeMemcacheMiddleware):
+ try:
+ app = loadapp(proxy_conf, global_conf=config)
+ except Exception as e:
+ raise InProcessException(e)
+
+ nl = utils.NullLogger()
+ prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
+ acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
+ acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
+ con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
+ con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)
+
+ objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl)
+ for objsrv in objsrvs]
+
+ global _test_coros
+ _test_coros = \
+ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa)
+
+ # Create accounts "test" and "test2"
+ def create_account(act):
+ ts = utils.normalize_timestamp(time())
+ account_ring = Ring(_testdir, ring_name='account')
+ partition, nodes = account_ring.get_nodes(act)
+ for node in nodes:
+ # Note: we are just using the http_connect method in the object
+ # controller here to talk to the account server nodes.
+ conn = swift.proxy.controllers.obj.http_connect(
+ node['ip'], node['port'], node['device'], partition, 'PUT',
+ '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
+ resp = conn.getresponse()
+ assert(resp.status == 201)
+
+ create_account('AUTH_test')
+ create_account('AUTH_test2')
+
+cluster_info = {}
+
+
+def get_cluster_info():
+ # The fallback constraints used for testing will come from the current
+ # effective constraints.
+ eff_constraints = dict(constraints.EFFECTIVE_CONSTRAINTS)
+
+ # We'll update those constraints based on what the /info API provides, if
+ # anything.
+ global cluster_info
+ try:
+ conn = Connection(config)
+ conn.authenticate()
+ cluster_info.update(conn.cluster_info())
+ except (ResponseError, socket.error):
+ # Failed to get cluster_information via /info API, so fall back on
+ # test.conf data
+ pass
+ else:
+ try:
+ eff_constraints.update(cluster_info['swift'])
+ except KeyError:
+ # Most likely the swift cluster has "expose_info = false" set
+ # in its proxy-server.conf file, so we'll just do the best we
+ # can.
+ print >>sys.stderr, "** Swift Cluster not exposing /info **"
+
+ # Finally, we'll allow any constraint present in the swift-constraints
+ # section of test.conf to override everything. Note that only those
+ # constraints defined in the constraints module are converted to integers.
+ test_constraints = get_config('swift-constraints')
+ for k in constraints.DEFAULT_CONSTRAINTS:
+ try:
+ test_constraints[k] = int(test_constraints[k])
+ except KeyError:
+ pass
+ except ValueError:
+ print >>sys.stderr, "Invalid constraint value: %s = %s" % (
+ k, test_constraints[k])
+ eff_constraints.update(test_constraints)
+
+ # Just make it look like these constraints were loaded from a /info call,
+ # even if the /info call failed, or when they are overridden by values
+ # from the swift-constraints section of test.conf
+ cluster_info['swift'] = eff_constraints
+
+
+def setup_package():
+ in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS')
+ if in_process_env is not None:
+ use_in_process = utils.config_true_value(in_process_env)
+ else:
+ use_in_process = None
+
+ global in_process
+
+ if use_in_process:
+ # Explicitly set to True, so barrel on ahead with in-process
+ # functional test setup.
+ in_process = True
+ # NOTE: No attempt is made to a read local test.conf file.
+ else:
+ if use_in_process is None:
+ # Not explicitly set, default to using in-process functional tests
+ # if the test.conf file is not found, or does not provide a usable
+ # configuration.
+ config.update(get_config('func_test'))
+ if config:
+ in_process = False
+ else:
+ in_process = True
+ else:
+ # Explicitly set to False, do not attempt to use in-process
+ # functional tests, be sure we attempt to read from local
+ # test.conf file.
+ in_process = False
+ config.update(get_config('func_test'))
+
+ if in_process:
+ in_mem_obj_env = os.environ.get('SWIFT_TEST_IN_MEMORY_OBJ')
+ in_mem_obj = utils.config_true_value(in_mem_obj_env)
+ try:
+ in_process_setup(the_object_server=(
+ mem_object_server if in_mem_obj else object_server))
+ except InProcessException as exc:
+ print >> sys.stderr, ('Exception during in-process setup: %s'
+ % str(exc))
+ raise
+
+ global web_front_end
+ web_front_end = config.get('web_front_end', 'integral')
+ global normalized_urls
+ normalized_urls = config.get('normalized_urls', False)
+
+ global orig_collate
+ orig_collate = locale.setlocale(locale.LC_COLLATE)
+ locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C'))
+
+ global insecure
+ insecure = config_true_value(config.get('insecure', False))
+
+ global swift_test_auth_version
+ global swift_test_auth
+ global swift_test_user
+ global swift_test_key
+ global swift_test_tenant
+ global swift_test_perm
+ global swift_test_domain
+ global swift_test_service_prefix
+
+ swift_test_service_prefix = None
+
+ if config:
+ swift_test_auth_version = str(config.get('auth_version', '1'))
+
+ swift_test_auth = 'http'
+ if config_true_value(config.get('auth_ssl', 'no')):
+ swift_test_auth = 'https'
+ if 'auth_prefix' not in config:
+ config['auth_prefix'] = '/'
+ try:
+ suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % config
+ swift_test_auth += suffix
+ except KeyError:
+ pass # skip
+
+ if 'service_prefix' in config:
+ swift_test_service_prefix = utils.append_underscore(
+ config['service_prefix'])
+
+ if swift_test_auth_version == "1":
+ swift_test_auth += 'v1.0'
+
+ try:
+ if 'account' in config:
+ swift_test_user[0] = '%(account)s:%(username)s' % config
+ else:
+ swift_test_user[0] = '%(username)s' % config
+ swift_test_key[0] = config['password']
+ except KeyError:
+ # bad config, no account/username configured, tests cannot be
+ # run
+ pass
+ try:
+ swift_test_user[1] = '%s%s' % (
+ '%s:' % config['account2'] if 'account2' in config else '',
+ config['username2'])
+ swift_test_key[1] = config['password2']
+ except KeyError:
+ pass # old config, no second account tests can be run
+ try:
+ swift_test_user[2] = '%s%s' % (
+ '%s:' % config['account'] if 'account'
+ in config else '', config['username3'])
+ swift_test_key[2] = config['password3']
+ except KeyError:
+ pass # old config, no third account tests can be run
+ try:
+ swift_test_user[4] = '%s%s' % (
+ '%s:' % config['account5'], config['username5'])
+ swift_test_key[4] = config['password5']
+ swift_test_tenant[4] = config['account5']
+ except KeyError:
+ pass # no service token tests can be run
+
+ for _ in range(3):
+ swift_test_perm[_] = swift_test_user[_]
+
+ else:
+ swift_test_user[0] = config['username']
+ swift_test_tenant[0] = config['account']
+ swift_test_key[0] = config['password']
+ swift_test_user[1] = config['username2']
+ swift_test_tenant[1] = config['account2']
+ swift_test_key[1] = config['password2']
+ swift_test_user[2] = config['username3']
+ swift_test_tenant[2] = config['account']
+ swift_test_key[2] = config['password3']
+ if 'username4' in config:
+ swift_test_user[3] = config['username4']
+ swift_test_tenant[3] = config['account4']
+ swift_test_key[3] = config['password4']
+ swift_test_domain[3] = config['domain4']
+ if 'username5' in config:
+ swift_test_user[4] = config['username5']
+ swift_test_tenant[4] = config['account5']
+ swift_test_key[4] = config['password5']
+
+ for _ in range(5):
+ swift_test_perm[_] = swift_test_tenant[_] + ':' \
+ + swift_test_user[_]
+
+ global skip
+ skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
+ if skip:
+ print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'
+
+ global skip2
+ skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
+ if not skip and skip2:
+ print >>sys.stderr, \
+ 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS' \
+ ' DUE TO NO CONFIG FOR THEM'
+
+ global skip3
+ skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
+ if not skip and skip3:
+ print >>sys.stderr, \
+ 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
+
+ global skip_if_not_v3
+ skip_if_not_v3 = (swift_test_auth_version != '3'
+ or not all([not skip,
+ swift_test_user[3],
+ swift_test_key[3]]))
+ if not skip and skip_if_not_v3:
+ print >>sys.stderr, \
+ 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3'
+
+ global skip_service_tokens
+ skip_service_tokens = not all([not skip, swift_test_user[4],
+ swift_test_key[4], swift_test_tenant[4],
+ swift_test_service_prefix])
+ if not skip and skip_service_tokens:
+ print >>sys.stderr, \
+ 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS'
+
+ get_cluster_info()
+
+
+def teardown_package():
+ global orig_collate
+ locale.setlocale(locale.LC_COLLATE, orig_collate)
+
+ # clean up containers and objects left behind after running tests
+ conn = Connection(config)
+ conn.authenticate()
+ account = Account(conn, config.get('account', config['username']))
+ account.delete_containers()
+
+ global in_process
+ if in_process:
+ try:
+ for server in _test_coros:
+ server.kill()
+ except Exception:
+ pass
+ try:
+ rmtree(os.path.dirname(_testdir))
+ except Exception:
+ pass
+ utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = \
+ orig_hash_path_suff_pref
+ utils.SWIFT_CONF_FILE = orig_swift_conf_name
+ constraints.reload_constraints()
+
+
+class AuthError(Exception):
+ pass
+
+
+class InternalServerError(Exception):
+ pass
+
+
+url = [None, None, None, None, None]
+token = [None, None, None, None, None]
+service_token = [None, None, None, None, None]
+parsed = [None, None, None, None, None]
+conn = [None, None, None, None, None]
+
+
+def connection(url):
+ if has_insecure:
+ return http_connection(url, insecure=insecure)
+ return http_connection(url)
+
+
+def get_url_token(user_index, os_options):
+ authargs = dict(snet=False,
+ tenant_name=swift_test_tenant[user_index],
+ auth_version=swift_test_auth_version,
+ os_options=os_options,
+ insecure=insecure)
+ return get_auth(swift_test_auth,
+ swift_test_user[user_index],
+ swift_test_key[user_index],
+ **authargs)
+
+
+def retry(func, *args, **kwargs):
+ """
+ You can use the kwargs to override:
+ 'retries' (default: 5)
+ 'use_account' (default: 1) - which user's token to pass
+ 'url_account' (default: matches 'use_account') - which user's storage URL
+ 'resource' (default: url[url_account] - URL to connect to; retry()
+ will interpolate the variable :storage_url: if present
+ 'service_user' - add a service token from this user (1 indexed)
+ """
+ global url, token, service_token, parsed, conn
+ retries = kwargs.get('retries', 5)
+ attempts, backoff = 0, 1
+
+ # use account #1 by default; turn user's 1-indexed account into 0-indexed
+ use_account = kwargs.pop('use_account', 1) - 1
+ service_user = kwargs.pop('service_user', None)
+ if service_user:
+ service_user -= 1 # 0-index
+
+ # access our own account by default
+ url_account = kwargs.pop('url_account', use_account + 1) - 1
+ os_options = {'user_domain_name': swift_test_domain[use_account],
+ 'project_domain_name': swift_test_domain[use_account]}
+ while attempts <= retries:
+ auth_failure = False
+ attempts += 1
+ try:
+ if not url[use_account] or not token[use_account]:
+ url[use_account], token[use_account] = get_url_token(
+ use_account, os_options)
+ parsed[use_account] = conn[use_account] = None
+ if not parsed[use_account] or not conn[use_account]:
+ parsed[use_account], conn[use_account] = \
+ connection(url[use_account])
+
+ # default resource is the account url[url_account]
+ resource = kwargs.pop('resource', '%(storage_url)s')
+ template_vars = {'storage_url': url[url_account]}
+ parsed_result = urlparse(resource % template_vars)
+ if isinstance(service_user, int):
+ if not service_token[service_user]:
+ dummy, service_token[service_user] = get_url_token(
+ service_user, os_options)
+ kwargs['service_token'] = service_token[service_user]
+ return func(url[url_account], token[use_account],
+ parsed_result, conn[url_account],
+ *args, **kwargs)
+ except (socket.error, HTTPException):
+ if attempts > retries:
+ raise
+ parsed[use_account] = conn[use_account] = None
+ if service_user:
+ service_token[service_user] = None
+ except AuthError:
+ auth_failure = True
+ url[use_account] = token[use_account] = None
+ if service_user:
+ service_token[service_user] = None
+ except InternalServerError:
+ pass
+ if attempts <= retries:
+ if not auth_failure:
+ sleep(backoff)
+ backoff *= 2
+ raise Exception('No result after %s retries.' % retries)
+
+
+def check_response(conn):
+ resp = conn.getresponse()
+ if resp.status == 401:
+ resp.read()
+ raise AuthError()
+ elif resp.status // 100 == 5:
+ resp.read()
+ raise InternalServerError()
+ return resp
+
+
+def load_constraint(name):
+ global cluster_info
+ try:
+ c = cluster_info['swift'][name]
+ except KeyError:
+ raise SkipTest("Missing constraint: %s" % name)
+ if not isinstance(c, int):
+ raise SkipTest("Bad value, %r, for constraint: %s" % (c, name))
+ return c
+
+
+def get_storage_policy_from_cluster_info(info):
+ policies = info['swift'].get('policies', {})
+ default_policy = []
+ non_default_policies = []
+ for p in policies:
+ if p.get('default', {}):
+ default_policy.append(p)
+ else:
+ non_default_policies.append(p)
+ return default_policy, non_default_policies
+
+
+def reset_acl():
+ def post(url, token, parsed, conn):
+ conn.request('POST', parsed.path, '', {
+ 'X-Auth-Token': token,
+ 'X-Account-Access-Control': '{}'
+ })
+ return check_response(conn)
+ resp = retry(post, use_account=1)
+ resp.read()
+
+
+def requires_acls(f):
+ @functools.wraps(f)
+ def wrapper(*args, **kwargs):
+ global skip, cluster_info
+ if skip or not cluster_info:
+ raise SkipTest('Requires account ACLs')
+ # Determine whether this cluster has account ACLs; if not, skip test
+ if not cluster_info.get('tempauth', {}).get('account_acls'):
+ raise SkipTest('Requires account ACLs')
+ if swift_test_auth_version != '1':
+ # remove when keystoneauth supports account acls
+ raise SkipTest('Requires account ACLs')
+ reset_acl()
+ try:
+ rv = f(*args, **kwargs)
+ finally:
+ reset_acl()
+ return rv
+ return wrapper
+
+
+class FunctionalStoragePolicyCollection(object):
+
+ def __init__(self, policies):
+ self._all = policies
+ self.default = None
+ for p in self:
+ if p.get('default', False):
+ assert self.default is None, 'Found multiple default ' \
+ 'policies %r and %r' % (self.default, p)
+ self.default = p
+
+ @classmethod
+ def from_info(cls, info=None):
+ if not (info or cluster_info):
+ get_cluster_info()
+ info = info or cluster_info
+ try:
+ policy_info = info['swift']['policies']
+ except KeyError:
+ raise AssertionError('Did not find any policy info in %r' % info)
+ policies = cls(policy_info)
+ assert policies.default, \
+ 'Did not find default policy in %r' % policy_info
+ return policies
+
+ def __len__(self):
+ return len(self._all)
+
+ def __iter__(self):
+ return iter(self._all)
+
+ def __getitem__(self, index):
+ return self._all[index]
+
+ def filter(self, **kwargs):
+ return self.__class__([p for p in self if all(
+ p.get(k) == v for k, v in kwargs.items())])
+
+ def exclude(self, **kwargs):
+ return self.__class__([p for p in self if all(
+ p.get(k) != v for k, v in kwargs.items())])
+
+ def select(self):
+ return random.choice(self)
+
+
+def requires_policies(f):
+ @functools.wraps(f)
+ def wrapper(self, *args, **kwargs):
+ if skip:
+ raise SkipTest
+ try:
+ self.policies = FunctionalStoragePolicyCollection.from_info()
+ except AssertionError:
+ raise SkipTest("Unable to determine available policies")
+ if len(self.policies) < 2:
+ raise SkipTest("Multiple policies not enabled")
+ return f(self, *args, **kwargs)
+
+ return wrapper
diff --git a/test/functional/gluster_swift_tests.py b/test/functional/gluster_swift_tests.py
index b4514c9..2ffb841 100644
--- a/test/functional/gluster_swift_tests.py
+++ b/test/functional/gluster_swift_tests.py
@@ -19,8 +19,9 @@ import random
import os,sys,re,hashlib
from nose import SkipTest
-from test.functional.tests import config, locale, Base, Base2, Utils, \
+from test.functional.tests import Base, Base2, Utils, \
TestFileEnv
+from test.functional import config, locale
from test.functional.swift_test_client import Account, Connection, File, \
ResponseError
diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py
index 27e025b..5c0ab87 100644
--- a/test/functional/swift_test_client.py
+++ b/test/functional/swift_test_client.py
@@ -26,10 +26,16 @@ import simplejson as json
from nose import SkipTest
from xml.dom import minidom
+
from swiftclient import get_auth
+from swift.common import constraints
+from swift.common.utils import config_true_value
+
from test import safe_repr
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
class AuthenticationFailed(Exception):
pass
@@ -103,11 +109,13 @@ class Connection(object):
def __init__(self, config):
for key in 'auth_host auth_port auth_ssl username password'.split():
if key not in config:
- raise SkipTest
+ raise SkipTest(
+ "Missing required configuration parameter: %s" % key)
self.auth_host = config['auth_host']
self.auth_port = int(config['auth_port'])
self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1')
+ self.insecure = config_true_value(config.get('insecure', 'false'))
self.auth_prefix = config.get('auth_prefix', '/')
self.auth_version = str(config.get('auth_version', '1'))
@@ -117,6 +125,7 @@ class Connection(object):
self.storage_host = None
self.storage_port = None
+ self.storage_url = None
self.conn_class = None
@@ -145,10 +154,11 @@ class Connection(object):
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
+ authargs = dict(snet=False, tenant_name=self.account,
+ auth_version=self.auth_version, os_options={},
+ insecure=self.insecure)
(storage_url, storage_token) = get_auth(
- auth_url, auth_user, self.password, snet=False,
- tenant_name=self.account, auth_version=self.auth_version,
- os_options={})
+ auth_url, auth_user, self.password, **authargs)
if not (storage_url and storage_token):
raise AuthenticationFailed()
@@ -172,8 +182,14 @@ class Connection(object):
# unicode and this would cause troubles when doing
# no_safe_quote query.
self.storage_url = str('/%s/%s' % (x[3], x[4]))
-
- self.storage_token = storage_token
+ self.account_name = str(x[4])
+ self.auth_user = auth_user
+ # With v2 keystone, storage_token is unicode.
+ # We want it to be string otherwise this would cause
+ # troubles when doing query with already encoded
+ # non ascii characters in its headers.
+ self.storage_token = str(storage_token)
+ self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_url, self.storage_token
@@ -184,7 +200,7 @@ class Connection(object):
"""
status = self.make_request('GET', '/info',
cfg={'absolute_path': True})
- if status == 404:
+ if status // 100 == 4:
return {}
if not 200 <= status <= 299:
raise ResponseError(self.response, 'GET', '/info')
@@ -195,7 +211,12 @@ class Connection(object):
port=self.storage_port)
#self.connection.set_debuglevel(3)
- def make_path(self, path=[], cfg={}):
+ def make_path(self, path=None, cfg=None):
+ if path is None:
+ path = []
+ if cfg is None:
+ cfg = {}
+
if cfg.get('version_only_path'):
return '/' + self.storage_url.split('/')[1]
@@ -208,7 +229,9 @@ class Connection(object):
else:
return self.storage_url
- def make_headers(self, hdrs, cfg={}):
+ def make_headers(self, hdrs, cfg=None):
+ if cfg is None:
+ cfg = {}
headers = {}
if not cfg.get('no_auth_token'):
@@ -218,8 +241,16 @@ class Connection(object):
headers.update(hdrs)
return headers
- def make_request(self, method, path=[], data='', hdrs={}, parms={},
- cfg={}):
+ def make_request(self, method, path=None, data='', hdrs=None, parms=None,
+ cfg=None):
+ if path is None:
+ path = []
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
if not cfg.get('absolute_path'):
# Set absolute_path=True to make a request to exactly the given
# path, not storage path + given path. Useful for
@@ -277,7 +308,14 @@ class Connection(object):
'Attempts: %s, Failures: %s' %
(request, len(fail_messages), fail_messages))
- def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False):
+ def put_start(self, path, hdrs=None, parms=None, cfg=None, chunked=False):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
self.http_connect()
path = self.make_path(path, cfg)
@@ -322,7 +360,10 @@ class Base(object):
def __str__(self):
return self.name
- def header_fields(self, required_fields, optional_fields=()):
+ def header_fields(self, required_fields, optional_fields=None):
+ if optional_fields is None:
+ optional_fields = ()
+
headers = dict(self.conn.response.getheaders())
ret = {}
@@ -352,7 +393,11 @@ class Account(Base):
self.conn = conn
self.name = str(name)
- def update_metadata(self, metadata={}, cfg={}):
+ def update_metadata(self, metadata=None, cfg=None):
+ if metadata is None:
+ metadata = {}
+ if cfg is None:
+ cfg = {}
headers = dict(("X-Account-Meta-%s" % k, v)
for k, v in metadata.items())
@@ -365,7 +410,14 @@ class Account(Base):
def container(self, container_name):
return Container(self.conn, self.name, container_name)
- def containers(self, hdrs={}, parms={}, cfg={}):
+ def containers(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
format_type = parms.get('format', None)
if format_type not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format_type)
@@ -411,7 +463,13 @@ class Account(Base):
return listing_empty(self.containers)
- def info(self, hdrs={}, parms={}, cfg={}):
+ def info(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
parms=parms, cfg=cfg) != 204:
@@ -435,11 +493,21 @@ class Container(Base):
self.account = str(account)
self.name = str(name)
- def create(self, hdrs={}, parms={}, cfg={}):
+ def create(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
return self.conn.make_request('PUT', self.path, hdrs=hdrs,
parms=parms, cfg=cfg) in (201, 202)
- def delete(self, hdrs={}, parms={}):
+ def delete(self, hdrs=None, parms=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
return self.conn.make_request('DELETE', self.path, hdrs=hdrs,
parms=parms) == 204
@@ -457,7 +525,13 @@ class Container(Base):
def file(self, file_name):
return File(self.conn, self.account, self.name, file_name)
- def files(self, hdrs={}, parms={}, cfg={}):
+ def files(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
format_type = parms.get('format', None)
if format_type not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format_type)
@@ -507,14 +581,23 @@ class Container(Base):
raise ResponseError(self.conn.response, 'GET',
self.conn.make_path(self.path))
- def info(self, hdrs={}, parms={}, cfg={}):
+ def info(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
self.conn.make_request('HEAD', self.path, hdrs=hdrs,
parms=parms, cfg=cfg)
if self.conn.response.status == 204:
required_fields = [['bytes_used', 'x-container-bytes-used'],
['object_count', 'x-container-object-count']]
- optional_fields = [['versions', 'x-versions-location']]
+ optional_fields = [
+ ['versions', 'x-versions-location'],
+ ['tempurl_key', 'x-container-meta-temp-url-key'],
+ ['tempurl_key2', 'x-container-meta-temp-url-key-2']]
return self.header_fields(required_fields, optional_fields)
@@ -538,7 +621,9 @@ class File(Base):
self.size = None
self.metadata = {}
- def make_headers(self, cfg={}):
+ def make_headers(self, cfg=None):
+ if cfg is None:
+ cfg = {}
headers = {}
if not cfg.get('no_content_length'):
if cfg.get('set_content_length'):
@@ -580,7 +665,13 @@ class File(Base):
data.seek(0)
return checksum.hexdigest()
- def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}):
+ def copy(self, dest_cont, dest_file, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
if 'destination' in cfg:
headers = {'Destination': cfg['destination']}
elif cfg.get('no_destination'):
@@ -595,7 +686,37 @@ class File(Base):
return self.conn.make_request('COPY', self.path, hdrs=headers,
parms=parms) == 201
- def delete(self, hdrs={}, parms={}):
+ def copy_account(self, dest_account, dest_cont, dest_file,
+ hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+ if 'destination' in cfg:
+ headers = {'Destination': cfg['destination']}
+ elif cfg.get('no_destination'):
+ headers = {}
+ else:
+ headers = {'Destination-Account': dest_account,
+ 'Destination': '%s/%s' % (dest_cont, dest_file)}
+ headers.update(hdrs)
+
+ if 'Destination-Account' in headers:
+ headers['Destination-Account'] = \
+ urllib.quote(headers['Destination-Account'])
+ if 'Destination' in headers:
+ headers['Destination'] = urllib.quote(headers['Destination'])
+
+ return self.conn.make_request('COPY', self.path, hdrs=headers,
+ parms=parms) == 201
+
+ def delete(self, hdrs=None, parms=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
if self.conn.make_request('DELETE', self.path, hdrs=hdrs,
parms=parms) != 204:
@@ -604,7 +725,13 @@ class File(Base):
return True
- def info(self, hdrs={}, parms={}, cfg={}):
+ def info(self, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
parms=parms, cfg=cfg) != 200:
@@ -615,8 +742,8 @@ class File(Base):
['content_type', 'content-type'],
['last_modified', 'last-modified'],
['etag', 'etag']]
-
- optional_fields = [['x_delete_at', 'x-delete-at'],
+ optional_fields = [['x_object_manifest', 'x-object-manifest'],
+ ['x_delete_at', 'x-delete-at'],
['x_delete_after', 'x-delete-after']]
header_fields = self.header_fields(fields,
@@ -624,7 +751,11 @@ class File(Base):
header_fields['etag'] = header_fields['etag'].strip('"')
return header_fields
- def initialize(self, hdrs={}, parms={}):
+ def initialize(self, hdrs=None, parms=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
if not self.name:
return False
@@ -669,7 +800,11 @@ class File(Base):
return data
def read(self, size=-1, offset=0, hdrs=None, buffer=None,
- callback=None, cfg={}, parms={}):
+ callback=None, cfg=None, parms=None):
+ if cfg is None:
+ cfg = {}
+ if parms is None:
+ parms = {}
if size > 0:
range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1)
@@ -726,7 +861,12 @@ class File(Base):
finally:
fobj.close()
- def sync_metadata(self, metadata={}, cfg={}):
+ def sync_metadata(self, metadata=None, cfg=None):
+ if metadata is None:
+ metadata = {}
+ if cfg is None:
+ cfg = {}
+
self.metadata.update(metadata)
if self.metadata:
@@ -737,6 +877,7 @@ class File(Base):
cfg.get('set_content_length')
else:
headers['Content-Length'] = 0
+
self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
if self.conn.response.status not in (201, 202):
@@ -745,7 +886,14 @@ class File(Base):
return True
- def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}):
+ def chunked_write(self, data=None, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
if data is not None and self.chunked_write_in_progress:
self.conn.put_data(data, True)
elif data is not None:
@@ -764,8 +912,15 @@ class File(Base):
else:
raise RuntimeError
- def write(self, data='', hdrs={}, parms={}, callback=None, cfg={},
+ def write(self, data='', hdrs=None, parms=None, callback=None, cfg=None,
return_resp=False):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
block_size = 2 ** 20
if isinstance(data, file):
@@ -786,13 +941,15 @@ class File(Base):
transferred = 0
buff = data.read(block_size)
+ buff_len = len(buff)
try:
- while len(buff) > 0:
+ while buff_len > 0:
self.conn.put_data(buff)
- buff = data.read(block_size)
- transferred += len(buff)
+ transferred += buff_len
if callable(callback):
callback(transferred, self.size)
+ buff = data.read(block_size)
+ buff_len = len(buff)
self.conn.put_end()
except socket.timeout as err:
@@ -814,7 +971,14 @@ class File(Base):
return True
- def write_random(self, size=None, hdrs={}, parms={}, cfg={}):
+ def write_random(self, size=None, hdrs=None, parms=None, cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
data = self.random_data(size)
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
raise ResponseError(self.conn.response, 'PUT',
@@ -822,7 +986,15 @@ class File(Base):
self.md5 = self.compute_md5sum(StringIO.StringIO(data))
return data
- def write_random_return_resp(self, size=None, hdrs={}, parms={}, cfg={}):
+ def write_random_return_resp(self, size=None, hdrs=None, parms=None,
+ cfg=None):
+ if hdrs is None:
+ hdrs = {}
+ if parms is None:
+ parms = {}
+ if cfg is None:
+ cfg = {}
+
data = self.random_data(size)
resp = self.write(data, hdrs=hdrs, parms=parms, cfg=cfg,
return_resp=True)
diff --git a/test/functional/test_account.py b/test/functional/test_account.py
index 1cc61bc..30a8e74 100755
--- a/test/functional/test_account.py
+++ b/test/functional/test_account.py
@@ -21,13 +21,11 @@ from uuid import uuid4
from nose import SkipTest
from string import letters
-from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
- MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
from swift.common.middleware.acl import format_acl
-from swift_testing import (check_response, retry, skip, skip2, skip3,
- web_front_end, requires_acls)
-import swift_testing
-from test.functional.tests import load_constraint
+
+from test.functional import check_response, retry, requires_acls, \
+ load_constraint
+import test.functional as tf
class TestAccount(unittest.TestCase):
@@ -69,7 +67,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status // 100, 2)
def test_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, value):
@@ -109,6 +107,9 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.getheader('x-account-meta-test'), 'Value')
def test_invalid_acls(self):
+ if tf.skip:
+ raise SkipTest
+
def post(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('POST', parsed.path, '', new_headers)
@@ -145,7 +146,7 @@ class TestAccount(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
- acl_user = swift_testing.swift_test_user[1]
+ acl_user = tf.swift_test_user[1]
acl = {'admin': [acl_user], 'invalid_key': 'invalid_value'}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
@@ -173,7 +174,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_read_only_acl(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -191,7 +192,7 @@ class TestAccount(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read access
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
@@ -224,7 +225,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_read_write_acl(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -242,7 +243,7 @@ class TestAccount(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-write access
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
@@ -265,7 +266,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_admin_acl(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -283,7 +284,7 @@ class TestAccount(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant admin access
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
acl_json_str = format_acl(version=2, acl_dict=acl)
headers = {'x-account-access-control': acl_json_str}
@@ -323,7 +324,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_protected_tempurl(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -335,7 +336,7 @@ class TestAccount(unittest.TestCase):
conn.request('POST', parsed.path, '', new_headers)
return check_response(conn)
- # add a account metadata, and temp-url-key to account
+ # add an account metadata, and temp-url-key to account
value = str(uuid4())
headers = {
'x-account-meta-temp-url-key': 'secret',
@@ -346,7 +347,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204)
# grant read-only access to tester3
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
acl_json_str = format_acl(version=2, acl_dict=acl)
headers = {'x-account-access-control': acl_json_str}
@@ -364,7 +365,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None)
# grant read-write access to tester3
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
acl_json_str = format_acl(version=2, acl_dict=acl)
headers = {'x-account-access-control': acl_json_str}
@@ -382,7 +383,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None)
# grant admin access to tester3
- acl_user = swift_testing.swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
acl_json_str = format_acl(version=2, acl_dict=acl)
headers = {'x-account-access-control': acl_json_str}
@@ -417,7 +418,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_account_acls(self):
- if skip2:
+ if tf.skip2:
raise SkipTest
def post(url, token, parsed, conn, headers):
@@ -464,7 +465,7 @@ class TestAccount(unittest.TestCase):
# User1 is swift_owner of their own account, so they can POST an
# ACL -- let's do this and make User2 (test_user[1]) an admin
- acl_user = swift_testing.swift_test_user[1]
+ acl_user = tf.swift_test_user[1]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
@@ -541,7 +542,7 @@ class TestAccount(unittest.TestCase):
@requires_acls
def test_swift_account_acls(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, headers):
@@ -604,7 +605,7 @@ class TestAccount(unittest.TestCase):
resp.read()
def test_swift_prohibits_garbage_account_acls(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, headers):
@@ -671,7 +672,7 @@ class TestAccount(unittest.TestCase):
resp.read()
def test_unicode_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
@@ -684,7 +685,7 @@ class TestAccount(unittest.TestCase):
return check_response(conn)
uni_key = u'X-Account-Meta-uni\u0E12'
uni_value = u'uni\u0E12'
- if (web_front_end == 'integral'):
+ if (tf.web_front_end == 'integral'):
resp = retry(post, uni_key, '1')
resp.read()
self.assertTrue(resp.status in (201, 204))
@@ -700,7 +701,7 @@ class TestAccount(unittest.TestCase):
self.assert_(resp.status in (200, 204), resp.status)
self.assertEqual(resp.getheader('X-Account-Meta-uni'),
uni_value.encode('utf-8'))
- if (web_front_end == 'integral'):
+ if (tf.web_front_end == 'integral'):
resp = retry(post, uni_key, uni_value)
resp.read()
self.assertEqual(resp.status, 204)
@@ -711,7 +712,7 @@ class TestAccount(unittest.TestCase):
uni_value.encode('utf-8'))
def test_multi_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
@@ -740,7 +741,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.getheader('x-account-meta-two'), '2')
def test_bad_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -750,27 +751,31 @@ class TestAccount(unittest.TestCase):
return check_response(conn)
resp = retry(post,
- {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
+ {'X-Account-Meta-' + (
+ 'k' * self.max_meta_name_length): 'v'})
resp.read()
self.assertEqual(resp.status, 204)
resp = retry(
post,
- {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
+ {'X-Account-Meta-' + ('k' * (
+ self.max_meta_name_length + 1)): 'v'})
resp.read()
self.assertEqual(resp.status, 400)
resp = retry(post,
- {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
+ {'X-Account-Meta-Too-Long': (
+ 'k' * self.max_meta_value_length)})
resp.read()
self.assertEqual(resp.status, 204)
resp = retry(
post,
- {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
+ {'X-Account-Meta-Too-Long': 'k' * (
+ self.max_meta_value_length + 1)})
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata2(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -785,20 +790,20 @@ class TestAccount(unittest.TestCase):
resp = retry(post, headers)
headers = {}
- for x in xrange(MAX_META_COUNT):
+ for x in xrange(self.max_meta_count):
headers['X-Account-Meta-%d' % x] = 'v'
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
headers = {}
- for x in xrange(MAX_META_COUNT + 1):
+ for x in xrange(self.max_meta_count + 1):
headers['X-Account-Meta-%d' % x] = 'v'
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata3(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -807,31 +812,55 @@ class TestAccount(unittest.TestCase):
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
- # TODO: Find the test that adds these and remove them.
- headers = {'x-remove-account-meta-temp-url-key': 'remove',
- 'x-remove-account-meta-temp-url-key-2': 'remove'}
- resp = retry(post, headers)
-
headers = {}
- header_value = 'k' * MAX_META_VALUE_LENGTH
+ header_value = 'k' * self.max_meta_value_length
size = 0
x = 0
- while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
- size += 4 + MAX_META_VALUE_LENGTH
+ while size < (self.max_meta_overall_size - 4
+ - self.max_meta_value_length):
+ size += 4 + self.max_meta_value_length
headers['X-Account-Meta-%04d' % x] = header_value
x += 1
- if MAX_META_OVERALL_SIZE - size > 1:
+ if self.max_meta_overall_size - size > 1:
headers['X-Account-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size - 1)
+ 'v' * (self.max_meta_overall_size - size - 1)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
headers['X-Account-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size)
+ 'v' * (self.max_meta_overall_size - size)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
+class TestAccountInNonDefaultDomain(unittest.TestCase):
+ def setUp(self):
+ if tf.skip or tf.skip2 or tf.skip_if_not_v3:
+ raise SkipTest('AUTH VERSION 3 SPECIFIC TEST')
+
+ def test_project_domain_id_header(self):
+ # make sure account exists (assumes account auto create)
+ def post(url, token, parsed, conn):
+ conn.request('POST', parsed.path, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(post, use_account=4)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # account in non-default domain should have a project domain id
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(head, use_account=4)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+ self.assertTrue('X-Account-Project-Domain-Id' in resp.headers)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/functional/test_container.py b/test/functional/test_container.py
index 91702e9..d7896a4 100755
--- a/test/functional/test_container.py
+++ b/test/functional/test_container.py
@@ -20,19 +20,19 @@ import unittest
from nose import SkipTest
from uuid import uuid4
-from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
- MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
-
-from swift_testing import check_response, retry, skip, skip2, skip3, \
- swift_test_perm, web_front_end, requires_acls, swift_test_user
+from test.functional import check_response, retry, requires_acls, \
+ load_constraint, requires_policies
+import test.functional as tf
class TestContainer(unittest.TestCase):
def setUp(self):
- if skip:
+ if tf.skip:
raise SkipTest
self.name = uuid4().hex
+ # this container isn't created by default, but will be cleaned up
+ self.container = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.name, '',
@@ -43,44 +43,58 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
+ self.max_meta_count = load_constraint('max_meta_count')
+ self.max_meta_name_length = load_constraint('max_meta_name_length')
+ self.max_meta_overall_size = load_constraint('max_meta_overall_size')
+ self.max_meta_value_length = load_constraint('max_meta_value_length')
+
def tearDown(self):
- if skip:
+ if tf.skip:
raise SkipTest
- def get(url, token, parsed, conn):
- conn.request('GET', parsed.path + '/' + self.name + '?format=json',
- '', {'X-Auth-Token': token})
- return check_response(conn)
-
- def delete(url, token, parsed, conn, obj):
- conn.request('DELETE',
- '/'.join([parsed.path, self.name, obj['name']]), '',
- {'X-Auth-Token': token})
- return check_response(conn)
-
- while True:
- resp = retry(get)
- body = resp.read()
- self.assert_(resp.status // 100 == 2, resp.status)
- objs = json.loads(body)
- if not objs:
- break
- for obj in objs:
- resp = retry(delete, obj)
- resp.read()
- self.assertEqual(resp.status, 204)
-
- def delete(url, token, parsed, conn):
- conn.request('DELETE', parsed.path + '/' + self.name, '',
+ def get(url, token, parsed, conn, container):
+ conn.request(
+ 'GET', parsed.path + '/' + container + '?format=json', '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ def delete(url, token, parsed, conn, container, obj):
+ conn.request(
+ 'DELETE', '/'.join([parsed.path, container, obj['name']]), '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ for container in (self.name, self.container):
+ while True:
+ resp = retry(get, container)
+ body = resp.read()
+ if resp.status == 404:
+ break
+ self.assert_(resp.status // 100 == 2, resp.status)
+ objs = json.loads(body)
+ if not objs:
+ break
+ for obj in objs:
+ resp = retry(delete, container, obj)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ def delete(url, token, parsed, conn, container):
+ conn.request('DELETE', parsed.path + '/' + container, '',
{'X-Auth-Token': token})
return check_response(conn)
- resp = retry(delete)
+ resp = retry(delete, self.name)
resp.read()
self.assertEqual(resp.status, 204)
+ # container may have not been created
+ resp = retry(delete, self.container)
+ resp.read()
+ self.assert_(resp.status in (204, 404))
+
def test_multi_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
@@ -110,7 +124,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('x-container-meta-two'), '2')
def test_unicode_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
@@ -125,7 +139,7 @@ class TestContainer(unittest.TestCase):
uni_key = u'X-Container-Meta-uni\u0E12'
uni_value = u'uni\u0E12'
- if (web_front_end == 'integral'):
+ if (tf.web_front_end == 'integral'):
resp = retry(post, uni_key, '1')
resp.read()
self.assertEqual(resp.status, 204)
@@ -141,7 +155,7 @@ class TestContainer(unittest.TestCase):
self.assert_(resp.status in (200, 204), resp.status)
self.assertEqual(resp.getheader('X-Container-Meta-uni'),
uni_value.encode('utf-8'))
- if (web_front_end == 'integral'):
+ if (tf.web_front_end == 'integral'):
resp = retry(post, uni_key, uni_value)
resp.read()
self.assertEqual(resp.status, 204)
@@ -152,7 +166,7 @@ class TestContainer(unittest.TestCase):
uni_value.encode('utf-8'))
def test_PUT_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn, name, value):
@@ -209,7 +223,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204)
def test_POST_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, value):
@@ -249,7 +263,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('x-container-meta-test'), 'Value')
def test_PUT_bad_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn, name, extra_headers):
@@ -266,7 +280,7 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
resp = retry(
put, name,
- {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
+ {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'})
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(delete, name)
@@ -275,7 +289,8 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
resp = retry(
put, name,
- {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
+ {'X-Container-Meta-' + (
+ 'k' * (self.max_meta_name_length + 1)): 'v'})
resp.read()
self.assertEqual(resp.status, 400)
resp = retry(delete, name)
@@ -285,7 +300,7 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
resp = retry(
put, name,
- {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
+ {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length})
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(delete, name)
@@ -294,7 +309,8 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
resp = retry(
put, name,
- {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
+ {'X-Container-Meta-Too-Long': 'k' * (
+ self.max_meta_value_length + 1)})
resp.read()
self.assertEqual(resp.status, 400)
resp = retry(delete, name)
@@ -303,7 +319,7 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
headers = {}
- for x in xrange(MAX_META_COUNT):
+ for x in xrange(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(put, name, headers)
resp.read()
@@ -313,7 +329,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204)
name = uuid4().hex
headers = {}
- for x in xrange(MAX_META_COUNT + 1):
+ for x in xrange(self.max_meta_count + 1):
headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(put, name, headers)
resp.read()
@@ -324,16 +340,17 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex
headers = {}
- header_value = 'k' * MAX_META_VALUE_LENGTH
+ header_value = 'k' * self.max_meta_value_length
size = 0
x = 0
- while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
- size += 4 + MAX_META_VALUE_LENGTH
+ while size < (self.max_meta_overall_size - 4
+ - self.max_meta_value_length):
+ size += 4 + self.max_meta_value_length
headers['X-Container-Meta-%04d' % x] = header_value
x += 1
- if MAX_META_OVERALL_SIZE - size > 1:
+ if self.max_meta_overall_size - size > 1:
headers['X-Container-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size - 1)
+ 'v' * (self.max_meta_overall_size - size - 1)
resp = retry(put, name, headers)
resp.read()
self.assertEqual(resp.status, 201)
@@ -342,7 +359,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204)
name = uuid4().hex
headers['X-Container-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size)
+ 'v' * (self.max_meta_overall_size - size)
resp = retry(put, name, headers)
resp.read()
self.assertEqual(resp.status, 400)
@@ -351,7 +368,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 404)
def test_POST_bad_metadata(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -362,28 +379,30 @@ class TestContainer(unittest.TestCase):
resp = retry(
post,
- {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
+ {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'})
resp.read()
self.assertEqual(resp.status, 204)
resp = retry(
post,
- {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
+ {'X-Container-Meta-' + (
+ 'k' * (self.max_meta_name_length + 1)): 'v'})
resp.read()
self.assertEqual(resp.status, 400)
resp = retry(
post,
- {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
+ {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length})
resp.read()
self.assertEqual(resp.status, 204)
resp = retry(
post,
- {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
+ {'X-Container-Meta-Too-Long': 'k' * (
+ self.max_meta_value_length + 1)})
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata2(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -393,20 +412,20 @@ class TestContainer(unittest.TestCase):
return check_response(conn)
headers = {}
- for x in xrange(MAX_META_COUNT):
+ for x in xrange(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
headers = {}
- for x in xrange(MAX_META_COUNT + 1):
+ for x in xrange(self.max_meta_count + 1):
headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata3(self):
- if skip:
+ if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
@@ -416,27 +435,28 @@ class TestContainer(unittest.TestCase):
return check_response(conn)
headers = {}
- header_value = 'k' * MAX_META_VALUE_LENGTH
+ header_value = 'k' * self.max_meta_value_length
size = 0
x = 0
- while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
- size += 4 + MAX_META_VALUE_LENGTH
+ while size < (self.max_meta_overall_size - 4
+ - self.max_meta_value_length):
+ size += 4 + self.max_meta_value_length
headers['X-Container-Meta-%04d' % x] = header_value
x += 1
- if MAX_META_OVERALL_SIZE - size > 1:
+ if self.max_meta_overall_size - size > 1:
headers['X-Container-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size - 1)
+ 'v' * (self.max_meta_overall_size - size - 1)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
headers['X-Container-Meta-k'] = \
- 'v' * (MAX_META_OVERALL_SIZE - size)
+ 'v' * (self.max_meta_overall_size - size)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
def test_public_container(self):
- if skip:
+ if tf.skip:
raise SkipTest
def get(url, token, parsed, conn):
@@ -477,7 +497,7 @@ class TestContainer(unittest.TestCase):
self.assert_(str(err).startswith('No result after '), err)
def test_cross_account_container(self):
- if skip or skip2:
+ if tf.skip or tf.skip2:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
@@ -505,8 +525,8 @@ class TestContainer(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
- 'X-Container-Read': swift_test_perm[1],
- 'X-Container-Write': swift_test_perm[1]})
+ 'X-Container-Read': tf.swift_test_perm[1],
+ 'X-Container-Write': tf.swift_test_perm[1]})
return check_response(conn)
resp = retry(post)
@@ -533,7 +553,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 403)
def test_cross_account_public_container(self):
- if skip or skip2:
+ if tf.skip or tf.skip2:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
@@ -586,7 +606,7 @@ class TestContainer(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
- 'X-Container-Write': swift_test_perm[1]})
+ 'X-Container-Write': tf.swift_test_perm[1]})
return check_response(conn)
resp = retry(post)
@@ -602,7 +622,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 201)
def test_nonadmin_user(self):
- if skip or skip3:
+ if tf.skip or tf.skip3:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
@@ -630,7 +650,7 @@ class TestContainer(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
- 'X-Container-Read': swift_test_perm[2]})
+ 'X-Container-Read': tf.swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
@@ -655,7 +675,7 @@ class TestContainer(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
- 'X-Container-Write': swift_test_perm[2]})
+ 'X-Container-Write': tf.swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
@@ -672,7 +692,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_read_only_acl_listings(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -695,7 +715,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-only access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -725,7 +745,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_read_only_acl_metadata(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn, name):
@@ -760,7 +780,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-only access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -782,7 +802,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_read_write_acl_listings(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -810,7 +830,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-write access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post, headers=headers, use_account=1)
@@ -853,7 +873,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_read_write_acl_metadata(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn, name):
@@ -888,7 +908,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-write access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -924,7 +944,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_admin_acl_listing(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn):
@@ -952,7 +972,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant admin access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post, headers=headers, use_account=1)
@@ -995,7 +1015,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_admin_acl_metadata(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn, name):
@@ -1030,7 +1050,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1066,7 +1086,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_protected_container_sync(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn, name):
@@ -1100,7 +1120,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# grant read-only access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1122,7 +1142,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 403)
# grant read-write access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1160,7 +1180,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret')
# grant admin access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1188,7 +1208,7 @@ class TestContainer(unittest.TestCase):
@requires_acls
def test_protected_container_acl(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get(url, token, parsed, conn, name):
@@ -1224,7 +1244,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# grant read-only access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1250,7 +1270,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 403)
# grant read-write access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1292,7 +1312,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe')
# grant admin access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -1322,7 +1342,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.getheader('X-Container-Read'), '.r:*')
def test_long_name_content_type(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn):
@@ -1338,7 +1358,7 @@ class TestContainer(unittest.TestCase):
'text/html; charset=UTF-8')
def test_null_name(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn):
@@ -1347,12 +1367,343 @@ class TestContainer(unittest.TestCase):
return check_response(conn)
resp = retry(put)
- if (web_front_end == 'apache2'):
+ if (tf.web_front_end == 'apache2'):
self.assertEqual(resp.status, 404)
else:
self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL')
self.assertEqual(resp.status, 412)
+ def test_create_container_gets_default_policy_by_default(self):
+ try:
+ default_policy = \
+ tf.FunctionalStoragePolicyCollection.from_info().default
+ except AssertionError:
+ raise SkipTest()
+
+ def put(url, token, parsed, conn):
+ conn.request('PUT', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(put)
+ resp.read()
+ self.assertEqual(resp.status // 100, 2)
+
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'),
+ default_policy['name'])
+
+ def test_error_invalid_storage_policy_name(self):
+ def put(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('PUT', parsed.path + '/' + self.container, '',
+ new_headers)
+ return check_response(conn)
+
+ # create
+ resp = retry(put, {'X-Storage-Policy': uuid4().hex})
+ resp.read()
+ self.assertEqual(resp.status, 400)
+
+ @requires_policies
+ def test_create_non_default_storage_policy_container(self):
+ policy = self.policies.exclude(default=True).select()
+
+ def put(url, token, parsed, conn, headers=None):
+ base_headers = {'X-Auth-Token': token}
+ if headers:
+ base_headers.update(headers)
+ conn.request('PUT', parsed.path + '/' + self.container, '',
+ base_headers)
+ return check_response(conn)
+ headers = {'X-Storage-Policy': policy['name']}
+ resp = retry(put, headers=headers)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'),
+ policy['name'])
+
+ # and test recreate with-out specifying Storage Policy
+ resp = retry(put)
+ resp.read()
+ self.assertEqual(resp.status, 202)
+ # should still be original storage policy
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'),
+ policy['name'])
+
+ # delete it
+ def delete(url, token, parsed, conn):
+ conn.request('DELETE', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(delete)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # verify no policy header
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'), None)
+
+ @requires_policies
+ def test_conflict_change_storage_policy_with_put(self):
+ def put(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('PUT', parsed.path + '/' + self.container, '',
+ new_headers)
+ return check_response(conn)
+
+ # create
+ policy = self.policies.select()
+ resp = retry(put, {'X-Storage-Policy': policy['name']})
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ # can't change it
+ other_policy = self.policies.exclude(name=policy['name']).select()
+ resp = retry(put, {'X-Storage-Policy': other_policy['name']})
+ resp.read()
+ self.assertEqual(resp.status, 409)
+
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ # still original policy
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'),
+ policy['name'])
+
+ @requires_policies
+ def test_noop_change_storage_policy_with_post(self):
+ def put(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('PUT', parsed.path + '/' + self.container, '',
+ new_headers)
+ return check_response(conn)
+
+ # create
+ policy = self.policies.select()
+ resp = retry(put, {'X-Storage-Policy': policy['name']})
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ def post(url, token, parsed, conn, headers):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('POST', parsed.path + '/' + self.container, '',
+ new_headers)
+ return check_response(conn)
+ # attempt update
+ for header in ('X-Storage-Policy', 'X-Storage-Policy-Index'):
+ other_policy = self.policies.exclude(name=policy['name']).select()
+ resp = retry(post, {header: other_policy['name']})
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ def head(url, token, parsed, conn):
+ conn.request('HEAD', parsed.path + '/' + self.container, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ # still original policy
+ resp = retry(head)
+ resp.read()
+ headers = dict((k.lower(), v) for k, v in resp.getheaders())
+ self.assertEquals(headers.get('x-storage-policy'),
+ policy['name'])
+
+
+class BaseTestContainerACLs(unittest.TestCase):
+ # subclasses can change the account in which container
+ # is created/deleted by setUp/tearDown
+ account = 1
+
+ def _get_account(self, url, token, parsed, conn):
+ return parsed.path
+
+ def _get_tenant_id(self, url, token, parsed, conn):
+ account = parsed.path
+ return account.replace('/v1/AUTH_', '', 1)
+
+ def setUp(self):
+ if tf.skip or tf.skip2 or tf.skip_if_not_v3:
+ raise SkipTest('AUTH VERSION 3 SPECIFIC TEST')
+ self.name = uuid4().hex
+
+ def put(url, token, parsed, conn):
+ conn.request('PUT', parsed.path + '/' + self.name, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(put, use_account=self.account)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ def tearDown(self):
+ if tf.skip or tf.skip2 or tf.skip_if_not_v3:
+ raise SkipTest
+
+ def get(url, token, parsed, conn):
+ conn.request('GET', parsed.path + '/' + self.name + '?format=json',
+ '', {'X-Auth-Token': token})
+ return check_response(conn)
+
+ def delete(url, token, parsed, conn, obj):
+ conn.request('DELETE',
+ '/'.join([parsed.path, self.name, obj['name']]), '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ while True:
+ resp = retry(get, use_account=self.account)
+ body = resp.read()
+ self.assert_(resp.status // 100 == 2, resp.status)
+ objs = json.loads(body)
+ if not objs:
+ break
+ for obj in objs:
+ resp = retry(delete, obj, use_account=self.account)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ def delete(url, token, parsed, conn):
+ conn.request('DELETE', parsed.path + '/' + self.name, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(delete, use_account=self.account)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ def _assert_cross_account_acl_granted(self, granted, grantee_account, acl):
+ '''
+ Check whether a given container ACL is granted when a user specified
+ by account_b attempts to access a container.
+ '''
+ # Obtain the first account's string
+ first_account = retry(self._get_account, use_account=self.account)
+
+ # Ensure we can't access the container with the grantee account
+ def get2(url, token, parsed, conn):
+ conn.request('GET', first_account + '/' + self.name, '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(get2, use_account=grantee_account)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ def put2(url, token, parsed, conn):
+ conn.request('PUT', first_account + '/' + self.name + '/object',
+ 'test object', {'X-Auth-Token': token})
+ return check_response(conn)
+
+ resp = retry(put2, use_account=grantee_account)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ # Post ACL to the container
+ def post(url, token, parsed, conn):
+ conn.request('POST', parsed.path + '/' + self.name, '',
+ {'X-Auth-Token': token,
+ 'X-Container-Read': acl,
+ 'X-Container-Write': acl})
+ return check_response(conn)
+
+ resp = retry(post, use_account=self.account)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # Check access to container from grantee account with ACL in place
+ resp = retry(get2, use_account=grantee_account)
+ resp.read()
+ expected = 204 if granted else 403
+ self.assertEqual(resp.status, expected)
+
+ resp = retry(put2, use_account=grantee_account)
+ resp.read()
+ expected = 201 if granted else 403
+ self.assertEqual(resp.status, expected)
+
+ # Make the container private again
+ def post(url, token, parsed, conn):
+ conn.request('POST', parsed.path + '/' + self.name, '',
+ {'X-Auth-Token': token, 'X-Container-Read': '',
+ 'X-Container-Write': ''})
+ return check_response(conn)
+
+ resp = retry(post, use_account=self.account)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
+ # Ensure we can't access the container with the grantee account again
+ resp = retry(get2, use_account=grantee_account)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ resp = retry(put2, use_account=grantee_account)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+
+class TestContainerACLsAccount1(BaseTestContainerACLs):
+ def test_cross_account_acl_names_with_user_in_non_default_domain(self):
+ # names in acls are disallowed when grantee is in a non-default domain
+ acl = '%s:%s' % (tf.swift_test_tenant[3], tf.swift_test_user[3])
+ self._assert_cross_account_acl_granted(False, 4, acl)
+
+ def test_cross_account_acl_ids_with_user_in_non_default_domain(self):
+ # ids are allowed in acls when grantee is in a non-default domain
+ tenant_id = retry(self._get_tenant_id, use_account=4)
+ acl = '%s:%s' % (tenant_id, '*')
+ self._assert_cross_account_acl_granted(True, 4, acl)
+
+ def test_cross_account_acl_names_in_default_domain(self):
+ # names are allowed in acls when grantee and project are in
+ # the default domain
+ acl = '%s:%s' % (tf.swift_test_tenant[1], tf.swift_test_user[1])
+ self._assert_cross_account_acl_granted(True, 2, acl)
+
+ def test_cross_account_acl_ids_in_default_domain(self):
+ # ids are allowed in acls when grantee and project are in
+ # the default domain
+ tenant_id = retry(self._get_tenant_id, use_account=2)
+ acl = '%s:%s' % (tenant_id, '*')
+ self._assert_cross_account_acl_granted(True, 2, acl)
+
+
+class TestContainerACLsAccount4(BaseTestContainerACLs):
+ account = 4
+
+ def test_cross_account_acl_names_with_project_in_non_default_domain(self):
+ # names in acls are disallowed when project is in a non-default domain
+ acl = '%s:%s' % (tf.swift_test_tenant[0], tf.swift_test_user[0])
+ self._assert_cross_account_acl_granted(False, 1, acl)
+
+ def test_cross_account_acl_ids_with_project_in_non_default_domain(self):
+ # ids are allowed in acls when project is in a non-default domain
+ tenant_id = retry(self._get_tenant_id, use_account=1)
+ acl = '%s:%s' % (tenant_id, '*')
+ self._assert_cross_account_acl_granted(True, 1, acl)
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/functional/test_object.py b/test/functional/test_object.py
index 675de30..e74a7f6 100755
--- a/test/functional/test_object.py
+++ b/test/functional/test_object.py
@@ -21,24 +21,22 @@ from uuid import uuid4
from swift.common.utils import json
-from swift_testing import check_response, retry, skip, skip3, \
- swift_test_perm, web_front_end, requires_acls, swift_test_user
+from test.functional import check_response, retry, requires_acls, \
+ requires_policies
+import test.functional as tf
class TestObject(unittest.TestCase):
def setUp(self):
- if skip:
+ if tf.skip:
raise SkipTest
self.container = uuid4().hex
- def put(url, token, parsed, conn):
- conn.request('PUT', parsed.path + '/' + self.container, '',
- {'X-Auth-Token': token})
- return check_response(conn)
- resp = retry(put)
- resp.read()
- self.assertEqual(resp.status, 201)
+ self.containers = []
+ self._create_container(self.container)
+ self._create_container(self.container, use_account=2)
+
self.obj = uuid4().hex
def put(url, token, parsed, conn):
@@ -50,40 +48,65 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
+ def _create_container(self, name=None, headers=None, use_account=1):
+ if not name:
+ name = uuid4().hex
+ self.containers.append(name)
+ headers = headers or {}
+
+ def put(url, token, parsed, conn, name):
+ new_headers = dict({'X-Auth-Token': token}, **headers)
+ conn.request('PUT', parsed.path + '/' + name, '',
+ new_headers)
+ return check_response(conn)
+ resp = retry(put, name, use_account=use_account)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+ return name
+
def tearDown(self):
- if skip:
+ if tf.skip:
raise SkipTest
- def delete(url, token, parsed, conn, obj):
- conn.request('DELETE',
- '%s/%s/%s' % (parsed.path, self.container, obj),
- '', {'X-Auth-Token': token})
+ # get list of objects in container
+ def get(url, token, parsed, conn, container):
+ conn.request(
+ 'GET', parsed.path + '/' + container + '?format=json', '',
+ {'X-Auth-Token': token})
return check_response(conn)
- # get list of objects in container
- def list(url, token, parsed, conn):
- conn.request('GET',
- '%s/%s' % (parsed.path, self.container),
- '', {'X-Auth-Token': token})
+ # delete an object
+ def delete(url, token, parsed, conn, container, obj):
+ conn.request(
+ 'DELETE', '/'.join([parsed.path, container, obj['name']]), '',
+ {'X-Auth-Token': token})
return check_response(conn)
- resp = retry(list)
- object_listing = resp.read()
- self.assertEqual(resp.status, 200)
- # iterate over object listing and delete all objects
- for obj in object_listing.splitlines():
- resp = retry(delete, obj)
- resp.read()
- self.assertEqual(resp.status, 204)
+ for container in self.containers:
+ while True:
+ resp = retry(get, container)
+ body = resp.read()
+ if resp.status == 404:
+ break
+ self.assert_(resp.status // 100 == 2, resp.status)
+ objs = json.loads(body)
+ if not objs:
+ break
+ for obj in objs:
+ resp = retry(delete, container, obj)
+ resp.read()
+ self.assertEqual(resp.status, 204)
# delete the container
- def delete(url, token, parsed, conn):
- conn.request('DELETE', parsed.path + '/' + self.container, '',
+ def delete(url, token, parsed, conn, name):
+ conn.request('DELETE', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
- resp = retry(delete)
- resp.read()
- self.assertEqual(resp.status, 204)
+
+ for container in self.containers:
+ resp = retry(delete, container)
+ resp.read()
+ self.assert_(resp.status in (204, 404))
def test_if_none_match(self):
def put(url, token, parsed, conn):
@@ -111,8 +134,47 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEquals(resp.status, 400)
+ def test_non_integer_x_delete_after(self):
+ def put(url, token, parsed, conn):
+ conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
+ 'non_integer_x_delete_after'),
+ '', {'X-Auth-Token': token,
+ 'Content-Length': '0',
+ 'X-Delete-After': '*'})
+ return check_response(conn)
+ resp = retry(put)
+ body = resp.read()
+ self.assertEquals(resp.status, 400)
+ self.assertEqual(body, 'Non-integer X-Delete-After')
+
+ def test_non_integer_x_delete_at(self):
+ def put(url, token, parsed, conn):
+ conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
+ 'non_integer_x_delete_at'),
+ '', {'X-Auth-Token': token,
+ 'Content-Length': '0',
+ 'X-Delete-At': '*'})
+ return check_response(conn)
+ resp = retry(put)
+ body = resp.read()
+ self.assertEquals(resp.status, 400)
+ self.assertEqual(body, 'Non-integer X-Delete-At')
+
+ def test_x_delete_at_in_the_past(self):
+ def put(url, token, parsed, conn):
+ conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
+ 'x_delete_at_in_the_past'),
+ '', {'X-Auth-Token': token,
+ 'Content-Length': '0',
+ 'X-Delete-At': '0'})
+ return check_response(conn)
+ resp = retry(put)
+ body = resp.read()
+ self.assertEquals(resp.status, 400)
+ self.assertEqual(body, 'X-Delete-At in past')
+
def test_copy_object(self):
- if skip:
+ if tf.skip:
raise SkipTest
source = '%s/%s' % (self.container, self.obj)
@@ -185,8 +247,118 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 204)
+ def test_copy_between_accounts(self):
+ if tf.skip:
+ raise SkipTest
+
+ source = '%s/%s' % (self.container, self.obj)
+ dest = '%s/%s' % (self.container, 'test_copy')
+
+ # get contents of source
+ def get_source(url, token, parsed, conn):
+ conn.request('GET',
+ '%s/%s' % (parsed.path, source),
+ '', {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(get_source)
+ source_contents = resp.read()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual(source_contents, 'test')
+
+ acct = tf.parsed[0].path.split('/', 2)[2]
+
+ # copy source to dest with X-Copy-From-Account
+ def put(url, token, parsed, conn):
+ conn.request('PUT', '%s/%s' % (parsed.path, dest), '',
+ {'X-Auth-Token': token,
+ 'Content-Length': '0',
+ 'X-Copy-From-Account': acct,
+ 'X-Copy-From': source})
+ return check_response(conn)
+ # try to put, will not succeed
+ # user does not have permissions to read from source
+ resp = retry(put, use_account=2)
+ self.assertEqual(resp.status, 403)
+
+ # add acl to allow reading from source
+ def post(url, token, parsed, conn):
+ conn.request('POST', '%s/%s' % (parsed.path, self.container), '',
+ {'X-Auth-Token': token,
+ 'X-Container-Read': tf.swift_test_perm[1]})
+ return check_response(conn)
+ resp = retry(post)
+ self.assertEqual(resp.status, 204)
+
+ # retry previous put, now should succeed
+ resp = retry(put, use_account=2)
+ self.assertEqual(resp.status, 201)
+
+ # contents of dest should be the same as source
+ def get_dest(url, token, parsed, conn):
+ conn.request('GET',
+ '%s/%s' % (parsed.path, dest),
+ '', {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(get_dest, use_account=2)
+ dest_contents = resp.read()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual(dest_contents, source_contents)
+
+ # delete the copy
+ def delete(url, token, parsed, conn):
+ conn.request('DELETE', '%s/%s' % (parsed.path, dest), '',
+ {'X-Auth-Token': token})
+ return check_response(conn)
+ resp = retry(delete, use_account=2)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+ # verify dest does not exist
+ resp = retry(get_dest, use_account=2)
+ resp.read()
+ self.assertEqual(resp.status, 404)
+
+ acct_dest = tf.parsed[1].path.split('/', 2)[2]
+
+ # copy source to dest with COPY
+ def copy(url, token, parsed, conn):
+ conn.request('COPY', '%s/%s' % (parsed.path, source), '',
+ {'X-Auth-Token': token,
+ 'Destination-Account': acct_dest,
+ 'Destination': dest})
+ return check_response(conn)
+ # try to copy, will not succeed
+ # user does not have permissions to write to destination
+ resp = retry(copy)
+ resp.read()
+ self.assertEqual(resp.status, 403)
+
+ # add acl to allow write to destination
+ def post(url, token, parsed, conn):
+ conn.request('POST', '%s/%s' % (parsed.path, self.container), '',
+ {'X-Auth-Token': token,
+ 'X-Container-Write': tf.swift_test_perm[0]})
+ return check_response(conn)
+ resp = retry(post, use_account=2)
+ self.assertEqual(resp.status, 204)
+
+ # now copy will succeed
+ resp = retry(copy)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ # contents of dest should be the same as source
+ resp = retry(get_dest, use_account=2)
+ dest_contents = resp.read()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual(dest_contents, source_contents)
+
+ # delete the copy
+ resp = retry(delete, use_account=2)
+ resp.read()
+ self.assertEqual(resp.status, 204)
+
def test_public_object(self):
- if skip:
+ if tf.skip:
raise SkipTest
def get(url, token, parsed, conn):
@@ -225,7 +397,7 @@ class TestObject(unittest.TestCase):
self.assert_(str(err).startswith('No result after '))
def test_private_object(self):
- if skip or skip3:
+ if tf.skip or tf.skip3:
raise SkipTest
# Ensure we can't access the object with the third account
@@ -245,8 +417,8 @@ class TestObject(unittest.TestCase):
conn.request('PUT', '%s/%s' % (
parsed.path, shared_container), '',
{'X-Auth-Token': token,
- 'X-Container-Read': swift_test_perm[2],
- 'X-Container-Write': swift_test_perm[2]})
+ 'X-Container-Read': tf.swift_test_perm[2],
+ 'X-Container-Write': tf.swift_test_perm[2]})
return check_response(conn)
resp = retry(put)
resp.read()
@@ -319,8 +491,8 @@ class TestObject(unittest.TestCase):
@requires_acls
def test_read_only(self):
- if skip3:
- raise SkipTest
+ if tf.skip3:
+ raise tf.SkipTest
def get_listing(url, token, parsed, conn):
conn.request('GET', '%s/%s' % (parsed.path, self.container), '',
@@ -361,7 +533,7 @@ class TestObject(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-only access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -400,7 +572,7 @@ class TestObject(unittest.TestCase):
@requires_acls
def test_read_write(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get_listing(url, token, parsed, conn):
@@ -442,7 +614,7 @@ class TestObject(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant read-write access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -481,7 +653,7 @@ class TestObject(unittest.TestCase):
@requires_acls
def test_admin(self):
- if skip3:
+ if tf.skip3:
raise SkipTest
def get_listing(url, token, parsed, conn):
@@ -523,7 +695,7 @@ class TestObject(unittest.TestCase):
self.assertEquals(resp.status, 403)
# grant admin access
- acl_user = swift_test_user[2]
+ acl_user = tf.swift_test_user[2]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': json.dumps(acl)}
resp = retry(post_account, headers=headers, use_account=1)
@@ -561,7 +733,7 @@ class TestObject(unittest.TestCase):
self.assert_(self.obj not in listing)
def test_manifest(self):
- if skip:
+ if tf.skip:
raise SkipTest
# Data for the object segments
segments1 = ['one', 'two', 'three', 'four', 'five']
@@ -672,7 +844,7 @@ class TestObject(unittest.TestCase):
self.assertEqual(resp.read(), ''.join(segments2))
self.assertEqual(resp.status, 200)
- if not skip3:
+ if not tf.skip3:
# Ensure we can't access the manifest with the third account
def get(url, token, parsed, conn):
@@ -687,7 +859,7 @@ class TestObject(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', '%s/%s' % (parsed.path, self.container),
'', {'X-Auth-Token': token,
- 'X-Container-Read': swift_test_perm[2]})
+ 'X-Container-Read': tf.swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
resp.read()
@@ -745,7 +917,7 @@ class TestObject(unittest.TestCase):
self.assertEqual(resp.read(), ''.join(segments3))
self.assertEqual(resp.status, 200)
- if not skip3:
+ if not tf.skip3:
# Ensure we can't access the manifest with the third account
# (because the segments are in a protected container even if the
@@ -763,7 +935,7 @@ class TestObject(unittest.TestCase):
def post(url, token, parsed, conn):
conn.request('POST', '%s/%s' % (parsed.path, acontainer),
'', {'X-Auth-Token': token,
- 'X-Container-Read': swift_test_perm[2]})
+ 'X-Container-Read': tf.swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
resp.read()
@@ -831,7 +1003,7 @@ class TestObject(unittest.TestCase):
self.assertEqual(resp.status, 204)
def test_delete_content_type(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn):
@@ -853,7 +1025,7 @@ class TestObject(unittest.TestCase):
'text/html; charset=UTF-8')
def test_delete_if_delete_at_bad(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn):
@@ -875,7 +1047,7 @@ class TestObject(unittest.TestCase):
self.assertEqual(resp.status, 400)
def test_null_name(self):
- if skip:
+ if tf.skip:
raise SkipTest
def put(url, token, parsed, conn):
@@ -884,23 +1056,20 @@ class TestObject(unittest.TestCase):
self.container), 'test', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
- if (web_front_end == 'apache2'):
+ if (tf.web_front_end == 'apache2'):
self.assertEqual(resp.status, 404)
else:
self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL')
self.assertEqual(resp.status, 412)
def test_cors(self):
- if skip:
+ if tf.skip:
raise SkipTest
- def is_strict_mode(url, token, parsed, conn):
- conn.request('GET', '/info')
- resp = conn.getresponse()
- if resp.status // 100 == 2:
- info = json.loads(resp.read())
- return info.get('swift', {}).get('strict_cors_mode', False)
- return False
+ try:
+ strict_cors = tf.cluster_info['swift']['strict_cors_mode']
+ except KeyError:
+ raise SkipTest("cors mode is unknown")
def put_cors_cont(url, token, parsed, conn, orig):
conn.request(
@@ -924,8 +1093,6 @@ class TestObject(unittest.TestCase):
'', headers)
return conn.getresponse()
- strict_cors = retry(is_strict_mode)
-
resp = retry(put_cors_cont, '*')
resp.read()
self.assertEquals(resp.status // 100, 2)
@@ -1001,6 +1168,64 @@ class TestObject(unittest.TestCase):
self.assertEquals(headers.get('access-control-allow-origin'),
'http://m.com')
+ @requires_policies
+ def test_cross_policy_copy(self):
+ # create container in first policy
+ policy = self.policies.select()
+ container = self._create_container(
+ headers={'X-Storage-Policy': policy['name']})
+ obj = uuid4().hex
+
+ # create a container in second policy
+ other_policy = self.policies.exclude(name=policy['name']).select()
+ other_container = self._create_container(
+ headers={'X-Storage-Policy': other_policy['name']})
+ other_obj = uuid4().hex
+
+ def put_obj(url, token, parsed, conn, container, obj):
+ # to keep track of things, use the original path as the body
+ content = '%s/%s' % (container, obj)
+ path = '%s/%s' % (parsed.path, content)
+ conn.request('PUT', path, content, {'X-Auth-Token': token})
+ return check_response(conn)
+
+ # create objects
+ for c, o in zip((container, other_container), (obj, other_obj)):
+ resp = retry(put_obj, c, o)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ def put_copy_from(url, token, parsed, conn, container, obj, source):
+ dest_path = '%s/%s/%s' % (parsed.path, container, obj)
+ conn.request('PUT', dest_path, '',
+ {'X-Auth-Token': token,
+ 'Content-Length': '0',
+ 'X-Copy-From': source})
+ return check_response(conn)
+
+ copy_requests = (
+ (container, other_obj, '%s/%s' % (other_container, other_obj)),
+ (other_container, obj, '%s/%s' % (container, obj)),
+ )
+
+ # copy objects
+ for c, o, source in copy_requests:
+ resp = retry(put_copy_from, c, o, source)
+ resp.read()
+ self.assertEqual(resp.status, 201)
+
+ def get_obj(url, token, parsed, conn, container, obj):
+ path = '%s/%s/%s' % (parsed.path, container, obj)
+ conn.request('GET', path, '', {'X-Auth-Token': token})
+ return check_response(conn)
+
+ # validate contents, contents should be source
+ validate_requests = copy_requests
+ for c, o, body in validate_requests:
+ resp = retry(get_obj, c, o)
+ self.assertEqual(resp.status, 200)
+ self.assertEqual(body, resp.read())
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/functional/tests.py b/test/functional/tests.py
index b8633b0..daa8897 100644
--- a/test/functional/tests.py
+++ b/test/functional/tests.py
@@ -14,10 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Modifications by Red Hat, Inc.
-
from datetime import datetime
-import os
import hashlib
import hmac
import json
@@ -25,72 +22,25 @@ import locale
import random
import StringIO
import time
-import threading
+import os
import unittest
import urllib
import uuid
+from copy import deepcopy
+import eventlet
from nose import SkipTest
-from ConfigParser import ConfigParser
+from swift.common.http import is_success, is_client_error
-from test import get_config
+from test.functional import normalized_urls, load_constraint, cluster_info
+from test.functional import check_response, retry
+import test.functional as tf
from test.functional.swift_test_client import Account, Connection, File, \
ResponseError
-from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \
- MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
- MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \
- MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_HEADER_SIZE
from gluster.swift.common.constraints import \
set_object_name_component_length, get_object_name_component_length
-default_constraints = dict((
- ('max_file_size', MAX_FILE_SIZE),
- ('max_meta_name_length', MAX_META_NAME_LENGTH),
- ('max_meta_value_length', MAX_META_VALUE_LENGTH),
- ('max_meta_count', MAX_META_COUNT),
- ('max_meta_overall_size', MAX_META_OVERALL_SIZE),
- ('max_object_name_length', MAX_OBJECT_NAME_LENGTH),
- ('container_listing_limit', CONTAINER_LISTING_LIMIT),
- ('account_listing_limit', ACCOUNT_LISTING_LIMIT),
- ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH),
- ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH),
- ('max_header_size', MAX_HEADER_SIZE)))
-constraints_conf = ConfigParser()
-conf_exists = constraints_conf.read('/etc/swift/swift.conf')
-# Constraints are set first from the test config, then from
-# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist,
-# then limit test coverage. This allows SAIO tests to work fine but
-# requires remote functional testing to know something about the cluster
-# that is being tested.
-config = get_config('func_test')
-for k in default_constraints:
- if k in config:
- # prefer what's in test.conf
- config[k] = int(config[k])
- elif conf_exists:
- # swift.conf exists, so use what's defined there (or swift defaults)
- # This normally happens when the test is running locally to the cluster
- # as in a SAIO.
- config[k] = default_constraints[k]
- else:
- # .functests don't know what the constraints of the tested cluster are,
- # so the tests can't reliably pass or fail. Therefore, skip those
- # tests.
- config[k] = '%s constraint is not defined' % k
-
-web_front_end = config.get('web_front_end', 'integral')
-normalized_urls = config.get('normalized_urls', False)
set_object_name_component_length()
-
-def load_constraint(name):
- c = config[name]
- if not isinstance(c, int):
- raise SkipTest(c)
- return c
-
-locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C'))
-
-
def create_limit_filename(name_limit):
"""
Convert a split a large object name with
@@ -116,42 +66,6 @@ def create_limit_filename(name_limit):
return "".join(filename_list)
-def chunks(s, length=3):
- i, j = 0, length
- while i < len(s):
- yield s[i:j]
- i, j = j, j + length
-
-
-def timeout(seconds, method, *args, **kwargs):
- class TimeoutThread(threading.Thread):
- def __init__(self, method, *args, **kwargs):
- threading.Thread.__init__(self)
-
- self.method = method
- self.args = args
- self.kwargs = kwargs
- self.exception = None
-
- def run(self):
- try:
- self.method(*self.args, **self.kwargs)
- except Exception as e:
- self.exception = e
-
- t = TimeoutThread(method, *args, **kwargs)
- t.start()
- t.join(seconds)
-
- if t.exception:
- raise t.exception
-
- if t.isAlive():
- t._Thread__stop()
- return True
- return False
-
-
class Utils(object):
@classmethod
def create_ascii_name(cls, length=None):
@@ -207,10 +121,10 @@ class Base2(object):
class TestAccountEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.containers = []
@@ -386,6 +300,28 @@ class TestAccount(Base):
self.assertEqual(sorted(containers, cmp=locale.strcoll),
containers)
+ def testQuotedWWWAuthenticateHeader(self):
+ # check that the www-authenticate header value with the swift realm
+ # is correctly quoted.
+ conn = Connection(tf.config)
+ conn.authenticate()
+ inserted_html = '<b>Hello World'
+ hax = 'AUTH_haxx"\nContent-Length: %d\n\n%s' % (len(inserted_html),
+ inserted_html)
+ quoted_hax = urllib.quote(hax)
+ conn.connection.request('GET', '/v1/' + quoted_hax, None, {})
+ resp = conn.connection.getresponse()
+ resp_headers = dict(resp.getheaders())
+ self.assertTrue('www-authenticate' in resp_headers,
+ 'www-authenticate not found in %s' % resp_headers)
+ actual = resp_headers['www-authenticate']
+ expected = 'Swift realm="%s"' % quoted_hax
+ # other middleware e.g. auth_token may also set www-authenticate
+ # headers in which case actual values will be a comma separated list.
+ # check that expected value is among the actual values
+ self.assertTrue(expected in actual,
+ '%s not found in %s' % (expected, actual))
+
class TestAccountUTF8(Base2, TestAccount):
set_up = False
@@ -394,10 +330,10 @@ class TestAccountUTF8(Base2, TestAccount):
class TestAccountNoContainersEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
@@ -423,10 +359,10 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
class TestContainerEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
@@ -714,11 +650,11 @@ class TestContainerUTF8(Base2, TestContainer):
class TestContainerPathsEnv(object):
@classmethod
def setUp(cls):
- raise SkipTest('Objects ending in / are not supported')
- cls.conn = Connection(config)
+ raise SkipTest('Objects ending in / are not supported')
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.file_size = 8
@@ -894,11 +830,24 @@ class TestContainerPaths(Base):
class TestFileEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
+ # creating another account and connection
+ # for account to account copy tests
+ config2 = deepcopy(tf.config)
+ config2['account'] = tf.config['account2']
+ config2['username'] = tf.config['username2']
+ config2['password'] = tf.config['password2']
+ cls.conn2 = Connection(config2)
+ cls.conn2.authenticate()
+
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
+ cls.account2 = cls.conn2.get_account()
+ cls.account2.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
@@ -952,6 +901,62 @@ class TestFile(Base):
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
+ def testCopyAccount(self):
+ # makes sure to test encoded characters
+ source_filename = 'dealde%2Fl04 011e%204c8df/flash.png'
+ file_item = self.env.container.file(source_filename)
+
+ metadata = {Utils.create_ascii_name(): Utils.create_name()}
+
+ data = file_item.write_random()
+ file_item.sync_metadata(metadata)
+
+ dest_cont = self.env.account.container(Utils.create_name())
+ self.assert_(dest_cont.create())
+
+ acct = self.env.conn.account_name
+ # copy both from within and across containers
+ for cont in (self.env.container, dest_cont):
+ # copy both with and without initial slash
+ for prefix in ('', '/'):
+ dest_filename = Utils.create_name()
+
+ file_item = self.env.container.file(source_filename)
+ file_item.copy_account(acct,
+ '%s%s' % (prefix, cont),
+ dest_filename)
+
+ self.assert_(dest_filename in cont.files())
+
+ file_item = cont.file(dest_filename)
+
+ self.assert_(data == file_item.read())
+ self.assert_(file_item.initialize())
+ self.assert_(metadata == file_item.metadata)
+
+ dest_cont = self.env.account2.container(Utils.create_name())
+ self.assert_(dest_cont.create(hdrs={
+ 'X-Container-Write': self.env.conn.user_acl
+ }))
+
+ acct = self.env.conn2.account_name
+ # copy both with and without initial slash
+ for prefix in ('', '/'):
+ dest_filename = Utils.create_name()
+
+ file_item = self.env.container.file(source_filename)
+ file_item.copy_account(acct,
+ '%s%s' % (prefix, dest_cont),
+ dest_filename)
+
+ self.assert_(dest_filename in dest_cont.files())
+
+ file_item = dest_cont.file(dest_filename)
+
+ self.assert_(data == file_item.read())
+ self.assert_(file_item.initialize())
+ self.assert_(metadata == file_item.metadata)
+
def testCopy404s(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@@ -990,6 +995,77 @@ class TestFile(Base):
'%s%s' % (prefix, Utils.create_name()),
Utils.create_name()))
+ def testCopyAccount404s(self):
+ acct = self.env.conn.account_name
+ acct2 = self.env.conn2.account_name
+ source_filename = Utils.create_name()
+ file_item = self.env.container.file(source_filename)
+ file_item.write_random()
+
+ dest_cont = self.env.account.container(Utils.create_name())
+ self.assert_(dest_cont.create(hdrs={
+ 'X-Container-Read': self.env.conn2.user_acl
+ }))
+ dest_cont2 = self.env.account2.container(Utils.create_name())
+ self.assert_(dest_cont2.create(hdrs={
+ 'X-Container-Write': self.env.conn.user_acl,
+ 'X-Container-Read': self.env.conn.user_acl
+ }))
+
+ for acct, cont in ((acct, dest_cont), (acct2, dest_cont2)):
+ for prefix in ('', '/'):
+ # invalid source container
+ source_cont = self.env.account.container(Utils.create_name())
+ file_item = source_cont.file(source_filename)
+ self.assert_(not file_item.copy_account(
+ acct,
+ '%s%s' % (prefix, self.env.container),
+ Utils.create_name()))
+ if acct == acct2:
+ # there is no such source container
+ # and foreign user can have no permission to read it
+ self.assert_status(403)
+ else:
+ self.assert_status(404)
+
+ self.assert_(not file_item.copy_account(
+ acct,
+ '%s%s' % (prefix, cont),
+ Utils.create_name()))
+ self.assert_status(404)
+
+ # invalid source object
+ file_item = self.env.container.file(Utils.create_name())
+ self.assert_(not file_item.copy_account(
+ acct,
+ '%s%s' % (prefix, self.env.container),
+ Utils.create_name()))
+ if acct == acct2:
+ # there is no such object
+ # and foreign user can have no permission to read it
+ self.assert_status(403)
+ else:
+ self.assert_status(404)
+
+ self.assert_(not file_item.copy_account(
+ acct,
+ '%s%s' % (prefix, cont),
+ Utils.create_name()))
+ self.assert_status(404)
+
+ # invalid destination container
+ file_item = self.env.container.file(source_filename)
+ self.assert_(not file_item.copy_account(
+ acct,
+ '%s%s' % (prefix, Utils.create_name()),
+ Utils.create_name()))
+ if acct == acct2:
+ # there is no such destination container
+ # and foreign user can have no permission to write there
+ self.assert_status(403)
+ else:
+ self.assert_status(404)
+
def testCopyNoDestinationHeader(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@@ -1044,6 +1120,49 @@ class TestFile(Base):
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
+ def testCopyFromAccountHeader(self):
+ acct = self.env.conn.account_name
+ src_cont = self.env.account.container(Utils.create_name())
+ self.assert_(src_cont.create(hdrs={
+ 'X-Container-Read': self.env.conn2.user_acl
+ }))
+ source_filename = Utils.create_name()
+ file_item = src_cont.file(source_filename)
+
+ metadata = {}
+ for i in range(1):
+ metadata[Utils.create_ascii_name()] = Utils.create_name()
+ file_item.metadata = metadata
+
+ data = file_item.write_random()
+
+ dest_cont = self.env.account.container(Utils.create_name())
+ self.assert_(dest_cont.create())
+ dest_cont2 = self.env.account2.container(Utils.create_name())
+ self.assert_(dest_cont2.create(hdrs={
+ 'X-Container-Write': self.env.conn.user_acl
+ }))
+
+ for cont in (src_cont, dest_cont, dest_cont2):
+ # copy both with and without initial slash
+ for prefix in ('', '/'):
+ dest_filename = Utils.create_name()
+
+ file_item = cont.file(dest_filename)
+ file_item.write(hdrs={'X-Copy-From-Account': acct,
+ 'X-Copy-From': '%s%s/%s' % (
+ prefix,
+ src_cont.name,
+ source_filename)})
+
+ self.assert_(dest_filename in cont.files())
+
+ file_item = cont.file(dest_filename)
+
+ self.assert_(data == file_item.read())
+ self.assert_(file_item.initialize())
+ self.assert_(metadata == file_item.metadata)
+
def testCopyFromHeader404s(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@@ -1075,6 +1194,52 @@ class TestFile(Base):
self.env.container.name, source_filename)})
self.assert_status(404)
+ def testCopyFromAccountHeader404s(self):
+ acct = self.env.conn2.account_name
+ src_cont = self.env.account2.container(Utils.create_name())
+ self.assert_(src_cont.create(hdrs={
+ 'X-Container-Read': self.env.conn.user_acl
+ }))
+ source_filename = Utils.create_name()
+ file_item = src_cont.file(source_filename)
+ file_item.write_random()
+ dest_cont = self.env.account.container(Utils.create_name())
+ self.assert_(dest_cont.create())
+
+ for prefix in ('', '/'):
+ # invalid source container
+ file_item = dest_cont.file(Utils.create_name())
+ self.assertRaises(ResponseError, file_item.write,
+ hdrs={'X-Copy-From-Account': acct,
+ 'X-Copy-From': '%s%s/%s' %
+ (prefix,
+ Utils.create_name(),
+ source_filename)})
+ # looks like cached responses leak "not found"
+ # to un-authorized users, not going to fix it now, but...
+ self.assert_status([403, 404])
+
+ # invalid source object
+ file_item = self.env.container.file(Utils.create_name())
+ self.assertRaises(ResponseError, file_item.write,
+ hdrs={'X-Copy-From-Account': acct,
+ 'X-Copy-From': '%s%s/%s' %
+ (prefix,
+ src_cont,
+ Utils.create_name())})
+ self.assert_status(404)
+
+ # invalid destination container
+ dest_cont = self.env.account.container(Utils.create_name())
+ file_item = dest_cont.file(Utils.create_name())
+ self.assertRaises(ResponseError, file_item.write,
+ hdrs={'X-Copy-From-Account': acct,
+ 'X-Copy-From': '%s%s/%s' %
+ (prefix,
+ src_cont,
+ source_filename)})
+ self.assert_status(404)
+
def testNameLimit(self):
limit = load_constraint('max_object_name_length')
@@ -1191,7 +1356,12 @@ class TestFile(Base):
self.assertEqual(file_types, file_types_read)
def testRangedGets(self):
- file_length = 10000
+ # We set the file_length to a strange multiple here. This is to check
+ # that ranges still work in the EC case when the requested range
+ # spans EC segment boundaries. The 1 MiB base value is chosen because
+ # that's a common EC segment size. The 1.33 multiple is to ensure we
+ # aren't aligned on segment boundaries
+ file_length = int(1048576 * 1.33)
range_size = file_length / 10
file_item = self.env.container.file(Utils.create_name())
data = file_item.write_random(file_length)
@@ -1254,6 +1424,15 @@ class TestFile(Base):
limit = load_constraint('max_file_size')
tsecs = 3
+ def timeout(seconds, method, *args, **kwargs):
+ try:
+ with eventlet.Timeout(seconds):
+ method(*args, **kwargs)
+ except eventlet.Timeout:
+ return True
+ else:
+ return False
+
for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1,
limit + 10, limit + 100):
@@ -1295,6 +1474,16 @@ class TestFile(Base):
cfg={'no_content_length': True})
self.assert_status(400)
+ # no content-length
+ self.assertRaises(ResponseError, file_item.write_random, file_length,
+ cfg={'no_content_length': True})
+ self.assert_status(411)
+
+ self.assertRaises(ResponseError, file_item.write_random, file_length,
+ hdrs={'transfer-encoding': 'gzip,chunked'},
+ cfg={'no_content_length': True})
+ self.assert_status(501)
+
# bad request types
#for req in ('LICK', 'GETorHEAD_base', 'container_info',
# 'best_response'):
@@ -1565,8 +1754,16 @@ class TestFile(Base):
self.assertEqual(etag, header_etag)
def testChunkedPut(self):
- if (web_front_end == 'apache2'):
- raise SkipTest()
+ if (tf.web_front_end == 'apache2'):
+ raise SkipTest("Chunked PUT can only be tested with apache2 web"
+ " front end")
+
+ def chunks(s, length=3):
+ i, j = 0, length
+ while i < len(s):
+ yield s[i:j]
+ i, j = j, j + length
+
data = File.random_data(10000)
etag = File.compute_md5sum(data)
@@ -1590,10 +1787,10 @@ class TestFileUTF8(Base2, TestFile):
class TestDloEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
@@ -1657,6 +1854,9 @@ class TestDlo(Base):
file_item = self.env.container.file('man1')
file_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(file_contents, "man1-contents")
+ self.assertEqual(file_item.info()['x_object_manifest'],
+ "%s/%s/seg_lower" %
+ (self.env.container.name, self.env.segment_prefix))
def test_get_range(self):
file_item = self.env.container.file('man1')
@@ -1691,9 +1891,38 @@ class TestDlo(Base):
self.assertEqual(
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
+ # The copied object must not have X-Object-Manifest
+ self.assertTrue("x_object_manifest" not in file_item.info())
+
+ def test_copy_account(self):
+ # dlo use same account and same container only
+ acct = self.env.conn.account_name
+ # Adding a new segment, copying the manifest, and then deleting the
+ # segment proves that the new object is really the concatenated
+ # segments and not just a manifest.
+ f_segment = self.env.container.file("%s/seg_lowerf" %
+ (self.env.segment_prefix))
+ f_segment.write('ffffffffff')
+ try:
+ man1_item = self.env.container.file('man1')
+ man1_item.copy_account(acct,
+ self.env.container.name,
+ "copied-man1")
+ finally:
+ # try not to leave this around for other tests to stumble over
+ f_segment.delete()
+
+ file_item = self.env.container.file('copied-man1')
+ file_contents = file_item.read()
+ self.assertEqual(
+ file_contents,
+ "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
+ # The copied object must not have X-Object-Manifest
+ self.assertTrue("x_object_manifest" not in file_item.info())
def test_copy_manifest(self):
- # Copying the manifest should result in another manifest
+ # Copying the manifest with multipart-manifest=get query string
+ # should result in another manifest
try:
man1_item = self.env.container.file('man1')
man1_item.copy(self.env.container.name, "copied-man1",
@@ -1707,10 +1936,57 @@ class TestDlo(Base):
self.assertEqual(
copied_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
+ self.assertEqual(man1_item.info()['x_object_manifest'],
+ copied.info()['x_object_manifest'])
finally:
# try not to leave this around for other tests to stumble over
self.env.container.file("copied-man1").delete()
+ def test_dlo_if_match_get(self):
+ manifest = self.env.container.file("man1")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.read,
+ hdrs={'If-Match': 'not-%s' % etag})
+ self.assert_status(412)
+
+ manifest.read(hdrs={'If-Match': etag})
+ self.assert_status(200)
+
+ def test_dlo_if_none_match_get(self):
+ manifest = self.env.container.file("man1")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.read,
+ hdrs={'If-None-Match': etag})
+ self.assert_status(304)
+
+ manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
+ self.assert_status(200)
+
+ def test_dlo_if_match_head(self):
+ manifest = self.env.container.file("man1")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.info,
+ hdrs={'If-Match': 'not-%s' % etag})
+ self.assert_status(412)
+
+ manifest.info(hdrs={'If-Match': etag})
+ self.assert_status(200)
+
+ def test_dlo_if_none_match_head(self):
+ manifest = self.env.container.file("man1")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.info,
+ hdrs={'If-None-Match': etag})
+ self.assert_status(304)
+
+ manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
+ self.assert_status(200)
+
+
class TestDloUTF8(Base2, TestDlo):
set_up = False
@@ -1718,10 +1994,10 @@ class TestDloUTF8(Base2, TestDlo):
class TestFileComparisonEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
@@ -1773,19 +2049,25 @@ class TestFileComparison(Base):
for file_item in self.env.files:
hdrs = {'If-Modified-Since': self.env.time_old_f1}
self.assert_(file_item.read(hdrs=hdrs))
+ self.assert_(file_item.info(hdrs=hdrs))
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
+ self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
+ self.assert_status(304)
def testIfUnmodifiedSince(self):
for file_item in self.env.files:
hdrs = {'If-Unmodified-Since': self.env.time_new}
self.assert_(file_item.read(hdrs=hdrs))
+ self.assert_(file_item.info(hdrs=hdrs))
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
+ self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
+ self.assert_status(412)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:
@@ -1835,17 +2117,24 @@ class TestSloEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
+ config2 = deepcopy(tf.config)
+ config2['account'] = tf.config['account2']
+ config2['username'] = tf.config['username2']
+ config2['password'] = tf.config['password2']
+ cls.conn2 = Connection(config2)
+ cls.conn2.authenticate()
+ cls.account2 = cls.conn2.get_account()
+ cls.account2.delete_containers()
if cls.slo_enabled is None:
- cluster_info = cls.conn.cluster_info()
cls.slo_enabled = 'slo' in cluster_info
if not cls.slo_enabled:
return
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
@@ -1911,7 +2200,7 @@ class TestSlo(Base):
set_up = False
def setUp(self):
- raise SkipTest("SLO not enabled yet in gluster-swift")
+ raise SkipTest("SLO not enabled yet in gluster-swift")
super(TestSlo, self).setUp()
if self.env.slo_enabled is False:
raise SkipTest("SLO not enabled")
@@ -2021,6 +2310,29 @@ class TestSlo(Base):
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
+ def test_slo_copy_account(self):
+ acct = self.env.conn.account_name
+ # same account copy
+ file_item = self.env.container.file("manifest-abcde")
+ file_item.copy_account(acct, self.env.container.name, "copied-abcde")
+
+ copied = self.env.container.file("copied-abcde")
+ copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+ self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
+
+ # copy to different account
+ acct = self.env.conn2.account_name
+ dest_cont = self.env.account2.container(Utils.create_name())
+ self.assert_(dest_cont.create(hdrs={
+ 'X-Container-Write': self.env.conn.user_acl
+ }))
+ file_item = self.env.container.file("manifest-abcde")
+ file_item.copy_account(acct, dest_cont, "copied-abcde")
+
+ copied = dest_cont.file("copied-abcde")
+ copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+ self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
+
def test_slo_copy_the_manifest(self):
file_item = self.env.container.file("manifest-abcde")
file_item.copy(self.env.container.name, "copied-abcde-manifest-only",
@@ -2033,6 +2345,40 @@ class TestSlo(Base):
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
+ def test_slo_copy_the_manifest_account(self):
+ acct = self.env.conn.account_name
+ # same account
+ file_item = self.env.container.file("manifest-abcde")
+ file_item.copy_account(acct,
+ self.env.container.name,
+ "copied-abcde-manifest-only",
+ parms={'multipart-manifest': 'get'})
+
+ copied = self.env.container.file("copied-abcde-manifest-only")
+ copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+ try:
+ json.loads(copied_contents)
+ except ValueError:
+ self.fail("COPY didn't copy the manifest (invalid json on GET)")
+
+ # different account
+ acct = self.env.conn2.account_name
+ dest_cont = self.env.account2.container(Utils.create_name())
+ self.assert_(dest_cont.create(hdrs={
+ 'X-Container-Write': self.env.conn.user_acl
+ }))
+ file_item.copy_account(acct,
+ dest_cont,
+ "copied-abcde-manifest-only",
+ parms={'multipart-manifest': 'get'})
+
+ copied = dest_cont.file("copied-abcde-manifest-only")
+ copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+ try:
+ json.loads(copied_contents)
+ except ValueError:
+ self.fail("COPY didn't copy the manifest (invalid json on GET)")
+
def test_slo_get_the_manifest(self):
manifest = self.env.container.file("manifest-abcde")
got_body = manifest.read(parms={'multipart-manifest': 'get'})
@@ -2051,6 +2397,50 @@ class TestSlo(Base):
self.assertEqual('application/json; charset=utf-8',
got_info['content_type'])
+ def test_slo_if_match_get(self):
+ manifest = self.env.container.file("manifest-abcde")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.read,
+ hdrs={'If-Match': 'not-%s' % etag})
+ self.assert_status(412)
+
+ manifest.read(hdrs={'If-Match': etag})
+ self.assert_status(200)
+
+ def test_slo_if_none_match_get(self):
+ manifest = self.env.container.file("manifest-abcde")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.read,
+ hdrs={'If-None-Match': etag})
+ self.assert_status(304)
+
+ manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
+ self.assert_status(200)
+
+ def test_slo_if_match_head(self):
+ manifest = self.env.container.file("manifest-abcde")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.info,
+ hdrs={'If-Match': 'not-%s' % etag})
+ self.assert_status(412)
+
+ manifest.info(hdrs={'If-Match': etag})
+ self.assert_status(200)
+
+ def test_slo_if_none_match_head(self):
+ manifest = self.env.container.file("manifest-abcde")
+ etag = manifest.info()['etag']
+
+ self.assertRaises(ResponseError, manifest.info,
+ hdrs={'If-None-Match': etag})
+ self.assert_status(304)
+
+ manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
+ self.assert_status(200)
+
class TestSloUTF8(Base2, TestSlo):
set_up = False
@@ -2061,11 +2451,19 @@ class TestObjectVersioningEnv(object):
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
- cls.account = Account(cls.conn, config.get('account',
- config['username']))
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
+
+ # Second connection for ACL tests
+ config2 = deepcopy(tf.config)
+ config2['account'] = tf.config['account2']
+ config2['username'] = tf.config['username2']
+ config2['password'] = tf.config['password2']
+ cls.conn2 = Connection(config2)
+ cls.conn2.authenticate()
# avoid getting a prefix that stops halfway through an encoded
# character
@@ -2085,6 +2483,69 @@ class TestObjectVersioningEnv(object):
cls.versioning_enabled = 'versions' in container_info
+class TestCrossPolicyObjectVersioningEnv(object):
+ # tri-state: None initially, then True/False
+ versioning_enabled = None
+ multiple_policies_enabled = None
+ policies = None
+
+ @classmethod
+ def setUp(cls):
+ cls.conn = Connection(tf.config)
+ cls.conn.authenticate()
+
+ if cls.multiple_policies_enabled is None:
+ try:
+ cls.policies = tf.FunctionalStoragePolicyCollection.from_info()
+ except AssertionError:
+ pass
+
+ if cls.policies and len(cls.policies) > 1:
+ cls.multiple_policies_enabled = True
+ else:
+ cls.multiple_policies_enabled = False
+ # We have to lie here that versioning is enabled. We actually
+ # don't know, but it does not matter. We know these tests cannot
+ # run without multiple policies present. If multiple policies are
+ # present, we won't be setting this field to any value, so it
+ # should all still work.
+ cls.versioning_enabled = True
+ return
+
+ policy = cls.policies.select()
+ version_policy = cls.policies.exclude(name=policy['name']).select()
+
+ cls.account = Account(cls.conn, tf.config.get('account',
+ tf.config['username']))
+
+ # Second connection for ACL tests
+ config2 = deepcopy(tf.config)
+ config2['account'] = tf.config['account2']
+ config2['username'] = tf.config['username2']
+ config2['password'] = tf.config['password2']
+ cls.conn2 = Connection(config2)
+ cls.conn2.authenticate()
+
+ # avoid getting a prefix that stops halfway through an encoded
+ # character
+ prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
+
+ cls.versions_container = cls.account.container(prefix + "-versions")
+ if not cls.versions_container.create(
+ {'X-Storage-Policy': policy['name']}):
+ raise ResponseError(cls.conn.response)
+
+ cls.container = cls.account.container(prefix + "-objs")
+ if not cls.container.create(
+ hdrs={'X-Versions-Location': cls.versions_container.name,
+ 'X-Storage-Policy': version_policy['name']}):
+ raise ResponseError(cls.conn.response)
+
+ container_info = cls.container.info()
+ # if versioning is off, then X-Versions-Location won't persist
+ cls.versioning_enabled = 'versions' in container_info
+
+
class TestObjectVersioning(Base):
env = TestObjectVersioningEnv
set_up = False
@@ -2099,6 +2560,15 @@ class TestObjectVersioning(Base):
"Expected versioning_enabled to be True/False, got %r" %
(self.env.versioning_enabled,))
+ def tearDown(self):
+ super(TestObjectVersioning, self).tearDown()
+ try:
+ # delete versions first!
+ self.env.versions_container.delete_files()
+ self.env.container.delete_files()
+ except ResponseError:
+ pass
+
def test_overwriting(self):
container = self.env.container
versions_container = self.env.versions_container
@@ -2130,31 +2600,100 @@ class TestObjectVersioning(Base):
versioned_obj.delete()
self.assertRaises(ResponseError, versioned_obj.read)
+ def test_versioning_dlo(self):
+ raise SkipTest('SOF incompatible test')
+ container = self.env.container
+ versions_container = self.env.versions_container
+ obj_name = Utils.create_name()
+
+ for i in ('1', '2', '3'):
+ time.sleep(.01) # guarantee that the timestamp changes
+ obj_name_seg = obj_name + '/' + i
+ versioned_obj = container.file(obj_name_seg)
+ versioned_obj.write(i)
+ versioned_obj.write(i + i)
+
+ self.assertEqual(3, versions_container.info()['object_count'])
+
+ man_file = container.file(obj_name)
+ man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
+ (self.env.container.name, obj_name)})
+
+ # guarantee that the timestamp changes
+ time.sleep(.01)
+
+ # write manifest file again
+ man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
+ (self.env.container.name, obj_name)})
+
+ self.assertEqual(3, versions_container.info()['object_count'])
+ self.assertEqual("112233", man_file.read())
+
+ def test_versioning_check_acl(self):
+ container = self.env.container
+ versions_container = self.env.versions_container
+ versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'})
+
+ obj_name = Utils.create_name()
+ versioned_obj = container.file(obj_name)
+ versioned_obj.write("aaaaa")
+ self.assertEqual("aaaaa", versioned_obj.read())
+
+ versioned_obj.write("bbbbb")
+ self.assertEqual("bbbbb", versioned_obj.read())
+
+ # Use token from second account and try to delete the object
+ org_token = self.env.account.conn.storage_token
+ self.env.account.conn.storage_token = self.env.conn2.storage_token
+ try:
+ self.assertRaises(ResponseError, versioned_obj.delete)
+ finally:
+ self.env.account.conn.storage_token = org_token
+
+ # Verify with token from first account
+ self.assertEqual("bbbbb", versioned_obj.read())
+
+ versioned_obj.delete()
+ self.assertEqual("aaaaa", versioned_obj.read())
+
class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
set_up = False
+class TestCrossPolicyObjectVersioning(TestObjectVersioning):
+ env = TestCrossPolicyObjectVersioningEnv
+ set_up = False
+
+ def setUp(self):
+ super(TestCrossPolicyObjectVersioning, self).setUp()
+ if self.env.multiple_policies_enabled is False:
+ raise SkipTest('Cross policy test requires multiple policies')
+ elif self.env.multiple_policies_enabled is not True:
+ # just some sanity checking
+ raise Exception("Expected multiple_policies_enabled "
+ "to be True/False, got %r" % (
+ self.env.versioning_enabled,))
+
+
class TestTempurlEnv(object):
tempurl_enabled = None # tri-state: None initially, then True/False
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.tempurl_enabled is None:
- cluster_info = cls.conn.cluster_info()
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
- cls.tempurl_methods = cluster_info['tempurl']['methods']
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account = Account(
- cls.conn, config.get('account', config['username']))
+ cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
@@ -2219,6 +2758,59 @@ class TestTempurl(Base):
contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True})
self.assertEqual(contents, "obj contents")
+ def test_GET_DLO_inside_container(self):
+ seg1 = self.env.container.file(
+ "get-dlo-inside-seg1" + Utils.create_name())
+ seg2 = self.env.container.file(
+ "get-dlo-inside-seg2" + Utils.create_name())
+ seg1.write("one fish two fish ")
+ seg2.write("red fish blue fish")
+
+ manifest = self.env.container.file("manifest" + Utils.create_name())
+ manifest.write(
+ '',
+ hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
+ (self.env.container.name,)})
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(manifest.path),
+ self.env.tempurl_key)
+ parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
+ self.assertEqual(contents, "one fish two fish red fish blue fish")
+
+ def test_GET_DLO_outside_container(self):
+ seg1 = self.env.container.file(
+ "get-dlo-outside-seg1" + Utils.create_name())
+ seg2 = self.env.container.file(
+ "get-dlo-outside-seg2" + Utils.create_name())
+ seg1.write("one fish two fish ")
+ seg2.write("red fish blue fish")
+
+ container2 = self.env.account.container(Utils.create_name())
+ container2.create()
+
+ manifest = container2.file("manifest" + Utils.create_name())
+ manifest.write(
+ '',
+ hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
+ (self.env.container.name,)})
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(manifest.path),
+ self.env.tempurl_key)
+ parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ # cross container tempurl works fine for account tempurl key
+ contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
+ self.assertEqual(contents, "one fish two fish red fish blue fish")
+ self.assert_status([200])
+
def test_PUT(self):
new_obj = self.env.container.file(Utils.create_name())
@@ -2237,6 +2829,42 @@ class TestTempurl(Base):
self.assert_(new_obj.info(parms=put_parms,
cfg={'no_auth_token': True}))
+ def test_PUT_manifest_access(self):
+ new_obj = self.env.container.file(Utils.create_name())
+
+ # give out a signature which allows a PUT to new_obj
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'PUT', expires, self.env.conn.make_path(new_obj.path),
+ self.env.tempurl_key)
+ put_parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ # try to create manifest pointing to some random container
+ try:
+ new_obj.write('', {
+ 'x-object-manifest': '%s/foo' % 'some_random_container'
+ }, parms=put_parms, cfg={'no_auth_token': True})
+ except ResponseError as e:
+ self.assertEqual(e.status, 400)
+ else:
+ self.fail('request did not error')
+
+ # create some other container
+ other_container = self.env.account.container(Utils.create_name())
+ if not other_container.create():
+ raise ResponseError(self.conn.response)
+
+ # try to create manifest pointing to new container
+ try:
+ new_obj.write('', {
+ 'x-object-manifest': '%s/foo' % other_container
+ }, parms=put_parms, cfg={'no_auth_token': True})
+ except ResponseError as e:
+ self.assertEqual(e.status, 400)
+ else:
+ self.fail('request did not error')
+
def test_HEAD(self):
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
@@ -2310,22 +2938,288 @@ class TestTempurlUTF8(Base2, TestTempurl):
set_up = False
+class TestContainerTempurlEnv(object):
+ tempurl_enabled = None # tri-state: None initially, then True/False
+
+ @classmethod
+ def setUp(cls):
+ cls.conn = Connection(tf.config)
+ cls.conn.authenticate()
+
+ if cls.tempurl_enabled is None:
+ cls.tempurl_enabled = 'tempurl' in cluster_info
+ if not cls.tempurl_enabled:
+ return
+
+ cls.tempurl_key = Utils.create_name()
+ cls.tempurl_key2 = Utils.create_name()
+
+ cls.account = Account(
+ cls.conn, tf.config.get('account', tf.config['username']))
+ cls.account.delete_containers()
+
+ # creating another account and connection
+ # for ACL tests
+ config2 = deepcopy(tf.config)
+ config2['account'] = tf.config['account2']
+ config2['username'] = tf.config['username2']
+ config2['password'] = tf.config['password2']
+ cls.conn2 = Connection(config2)
+ cls.conn2.authenticate()
+ cls.account2 = Account(
+ cls.conn2, config2.get('account', config2['username']))
+ cls.account2 = cls.conn2.get_account()
+
+ cls.container = cls.account.container(Utils.create_name())
+ if not cls.container.create({
+ 'x-container-meta-temp-url-key': cls.tempurl_key,
+ 'x-container-meta-temp-url-key-2': cls.tempurl_key2,
+ 'x-container-read': cls.account2.name}):
+ raise ResponseError(cls.conn.response)
+
+ cls.obj = cls.container.file(Utils.create_name())
+ cls.obj.write("obj contents")
+ cls.other_obj = cls.container.file(Utils.create_name())
+ cls.other_obj.write("other obj contents")
+
+
+class TestContainerTempurl(Base):
+ env = TestContainerTempurlEnv
+ set_up = False
+
+ def setUp(self):
+ super(TestContainerTempurl, self).setUp()
+ if self.env.tempurl_enabled is False:
+ raise SkipTest("TempURL not enabled")
+ elif self.env.tempurl_enabled is not True:
+ # just some sanity checking
+ raise Exception(
+ "Expected tempurl_enabled to be True/False, got %r" %
+ (self.env.tempurl_enabled,))
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(self.env.obj.path),
+ self.env.tempurl_key)
+ self.obj_tempurl_parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ def tempurl_sig(self, method, expires, path, key):
+ return hmac.new(
+ key,
+ '%s\n%s\n%s' % (method, expires, urllib.unquote(path)),
+ hashlib.sha1).hexdigest()
+
+ def test_GET(self):
+ contents = self.env.obj.read(
+ parms=self.obj_tempurl_parms,
+ cfg={'no_auth_token': True})
+ self.assertEqual(contents, "obj contents")
+
+ # GET tempurls also allow HEAD requests
+ self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms,
+ cfg={'no_auth_token': True}))
+
+ def test_GET_with_key_2(self):
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(self.env.obj.path),
+ self.env.tempurl_key2)
+ parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True})
+ self.assertEqual(contents, "obj contents")
+
+ def test_PUT(self):
+ new_obj = self.env.container.file(Utils.create_name())
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'PUT', expires, self.env.conn.make_path(new_obj.path),
+ self.env.tempurl_key)
+ put_parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ new_obj.write('new obj contents',
+ parms=put_parms, cfg={'no_auth_token': True})
+ self.assertEqual(new_obj.read(), "new obj contents")
+
+ # PUT tempurls also allow HEAD requests
+ self.assert_(new_obj.info(parms=put_parms,
+ cfg={'no_auth_token': True}))
+
+ def test_HEAD(self):
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'HEAD', expires, self.env.conn.make_path(self.env.obj.path),
+ self.env.tempurl_key)
+ head_parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ self.assert_(self.env.obj.info(parms=head_parms,
+ cfg={'no_auth_token': True}))
+ # HEAD tempurls don't allow PUT or GET requests, despite the fact that
+ # PUT and GET tempurls both allow HEAD requests
+ self.assertRaises(ResponseError, self.env.other_obj.read,
+ cfg={'no_auth_token': True},
+ parms=self.obj_tempurl_parms)
+ self.assert_status([401])
+
+ self.assertRaises(ResponseError, self.env.other_obj.write,
+ 'new contents',
+ cfg={'no_auth_token': True},
+ parms=self.obj_tempurl_parms)
+ self.assert_status([401])
+
+ def test_different_object(self):
+ contents = self.env.obj.read(
+ parms=self.obj_tempurl_parms,
+ cfg={'no_auth_token': True})
+ self.assertEqual(contents, "obj contents")
+
+ self.assertRaises(ResponseError, self.env.other_obj.read,
+ cfg={'no_auth_token': True},
+ parms=self.obj_tempurl_parms)
+ self.assert_status([401])
+
+ def test_changing_sig(self):
+ contents = self.env.obj.read(
+ parms=self.obj_tempurl_parms,
+ cfg={'no_auth_token': True})
+ self.assertEqual(contents, "obj contents")
+
+ parms = self.obj_tempurl_parms.copy()
+ if parms['temp_url_sig'][0] == 'a':
+ parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:]
+ else:
+ parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:]
+
+ self.assertRaises(ResponseError, self.env.obj.read,
+ cfg={'no_auth_token': True},
+ parms=parms)
+ self.assert_status([401])
+
+ def test_changing_expires(self):
+ contents = self.env.obj.read(
+ parms=self.obj_tempurl_parms,
+ cfg={'no_auth_token': True})
+ self.assertEqual(contents, "obj contents")
+
+ parms = self.obj_tempurl_parms.copy()
+ if parms['temp_url_expires'][-1] == '0':
+ parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1'
+ else:
+ parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0'
+
+ self.assertRaises(ResponseError, self.env.obj.read,
+ cfg={'no_auth_token': True},
+ parms=parms)
+ self.assert_status([401])
+
+ def test_tempurl_keys_visible_to_account_owner(self):
+ if not tf.cluster_info.get('tempauth'):
+ raise SkipTest('TEMP AUTH SPECIFIC TEST')
+ metadata = self.env.container.info()
+ self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key)
+ self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2)
+
+ def test_tempurl_keys_hidden_from_acl_readonly(self):
+ if not tf.cluster_info.get('tempauth'):
+ raise SkipTest('TEMP AUTH SPECIFIC TEST')
+ original_token = self.env.container.conn.storage_token
+ self.env.container.conn.storage_token = self.env.conn2.storage_token
+ metadata = self.env.container.info()
+ self.env.container.conn.storage_token = original_token
+
+ self.assertTrue('tempurl_key' not in metadata,
+ 'Container TempURL key found, should not be visible '
+ 'to readonly ACLs')
+ self.assertTrue('tempurl_key2' not in metadata,
+ 'Container TempURL key-2 found, should not be visible '
+ 'to readonly ACLs')
+
+ def test_GET_DLO_inside_container(self):
+ seg1 = self.env.container.file(
+ "get-dlo-inside-seg1" + Utils.create_name())
+ seg2 = self.env.container.file(
+ "get-dlo-inside-seg2" + Utils.create_name())
+ seg1.write("one fish two fish ")
+ seg2.write("red fish blue fish")
+
+ manifest = self.env.container.file("manifest" + Utils.create_name())
+ manifest.write(
+ '',
+ hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
+ (self.env.container.name,)})
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(manifest.path),
+ self.env.tempurl_key)
+ parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
+ self.assertEqual(contents, "one fish two fish red fish blue fish")
+
+ def test_GET_DLO_outside_container(self):
+ container2 = self.env.account.container(Utils.create_name())
+ container2.create()
+ seg1 = container2.file(
+ "get-dlo-outside-seg1" + Utils.create_name())
+ seg2 = container2.file(
+ "get-dlo-outside-seg2" + Utils.create_name())
+ seg1.write("one fish two fish ")
+ seg2.write("red fish blue fish")
+
+ manifest = self.env.container.file("manifest" + Utils.create_name())
+ manifest.write(
+ '',
+ hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
+ (container2.name,)})
+
+ expires = int(time.time()) + 86400
+ sig = self.tempurl_sig(
+ 'GET', expires, self.env.conn.make_path(manifest.path),
+ self.env.tempurl_key)
+ parms = {'temp_url_sig': sig,
+ 'temp_url_expires': str(expires)}
+
+ # cross container tempurl does not work for container tempurl key
+ try:
+ manifest.read(parms=parms, cfg={'no_auth_token': True})
+ except ResponseError as e:
+ self.assertEqual(e.status, 401)
+ else:
+ self.fail('request did not error')
+ try:
+ manifest.info(parms=parms, cfg={'no_auth_token': True})
+ except ResponseError as e:
+ self.assertEqual(e.status, 401)
+ else:
+ self.fail('request did not error')
+
+
+class TestContainerTempurlUTF8(Base2, TestContainerTempurl):
+ set_up = False
+
+
class TestSloTempurlEnv(object):
enabled = None # tri-state: None initially, then True/False
@classmethod
def setUp(cls):
- cls.conn = Connection(config)
+ cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.enabled is None:
- cluster_info = cls.conn.cluster_info()
cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info
cls.tempurl_key = Utils.create_name()
cls.account = Account(
- cls.conn, config.get('account', config['username']))
+ cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({'temp-url-key': cls.tempurl_key})
@@ -2398,5 +3292,174 @@ class TestSloTempurlUTF8(Base2, TestSloTempurl):
set_up = False
+class TestServiceToken(unittest.TestCase):
+
+ def setUp(self):
+ if tf.skip_service_tokens:
+ raise SkipTest
+
+ self.SET_TO_USERS_TOKEN = 1
+ self.SET_TO_SERVICE_TOKEN = 2
+
+ # keystoneauth and tempauth differ in allowing PUT account
+ # Even if keystoneauth allows it, the proxy-server uses
+ # allow_account_management to decide if accounts can be created
+ self.put_account_expect = is_client_error
+ if tf.swift_test_auth_version != '1':
+ if cluster_info.get('swift').get('allow_account_management'):
+ self.put_account_expect = is_success
+
+ def _scenario_generator(self):
+ paths = ((None, None), ('c', None), ('c', 'o'))
+ for path in paths:
+ for method in ('PUT', 'POST', 'HEAD', 'GET', 'OPTIONS'):
+ yield method, path[0], path[1]
+ for path in reversed(paths):
+ yield 'DELETE', path[0], path[1]
+
+ def _assert_is_authed_response(self, method, container, object, resp):
+ resp.read()
+ expect = is_success
+ if method == 'DELETE' and not container:
+ expect = is_client_error
+ if method == 'PUT' and not container:
+ expect = self.put_account_expect
+ self.assertTrue(expect(resp.status), 'Unexpected %s for %s %s %s'
+ % (resp.status, method, container, object))
+
+ def _assert_not_authed_response(self, method, container, object, resp):
+ resp.read()
+ expect = is_client_error
+ if method == 'OPTIONS':
+ expect = is_success
+ self.assertTrue(expect(resp.status), 'Unexpected %s for %s %s %s'
+ % (resp.status, method, container, object))
+
+ def prepare_request(self, method, use_service_account=False,
+ container=None, obj=None, body=None, headers=None,
+ x_auth_token=None,
+ x_service_token=None, dbg=False):
+ """
+ Setup for making the request
+
+ When retry() calls the do_request() function, it calls it the
+ test user's token, the parsed path, a connection and (optionally)
+ a token from the test service user. We save options here so that
+ do_request() can make the appropriate request.
+
+ :param method: The operation (e.g'. 'HEAD')
+ :param use_service_account: Optional. Set True to change the path to
+ be the service account
+ :param container: Optional. Adds a container name to the path
+ :param obj: Optional. Adds an object name to the path
+ :param body: Optional. Adds a body (string) in the request
+ :param headers: Optional. Adds additional headers.
+ :param x_auth_token: Optional. Default is SET_TO_USERS_TOKEN. One of:
+ SET_TO_USERS_TOKEN Put the test user's token in
+ X-Auth-Token
+ SET_TO_SERVICE_TOKEN Put the service token in X-Auth-Token
+ :param x_service_token: Optional. Default is to not set X-Service-Token
+ to any value. If specified, is one of following:
+ SET_TO_USERS_TOKEN Put the test user's token in
+ X-Service-Token
+ SET_TO_SERVICE_TOKEN Put the service token in
+ X-Service-Token
+ :param dbg: Optional. Set true to check request arguments
+ """
+ self.method = method
+ self.use_service_account = use_service_account
+ self.container = container
+ self.obj = obj
+ self.body = body
+ self.headers = headers
+ if x_auth_token:
+ self.x_auth_token = x_auth_token
+ else:
+ self.x_auth_token = self.SET_TO_USERS_TOKEN
+ self.x_service_token = x_service_token
+ self.dbg = dbg
+
+ def do_request(self, url, token, parsed, conn, service_token=''):
+ if self.use_service_account:
+ path = self._service_account(parsed.path)
+ else:
+ path = parsed.path
+ if self.container:
+ path += '/%s' % self.container
+ if self.obj:
+ path += '/%s' % self.obj
+ headers = {}
+ if self.body:
+ headers.update({'Content-Length': len(self.body)})
+ if self.headers:
+ headers.update(self.headers)
+ if self.x_auth_token == self.SET_TO_USERS_TOKEN:
+ headers.update({'X-Auth-Token': token})
+ elif self.x_auth_token == self.SET_TO_SERVICE_TOKEN:
+ headers.update({'X-Auth-Token': service_token})
+ if self.x_service_token == self.SET_TO_USERS_TOKEN:
+ headers.update({'X-Service-Token': token})
+ elif self.x_service_token == self.SET_TO_SERVICE_TOKEN:
+ headers.update({'X-Service-Token': service_token})
+ if self.dbg:
+ print('DEBUG: conn.request: method:%s path:%s'
+ ' body:%s headers:%s' % (self.method, path, self.body,
+ headers))
+ conn.request(self.method, path, self.body, headers=headers)
+ return check_response(conn)
+
+ def _service_account(self, path):
+ parts = path.split('/', 3)
+ account = parts[2]
+ try:
+ project_id = account[account.index('_') + 1:]
+ except ValueError:
+ project_id = account
+ parts[2] = '%s%s' % (tf.swift_test_service_prefix, project_id)
+ return '/'.join(parts)
+
+ def test_user_access_own_auth_account(self):
+ # This covers ground tested elsewhere (tests a user doing HEAD
+ # on own account). However, if this fails, none of the remaining
+ # tests will work
+ self.prepare_request('HEAD')
+ resp = retry(self.do_request)
+ resp.read()
+ self.assert_(resp.status in (200, 204), resp.status)
+
+ def test_user_cannot_access_service_account(self):
+ for method, container, obj in self._scenario_generator():
+ self.prepare_request(method, use_service_account=True,
+ container=container, obj=obj)
+ resp = retry(self.do_request)
+ self._assert_not_authed_response(method, container, obj, resp)
+
+ def test_service_user_denied_with_x_auth_token(self):
+ for method, container, obj in self._scenario_generator():
+ self.prepare_request(method, use_service_account=True,
+ container=container, obj=obj,
+ x_auth_token=self.SET_TO_SERVICE_TOKEN)
+ resp = retry(self.do_request, service_user=5)
+ self._assert_not_authed_response(method, container, obj, resp)
+
+ def test_service_user_denied_with_x_service_token(self):
+ for method, container, obj in self._scenario_generator():
+ self.prepare_request(method, use_service_account=True,
+ container=container, obj=obj,
+ x_auth_token=self.SET_TO_SERVICE_TOKEN,
+ x_service_token=self.SET_TO_SERVICE_TOKEN)
+ resp = retry(self.do_request, service_user=5)
+ self._assert_not_authed_response(method, container, obj, resp)
+
+ def test_user_plus_service_can_access_service_account(self):
+ for method, container, obj in self._scenario_generator():
+ self.prepare_request(method, use_service_account=True,
+ container=container, obj=obj,
+ x_auth_token=self.SET_TO_USERS_TOKEN,
+ x_service_token=self.SET_TO_SERVICE_TOKEN)
+ resp = retry(self.do_request, service_user=5)
+ self._assert_is_authed_response(method, container, obj, resp)
+
+
if __name__ == '__main__':
unittest.main()