diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/functional/__init__.py | 0 | ||||
| -rw-r--r-- | test/functional/swift_test_client.py | 736 | ||||
| -rw-r--r-- | test/functional/tests.py | 1617 | ||||
| -rw-r--r-- | test/functionalnosetests/__init__.py | 0 | ||||
| -rw-r--r-- | test/functionalnosetests/swift_testing.py | 175 | ||||
| -rwxr-xr-x | test/functionalnosetests/test_account.py | 152 | ||||
| -rwxr-xr-x | test/functionalnosetests/test_container.py | 573 | ||||
| -rwxr-xr-x | test/functionalnosetests/test_object.py | 600 |
8 files changed, 3853 insertions, 0 deletions
diff --git a/test/functional/__init__.py b/test/functional/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/functional/__init__.py diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py new file mode 100644 index 0000000..a6d8aec --- /dev/null +++ b/test/functional/swift_test_client.py @@ -0,0 +1,736 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import httplib +import os +import random +import socket +import StringIO +import time +import urllib + +import simplejson as json + +from nose import SkipTest +from xml.dom import minidom +from swiftclient import get_auth + + +class AuthenticationFailed(Exception): + pass + + +class RequestError(Exception): + pass + + +class ResponseError(Exception): + def __init__(self, response): + self.status = response.status + self.reason = response.reason + Exception.__init__(self) + + def __str__(self): + return '%d: %s' % (self.status, self.reason) + + def __repr__(self): + return '%d: %s' % (self.status, self.reason) + + +def listing_empty(method): + for i in xrange(0, 6): + if len(method()) == 0: + return True + + time.sleep(2 ** i) + + return False + + +def listing_items(method): + marker = None + once = True + items = [] + + while once or items: + for i in items: + yield i + + if once or marker: + if marker: + items = method(parms={'marker': marker}) + else: + items = method() + + if len(items) == 10000: + marker = items[-1] + else: + marker = None + + once = False + else: + items = [] + + +class Connection(object): + def __init__(self, config): + for key in 'auth_host auth_port auth_ssl username password'.split(): + if key not in config: + raise SkipTest + + self.auth_host = config['auth_host'] + self.auth_port = int(config['auth_port']) + self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1') + self.auth_prefix = config.get('auth_prefix', '/') + self.auth_version = str(config.get('auth_version', '1')) + + self.account = config.get('account') + self.username = config['username'] + self.password = config['password'] + + self.storage_host = None + self.storage_port = None + + self.conn_class = None + + def get_account(self): + return Account(self, self.account) + + def authenticate(self, clone_conn=None): + if clone_conn: + self.conn_class = clone_conn.conn_class + self.storage_host = clone_conn.storage_host + self.storage_url = clone_conn.storage_url + self.storage_port = clone_conn.storage_port + self.storage_token = clone_conn.storage_token + return + + if self.auth_version == "1": + auth_path = '%sv1.0' % (self.auth_prefix) + if self.account: + auth_user = '%s:%s' % (self.account, self.username) + else: + auth_user = self.username + else: + auth_user = self.username + auth_path = self.auth_prefix + auth_scheme = 'https://' if self.auth_ssl else 'http://' + auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) + auth_url = auth_scheme + auth_netloc + auth_path + + (storage_url, storage_token) = get_auth(auth_url, + auth_user, self.password, + snet=False, + tenant_name=self.account, + auth_version=self.auth_version, + os_options={}) + + if not (storage_url and storage_token): + raise AuthenticationFailed() + + x = storage_url.split('/') + + if x[0] == 'http:': + self.conn_class = httplib.HTTPConnection + self.storage_port = 80 + elif x[0] == 'https:': + self.conn_class = httplib.HTTPSConnection + self.storage_port = 443 + else: + raise ValueError('unexpected protocol %s' % (x[0])) + + self.storage_host = x[2].split(':')[0] + if ':' in x[2]: + self.storage_port = int(x[2].split(':')[1]) + self.storage_url = '/%s/%s' % (x[3], x[4]) + + self.storage_token = storage_token + + self.http_connect() + return self.storage_url, self.storage_token + + def http_connect(self): + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + + def make_path(self, path=[], cfg={}): + if cfg.get('version_only_path'): + return '/' + self.storage_url.split('/')[1] + + if path: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_path_quote'): + quote = lambda x: x + return '%s/%s' % (self.storage_url, + '/'.join([quote(i) for i in path])) + else: + return self.storage_url + + def make_headers(self, hdrs, cfg={}): + headers = {} + + if not cfg.get('no_auth_token'): + headers['X-Auth-Token'] = self.storage_token + + if isinstance(hdrs, dict): + headers.update(hdrs) + return headers + + def make_request(self, method, path=[], data='', hdrs={}, parms={}, + cfg={}): + path = self.make_path(path, cfg=cfg) + headers = self.make_headers(hdrs, cfg=cfg) + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) + for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + else: + headers['Content-Length'] = len(data) + + def try_request(): + self.http_connect() + self.connection.request(method, path, data, headers) + return self.connection.getresponse() + + self.response = None + try_count = 0 + while try_count < 5: + try_count += 1 + + try: + self.response = try_request() + except httplib.HTTPException: + continue + + if self.response.status == 401: + self.authenticate() + continue + elif self.response.status == 503: + if try_count != 5: + time.sleep(5) + continue + + break + + if self.response: + return self.response.status + + raise RequestError('Unable to complete http request') + + def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): + self.http_connect() + + path = self.make_path(path, cfg) + headers = self.make_headers(hdrs, cfg=cfg) + + if chunked: + headers['Transfer-Encoding'] = 'chunked' + headers.pop('Content-Length', None) + + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) + for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + query_args = ['%s=%s' % (urllib.quote(x), + urllib.quote(str(y))) for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + self.connection.putrequest('PUT', path) + for key, value in headers.iteritems(): + self.connection.putheader(key, value) + self.connection.endheaders() + + def put_data(self, data, chunked=False): + if chunked: + self.connection.send('%x\r\n%s\r\n' % (len(data), data)) + else: + self.connection.send(data) + + def put_end(self, chunked=False): + if chunked: + self.connection.send('0\r\n\r\n') + + self.response = self.connection.getresponse() + self.connection.close() + return self.response.status + + +class Base: + def __str__(self): + return self.name + + def header_fields(self, fields): + headers = dict(self.conn.response.getheaders()) + ret = {} + for field in fields: + if field[1] not in headers: + raise ValueError("%s was not found in response header" % + (field[1])) + + try: + ret[field[0]] = int(headers[field[1]]) + except ValueError: + ret[field[0]] = headers[field[1]] + return ret + + +class Account(Base): + def __init__(self, conn, name): + self.conn = conn + self.name = str(name) + + def container(self, container_name): + return Container(self.conn, self.name, container_name) + + def containers(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format is None and 'format' in parms: + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + conts = json.loads(self.conn.response.read()) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + elif format == 'xml': + conts = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('container'): + cont = {} + for key in ['name', 'count', 'bytes']: + cont[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + conts.append(cont) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + else: + lines = self.conn.response.read().split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def delete_containers(self): + for c in listing_items(self.containers): + cont = self.container(c) + if not cont.delete_recursive(): + return False + + return listing_empty(self.containers) + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 204: + + raise ResponseError(self.conn.response) + + fields = [['object_count', 'x-account-object-count'], + ['container_count', 'x-account-container-count'], + ['bytes_used', 'x-account-bytes-used']] + + return self.header_fields(fields) + + @property + def path(self): + return [] + + +class Container(Base): + def __init__(self, conn, account, name): + self.conn = conn + self.account = str(account) + self.name = str(name) + + def create(self, hdrs={}, parms={}, cfg={}): + return self.conn.make_request('PUT', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) in (201, 202) + + def delete(self, hdrs={}, parms={}): + return self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) == 204 + + def delete_files(self): + for f in listing_items(self.files): + file = self.file(f) + if not file.delete(): + return False + + return listing_empty(self.files) + + def delete_recursive(self): + return self.delete_files() and self.delete() + + def file(self, file_name): + return File(self.conn, self.account, self.name, file_name) + + def files(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format is None and 'format' in parms: + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + files = json.loads(self.conn.response.read()) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + elif format == 'xml': + files = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('object'): + file = {} + for key in ['name', 'hash', 'bytes', 'content_type', + 'last_modified']: + + file[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + files.append(file) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + else: + content = self.conn.response.read() + if content: + lines = content.split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + else: + return [] + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def info(self, hdrs={}, parms={}, cfg={}): + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + + if self.conn.response.status == 204: + fields = [['bytes_used', 'x-container-bytes-used'], + ['object_count', 'x-container-object-count']] + + return self.header_fields(fields) + + raise ResponseError(self.conn.response) + + @property + def path(self): + return [self.name] + + +class File(Base): + def __init__(self, conn, account, container, name): + self.conn = conn + self.account = str(account) + self.container = str(container) + self.name = str(name) + + self.chunked_write_in_progress = False + self.content_type = None + self.size = None + self.metadata = {} + + def make_headers(self, cfg={}): + headers = {} + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + elif self.size: + headers['Content-Length'] = self.size + else: + headers['Content-Length'] = 0 + + if cfg.get('no_content_type'): + pass + elif self.content_type: + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = 'application/octet-stream' + + for key in self.metadata: + headers['X-Object-Meta-' + key] = self.metadata[key] + + return headers + + @classmethod + def compute_md5sum(cls, data): + block_size = 4096 + + if isinstance(data, str): + data = StringIO.StringIO(data) + + checksum = hashlib.md5() + buff = data.read(block_size) + while buff: + checksum.update(buff) + buff = data.read(block_size) + data.seek(0) + return checksum.hexdigest() + + def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}): + if 'destination' in cfg: + headers = {'Destination': cfg['destination']} + elif cfg.get('no_destination'): + headers = {} + else: + headers = {'Destination': '%s/%s' % (dest_cont, dest_file)} + headers.update(hdrs) + + if 'Destination' in headers: + headers['Destination'] = urllib.quote(headers['Destination']) + + return self.conn.make_request('COPY', self.path, hdrs=headers, + parms=parms) == 201 + + def delete(self, hdrs={}, parms={}): + if self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) != 204: + + raise ResponseError(self.conn.response) + + return True + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 200: + + raise ResponseError(self.conn.response) + + fields = [['content_length', 'content-length'], + ['content_type', 'content-type'], + ['last_modified', 'last-modified'], + ['etag', 'etag']] + + header_fields = self.header_fields(fields) + header_fields['etag'] = header_fields['etag'].strip('"') + return header_fields + + def initialize(self, hdrs={}, parms={}): + if not self.name: + return False + + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms) + if status == 404: + return False + elif (status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + if hdr[0].lower().startswith('x-object-meta-'): + self.metadata[hdr[0][14:]] = hdr[1] + if hdr[0].lower() == 'etag': + self.etag = hdr[1].strip('"') + if hdr[0].lower() == 'content-length': + self.size = int(hdr[1]) + if hdr[0].lower() == 'last-modified': + self.last_modified = hdr[1] + + return True + + def load_from_filename(self, filename, callback=None): + fobj = open(filename, 'rb') + self.write(fobj, callback=callback) + fobj.close() + + @property + def path(self): + return [self.container, self.name] + + @classmethod + def random_data(cls, size=None): + if size is None: + size = random.randint(1, 32768) + fd = open('/dev/urandom', 'r') + data = fd.read(size) + fd.close() + return data + + def read(self, size=-1, offset=0, hdrs=None, buffer=None, + callback=None, cfg={}): + + if size > 0: + range = 'bytes=%d-%d' % (offset, (offset + size) - 1) + if hdrs: + hdrs['Range'] = range + else: + hdrs = {'Range': range} + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + cfg=cfg) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + + if hasattr(buffer, 'write'): + scratch = self.conn.response.read(8192) + transferred = 0 + + while len(scratch) > 0: + buffer.write(scratch) + transferred += len(scratch) + if callable(callback): + callback(transferred, self.size) + scratch = self.conn.response.read(8192) + return None + else: + return self.conn.response.read() + + def read_md5(self): + status = self.conn.make_request('GET', self.path) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + checksum = hashlib.md5() + + scratch = self.conn.response.read(8192) + while len(scratch) > 0: + checksum.update(scratch) + scratch = self.conn.response.read(8192) + + return checksum.hexdigest() + + def save_to_filename(self, filename, callback=None): + try: + fobj = open(filename, 'wb') + self.read(buffer=fobj, callback=callback) + finally: + fobj.close() + + def sync_metadata(self, metadata={}, cfg={}): + self.metadata.update(metadata) + + if self.metadata: + headers = self.make_headers(cfg=cfg) + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = \ + cfg.get('set_content_length') + else: + headers['Content-Length'] = 0 + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + + if self.conn.response.status not in (201, 202): + raise ResponseError(self.conn.response) + + return True + + def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}): + if data is not None and self.chunked_write_in_progress: + self.conn.put_data(data, True) + elif data is not None: + self.chunked_write_in_progress = True + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, + cfg=cfg, chunked=True) + + self.conn.put_data(data, True) + elif self.chunked_write_in_progress: + self.chunked_write_in_progress = False + return self.conn.put_end(True) == 201 + else: + raise RuntimeError + + def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}): + block_size = 2 ** 20 + + if isinstance(data, file): + try: + data.flush() + data.seek(0) + except IOError: + pass + self.size = int(os.fstat(data.fileno())[6]) + else: + data = StringIO.StringIO(data) + self.size = data.len + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg) + + transferred = 0 + buff = data.read(block_size) + try: + while len(buff) > 0: + self.conn.put_data(buff) + buff = data.read(block_size) + transferred += len(buff) + if callable(callback): + callback(transferred, self.size) + + self.conn.put_end() + except socket.timeout, err: + raise err + + if (self.conn.response.status < 200) or \ + (self.conn.response.status > 299): + raise ResponseError(self.conn.response) + + self.md5 = self.compute_md5sum(data) + + return True + + def write_random(self, size=None, hdrs={}, parms={}, cfg={}): + data = self.random_data(size) + if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): + raise ResponseError(self.conn.response) + self.md5 = self.compute_md5sum(StringIO.StringIO(data)) + return data diff --git a/test/functional/tests.py b/test/functional/tests.py new file mode 100644 index 0000000..d6f8d70 --- /dev/null +++ b/test/functional/tests.py @@ -0,0 +1,1617 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import locale +import random +import StringIO +import time +import threading +import uuid +import unittest +from nose import SkipTest +from ConfigParser import ConfigParser + +from test import get_config +from test.functional.swift_test_client import Account, Connection, File, \ + ResponseError +from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ + MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ + MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ + MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH + +default_constraints = dict(( + ('max_file_size', MAX_FILE_SIZE), + ('max_meta_name_length', MAX_META_NAME_LENGTH), + ('max_meta_value_length', MAX_META_VALUE_LENGTH), + ('max_meta_count', MAX_META_COUNT), + ('max_meta_overall_size', MAX_META_OVERALL_SIZE), + ('max_object_name_length', MAX_OBJECT_NAME_LENGTH), + ('container_listing_limit', CONTAINER_LISTING_LIMIT), + ('account_listing_limit', ACCOUNT_LISTING_LIMIT), + ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), + ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) +constraints_conf = ConfigParser() +conf_exists = constraints_conf.read('/etc/swift/swift.conf') +# Constraints are set first from the test config, then from +# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist, +# then limit test coverage. This allows SAIO tests to work fine but +# requires remote funtional testing to know something about the cluster +# that is being tested. +config = get_config('func_test') +for k in default_constraints: + if k in config: + # prefer what's in test.conf + config[k] = int(config[k]) + elif conf_exists: + # swift.conf exists, so use what's defined there (or swift defaults) + # This normally happens when the test is running locally to the cluster + # as in a SAIO. + config[k] = default_constraints[k] + else: + # .functests don't know what the constraints of the tested cluster are, + # so the tests can't reliably pass or fail. Therefore, skip those + # tests. + config[k] = '%s constraint is not defined' % k + +web_front_end = config.get('web_front_end', 'integral') +normalized_urls = config.get('normalized_urls', False) + +def load_constraint(name): + c = config[name] + if not isinstance(c, int): + raise SkipTest(c) + return c + +locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) + + +def chunks(s, length=3): + i, j = 0, length + while i < len(s): + yield s[i:j] + i, j = j, j + length + + +def timeout(seconds, method, *args, **kwargs): + class TimeoutThread(threading.Thread): + def __init__(self, method, *args, **kwargs): + threading.Thread.__init__(self) + + self.method = method + self.args = args + self.kwargs = kwargs + self.exception = None + + def run(self): + try: + self.method(*self.args, **self.kwargs) + except Exception, e: + self.exception = e + + t = TimeoutThread(method, *args, **kwargs) + t.start() + t.join(seconds) + + if t.exception: + raise t.exception + + if t.isAlive(): + t._Thread__stop() + return True + return False + + +class Utils: + @classmethod + def create_ascii_name(cls, length=None): + return uuid.uuid4().hex + + @classmethod + def create_utf8_name(cls, length=None): + if length is None: + length = 15 + else: + length = int(length) + + utf8_chars = u'\uF10F\uD20D\uB30B\u9409\u8508\u5605\u3703\u1801'\ + u'\u0900\uF110\uD20E\uB30C\u940A\u8509\u5606\u3704'\ + u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ + u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ + u'\u5608\u3706\u1804\u0903\u03A9\u2603' + return ''.join([random.choice(utf8_chars) + for x in xrange(length)]).encode('utf-8') + + create_name = create_ascii_name + + +class Base(unittest.TestCase): + def setUp(self): + cls = type(self) + if not cls.set_up: + cls.env.setUp() + cls.set_up = True + + def assert_body(self, body): + response_body = self.env.conn.response.read() + self.assert_(response_body == body, + 'Body returned: %s' % (response_body)) + + def assert_status(self, status_or_statuses): + self.assert_(self.env.conn.response.status == status_or_statuses or + (hasattr(status_or_statuses, '__iter__') and + self.env.conn.response.status in status_or_statuses), + 'Status returned: %d Expected: %s' % + (self.env.conn.response.status, status_or_statuses)) + + +class Base2(object): + def setUp(self): + Utils.create_name = Utils.create_utf8_name + super(Base2, self).setUp() + + def tearDown(self): + Utils.create_name = Utils.create_ascii_name + + +class TestAccountEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.containers = [] + for i in range(10): + cont = cls.account.container(Utils.create_name()) + if not cont.create(): + raise ResponseError(cls.conn.response) + + cls.containers.append(cont) + + +class TestAccountDev(Base): + env = TestAccountEnv + set_up = False + + +class TestAccountDevUTF8(Base2, TestAccountDev): + set_up = False + + +class TestAccount(Base): + env = TestAccountEnv + set_up = False + + def testNoAuthToken(self): + self.assertRaises(ResponseError, self.env.account.info, + cfg={'no_auth_token': True}) + self.assert_status([401, 412]) + + self.assertRaises(ResponseError, self.env.account.containers, + cfg={'no_auth_token': True}) + self.assert_status([401, 412]) + + def testInvalidUTF8Path(self): + invalid_utf8 = Utils.create_utf8_name()[::-1] + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote': True})) + self.assert_status(412) + self.assert_body('Invalid UTF8 or contains NULL') + + def testVersionOnlyPath(self): + self.env.account.conn.make_request('PUT', + cfg={'version_only_path': True}) + self.assert_status(412) + self.assert_body('Bad URL') + + def testInvalidPath(self): + was_url = self.env.account.conn.storage_url + if (normalized_urls): + self.env.account.conn.storage_url = '/' + else: + self.env.account.conn.storage_url = "/%s" % was_url + self.env.account.conn.make_request('GET') + try: + self.assert_status(404) + finally: + self.env.account.conn.storage_url = was_url + + def testPUT(self): + self.env.account.conn.make_request('PUT') + self.assert_status([403, 405]) + + def testAccountHead(self): + try_count = 0 + while try_count < 5: + try_count += 1 + + info = self.env.account.info() + for field in ['object_count', 'container_count', 'bytes_used']: + self.assert_(info[field] >= 0) + + if info['container_count'] == len(self.env.containers): + break + + if try_count < 5: + time.sleep(1) + + self.assertEquals(info['container_count'], len(self.env.containers)) + self.assert_status(204) + + def testContainerSerializedInfo(self): + container_info = {} + for container in self.env.containers: + info = {'bytes': 0} + info['count'] = random.randint(10, 30) + for i in range(info['count']): + file = container.file(Utils.create_name()) + bytes = random.randint(1, 32768) + file.write_random(bytes) + info['bytes'] += bytes + + container_info[container.name] = info + + for format in ['json', 'xml']: + for a in self.env.account.containers(parms={'format': format}): + self.assert_(a['count'] >= 0) + self.assert_(a['bytes'] >= 0) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf-8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf-8') + + def testListingLimit(self): + limit = load_constraint('account_listing_limit') + for l in (1, 100, limit / 2, limit - 1, limit, limit + 1, limit * 2): + p = {'limit': l} + + if l <= limit: + self.assert_(len(self.env.account.containers(parms=p)) <= l) + self.assert_status(200) + else: + self.assertRaises(ResponseError, + self.env.account.containers, parms=p) + self.assert_status(412) + + def testContainerListing(self): + a = sorted([c.name for c in self.env.containers]) + + for format in [None, 'json', 'xml']: + b = self.env.account.containers(parms={'format': format}) + + if isinstance(b[0], dict): + b = [x['name'] for x in b] + + self.assertEquals(a, b) + + def testInvalidAuthToken(self): + hdrs = {'X-Auth-Token': 'bogus_auth_token'} + self.assertRaises(ResponseError, self.env.account.info, hdrs=hdrs) + self.assert_status(401) + + def testLastContainerMarker(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers({'format': format}) + self.assertEquals(len(containers), len(self.env.containers)) + self.assert_status(200) + + containers = self.env.account.containers( + parms={'format': format, 'marker': containers[-1]}) + self.assertEquals(len(containers), 0) + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + def testMarkerLimitContainerList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', + 'abc123', 'mnop', 'xyz']: + + limit = random.randint(2, 9) + containers = self.env.account.containers( + parms={'format': format, 'marker': marker, 'limit': limit}) + self.assert_(len(containers) <= limit) + if containers: + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assert_(locale.strcoll(containers[0], marker) > 0) + + def testContainersOrderedByName(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers( + parms={'format': format}) + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assertEquals(sorted(containers, cmp=locale.strcoll), + containers) + + +class TestAccountUTF8(Base2, TestAccount): + set_up = False + + +class TestAccountNoContainersEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + +class TestAccountNoContainers(Base): + env = TestAccountNoContainersEnv + set_up = False + + def testGetRequest(self): + for format in [None, 'json', 'xml']: + self.assert_(not self.env.account.containers( + parms={'format': format})) + + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + +class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): + set_up = False + + +class TestContainerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 10 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file.name) + + +class TestContainerDev(Base): + env = TestContainerEnv + set_up = False + + +class TestContainerDevUTF8(Base2, TestContainerDev): + set_up = False + + +class TestContainer(Base): + env = TestContainerEnv + set_up = False + + def testContainerNameLimit(self): + limit = load_constraint('max_container_name_length') + + for l in (limit - 100, limit - 10, limit - 1, limit, + limit + 1, limit + 10, limit + 100): + cont = self.env.account.container('a' * l) + if l <= limit: + self.assert_(cont.create()) + self.assert_status(201) + else: + self.assert_(not cont.create()) + self.assert_status(400) + + def testFileThenContainerDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + self.assert_(file.write_random()) + + self.assert_(file.delete()) + self.assert_status(204) + self.assert_(file.name not in cont.files()) + + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testFileListingLimitMarkerPrefix(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + files = sorted([Utils.create_name() for x in xrange(10)]) + for f in files: + file = cont.file(f) + self.assert_(file.write_random()) + + for i in xrange(len(files)): + f = files[i] + for j in xrange(1, len(files) - i): + self.assert_(cont.files(parms={'limit': j, 'marker': f}) == + files[i + 1: i + j + 1]) + self.assert_(cont.files(parms={'marker': f}) == files[i + 1:]) + self.assert_(cont.files(parms={'marker': f, 'prefix': f}) == []) + self.assert_(cont.files(parms={'prefix': f}) == [f]) + + def testPrefixAndLimit(self): + load_constraint('container_listing_limit') + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + prefix_file_count = 10 + limit_count = 2 + prefixs = ['alpha/', 'beta/', 'kappa/'] + prefix_files = {} + + all_files = [] + for prefix in prefixs: + prefix_files[prefix] = [] + + for i in range(prefix_file_count): + file = cont.file(prefix + Utils.create_name()) + file.write() + prefix_files[prefix].append(file.name) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'prefix': prefix}) + self.assertEquals(files, sorted(prefix_files[prefix])) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'limit': limit_count, + 'prefix': prefix}) + self.assertEquals(len(files), limit_count) + + for file in files: + self.assert_(file.startswith(prefix)) + + def testCreate(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.name in self.env.account.containers()) + + def testContainerFileListOnContainerThatDoesNotExist(self): + for format in [None, 'json', 'xml']: + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.files, + parms={'format': format}) + self.assert_status(404) + + def testUtf8Container(self): + valid_utf8 = Utils.create_utf8_name() + invalid_utf8 = valid_utf8[::-1] + container = self.env.account.container(valid_utf8) + self.assert_(container.create(cfg={'no_path_quote': True})) + self.assert_(container.name in self.env.account.containers()) + self.assertEquals(container.files(), []) + self.assert_(container.delete()) + + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote': True})) + self.assert_status(412) + self.assertRaises(ResponseError, container.files, + cfg={'no_path_quote': True}) + self.assert_status(412) + + def testCreateOnExisting(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.create()) + self.assert_status(202) + + def testSlashInName(self): + if Utils.create_name == Utils.create_utf8_name: + cont_name = list(unicode(Utils.create_name(), 'utf-8')) + else: + cont_name = list(Utils.create_name()) + + cont_name[random.randint(2, len(cont_name) - 2)] = '/' + cont_name = ''.join(cont_name) + + if Utils.create_name == Utils.create_utf8_name: + cont_name = cont_name.encode('utf-8') + + cont = self.env.account.container(cont_name) + self.assert_(not cont.create(cfg={'no_path_quote': True}), + 'created container with name %s' % (cont_name)) + self.assert_status(404) + self.assert_(cont.name not in self.env.account.containers()) + + def testDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testDeleteOnContainerThatDoesNotExist(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(not cont.delete()) + self.assert_status(404) + + def testDeleteOnContainerWithFiles(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random(self.env.file_size) + self.assert_(file.name in cont.files()) + self.assert_(not cont.delete()) + self.assert_status(409) + + def testFileCreateInContainerThatDoesNotExist(self): + file = File(self.env.conn, self.env.account, Utils.create_name(), + Utils.create_name()) + self.assertRaises(ResponseError, file.write) + self.assert_status(404) + + def testLastFileMarker(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files({'format': format}) + self.assertEquals(len(files), len(self.env.files)) + self.assert_status(200) + + files = self.env.container.files( + parms={'format': format, 'marker': files[-1]}) + self.assertEquals(len(files), 0) + + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + def testContainerFileList(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format}) + self.assert_status(200) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + for file in self.env.files: + self.assert_(file in files) + + for file in files: + self.assert_(file in self.env.files) + + def testMarkerLimitFileList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', + 'abc123', 'mnop', 'xyz']: + limit = random.randint(2, self.env.file_count - 1) + files = self.env.container.files(parms={'format': format, + 'marker': marker, + 'limit': limit}) + + if not files: + continue + + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + self.assert_(len(files) <= limit) + if files: + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assert_(locale.strcoll(files[0], marker) > 0) + + def testFileOrder(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format}) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assertEquals(sorted(files, cmp=locale.strcoll), files) + + def testContainerInfo(self): + info = self.env.container.info() + self.assert_status(204) + self.assertEquals(info['object_count'], self.env.file_count) + self.assertEquals(info['bytes_used'], + self.env.file_count * self.env.file_size) + + def testContainerInfoOnContainerThatDoesNotExist(self): + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.info) + self.assert_status(404) + + def testContainerFileListWithLimit(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format, + 'limit': 2}) + self.assertEquals(len(files), 2) + + def testTooLongName(self): + cont = self.env.account.container('x' * 257) + self.assert_(not cont.create(), + 'created container with name %s' % (cont.name)) + self.assert_status(400) + + def testContainerExistenceCachingProblem(self): + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + cont.files() + + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random() + + +class TestContainerUTF8(Base2, TestContainer): + set_up = False + + +class TestContainerPathsEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.file_size = 8 + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.files = [ + '/file1', + '/file A', + '/dir1/', + '/dir2/', + '/dir1/file2', + '/dir1/subdir1/', + '/dir1/subdir2/', + '/dir1/subdir1/file2', + '/dir1/subdir1/file3', + '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/', + '/dir1/subdir1/subsubdir1/file5', + '/dir1/subdir1/subsubdir1/file6', + '/dir1/subdir1/subsubdir1/file7', + '/dir1/subdir1/subsubdir1/file8', + '/dir1/subdir1/subsubdir2/', + '/dir1/subdir1/subsubdir2/file9', + '/dir1/subdir1/subsubdir2/file0', + 'file1', + 'dir1/', + 'dir2/', + 'dir1/file2', + 'dir1/subdir1/', + 'dir1/subdir2/', + 'dir1/subdir1/file2', + 'dir1/subdir1/file3', + 'dir1/subdir1/file4', + 'dir1/subdir1/subsubdir1/', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file6', + 'dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/subsubdir2/file9', + 'dir1/subdir1/subsubdir2/file0', + 'dir1/subdir with spaces/', + 'dir1/subdir with spaces/file B', + 'dir1/subdir+with{whatever/', + 'dir1/subdir+with{whatever/file D', + ] + + stored_files = set() + for f in cls.files: + file = cls.container.file(f) + if f.endswith('/'): + file.write(hdrs={'Content-Type': 'application/directory'}) + else: + file.write_random(cls.file_size, hdrs={'Content-Type': + 'application/directory'}) + if (normalized_urls): + nfile = '/'.join(filter(None, f.split('/'))) + if (f[-1] == '/'): + nfile += '/' + stored_files.add(nfile) + else: + stored_files.add(f) + cls.stored_files = sorted(stored_files) + + + + +class TestContainerPaths(Base): + env = TestContainerPathsEnv + set_up = False + + def testTraverseContainer(self): + found_files = [] + found_dirs = [] + + def recurse_path(path, count=0): + if count > 10: + raise ValueError('too deep recursion') + + for file in self.env.container.files(parms={'path': path}): + self.assert_(file.startswith(path)) + if file.endswith('/'): + recurse_path(file, count + 1) + found_dirs.append(file) + else: + found_files.append(file) + + recurse_path('') + for file in self.env.stored_files: + if file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + + found_files = [] + found_dirs = [] + recurse_path('/') + for file in self.env.stored_files: + if not file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + + def testContainerListing(self): + for format in (None, 'json', 'xml'): + files = self.env.container.files(parms={'format': format}) + + if isinstance(files[0], dict): + files = [str(x['name']) for x in files] + + self.assertEquals(files, self.env.stored_files) + + for format in ('json', 'xml'): + for file in self.env.container.files(parms={'format': format}): + self.assert_(int(file['bytes']) >= 0) + self.assert_('last_modified' in file) + if file['name'].endswith('/'): + self.assertEquals(file['content_type'], + 'application/directory') + + def testStructure(self): + def assert_listing(path, list): + files = self.env.container.files(parms={'path': path}) + self.assertEquals(sorted(list, cmp=locale.strcoll), files) + if not normalized_urls: + assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) + assert_listing('/dir1', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/subdir1', + ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2', + '/dir1/subdir1/file3', '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/']) + assert_listing('/dir1/subdir2', []) + assert_listing('', ['file1', 'dir1/', 'dir2/']) + else: + assert_listing('', ['file1', 'dir1/', 'dir2/', 'file A']) + assert_listing('dir1', ['dir1/file2', 'dir1/subdir1/', + 'dir1/subdir2/', 'dir1/subdir with spaces/', + 'dir1/subdir+with{whatever/']) + assert_listing('dir1/subdir1', + ['dir1/subdir1/file4', 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/file2', 'dir1/subdir1/file3', + 'dir1/subdir1/subsubdir1/']) + assert_listing('dir1/subdir1/subsubdir1', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir1/subsubdir1/', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir with spaces/', + ['dir1/subdir with spaces/file B']) + + +class TestFileEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_size = 128 + + +class TestFileDev(Base): + env = TestFileEnv + set_up = False + + +class TestFileDevUTF8(Base2, TestFileDev): + set_up = False + + +class TestFile(Base): + env = TestFileEnv + set_up = False + + def testCopy(self): + # makes sure to test encoded characters" + source_filename = 'dealde%2Fl04 011e%204c8df/flash.png' + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + data = file.write_random() + file.sync_metadata(metadata) + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = self.env.container.file(source_filename) + file.copy('%s%s' % (prefix, cont), dest_filename) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopy404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + for prefix in ('', '/'): + # invalid source container + source_cont = self.env.account.container(Utils.create_name()) + file = source_cont.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid destination container + file = self.env.container.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) + + def testCopyNoDestinationHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + file = self.env.container.file(source_filename) + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'no_destination': True})) + self.assert_status(412) + + def testCopyDestinationSlashProblems(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + # no slash + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'destination': Utils.create_name()})) + self.assert_status(412) + + def testCopyFromHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + file.metadata = metadata + + data = file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = cont.file(dest_filename) + file.write(hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + self.env.container.name, source_filename)}) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopyFromHeader404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + for prefix in ('', '/'): + # invalid source container + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + Utils.create_name(), source_filename)}) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + self.env.container.name, Utils.create_name())}) + self.assert_status(404) + + # invalid destination container + dest_cont = self.env.account.container(Utils.create_name()) + file = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + self.env.container.name, source_filename)}) + self.assert_status(404) + + def testNameLimit(self): + limit = load_constraint('max_object_name_length') + + for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): + file = self.env.container.file('a' * l) + + if l <= limit: + self.assert_(file.write()) + self.assert_status(201) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testQuestionMarkInName(self): + if Utils.create_name == Utils.create_ascii_name: + file_name = list(Utils.create_name()) + file_name[random.randint(2, len(file_name) - 2)] = '?' + file_name = "".join(file_name) + else: + file_name = Utils.create_name(6) + '?' + Utils.create_name(6) + + file = self.env.container.file(file_name) + self.assert_(file.write(cfg={'no_path_quote': True})) + self.assert_(file_name not in self.env.container.files()) + self.assert_(file_name.split('?')[0] in self.env.container.files()) + + def testDeleteThen404s(self): + file = self.env.container.file(Utils.create_name()) + self.assert_(file.write_random()) + self.assert_status(201) + + self.assert_(file.delete()) + self.assert_status(204) + + file.metadata = {Utils.create_ascii_name(): Utils.create_name()} + + for method in (file.info, file.read, file.sync_metadata, + file.delete): + self.assertRaises(ResponseError, method) + self.assert_status(404) + + def testBlankMetadataName(self): + file = self.env.container.file(Utils.create_name()) + file.metadata = {'': Utils.create_name()} + self.assertRaises(ResponseError, file.write_random) + self.assert_status(400) + + def testMetadataNumberLimit(self): + number_limit = load_constraint('max_meta_count') + size_limit = load_constraint('max_meta_overall_size') + + for i in (number_limit - 10, number_limit - 1, number_limit, + number_limit + 1, number_limit + 10, number_limit + 100): + + j = size_limit / (i * 2) + + size = 0 + metadata = {} + while len(metadata.keys()) < i: + key = Utils.create_ascii_name() + val = Utils.create_name() + + if len(key) > j: + key = key[:j] + val = val[:j] + + size += len(key) + len(val) + metadata[key] = val + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if i <= number_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + self.assert_status((201, 202)) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testContentTypeGuessing(self): + file_types = {'wav': 'audio/x-wav', 'txt': 'text/plain', + 'zip': 'application/zip'} + + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + for i in file_types.keys(): + file = container.file(Utils.create_name() + '.' + i) + file.write('', cfg={'no_content_type': True}) + + file_types_read = {} + for i in container.files(parms={'format': 'json'}): + file_types_read[i['name'].split('.')[1]] = i['content_type'] + + self.assertEquals(file_types, file_types_read) + + def testRangedGets(self): + file_length = 10000 + range_size = file_length / 10 + file = self.env.container.file(Utils.create_name()) + data = file.write_random(file_length) + + for i in range(0, file_length, range_size): + range_string = 'bytes=%d-%d' % (i, i + range_size - 1) + hdrs = {'Range': range_string} + self.assert_(data[i: i + range_size] == file.read(hdrs=hdrs), + range_string) + + range_string = 'bytes=-%d' % (i) + hdrs = {'Range': range_string} + if i == 0: + # RFC 2616 14.35.1 + # "If a syntactically valid byte-range-set includes ... at + # least one suffix-byte-range-spec with a NON-ZERO + # suffix-length, then the byte-range-set is satisfiable. + # Otherwise, the byte-range-set is unsatisfiable. + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(416) + else: + self.assertEquals(file.read(hdrs=hdrs), data[-i:]) + + range_string = 'bytes=%d-' % (i) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[i - file_length:], + range_string) + + range_string = 'bytes=%d-%d' % (file_length + 1000, file_length + 2000) + hdrs = {'Range': range_string} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(416) + + range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[-1000:], range_string) + + hdrs = {'Range': '0-4'} + self.assert_(file.read(hdrs=hdrs) == data, range_string) + + # RFC 2616 14.35.1 + # "If the entity is shorter than the specified suffix-length, the + # entire entity-body is used." + range_string = 'bytes=-%d' % (file_length + 10) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data, range_string) + + def testRangedGetsWithLWSinHeader(self): + #Skip this test until webob 1.2 can tolerate LWS in Range header. + file_length = 10000 + range_size = file_length / 10 + file = self.env.container.file(Utils.create_name()) + data = file.write_random(file_length) + + for r in ('BYTES=0-999', 'bytes = 0-999', 'BYTES = 0 - 999', + 'bytes = 0 - 999', 'bytes=0 - 999', 'bytes=0-999 '): + + self.assert_(file.read(hdrs={'Range': r}) == data[0:1000]) + + def testFileSizeLimit(self): + limit = load_constraint('max_file_size') + tsecs = 3 + + for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1, + limit + 10, limit + 100): + + file = self.env.container.file(Utils.create_name()) + + if i <= limit: + self.assert_(timeout(tsecs, file.write, + cfg={'set_content_length': i})) + else: + self.assertRaises(ResponseError, timeout, tsecs, + file.write, cfg={'set_content_length': i}) + + def testNoContentLengthForPut(self): + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, 'testing', + cfg={'no_content_length': True}) + self.assert_status(411) + + def testDelete(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + self.assert_(file.name in self.env.container.files()) + self.assert_(file.delete()) + self.assert_(file.name not in self.env.container.files()) + + def testBadHeaders(self): + file_length = 100 + + # no content type on puts should be ok + file = self.env.container.file(Utils.create_name()) + file.write_random(file_length, cfg={'no_content_type': True}) + self.assert_status(201) + + # content length x + self.assertRaises(ResponseError, file.write_random, file_length, + hdrs={'Content-Length': 'X'}, + cfg={'no_content_length': True}) + self.assert_status(400) + + # bad request types + #for req in ('LICK', 'GETorHEAD_base', 'container_info', + # 'best_response'): + for req in ('LICK', 'GETorHEAD_base'): + self.env.account.conn.make_request(req) + self.assert_status(405) + + # bad range headers + self.assert_(len(file.read(hdrs={'Range': 'parsecs=8-12'})) == + file_length) + self.assert_status(200) + + def testMetadataLengthLimits(self): + key_limit = load_constraint('max_meta_name_length') + value_limit = load_constraint('max_meta_value_length') + lengths = [[key_limit, value_limit], [key_limit, value_limit + 1], + [key_limit + 1, value_limit], [key_limit, 0], + [key_limit, value_limit * 10], + [key_limit * 10, value_limit]] + + for l in lengths: + metadata = {'a' * l[0]: 'b' * l[1]} + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if l[0] <= key_limit and l[1] <= value_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testEtagWayoff(self): + file = self.env.container.file(Utils.create_name()) + hdrs = {'etag': 'reallylonganddefinitelynotavalidetagvalue'} + self.assertRaises(ResponseError, file.write_random, hdrs=hdrs) + self.assert_status(422) + + def testFileCreate(self): + for i in range(10): + file = self.env.container.file(Utils.create_name()) + data = file.write_random() + self.assert_status(201) + self.assert_(data == file.read()) + self.assert_status(200) + + def testHead(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random(self.env.file_size) + + md5 = file.md5 + + file = self.env.container.file(file_name) + info = file.info() + + self.assert_status(200) + self.assertEquals(info['content_length'], self.env.file_size) + self.assertEquals(info['etag'], md5) + self.assertEquals(info['content_type'], content_type) + self.assert_('last_modified' in info) + + def testDeleteOfFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + def testHeadOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + def testMetadataOnPost(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + for i in range(10): + metadata = {} + for i in range(10): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + file.metadata = metadata + self.assert_(file.sync_metadata()) + self.assert_status((201, 202)) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testGetContentType(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random() + + file = self.env.container.file(file_name) + file.read() + + self.assertEquals(content_type, file.content_type) + + def testGetOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + def testPostOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + def testMetadataOnPut(self): + for i in range(10): + metadata = {} + for j in range(10): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + file.write_random(self.env.file_size) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testSerialization(self): + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + files = [] + for i in (0, 1, 10, 100, 1000, 10000): + files.append({'name': Utils.create_name(), + 'content_type': Utils.create_name(), 'bytes': i}) + + write_time = time.time() + for f in files: + file = container.file(f['name']) + file.content_type = f['content_type'] + file.write_random(f['bytes']) + + f['hash'] = file.md5 + f['json'] = False + f['xml'] = False + write_time = time.time() - write_time + + for format in ['json', 'xml']: + for file in container.files(parms={'format': format}): + found = False + for f in files: + if f['name'] != file['name']: + continue + + self.assertEquals(file['content_type'], + f['content_type']) + self.assertEquals(int(file['bytes']), f['bytes']) + + d = datetime.strptime(file['last_modified'].split('.')[0], + "%Y-%m-%dT%H:%M:%S") + lm = time.mktime(d.timetuple()) + + if 'last_modified' in f: + self.assertEquals(f['last_modified'], lm) + else: + f['last_modified'] = lm + + f[format] = True + found = True + + self.assert_(found, 'Unexpected file %s found in ' + '%s listing' % (file['name'], format)) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf-8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf-8') + + lm_diff = max([f['last_modified'] for f in files]) -\ + min([f['last_modified'] for f in files]) + self.assert_(lm_diff < write_time + 1, 'Diff in last ' + 'modified times should be less than time to write files') + + for f in files: + for format in ['json', 'xml']: + self.assert_(f[format], 'File %s not found in %s listing' + % (f['name'], format)) + + def testStackedOverwrite(self): + file = self.env.container.file(Utils.create_name()) + + for i in range(1, 11): + data = file.write_random(512) + file.write(data) + + self.assert_(file.read() == data) + + def testTooLongName(self): + file = self.env.container.file('x' * 1025) + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testZeroByteFile(self): + file = self.env.container.file(Utils.create_name()) + + self.assert_(file.write('')) + self.assert_(file.name in self.env.container.files()) + self.assert_(file.read() == '') + + def testEtagResponse(self): + file = self.env.container.file(Utils.create_name()) + + data = StringIO.StringIO(file.write_random(512)) + etag = File.compute_md5sum(data) + + headers = dict(self.env.conn.response.getheaders()) + self.assert_('etag' in headers.keys()) + + header_etag = headers['etag'].strip('"') + self.assertEquals(etag, header_etag) + + def testChunkedPut(self): + if (web_front_end == 'apache2'): + raise SkipTest() + data = File.random_data(10000) + etag = File.compute_md5sum(data) + + for i in (1, 10, 100, 1000): + file = self.env.container.file(Utils.create_name()) + + for j in chunks(data, i): + file.chunked_write(j) + + self.assert_(file.chunked_write()) + self.assert_(data == file.read()) + + info = file.info() + self.assertEquals(etag, info['etag']) + + +class TestFileUTF8(Base2, TestFile): + set_up = False + + +class TestFileComparisonEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 20 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file) + + cls.time_old = time.asctime(time.localtime(time.time() - 86400)) + cls.time_new = time.asctime(time.localtime(time.time() + 86400)) + + +class TestFileComparison(Base): + env = TestFileComparisonEnv + set_up = False + + def testIfMatch(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus'} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfNoneMatch(self): + for file in self.env.files: + hdrs = {'If-None-Match': 'bogus'} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-None-Match': file.md5} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfModifiedSince(self): + for file in self.env.files: + hdrs = {'If-Modified-Since': self.env.time_old} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Modified-Since': self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfUnmodifiedSince(self): + for file in self.env.files: + hdrs = {'If-Unmodified-Since': self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Unmodified-Since': self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfMatchAndUnmodified(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5, + 'If-Unmodified-Since': self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus', + 'If-Unmodified-Since': self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + hdrs = {'If-Match': file.md5, + 'If-Unmodified-Since': self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + +class TestFileComparisonUTF8(Base2, TestFileComparison): + set_up = False + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/__init__.py b/test/functionalnosetests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/functionalnosetests/__init__.py diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py new file mode 100644 index 0000000..023a753 --- /dev/null +++ b/test/functionalnosetests/swift_testing.py @@ -0,0 +1,175 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import os +import socket +import sys +from time import sleep +from nose import SkipTest +from ConfigParser import MissingSectionHeaderError + +from test import get_config + +from swiftclient import get_auth, http_connection, HTTPException + +conf = get_config('func_test') +web_front_end = conf.get('web_front_end', 'integral') +normalized_urls = conf.get('normalized_urls', False) + +# If no conf was read, we will fall back to old school env vars +swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] +swift_test_tenant = ['', '', ''] +swift_test_perm = ['', '', ''] + +if conf: + swift_test_auth_version = str(conf.get('auth_version', '1')) + + swift_test_auth = 'http' + if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): + swift_test_auth = 'https' + if 'auth_prefix' not in conf: + conf['auth_prefix'] = '/' + try: + swift_test_auth += \ + '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf + except KeyError: + pass # skip + + if swift_test_auth_version == "1": + swift_test_auth += 'v1.0' + + if 'account' in conf: + swift_test_user[0] = '%(account)s:%(username)s' % conf + else: + swift_test_user[0] = '%(username)s' % conf + swift_test_key[0] = conf['password'] + try: + swift_test_user[1] = '%s%s' % \ + ('%s:' % conf['account2'] if 'account2' in conf else '', + conf['username2']) + swift_test_key[1] = conf['password2'] + except KeyError, err: + pass # old conf, no second account tests can be run + try: + swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account' + in conf else '', conf['username3']) + swift_test_key[2] = conf['password3'] + except KeyError, err: + pass # old conf, no third account tests can be run + + for _ in range(3): + swift_test_perm[_] = swift_test_user[_] + + else: + swift_test_user[0] = conf['username'] + swift_test_tenant[0] = conf['account'] + swift_test_key[0] = conf['password'] + swift_test_user[1] = conf['username2'] + swift_test_tenant[1] = conf['account2'] + swift_test_key[1] = conf['password2'] + swift_test_user[2] = conf['username3'] + swift_test_tenant[2] = conf['account'] + swift_test_key[2] = conf['password3'] + + for _ in range(3): + swift_test_perm[_] = swift_test_tenant[_] + ':' + swift_test_user[_] + +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) +if skip: + print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' + +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) +if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) +if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + + +class AuthError(Exception): + pass + + +class InternalServerError(Exception): + pass + + +url = [None, None, None] +token = [None, None, None] +parsed = [None, None, None] +conn = [None, None, None] + + +def retry(func, *args, **kwargs): + """ + You can use the kwargs to override the 'retries' (default: 5) and + 'use_account' (default: 1). + """ + global url, token, parsed, conn + retries = kwargs.get('retries', 5) + use_account = 1 + if 'use_account' in kwargs: + use_account = kwargs['use_account'] + del kwargs['use_account'] + use_account -= 1 + attempts = 0 + backoff = 1 + while attempts <= retries: + attempts += 1 + try: + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = \ + get_auth(swift_test_auth, swift_test_user[use_account], + swift_test_key[use_account], + snet=False, + tenant_name=swift_test_tenant[use_account], + auth_version=swift_test_auth_version, + os_options={}) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + http_connection(url[use_account]) + return func(url[use_account], token[use_account], + parsed[use_account], conn[use_account], *args, **kwargs) + except (socket.error, HTTPException): + if attempts > retries: + raise + parsed[use_account] = conn[use_account] = None + except AuthError, err: + url[use_account] = token[use_account] = None + continue + except InternalServerError, err: + pass + if attempts <= retries: + sleep(backoff) + backoff *= 2 + raise Exception('No result after %s retries.' % retries) + + +def check_response(conn): + resp = conn.getresponse() + if resp.status == 401: + resp.read() + raise AuthError() + elif resp.status // 100 == 5: + resp.read() + raise InternalServerError() + return resp diff --git a/test/functionalnosetests/test_account.py b/test/functionalnosetests/test_account.py new file mode 100755 index 0000000..ae6e3c4 --- /dev/null +++ b/test/functionalnosetests/test_account.py @@ -0,0 +1,152 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip + + +class TestAccount(unittest.TestCase): + + def test_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, 'X-Account-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, '') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + + def test_multi_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, 'X-Account-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + resp = retry(post, 'X-Account-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + self.assertEquals(resp.getheader('x-account-meta-two'), '2') + + def test_bad_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + resp = retry(post, + {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry(post, + {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Account-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py new file mode 100755 index 0000000..e92a86c --- /dev/null +++ b/test/functionalnosetests/test_container.py @@ -0,0 +1,573 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, skip2, skip3, \ + swift_test_perm, web_front_end + + +class TestContainer(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.name = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + while True: + resp = retry(get) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_multi_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, 'X-Container-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + resp = retry(post, 'X-Container-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + self.assertEquals(resp.getheader('x-container-meta-two'), '2') + + def test_PUT_metadata(self): + if skip: + raise SkipTest + def put(url, token, parsed, conn, name, value): + conn.request('PUT', parsed.path + '/' + name, '', + {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn, name): + conn.request('HEAD', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + name = uuid4().hex + resp = retry(put, name, 'Value') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + name = uuid4().hex + resp = retry(put, name, '') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + def test_POST_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + + def test_PUT_bad_metadata(self): + if skip: + raise SkipTest + def put(url, token, parsed, conn, name, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('PUT', parsed.path + '/' + name, '', headers) + return check_response(conn) + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + def test_POST_bad_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) + resp = retry(post, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry(post, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + def test_public_container(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 204) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + + def test_cross_account_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[1], + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now use the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can't access the container with the second account again + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + def test_cross_account_public_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container completely public + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the second account + def put2(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the second account + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 201) + + def test_nonadmin_user(self): + if skip or skip3: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the third account + def get3(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the third account + def put3(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the third account + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + def test_long_name_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + container_name = 'X' * 2048 + conn.request('PUT', '%s/%s' % (parsed.path, + container_name), 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 400) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/abc%%00def' % parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py new file mode 100755 index 0000000..168375d --- /dev/null +++ b/test/functionalnosetests/test_object.py @@ -0,0 +1,600 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, skip3, \ + swift_test_perm, web_front_end +from test import get_config + + +class TestObject(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + self.obj = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + self.obj), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '%s/%s/%s' % (parsed.path, self.container, obj), + '', {'X-Auth-Token': token}) + return check_response(conn) + + # get list of objects in container + def list(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(list) + object_listing = resp.read() + self.assertEquals(resp.status, 200) + + # iterate over object listing and delete all objects + for obj in object_listing.splitlines(): + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + + # delete the container + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_copy_object(self): + if skip: + raise SkipTest + + source = '%s/%s' % (self.container, self.obj) + dest = '%s/%s' % (self.container, 'test_copy') + + # get contents of source + def get_source(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, source), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_source) + source_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(source_contents, 'test') + + # copy source to dest with X-Copy-From + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': source}) + return check_response(conn) + resp = retry(put) + contents = resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + def get_dest(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, dest), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + # verify dest does not exist + resp = retry(get_dest) + resp.read() + self.assertEquals(resp.status, 404) + + # copy source to dest with COPY + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination': dest}) + return check_response(conn) + resp = retry(copy) + contents = resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_public_object(self): + if skip: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', + '%s/%s/%s' % (parsed.path, self.container, self.obj)) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 200) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + def test_private_object(self): + if skip or skip3: + raise SkipTest + + # Ensure we can't access the object with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/%s' % (parsed.path, self.container, + self.obj), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # create a shared container writable by account3 + shared_container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, + shared_container), '', + {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2], + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can not copy from private container + def copy(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, + shared_container, + 'private_object'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': '%s/%s' % (self.container, + self.obj)}) + return check_response(conn) + resp = retry(copy, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # verify third account can write "obj1" to shared container + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, shared_container, + 'obj1'), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can copy "obj1" to shared container + def copy2(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % (parsed.path, + shared_container, + 'obj1'), + '', {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, + 'obj1')}) + return check_response(conn) + resp = retry(copy2, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account STILL can not copy from private container + def copy3(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % (parsed.path, + self.container, + self.obj), + '', {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, + 'private_object')}) + return check_response(conn) + resp = retry(copy3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # clean up "obj1" + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/%s' % (parsed.path, shared_container, + 'obj1'), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + # clean up shared_container + def delete(url, token, parsed, conn): + conn.request('DELETE', + parsed.path + '/' + shared_container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_manifest(self): + if skip: + raise SkipTest + # Data for the object segments + segments1 = ['one', 'two', 'three', 'four', 'five'] + segments2 = ['six', 'seven', 'eight'] + segments3 = ['nine', 'ten', 'eleven'] + + # Upload the first set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments1/%s' % (parsed.path, + self.container, str(objnum)), segments1[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments1/' % self.container, + 'Content-Type': 'text/jibberish', 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should get all the segments as the body) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + self.assertEquals(resp.getheader('content-type'), 'text/jibberish') + + # Get with a range at the start of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=3-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1[1:])) + self.assertEquals(resp.status, 206) + + # Get with a range in the middle of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=5-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:]) + self.assertEquals(resp.status, 206) + + # Get with a full start and stop range + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=5-10'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:11]) + self.assertEquals(resp.status, 206) + + # Upload the second set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments2/%s' % (parsed.path, + self.container, str(objnum)), segments2[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should still be the first segments of course) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments2/' % self.container, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should be the second set of segments now) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + # Create another container for the third set of segments + acontainer = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + acontainer, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the third set of segments in the other container + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments3/%s' % (parsed.path, + acontainer, str(objnum)), segments3[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments3/' % acontainer, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest to ensure it's the third set of segments + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + # (because the segments are in a protected container even if the + # manifest itself is not). + + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, acontainer), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + # Delete the manifest + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the third set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments3/%s' % (parsed.path, + acontainer, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the second set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments2/%s' % (parsed.path, + self.container, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the first set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments1/%s' % (parsed.path, + self.container, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the extra container + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, acontainer), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_delete_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/hi' % (parsed.path, + self.container), 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/abc%%00def' % (parsed.path, + self.container), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main() |