diff options
Diffstat (limited to 'test/unit/proxy/test_server.py')
| -rw-r--r-- | test/unit/proxy/test_server.py | 3991 |
1 files changed, 3360 insertions, 631 deletions
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 1a59016..bdce41f 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,55 +15,67 @@ # limitations under the License. from __future__ import with_statement -import cPickle as pickle import logging +import math import os +import pickle import sys import unittest -import urlparse -from nose import SkipTest -from contextlib import contextmanager, nested, closing +from contextlib import closing, contextmanager, nested from gzip import GzipFile from shutil import rmtree +from StringIO import StringIO import gc import time +from textwrap import dedent from urllib import quote from hashlib import md5 -from tempfile import mkdtemp +from pyeclib.ec_iface import ECDriverError +from tempfile import mkdtemp, NamedTemporaryFile import weakref +import operator +import functools +from swift.obj import diskfile import re +import random +import urlparse +from nose import SkipTest import mock -from eventlet import sleep, spawn, wsgi, listen -import simplejson +from eventlet import sleep, spawn, wsgi, listen, Timeout +from swift.common.utils import hash_path, json, storage_directory, public import gluster.swift.common.Glusterfs as gfs gfs.RUN_DIR = mkdtemp() -from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ - fake_http_connect, FakeRing, FakeMemcache, debug_logger +from test.unit import ( + connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing, + FakeMemcache, debug_logger, patch_policies, write_fake_ring, + mocked_http_conn) +from swift.proxy.controllers.obj import ReplicatedObjectController from gluster.swift.proxy import server as proxy_server from gluster.swift.account import server as account_server from gluster.swift.container import server as container_server from gluster.swift.obj import server as object_server -from swift.common import ring from swift.common.middleware import proxy_logging from swift.common.middleware.acl import parse_acl, format_acl -from swift.common.exceptions import ChunkReadTimeout -from swift.common.constraints import MAX_META_NAME_LENGTH, \ - MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ - MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ - ACCOUNT_LISTING_LIMIT, CONTAINER_LISTING_LIMIT, MAX_OBJECT_NAME_LENGTH -from swift.common import utils +from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ + APIVersionError +from swift.common import utils, constraints +from swift.common.ring import RingData from swift.common.utils import mkdirs, normalize_timestamp, NullLogger -from swift.common.wsgi import monkey_patch_mimetools +from swift.common.wsgi import monkey_patch_mimetools, loadapp from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers -from swift.common.request_helpers import get_sys_meta_prefix +import swift.proxy.controllers.obj from swift.common.swob import Request, Response, HTTPUnauthorized, \ - HTTPException + HTTPException, HTTPForbidden, HeaderKeyDict +from swift.common import storage_policy +from swift.common.storage_policy import StoragePolicy, ECStoragePolicy, \ + StoragePolicyCollection, POLICIES +from swift.common.request_helpers import get_sys_meta_prefix # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -70,13 +83,15 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) STATIC_TIME = time.time() _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ - _testdir = _orig_SysLogHandler = None + _testdir = _orig_SysLogHandler = _orig_POLICIES = _test_POLICIES = None def do_setup(the_object_server): utils.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ - _orig_container_listing_limit, _test_coros, _orig_SysLogHandler + _orig_container_listing_limit, _test_coros, _orig_SysLogHandler, \ + _orig_POLICIES, _test_POLICIES + _orig_POLICIES = storage_policy._POLICIES _orig_SysLogHandler = utils.SysLogHandler utils.SysLogHandler = mock.MagicMock() monkey_patch_mimetools() @@ -86,12 +101,13 @@ def do_setup(the_object_server): _testdir = os.path.join(gfs.RUN_DIR, 'swift') mkdirs(_testdir) rmtree(_testdir) - mkdirs(os.path.join(_testdir, 'sda1')) - mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) - mkdirs(os.path.join(_testdir, 'sdb1')) - mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) + for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', + 'sdf1', 'sdg1', 'sdh1', 'sdi1'): + mkdirs(os.path.join(_testdir, drive, 'tmp')) mkdirs(os.path.join(_testdir, 'a')) + mkdirs(os.path.join(_testdir, 'a1')) mkdirs(os.path.join(_testdir, 'a', 'tmp')) + mkdirs(os.path.join(_testdir, 'a1', 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', @@ -103,43 +119,57 @@ def do_setup(the_object_server): con2lis = listen(('localhost', 0)) obj1lis = listen(('localhost', 0)) obj2lis = listen(('localhost', 0)) + obj3lis = listen(('localhost', 0)) + objsocks = [obj1lis, obj2lis, obj3lis] _test_sockets = \ - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: - pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}, # Gluster volume mapping to device - {'id': 1, 'zone': 1, 'device': 'a', 'ip': '127.0.0.1', + {'id': 0, 'zone': 0, 'device': 'a', 'ip': '127.0.0.1', + 'port': acc1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'a1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: - pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}, # Gluster volume mapping to device - {'id': 1, 'zone': 1, 'device': 'a', 'ip': '127.0.0.1', + {'id': 0, 'zone': 0, 'device': 'a', 'ip': '127.0.0.1', + 'port': con1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'a1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) object_ring_path = os.path.join(_testdir, 'object.ring.gz') with closing(GzipFile(object_ring_path, 'wb')) as f: - pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}, # Gluster volume mapping to device - {'id': 1, 'zone': 1, 'device': 'a', 'ip': '127.0.0.1', + {'id': 0, 'zone': 0, 'device': 'a', 'ip': '127.0.0.1', + 'port': obj1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'a1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}], 30), f) + prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(), logger=debug_logger('proxy')) + for policy in POLICIES: + # make sure all the rings are loaded + prosrv.get_object_ring(policy.idx) + # don't lose this one! + _test_POLICIES = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( @@ -152,8 +182,10 @@ def do_setup(the_object_server): conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) + obj3srv = the_object_server.ObjectController( + conf, logger=debug_logger('obj3')) _test_servers = \ - (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) + (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware(prosrv, conf, logger=prosrv.logger) @@ -164,9 +196,10 @@ def do_setup(the_object_server): con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) + obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl) _test_coros = \ - (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa) - # Gluster: ensure account exists + (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa) + # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: @@ -177,12 +210,24 @@ def do_setup(the_object_server): {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() - + assert(resp.status == 202) + # Gluster: ensure account exists + ts = normalize_timestamp(time.time()) + partition, nodes = prosrv.account_ring.get_nodes('a1') + for node in nodes: + conn = swift.proxy.controllers.obj.http_connect(node['ip'], + node['port'], + node['device'], + partition, 'PUT', + '/a1', + {'X-Timestamp': ts, + 'x-trans-id': 'test'}) + resp = conn.getresponse() # For GlusterFS the volume should have already been created since # accounts map to volumes. Expect a 202 instead of a 201 as for # OpenStack Swift's proxy unit test the account is explicitly created. assert(resp.status == 202) - # Create container + # Create containers, 1 per test policy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' @@ -193,6 +238,62 @@ def do_setup(the_object_server): exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) + # Create container in other account + # used for account-to-account tests + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( + exp, headers[:len(exp)]) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write( + 'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n' + #'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n' + # Gluster-Swift: No storage policies + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + assert headers[:len(exp)] == exp, \ + "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write( + 'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n' + #'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n' + # Gluster-Swift: No storage policies + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + assert headers[:len(exp)] == exp, \ + "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) + + +def unpatch_policies(f): + """ + This will unset a TestCase level patch_policies to use the module level + policies setup for the _test_servers instead. + + N.B. You should NEVER modify the _test_server policies or rings during a + test because they persist for the life of the entire module! + """ + raise SkipTest("Storage Policies are not supported") + @functools.wraps(f) + def wrapper(*args, **kwargs): + with patch_policies(_test_POLICIES): + return f(*args, **kwargs) + return wrapper def setup(): @@ -204,6 +305,7 @@ def teardown(): server.kill() rmtree(os.path.dirname(_testdir)) utils.SysLogHandler = _orig_SysLogHandler + storage_policy._POLICIES = _orig_POLICIES def sortHeaderNames(headerNames): @@ -217,6 +319,37 @@ def sortHeaderNames(headerNames): return ', '.join(headers) +def parse_headers_string(headers_str): + headers_dict = HeaderKeyDict() + for line in headers_str.split('\r\n'): + if ': ' in line: + header, value = line.split(': ', 1) + headers_dict[header] = value + return headers_dict + + +def node_error_count(proxy_app, ring_node): + # Reach into the proxy's internals to get the error count for a + # particular node + node_key = proxy_app._error_limit_node_key(ring_node) + return proxy_app._error_limiting.get(node_key, {}).get('errors', 0) + + +def node_last_error(proxy_app, ring_node): + # Reach into the proxy's internals to get the last error for a + # particular node + node_key = proxy_app._error_limit_node_key(ring_node) + return proxy_app._error_limiting.get(node_key, {}).get('last_error') + + +def set_node_errors(proxy_app, ring_node, value, last_error): + # Set the node's error count to value + node_key = proxy_app._error_limit_node_key(ring_node) + stats = proxy_app._error_limiting.setdefault(node_key, {}) + stats['errors'] = value + stats['last_error'] = last_error + + class FakeMemcacheReturnsNone(FakeMemcache): def get(self, key): @@ -231,6 +364,9 @@ def save_globals(): None) orig_account_info = getattr(swift.proxy.controllers.Controller, 'account_info', None) + orig_container_info = getattr(swift.proxy.controllers.Controller, + 'container_info', None) + try: yield True finally: @@ -239,6 +375,7 @@ def save_globals(): swift.proxy.controllers.obj.http_connect = orig_http_connect swift.proxy.controllers.account.http_connect = orig_http_connect swift.proxy.controllers.container.http_connect = orig_http_connect + swift.proxy.controllers.Controller.container_info = orig_container_info def set_http_connect(*args, **kwargs): @@ -250,6 +387,36 @@ def set_http_connect(*args, **kwargs): return new_connect +def _make_callback_func(calls): + def callback(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + context = {} + context['method'] = method + context['path'] = path + context['headers'] = headers or {} + calls.append(context) + return callback + + +def _limit_max_file_size(f): + """ + This will limit constraints.MAX_FILE_SIZE for the duration of the + wrapped function, based on whether MAX_FILE_SIZE exceeds the + sys.maxsize limit on the system running the tests. + + This allows successful testing on 32 bit systems. + """ + @functools.wraps(f) + def wrapper(*args, **kwargs): + test_max_file_size = constraints.MAX_FILE_SIZE + if constraints.MAX_FILE_SIZE >= sys.maxsize: + test_max_file_size = (2 ** 30 + 2) + with mock.patch.object(constraints, 'MAX_FILE_SIZE', + test_max_file_size): + return f(*args, **kwargs) + return wrapper + + # tests class TestController(unittest.TestCase): @@ -257,11 +424,9 @@ class TestController(unittest.TestCase): self.account_ring = FakeRing() self.container_ring = FakeRing() self.memcache = FakeMemcache() - app = proxy_server.Application(None, self.memcache, account_ring=self.account_ring, - container_ring=self.container_ring, - object_ring=FakeRing()) + container_ring=self.container_ring) self.controller = swift.proxy.controllers.Controller(app) class FakeReq(object): @@ -515,17 +680,43 @@ class TestController(unittest.TestCase): test(503, 503, 503) +@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestProxyServer(unittest.TestCase): + def test_get_object_ring(self): + baseapp = proxy_server.Application({}, + FakeMemcache(), + container_ring=FakeRing(), + account_ring=FakeRing()) + with patch_policies([ + StoragePolicy(0, 'a', False, object_ring=123), + StoragePolicy(1, 'b', True, object_ring=456), + StoragePolicy(2, 'd', False, object_ring=789) + ]): + # None means legacy so always use policy 0 + ring = baseapp.get_object_ring(None) + self.assertEqual(ring, 123) + ring = baseapp.get_object_ring('') + self.assertEqual(ring, 123) + ring = baseapp.get_object_ring('0') + self.assertEqual(ring, 123) + ring = baseapp.get_object_ring('1') + self.assertEqual(ring, 456) + ring = baseapp.get_object_ring('2') + self.assertEqual(ring, 789) + # illegal values + self.assertRaises(ValueError, baseapp.get_object_ring, '99') + self.assertRaises(ValueError, baseapp.get_object_ring, 'asdf') + def test_unhandled_exception(self): class MyApp(proxy_server.Application): def get_controller(self, path): - raise Exception('this shouldnt be caught') + raise Exception('this shouldn\'t be caught') app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), object_ring=FakeRing()) + container_ring=FakeRing()) req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) app.update_request(req) resp = app.handle_request(req) @@ -535,7 +726,6 @@ class TestProxyServer(unittest.TestCase): baseapp = proxy_server.Application({}, FakeMemcache(), container_ring=FakeRing(), - object_ring=FakeRing(), account_ring=FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a', environ={'REQUEST_METHOD': '__init__'})) @@ -545,8 +735,7 @@ class TestProxyServer(unittest.TestCase): baseapp = proxy_server.Application({}, FakeMemcache(), container_ring=FakeRing(), - account_ring=FakeRing(), - object_ring=FakeRing()) + account_ring=FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a', environ={'REQUEST_METHOD': '!invalid'})) self.assertEquals(resp.status, '405 Method Not Allowed') @@ -560,8 +749,7 @@ class TestProxyServer(unittest.TestCase): set_http_connect(200) app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing()) + container_ring=FakeRing()) req = Request.blank('/v1/a') req.environ['swift.authorize'] = authorize app.update_request(req) @@ -576,8 +764,7 @@ class TestProxyServer(unittest.TestCase): return HTTPUnauthorized(request=req) app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing()) + container_ring=FakeRing()) req = Request.blank('/v1/a') req.environ['swift.authorize'] = authorize app.update_request(req) @@ -589,8 +776,7 @@ class TestProxyServer(unittest.TestCase): try: baseapp = proxy_server.Application({'swift_dir': swift_dir}, FakeMemcache(), FakeLogger(), - FakeRing(), FakeRing(), - FakeRing()) + FakeRing(), FakeRing()) resp = baseapp.handle_request( Request.blank('/', environ={'CONTENT_LENGTH': '-1'})) self.assertEquals(resp.status, '400 Bad Request') @@ -602,15 +788,52 @@ class TestProxyServer(unittest.TestCase): finally: rmtree(swift_dir, ignore_errors=True) + def test_adds_transaction_id(self): + swift_dir = mkdtemp() + try: + logger = FakeLogger() + baseapp = proxy_server.Application({'swift_dir': swift_dir}, + FakeMemcache(), logger, + container_ring=FakeLogger(), + account_ring=FakeRing()) + baseapp.handle_request( + Request.blank('/info', + environ={'HTTP_X_TRANS_ID_EXTRA': 'sardine', + 'REQUEST_METHOD': 'GET'})) + # This is kind of a hokey way to get the transaction ID; it'd be + # better to examine response headers, but the catch_errors + # middleware is what sets the X-Trans-Id header, and we don't have + # that available here. + self.assertTrue(logger.txn_id.endswith('-sardine')) + finally: + rmtree(swift_dir, ignore_errors=True) + + def test_adds_transaction_id_length_limit(self): + swift_dir = mkdtemp() + try: + logger = FakeLogger() + baseapp = proxy_server.Application({'swift_dir': swift_dir}, + FakeMemcache(), logger, + container_ring=FakeLogger(), + account_ring=FakeRing()) + baseapp.handle_request( + Request.blank('/info', + environ={'HTTP_X_TRANS_ID_EXTRA': 'a' * 1000, + 'REQUEST_METHOD': 'GET'})) + self.assertTrue(logger.txn_id.endswith( + '-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + finally: + rmtree(swift_dir, ignore_errors=True) + def test_denied_host_header(self): swift_dir = mkdtemp() try: baseapp = proxy_server.Application({'swift_dir': swift_dir, 'deny_host_headers': 'invalid_host.com'}, - FakeMemcache(), FakeLogger(), - FakeRing(), FakeRing(), - FakeRing()) + FakeMemcache(), + container_ring=FakeLogger(), + account_ring=FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a/c/o', environ={'HTTP_HOST': 'invalid_host.com'})) @@ -622,7 +845,6 @@ class TestProxyServer(unittest.TestCase): baseapp = proxy_server.Application({'sorting_method': 'timing'}, FakeMemcache(), container_ring=FakeRing(), - object_ring=FakeRing(), account_ring=FakeRing()) self.assertEquals(baseapp.node_timings, {}) @@ -651,7 +873,6 @@ class TestProxyServer(unittest.TestCase): 'read_affinity': 'r1=1'}, FakeMemcache(), container_ring=FakeRing(), - object_ring=FakeRing(), account_ring=FakeRing()) nodes = [{'region': 2, 'zone': 1, 'ip': '127.0.0.1'}, @@ -665,22 +886,22 @@ class TestProxyServer(unittest.TestCase): def test_info_defaults(self): app = proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing()) + container_ring=FakeRing()) self.assertTrue(app.expose_info) self.assertTrue(isinstance(app.disallowed_sections, list)) - self.assertEqual(0, len(app.disallowed_sections)) + self.assertEqual(1, len(app.disallowed_sections)) + self.assertEqual(['swift.valid_api_versions'], + app.disallowed_sections) self.assertTrue(app.admin_key is None) def test_get_info_controller(self): - path = '/info' + req = Request.blank('/info') app = proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing()) + container_ring=FakeRing()) - controller, path_parts = app.get_controller(path) + controller, path_parts = app.get_controller(req) self.assertTrue('version' in path_parts) self.assertTrue(path_parts['version'] is None) @@ -690,20 +911,253 @@ class TestProxyServer(unittest.TestCase): self.assertEqual(controller.__name__, 'InfoController') + def test_error_limit_methods(self): + logger = debug_logger('test') + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + logger=logger) + node = app.container_ring.get_part_nodes(0)[0] + # error occurred + app.error_occurred(node, 'test msg') + self.assertTrue('test msg' in + logger.get_lines_for_level('error')[-1]) + self.assertEqual(1, node_error_count(app, node)) + + # exception occurred + try: + raise Exception('kaboom1!') + except Exception as e1: + app.exception_occurred(node, 'test1', 'test1 msg') + line = logger.get_lines_for_level('error')[-1] + self.assertTrue('test1 server' in line) + self.assertTrue('test1 msg' in line) + log_args, log_kwargs = logger.log_dict['error'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e1) + self.assertEqual(2, node_error_count(app, node)) + + # warning exception occurred + try: + raise Exception('kaboom2!') + except Exception as e2: + app.exception_occurred(node, 'test2', 'test2 msg', + level=logging.WARNING) + line = logger.get_lines_for_level('warning')[-1] + self.assertTrue('test2 server' in line) + self.assertTrue('test2 msg' in line) + log_args, log_kwargs = logger.log_dict['warning'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e2) + self.assertEqual(3, node_error_count(app, node)) + + # custom exception occurred + try: + raise Exception('kaboom3!') + except Exception as e3: + e3_info = sys.exc_info() + try: + raise Exception('kaboom4!') + except Exception: + pass + app.exception_occurred(node, 'test3', 'test3 msg', + level=logging.WARNING, exc_info=e3_info) + line = logger.get_lines_for_level('warning')[-1] + self.assertTrue('test3 server' in line) + self.assertTrue('test3 msg' in line) + log_args, log_kwargs = logger.log_dict['warning'][-1] + self.assertTrue(log_kwargs['exc_info']) + self.assertEqual(log_kwargs['exc_info'][1], e3) + self.assertEqual(4, node_error_count(app, node)) + + def test_valid_api_version(self): + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing()) + + # The version string is only checked for account, container and object + # requests; the raised APIVersionError returns a 404 to the client + for path in [ + '/v2/a', + '/v2/a/c', + '/v2/a/c/o']: + req = Request.blank(path) + self.assertRaises(APIVersionError, app.get_controller, req) + + # Default valid API versions are ok + for path in [ + '/v1/a', + '/v1/a/c', + '/v1/a/c/o', + '/v1.0/a', + '/v1.0/a/c', + '/v1.0/a/c/o']: + req = Request.blank(path) + controller, path_parts = app.get_controller(req) + self.assertTrue(controller is not None) + + # Ensure settings valid API version constraint works + for version in ["42", 42]: + try: + with NamedTemporaryFile() as f: + f.write('[swift-constraints]\n') + f.write('valid_api_versions = %s\n' % version) + f.flush() + with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name): + constraints.reload_constraints() + + req = Request.blank('/%s/a' % version) + controller, _ = app.get_controller(req) + self.assertTrue(controller is not None) + + # In this case v1 is invalid + req = Request.blank('/v1/a') + self.assertRaises(APIVersionError, app.get_controller, req) + finally: + constraints.reload_constraints() + + # Check that the valid_api_versions is not exposed by default + req = Request.blank('/info') + controller, path_parts = app.get_controller(req) + self.assertTrue('swift.valid_api_versions' in + path_parts.get('disallowed_sections')) + +@patch_policies([ + StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(1, 'one'), +]) +class TestProxyServerLoading(unittest.TestCase): + + def setUp(self): + self._orig_hash_suffix = utils.HASH_PATH_SUFFIX + utils.HASH_PATH_SUFFIX = 'endcap' + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + utils.HASH_PATH_SUFFIX = self._orig_hash_suffix + for policy in POLICIES: + policy.object_ring = None + + def test_load_policy_rings(self): + for policy in POLICIES: + self.assertFalse(policy.object_ring) + conf_path = os.path.join(self.tempdir, 'proxy-server.conf') + conf_body = """ + [DEFAULT] + swift_dir = %s + + [pipeline:main] + pipeline = catch_errors cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + + [filter:catch_errors] + use = egg:swift#catch_errors + """ % self.tempdir + with open(conf_path, 'w') as f: + f.write(dedent(conf_body)) + account_ring_path = os.path.join(self.tempdir, 'account.ring.gz') + write_fake_ring(account_ring_path) + container_ring_path = os.path.join(self.tempdir, 'container.ring.gz') + write_fake_ring(container_ring_path) + for policy in POLICIES: + object_ring_path = os.path.join(self.tempdir, + policy.ring_name + '.ring.gz') + write_fake_ring(object_ring_path) + app = loadapp(conf_path) + # find the end of the pipeline + while hasattr(app, 'app'): + app = app.app + + # validate loaded rings + self.assertEqual(app.account_ring.serialized_path, + account_ring_path) + self.assertEqual(app.container_ring.serialized_path, + container_ring_path) + for policy in POLICIES: + self.assertEqual(policy.object_ring, + app.get_object_ring(int(policy))) + + def test_missing_rings(self): + conf_path = os.path.join(self.tempdir, 'proxy-server.conf') + conf_body = """ + [DEFAULT] + swift_dir = %s + + [pipeline:main] + pipeline = catch_errors cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + + [filter:catch_errors] + use = egg:swift#catch_errors + """ % self.tempdir + with open(conf_path, 'w') as f: + f.write(dedent(conf_body)) + ring_paths = [ + os.path.join(self.tempdir, 'account.ring.gz'), + os.path.join(self.tempdir, 'container.ring.gz'), + ] + for policy in POLICIES: + self.assertFalse(policy.object_ring) + object_ring_path = os.path.join(self.tempdir, + policy.ring_name + '.ring.gz') + ring_paths.append(object_ring_path) + for policy in POLICIES: + self.assertFalse(policy.object_ring) + for ring_path in ring_paths: + self.assertFalse(os.path.exists(ring_path)) + self.assertRaises(IOError, loadapp, conf_path) + write_fake_ring(ring_path) + # all rings exist, app should load + loadapp(conf_path) + for policy in POLICIES: + self.assert_(policy.object_ring) + + +@patch_policies([StoragePolicy(0, 'zero', True, + object_ring=FakeRing(base_port=3000))]) class TestObjectController(unittest.TestCase): def setUp(self): - self.app = proxy_server.Application(None, FakeMemcache(), - logger=debug_logger('proxy-ut'), - account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing()) + self.app = proxy_server.Application( + None, FakeMemcache(), + logger=debug_logger('proxy-ut'), + account_ring=FakeRing(), + container_ring=FakeRing()) def tearDown(self): self.app.account_ring.set_replicas(3) self.app.container_ring.set_replicas(3) - self.app.object_ring.set_replicas(3) + for policy in POLICIES: + policy.object_ring = FakeRing(base_port=3000) + + def put_container(self, policy_name, container_name): + # Note: only works if called with unpatched policies + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: 0\r\n' + 'X-Storage-Token: t\r\n' + 'X-Storage-Policy: %s\r\n' + '\r\n' % (container_name, policy_name)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' + self.assertEqual(headers[:len(exp)], exp) def assert_status_map(self, method, statuses, expected, raise_exc=False): with save_globals(): @@ -717,7 +1171,10 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) - res = method(req) + try: + res = method(req) + except HTTPException as res: + pass self.assertEquals(res.status_int, expected) # repeat test @@ -727,9 +1184,181 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) - res = method(req) + try: + res = method(req) + except HTTPException as res: + pass self.assertEquals(res.status_int, expected) + @unpatch_policies + def test_policy_IO(self): + def check_file(policy, cont, devs, check_val): + partition, nodes = policy.object_ring.get_nodes('a', cont, 'o') + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + for dev in devs: + file = df_mgr.get_diskfile(dev, partition, 'a', + cont, 'o', + policy=policy) + if check_val is True: + file.open() + + prolis = _test_sockets[0] + prosrv = _test_servers[0] + + # check policy 0: put file on c, read it back, check loc on disk + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + obj = 'test_object0' + path = '/v1/a/c/o' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: text/plain\r\n' + '\r\n%s' % (path, str(len(obj)), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + req = Request.blank(path, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': + 'text/plain'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, obj) + + check_file(POLICIES[0], 'c', ['sda1', 'sdb1'], True) + check_file(POLICIES[0], 'c', ['sdc1', 'sdd1', 'sde1', 'sdf1'], False) + + # check policy 1: put file on c1, read it back, check loc on disk + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + path = '/v1/a/c1/o' + obj = 'test_object1' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: text/plain\r\n' + '\r\n%s' % (path, str(len(obj)), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + self.assertEqual(headers[:len(exp)], exp) + req = Request.blank(path, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': + 'text/plain'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, obj) + + check_file(POLICIES[1], 'c1', ['sdc1', 'sdd1'], True) + check_file(POLICIES[1], 'c1', ['sda1', 'sdb1', 'sde1', 'sdf1'], False) + + # check policy 2: put file on c2, read it back, check loc on disk + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + path = '/v1/a/c2/o' + obj = 'test_object2' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: text/plain\r\n' + '\r\n%s' % (path, str(len(obj)), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + self.assertEqual(headers[:len(exp)], exp) + req = Request.blank(path, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': + 'text/plain'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, obj) + + check_file(POLICIES[2], 'c2', ['sde1', 'sdf1'], True) + check_file(POLICIES[2], 'c2', ['sda1', 'sdb1', 'sdc1', 'sdd1'], False) + + @unpatch_policies + def test_policy_IO_override(self): + if hasattr(_test_servers[-1], '_filesystem'): + # ironically, the _filesystem attribute on the object server means + # the in-memory diskfile is in use, so this test does not apply + return + + prosrv = _test_servers[0] + + # validate container policy is 1 + req = Request.blank('/v1/a/c1', method='HEAD') + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 204) # sanity check + self.assertEqual(POLICIES[1].name, res.headers['x-storage-policy']) + + # check overrides: put it in policy 2 (not where the container says) + req = Request.blank( + '/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO("hello")}, + headers={'Content-Type': 'text/plain', + 'Content-Length': '5', + 'X-Backend-Storage-Policy-Index': '2'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 201) # sanity check + + # go to disk to make sure it's there + partition, nodes = prosrv.get_object_ring(2).get_nodes( + 'a', 'c1', 'wrong-o') + node = nodes[0] + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + df = df_mgr.get_diskfile(node['device'], partition, 'a', + 'c1', 'wrong-o', policy=POLICIES[2]) + with df.open(): + contents = ''.join(df.reader()) + self.assertEqual(contents, "hello") + + # can't get it from the normal place + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': 'text/plain'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 404) # sanity check + + # but we can get it from policy 2 + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': 'text/plain', + 'X-Backend-Storage-Policy-Index': '2'}) + + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, 'hello') + + # and we can delete it the same way + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Content-Type': 'text/plain', + 'X-Backend-Storage-Policy-Index': '2'}) + + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 204) + + df = df_mgr.get_diskfile(node['device'], partition, 'a', + 'c1', 'wrong-o', policy=POLICIES[2]) + try: + df.open() + except DiskFileNotExist as e: + self.assert_(float(e.timestamp) > 0) + else: + self.fail('did not raise DiskFileNotExist') + + @unpatch_policies def test_GET_newest_large_file(self): prolis = _test_sockets[0] prosrv = _test_servers[0] @@ -757,6 +1386,619 @@ class TestObjectController(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) + @unpatch_policies + def test_PUT_ec(self): + policy = POLICIES[3] + self.put_container("ec", "ec-con") + + obj = 'abCD' * 10 # small, so we don't get multiple EC stripes + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o1 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + ecd = policy.pyeclib_driver + expected_pieces = set(ecd.encode(obj)) + + # go to disk to make sure it's there and all erasure-coded + partition, nodes = policy.object_ring.get_nodes('a', 'ec-con', 'o1') + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + got_pieces = set() + got_indices = set() + got_durable = [] + for node_index, node in enumerate(nodes): + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'o1', + policy=policy) + with df.open(): + meta = df.get_metadata() + contents = ''.join(df.reader()) + got_pieces.add(contents) + + # check presence for a .durable file for the timestamp + durable_file = os.path.join( + _testdir, node['device'], storage_directory( + diskfile.get_data_dir(policy), + partition, hash_path('a', 'ec-con', 'o1')), + utils.Timestamp(df.timestamp).internal + '.durable') + + if os.path.isfile(durable_file): + got_durable.append(True) + + lmeta = dict((k.lower(), v) for k, v in meta.items()) + got_indices.add( + lmeta['x-object-sysmeta-ec-frag-index']) + + self.assertEqual( + lmeta['x-object-sysmeta-ec-etag'], + md5(obj).hexdigest()) + self.assertEqual( + lmeta['x-object-sysmeta-ec-content-length'], + str(len(obj))) + self.assertEqual( + lmeta['x-object-sysmeta-ec-segment-size'], + '4096') + self.assertEqual( + lmeta['x-object-sysmeta-ec-scheme'], + 'jerasure_rs_vand 2+1') + self.assertEqual( + lmeta['etag'], + md5(contents).hexdigest()) + + self.assertEqual(expected_pieces, got_pieces) + self.assertEqual(set(('0', '1', '2')), got_indices) + + # verify at least 2 puts made it all the way to the end of 2nd + # phase, ie at least 2 .durable statuses were written + num_durable_puts = sum(d is True for d in got_durable) + self.assertTrue(num_durable_puts >= 2) + + @unpatch_policies + def test_PUT_ec_multiple_segments(self): + ec_policy = POLICIES[3] + self.put_container("ec", "ec-con") + + pyeclib_header_size = len(ec_policy.pyeclib_driver.encode("")[0]) + segment_size = ec_policy.ec_segment_size + + # Big enough to have multiple segments. Also a multiple of the + # segment size to get coverage of that path too. + obj = 'ABC' * segment_size + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o2 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + # it's a 2+1 erasure code, so each fragment archive should be half + # the length of the object, plus three inline pyeclib metadata + # things (one per segment) + expected_length = (len(obj) / 2 + pyeclib_header_size * 3) + + partition, nodes = ec_policy.object_ring.get_nodes( + 'a', 'ec-con', 'o2') + + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + got_durable = [] + fragment_archives = [] + for node in nodes: + df = df_mgr.get_diskfile( + node['device'], partition, 'a', + 'ec-con', 'o2', policy=ec_policy) + with df.open(): + contents = ''.join(df.reader()) + fragment_archives.append(contents) + self.assertEqual(len(contents), expected_length) + + # check presence for a .durable file for the timestamp + durable_file = os.path.join( + _testdir, node['device'], storage_directory( + diskfile.get_data_dir(ec_policy), + partition, hash_path('a', 'ec-con', 'o2')), + utils.Timestamp(df.timestamp).internal + '.durable') + + if os.path.isfile(durable_file): + got_durable.append(True) + + # Verify that we can decode each individual fragment and that they + # are all the correct size + fragment_size = ec_policy.fragment_size + nfragments = int( + math.ceil(float(len(fragment_archives[0])) / fragment_size)) + + for fragment_index in range(nfragments): + fragment_start = fragment_index * fragment_size + fragment_end = (fragment_index + 1) * fragment_size + + try: + frags = [fa[fragment_start:fragment_end] + for fa in fragment_archives] + seg = ec_policy.pyeclib_driver.decode(frags) + except ECDriverError: + self.fail("Failed to decode fragments %d; this probably " + "means the fragments are not the sizes they " + "should be" % fragment_index) + + segment_start = fragment_index * segment_size + segment_end = (fragment_index + 1) * segment_size + + self.assertEqual(seg, obj[segment_start:segment_end]) + + # verify at least 2 puts made it all the way to the end of 2nd + # phase, ie at least 2 .durable statuses were written + num_durable_puts = sum(d is True for d in got_durable) + self.assertTrue(num_durable_puts >= 2) + + @unpatch_policies + def test_PUT_ec_object_etag_mismatch(self): + self.put_container("ec", "ec-con") + + obj = '90:6A:02:60:B1:08-96da3e706025537fc42464916427727e' + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/o3 HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: %s\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5('something else').hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 422' + self.assertEqual(headers[:len(exp)], exp) + + # nothing should have made it to disk on the object servers + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'o3') + conf = {'devices': _testdir, 'mount_check': 'false'} + + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'o3') + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + for node in nodes: + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'o3', policy=POLICIES[3]) + self.assertRaises(DiskFileNotExist, df.open) + + @unpatch_policies + def test_PUT_ec_fragment_archive_etag_mismatch(self): + self.put_container("ec", "ec-con") + + # Cause a hash mismatch by feeding one particular MD5 hasher some + # extra data. The goal here is to get exactly one of the hashers in + # an object server. + countdown = [1] + + def busted_md5_constructor(initial_str=""): + hasher = md5(initial_str) + if countdown[0] == 0: + hasher.update('wrong') + countdown[0] -= 1 + return hasher + + obj = 'uvarovite-esurience-cerated-symphysic' + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + with mock.patch('swift.obj.server.md5', busted_md5_constructor): + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/pimento HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: %s\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 503' # no quorum + self.assertEqual(headers[:len(exp)], exp) + + # 2/3 of the fragment archives should have landed on disk + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'pimento') + conf = {'devices': _testdir, 'mount_check': 'false'} + + partition, nodes = prosrv.get_object_ring(3).get_nodes( + 'a', 'ec-con', 'pimento') + conf = {'devices': _testdir, 'mount_check': 'false'} + + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + + found = 0 + for node in nodes: + df = df_mgr.get_diskfile(node['device'], partition, + 'a', 'ec-con', 'pimento', + policy=POLICIES[3]) + try: + df.open() + found += 1 + except DiskFileNotExist: + pass + self.assertEqual(found, 2) + + @unpatch_policies + def test_PUT_ec_if_none_match(self): + self.put_container("ec", "ec-con") + + obj = 'ananepionic-lepidophyllous-ropewalker-neglectful' + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/inm HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/inm HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'If-None-Match: *\r\n' + 'Etag: "%s"\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (md5(obj).hexdigest(), len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEqual(headers[:len(exp)], exp) + + @unpatch_policies + def test_GET_ec(self): + self.put_container("ec", "ec-con") + + obj = '0123456' * 11 * 17 + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/go-get-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'X-Object-Meta-Color: chartreuse\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/go-get-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + self.assertEqual('chartreuse', headers['X-Object-Meta-Color']) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + self.assertEqual(gotten_obj, obj) + + @unpatch_policies + def test_conditional_GET_ec(self): + self.put_container("ec", "ec-con") + + obj = 'this object has an etag and is otherwise unimportant' + etag = md5(obj).hexdigest() + not_etag = md5(obj + "blahblah").hexdigest() + + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/conditionals HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + for verb in ('GET', 'HEAD'): + # If-Match + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': not_etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 412) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-Match': "*"}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + # If-None-Match + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 304) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': not_etag}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + + req = Request.blank( + '/v1/a/ec-con/conditionals', + environ={'REQUEST_METHOD': verb}, + headers={'If-None-Match': "*"}) + resp = req.get_response(prosrv) + self.assertEqual(resp.status_int, 304) + + @unpatch_policies + def test_GET_ec_big(self): + self.put_container("ec", "ec-con") + + # our EC segment size is 4 KiB, so this is multiple (3) segments; + # we'll verify that with a sanity check + obj = 'a moose once bit my sister' * 400 + self.assertTrue( + len(obj) > POLICIES.get_by_name("ec").ec_segment_size * 2, + "object is too small for proper testing") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/big-obj-get HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/big-obj-get HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + # This may look like a redundant test, but when things fail, this + # has a useful failure message while the subsequent one spews piles + # of garbage and demolishes your terminal's scrollback buffer. + self.assertEqual(len(gotten_obj), len(obj)) + self.assertEqual(gotten_obj, obj) + + @unpatch_policies + def test_GET_ec_failure_handling(self): + self.put_container("ec", "ec-con") + + obj = 'look at this object; it is simply amazing ' * 500 + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/crash-test-dummy HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + def explodey_iter(inner_iter): + yield next(inner_iter) + raise Exception("doom ba doom") + + real_ec_app_iter = swift.proxy.controllers.obj.ECAppIter + + def explodey_ec_app_iter(path, policy, iterators, *a, **kw): + # Each thing in `iterators` here is a document-parts iterator, + # and we want to fail after getting a little into each part. + # + # That way, we ensure we've started streaming the response to + # the client when things go wrong. + return real_ec_app_iter( + path, policy, + [explodey_iter(i) for i in iterators], + *a, **kw) + + with mock.patch("swift.proxy.controllers.obj.ECAppIter", + explodey_ec_app_iter): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/crash-test-dummy HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + + gotten_obj = '' + try: + with Timeout(300): # don't hang the testrun when this fails + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + except Timeout: + self.fail("GET hung when connection failed") + + # Ensure we failed partway through, otherwise the mocks could + # get out of date without anyone noticing + self.assertTrue(0 < len(gotten_obj) < len(obj)) + + @unpatch_policies + def test_HEAD_ec(self): + self.put_container("ec", "ec-con") + + obj = '0123456' * 11 * 17 + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/go-head-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'X-Object-Meta-Color: chartreuse\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a/ec-con/go-head-it HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + headers = parse_headers_string(headers) + self.assertEqual(str(len(obj)), headers['Content-Length']) + self.assertEqual(md5(obj).hexdigest(), headers['Etag']) + self.assertEqual('chartreuse', headers['X-Object-Meta-Color']) + + @unpatch_policies + def test_GET_ec_404(self): + self.put_container("ec", "ec-con") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/yes-we-have-no-bananas HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEqual(headers[:len(exp)], exp) + + @unpatch_policies + def test_HEAD_ec_404(self): + self.put_container("ec", "ec-con") + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a/ec-con/yes-we-have-no-bananas HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEqual(headers[:len(exp)], exp) + def test_PUT_expect_header_zero_content_length(self): test_errors = [] @@ -768,13 +2010,22 @@ class TestObjectController(unittest.TestCase): 'server!') with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - # The (201, -4) tuples in there have the effect of letting the - # initial connect succeed, after which getexpect() gets called and - # then the -4 makes the response of that actually be 201 instead of - # 100. Perfectly straightforward. - set_http_connect(200, 200, (201, -4), (201, -4), (201, -4), + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') + # The (201, Exception('test')) tuples in there have the effect of + # changing the status of the initial expect response. The default + # expect response from FakeConn for 201 is 100. + # But the object server won't send a 100 continue line if the + # client doesn't send a expect 100 header (as is the case with + # zero byte PUTs as validated by this test), nevertheless the + # object controller calls getexpect without prejudice. In this + # case the status from the response shows up early in getexpect + # instead of having to wait until getresponse. The Exception is + # in there to ensure that the object controller also *uses* the + # result of getexpect instead of calling getresponse in which case + # our FakeConn will blow up. + success_codes = [(201, Exception('test'))] * 3 + set_http_connect(200, 200, *success_codes, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 @@ -795,9 +2046,14 @@ class TestObjectController(unittest.TestCase): 'non-zero byte PUT!') with save_globals(): - controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') - set_http_connect(200, 200, 201, 201, 201, + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') + # the (100, 201) tuples in there are just being extra explicit + # about the FakeConn returning the 100 Continue status when the + # object controller calls getexpect. Which is FakeConn's default + # for 201 if no expect_status is specified. + success_codes = [(100, 201)] * 3 + set_http_connect(200, 200, *success_codes, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 1 @@ -820,12 +2076,14 @@ class TestObjectController(unittest.TestCase): def is_r0(node): return node['region'] == 0 - self.app.object_ring.max_more_nodes = 100 + object_ring = self.app.get_object_ring(None) + object_ring.max_more_nodes = 100 self.app.write_affinity_is_local_fn = is_r0 self.app.write_affinity_node_count = lambda r: 3 controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') + ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') set_http_connect(200, 200, 201, 201, 201, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) @@ -854,14 +2112,16 @@ class TestObjectController(unittest.TestCase): def is_r0(node): return node['region'] == 0 - self.app.object_ring.max_more_nodes = 100 + object_ring = self.app.get_object_ring(None) + object_ring.max_more_nodes = 100 self.app.write_affinity_is_local_fn = is_r0 self.app.write_affinity_node_count = lambda r: 3 controller = \ - proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') + ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') self.app.error_limit( - self.app.object_ring.get_part_nodes(1)[0], 'test') + object_ring.get_part_nodes(1)[0], 'test') set_http_connect(200, 200, # account, container 201, 201, 201, # 3 working backends give_connect=test_connect) @@ -880,6 +2140,28 @@ class TestObjectController(unittest.TestCase): self.assertEqual(0, written_to[1][1] % 2) self.assertNotEqual(0, written_to[2][1] % 2) + @unpatch_policies + def test_PUT_no_etag_fallocate(self): + with mock.patch('swift.obj.diskfile.fallocate') as mock_fallocate: + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + obj = 'hemoleucocytic-surfactant' + fd.write('PUT /v1/a/c/o HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + # one for each obj server; this test has 2 + self.assertEqual(len(mock_fallocate.mock_calls), 2) + + @unpatch_policies def test_PUT_message_length_using_content_length(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -897,6 +2179,7 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) + @unpatch_policies def test_PUT_message_length_using_transfer_encoding(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -929,6 +2212,7 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) + @unpatch_policies def test_PUT_message_length_using_both(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -962,6 +2246,7 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) + @unpatch_policies def test_PUT_bad_message_length(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -995,6 +2280,7 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 400' self.assertEqual(headers[:len(exp)], exp) + @unpatch_policies def test_PUT_message_length_unsup_xfr_encoding(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1028,9 +2314,9 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 501' self.assertEqual(headers[:len(exp)], exp) + @unpatch_policies def test_PUT_message_length_too_large(self): - swift.proxy.controllers.obj.MAX_FILE_SIZE = 10 - try: + with mock.patch('swift.common.constraints.MAX_FILE_SIZE', 10): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -1045,9 +2331,8 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 413' self.assertEqual(headers[:len(exp)], exp) - finally: - swift.proxy.controllers.obj.MAX_FILE_SIZE = MAX_FILE_SIZE + @unpatch_policies def test_PUT_last_modified(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1107,7 +2392,7 @@ class TestObjectController(unittest.TestCase): if 'x-if-delete-at' in headers or 'X-If-Delete-At' in headers: test_errors.append('X-If-Delete-At in headers') - body = simplejson.dumps( + body = json.dumps( [{"name": "001o/1", "hash": "x", "bytes": 0, @@ -1115,11 +2400,12 @@ class TestObjectController(unittest.TestCase): "last_modified": "1970-01-01T00:00:01.000000"}]) body_iter = ('', '', body, '', '', '', '', '', '', '', '', '', '', '') with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') # HEAD HEAD GET GET HEAD GET GET GET PUT PUT # PUT DEL DEL DEL set_http_connect(200, 200, 200, 200, 200, 200, 200, 200, 201, 201, - 201, 200, 200, 200, + 201, 204, 204, 204, give_connect=test_connect, body_iter=body_iter, headers={'x-versions-location': 'foo'}) @@ -1131,10 +2417,133 @@ class TestObjectController(unittest.TestCase): controller.DELETE(req) self.assertEquals(test_errors, []) + @patch_policies([ + StoragePolicy(0, 'zero', False, object_ring=FakeRing()), + StoragePolicy(1, 'one', True, object_ring=FakeRing()) + ]) + def test_DELETE_on_expired_versioned_object(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() + methods = set() + authorize_call_count = [0] + + def test_connect(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + methods.add((method, path)) + + def fake_container_info(account, container, req): + return {'status': 200, 'sync_key': None, + 'meta': {}, 'cors': {'allow_origin': None, + 'expose_headers': None, + 'max_age': None}, + 'sysmeta': {}, 'read_acl': None, 'object_count': None, + 'write_acl': None, 'versions': 'foo', + 'partition': 1, 'bytes': None, 'storage_policy': '1', + 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, + 'id': 0, 'device': 'sda', 'port': 1000}, + {'zone': 1, 'ip': '10.0.0.1', 'region': 1, + 'id': 1, 'device': 'sdb', 'port': 1001}, + {'zone': 2, 'ip': '10.0.0.2', 'region': 0, + 'id': 2, 'device': 'sdc', 'port': 1002}]} + + def fake_list_iter(container, prefix, env): + object_list = [{'name': '1'}, {'name': '2'}, {'name': '3'}] + for obj in object_list: + yield obj + + def fake_authorize(req): + authorize_call_count[0] += 1 + return None # allow the request + + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') + controller.container_info = fake_container_info + controller._listing_iter = fake_list_iter + set_http_connect(404, 404, 404, # get for the previous version + 200, 200, 200, # get for the pre-previous + 201, 201, 201, # put move the pre-previous + 204, 204, 204, # delete for the pre-previous + give_connect=test_connect) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', + 'swift.authorize': fake_authorize}) + + self.app.memcache.store = {} + self.app.update_request(req) + controller.DELETE(req) + exp_methods = [('GET', '/a/foo/3'), + ('GET', '/a/foo/2'), + ('PUT', '/a/c/o'), + ('DELETE', '/a/foo/2')] + self.assertEquals(set(exp_methods), (methods)) + self.assertEquals(authorize_call_count[0], 2) + + @patch_policies([ + StoragePolicy(0, 'zero', False, object_ring=FakeRing()), + StoragePolicy(1, 'one', True, object_ring=FakeRing()) + ]) + def test_denied_DELETE_of_versioned_object(self): + """ + Verify that a request with read access to a versions container + is unable to cause any write operations on the versioned container. + """ + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() + methods = set() + authorize_call_count = [0] + + def test_connect(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + methods.add((method, path)) + + def fake_container_info(account, container, req): + return {'status': 200, 'sync_key': None, + 'meta': {}, 'cors': {'allow_origin': None, + 'expose_headers': None, + 'max_age': None}, + 'sysmeta': {}, 'read_acl': None, 'object_count': None, + 'write_acl': None, 'versions': 'foo', + 'partition': 1, 'bytes': None, 'storage_policy': '1', + 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, + 'id': 0, 'device': 'sda', 'port': 1000}, + {'zone': 1, 'ip': '10.0.0.1', 'region': 1, + 'id': 1, 'device': 'sdb', 'port': 1001}, + {'zone': 2, 'ip': '10.0.0.2', 'region': 0, + 'id': 2, 'device': 'sdc', 'port': 1002}]} + + def fake_list_iter(container, prefix, env): + object_list = [{'name': '1'}, {'name': '2'}, {'name': '3'}] + for obj in object_list: + yield obj + + def fake_authorize(req): + # deny write access + authorize_call_count[0] += 1 + return HTTPForbidden(req) # allow the request + + with save_globals(): + controller = ReplicatedObjectController(self.app, 'a', 'c', 'o') + controller.container_info = fake_container_info + # patching _listing_iter simulates request being authorized + # to list versions container + controller._listing_iter = fake_list_iter + set_http_connect(give_connect=test_connect) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE', + 'swift.authorize': fake_authorize}) + + self.app.memcache.store = {} + self.app.update_request(req) + resp = controller.DELETE(req) + self.assertEqual(403, resp.status_int) + self.assertFalse(methods, methods) + self.assertEquals(authorize_call_count[0], 1) + def test_PUT_auto_content_type(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_content_type(filename, expected): # The three responses here are for account_info() (HEAD to @@ -1170,8 +2579,7 @@ class TestObjectController(unittest.TestCase): fp.write('foo/bar foo\n') proxy_server.Application({'swift_dir': swift_dir}, FakeMemcache(), FakeLogger(), - FakeRing(), FakeRing(), - FakeRing()) + FakeRing(), FakeRing()) self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0], 'foo/bar') self.assertEquals(proxy_server.mimetypes.guess_type('blah.jpg')[0], @@ -1181,8 +2589,8 @@ class TestObjectController(unittest.TestCase): def test_PUT(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) @@ -1197,11 +2605,12 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 201, 201, 500), 201) test_status_map((200, 200, 204, 404, 404), 404) test_status_map((200, 200, 204, 500, 404), 503) + test_status_map((200, 200, 202, 202, 204), 204) def test_PUT_connect_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) @@ -1209,19 +2618,30 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) - res = controller.PUT(req) + try: + res = controller.PUT(req) + except HTTPException as res: + pass expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) - test_status_map((200, 200, 201, 201, -1), 201) - test_status_map((200, 200, 201, 201, -2), 201) # expect timeout - test_status_map((200, 200, 201, 201, -3), 201) # error limited - test_status_map((200, 200, 201, -1, -1), 503) - test_status_map((200, 200, 503, 503, -1), 503) + test_status_map((200, 200, 201, 201, -1), 201) # connect exc + # connect errors + test_status_map((200, 200, Timeout(), 201, 201, ), 201) + test_status_map((200, 200, 201, 201, Exception()), 201) + # expect errors + test_status_map((200, 200, (Timeout(), None), 201, 201), 201) + test_status_map((200, 200, (Exception(), None), 201, 201), 201) + # response errors + test_status_map((200, 200, (100, Timeout()), 201, 201), 201) + test_status_map((200, 200, (100, Exception()), 201, 201), 201) + test_status_map((200, 200, 507, 201, 201), 201) # error limited + test_status_map((200, 200, -1, 201, -1), 503) + test_status_map((200, 200, 503, -1, 503), 503) def test_PUT_send_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} @@ -1230,7 +2650,10 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, body='some data') self.app.update_request(req) - res = controller.PUT(req) + try: + res = controller.PUT(req) + except HTTPException as res: + pass expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 201, -1, 201), 201) @@ -1240,10 +2663,10 @@ class TestObjectController(unittest.TestCase): def test_PUT_max_size(self): with save_globals(): set_http_connect(201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ - 'Content-Length': str(MAX_FILE_SIZE + 1), + 'Content-Length': str(constraints.MAX_FILE_SIZE + 1), 'Content-Type': 'foo/bar'}) self.app.update_request(req) res = controller.PUT(req) @@ -1252,8 +2675,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_bad_content_type(self): with save_globals(): set_http_connect(201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ 'Content-Length': 0, 'Content-Type': 'foo/bar;swift_hey=45'}) self.app.update_request(req) @@ -1263,8 +2686,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_getresponse_exceptions(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} @@ -1272,7 +2695,10 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) - res = controller.PUT(req) + try: + res = controller.PUT(req) + except HTTPException as res: + pass expected = str(expected) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -1301,6 +2727,137 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 404, 500, 500), 503) test_status_map((200, 200, 404, 404, 404), 404) + @patch_policies([ + StoragePolicy(0, 'zero', is_default=True, object_ring=FakeRing()), + StoragePolicy(1, 'one', object_ring=FakeRing()), + ]) + def test_POST_backend_headers(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() + self.app.object_post_as_copy = False + self.app.sort_nodes = lambda nodes: nodes + backend_requests = [] + + def capture_requests(ip, port, method, path, headers, *args, + **kwargs): + backend_requests.append((method, path, headers)) + + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue'}) + + # we want the container_info response to says a policy index of 1 + resp_headers = {'X-Backend-Storage-Policy-Index': 1} + with mocked_http_conn( + 200, 200, 202, 202, 202, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 5) + + def check_request(req, method, path, headers=None): + req_method, req_path, req_headers = req + self.assertEqual(method, req_method) + # caller can ignore leading path parts + self.assertTrue(req_path.endswith(path), + 'expected path to end with %s, it was %s' % ( + path, req_path)) + headers = headers or {} + # caller can ignore some headers + for k, v in headers.items(): + self.assertEqual(req_headers[k], v) + account_request = backend_requests.pop(0) + check_request(account_request, method='HEAD', path='/sda/0/a') + container_request = backend_requests.pop(0) + check_request(container_request, method='HEAD', path='/sda/0/a/c') + # make sure backend requests included expected container headers + container_headers = {} + for request in backend_requests: + req_headers = request[2] + device = req_headers['x-container-device'] + host = req_headers['x-container-host'] + container_headers[device] = host + expectations = { + 'method': 'POST', + 'path': '/0/a/c/o', + 'headers': { + 'X-Container-Partition': '0', + 'Connection': 'close', + 'User-Agent': 'proxy-server %s' % os.getpid(), + 'Host': 'localhost:80', + 'Referer': 'POST http://localhost/v1/a/c/o', + 'X-Object-Meta-Color': 'Blue', + 'X-Backend-Storage-Policy-Index': '1' + }, + } + check_request(request, **expectations) + + expected = {} + for i, device in enumerate(['sda', 'sdb', 'sdc']): + expected[device] = '10.0.0.%d:100%d' % (i, i) + self.assertEqual(container_headers, expected) + + # and again with policy override + self.app.memcache.store = {} + backend_requests = [] + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue', + 'X-Backend-Storage-Policy-Index': 0}) + with mocked_http_conn( + 200, 200, 202, 202, 202, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 5) + for request in backend_requests[2:]: + expectations = { + 'method': 'POST', + 'path': '/0/a/c/o', # ignore device bit + 'headers': { + 'X-Object-Meta-Color': 'Blue', + 'X-Backend-Storage-Policy-Index': '0', + } + } + check_request(request, **expectations) + + # and this time with post as copy + self.app.object_post_as_copy = True + self.app.memcache.store = {} + backend_requests = [] + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue', + 'X-Backend-Storage-Policy-Index': 0}) + with mocked_http_conn( + 200, 200, 200, 200, 200, 201, 201, 201, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 8) + policy0 = {'X-Backend-Storage-Policy-Index': '0'} + policy1 = {'X-Backend-Storage-Policy-Index': '1'} + expected = [ + # account info + {'method': 'HEAD', 'path': '/0/a'}, + # container info + {'method': 'HEAD', 'path': '/0/a/c'}, + # x-newests + {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, + {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, + {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, + # new writes + {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, + {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, + {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, + ] + for request, expectations in zip(backend_requests, expected): + check_request(request, **expectations) + def test_POST_as_copy(self): with save_globals(): def test_status_map(statuses, expected): @@ -1333,9 +2890,9 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 204, 204, 204), 204) test_status_map((200, 200, 204, 204, 500), 204) test_status_map((200, 200, 204, 404, 404), 404) - test_status_map((200, 200, 204, 500, 404), 503) + test_status_map((200, 204, 500, 500, 404), 503) test_status_map((200, 200, 404, 404, 404), 404) - test_status_map((200, 200, 404, 404, 500), 404) + test_status_map((200, 200, 400, 400, 400), 400) def test_HEAD(self): with save_globals(): @@ -1388,6 +2945,8 @@ class TestObjectController(unittest.TestCase): None, None), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1') + test_status_map((200, 200, 404, 404, 200), 200, ('0', '0', None, + None, '1'), '1') def test_GET_newest(self): with save_globals(): @@ -1443,10 +3002,10 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_val_len(self): with save_globals(): - limit = MAX_META_VALUE_LENGTH + limit = constraints.MAX_META_VALUE_LENGTH self.app.object_post_as_copy = False - proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, @@ -1466,7 +3025,7 @@ class TestObjectController(unittest.TestCase): def test_POST_as_copy_meta_val_len(self): with save_globals(): - limit = MAX_META_VALUE_LENGTH + limit = constraints.MAX_META_VALUE_LENGTH set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, @@ -1486,7 +3045,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_key_len(self): with save_globals(): - limit = MAX_META_NAME_LENGTH + limit = constraints.MAX_META_NAME_LENGTH self.app.object_post_as_copy = False set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj @@ -1508,7 +3067,7 @@ class TestObjectController(unittest.TestCase): def test_POST_as_copy_meta_key_len(self): with save_globals(): - limit = MAX_META_NAME_LENGTH + limit = constraints.MAX_META_NAME_LENGTH set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj req = Request.blank( @@ -1529,7 +3088,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_count(self): with save_globals(): - limit = MAX_META_COUNT + limit = constraints.MAX_META_COUNT headers = dict( (('X-Object-Meta-' + str(i), 'a') for i in xrange(limit + 1))) headers.update({'Content-Type': 'foo/bar'}) @@ -1542,7 +3101,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_size(self): with save_globals(): - limit = MAX_META_OVERALL_SIZE + limit = constraints.MAX_META_OVERALL_SIZE count = limit / 256 # enough to cause the limit to be reached headers = dict( (('X-Object-Meta-' + str(i), 'a' * 256) @@ -1601,15 +3160,16 @@ class TestObjectController(unittest.TestCase): def test_client_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') - for dev in self.app.container_ring.devs.values(): + for dev in self.app.container_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 - self.app.object_ring.get_nodes('account') - for dev in self.app.object_ring.devs.values(): + object_ring = self.app.get_object_ring(None) + object_ring.get_nodes('account') + for dev in object_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 @@ -1635,7 +3195,7 @@ class TestObjectController(unittest.TestCase): # acct cont obj obj obj resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) - self.app.client_timeout = 0.1 + self.app.client_timeout = 0.05 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, @@ -1650,15 +3210,16 @@ class TestObjectController(unittest.TestCase): def test_client_disconnect(self): with save_globals(): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') - for dev in self.app.container_ring.devs.values(): + for dev in self.app.container_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 - self.app.object_ring.get_nodes('account') - for dev in self.app.object_ring.devs.values(): + object_ring = self.app.get_object_ring(None) + object_ring.get_nodes('account') + for dev in object_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 @@ -1684,15 +3245,16 @@ class TestObjectController(unittest.TestCase): def test_node_read_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') - for dev in self.app.container_ring.devs.values(): + for dev in self.app.container_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 - self.app.object_ring.get_nodes('account') - for dev in self.app.object_ring.devs.values(): + object_ring = self.app.get_object_ring(None) + object_ring.get_nodes('account') + for dev in object_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) @@ -1718,6 +3280,19 @@ class TestObjectController(unittest.TestCase): def test_node_read_timeout_retry(self): with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs: + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs: + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + object_ring = self.app.get_object_ring(None) + object_ring.get_nodes('account') + for dev in object_ring.devs: + dev['ip'] = '127.0.0.1' + dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) @@ -1775,15 +3350,16 @@ class TestObjectController(unittest.TestCase): def test_node_write_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') - for dev in self.app.container_ring.devs.values(): + for dev in self.app.container_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 - self.app.object_ring.get_nodes('account') - for dev in self.app.object_ring.devs.values(): + object_ring = self.app.get_object_ring(None) + object_ring.get_nodes('account') + for dev in object_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/v1/a/c/o', @@ -1806,87 +3382,149 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.app) self.assertEquals(resp.status_int, 503) + def test_node_request_setting(self): + baseapp = proxy_server.Application({'request_node_count': '3'}, + FakeMemcache(), + container_ring=FakeRing(), + account_ring=FakeRing()) + self.assertEquals(baseapp.request_node_count(3), 3) + def test_iter_nodes(self): with save_globals(): try: - self.app.object_ring.max_more_nodes = 2 - partition, nodes = self.app.object_ring.get_nodes('account', - 'container', - 'object') + object_ring = self.app.get_object_ring(None) + object_ring.max_more_nodes = 2 + partition, nodes = object_ring.get_nodes('account', + 'container', + 'object') collected_nodes = [] - for node in self.app.iter_nodes(self.app.object_ring, + for node in self.app.iter_nodes(object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) - self.app.object_ring.max_more_nodes = 20 + object_ring.max_more_nodes = 20 self.app.request_node_count = lambda r: 20 - partition, nodes = self.app.object_ring.get_nodes('account', - 'container', - 'object') + partition, nodes = object_ring.get_nodes('account', + 'container', + 'object') collected_nodes = [] - for node in self.app.iter_nodes(self.app.object_ring, + for node in self.app.iter_nodes(object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 9) + # zero error-limited primary nodes -> no handoff warnings self.app.log_handoffs = True self.app.logger = FakeLogger() - self.app.object_ring.max_more_nodes = 2 - partition, nodes = self.app.object_ring.get_nodes('account', - 'container', - 'object') + self.app.request_node_count = lambda r: 7 + object_ring.max_more_nodes = 20 + partition, nodes = object_ring.get_nodes('account', + 'container', + 'object') collected_nodes = [] - for node in self.app.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(object_ring, partition): collected_nodes.append(node) - self.assertEquals(len(collected_nodes), 5) - self.assertEquals( - self.app.logger.log_dict['warning'], - [(('Handoff requested (1)',), {}), - (('Handoff requested (2)',), {})]) + self.assertEquals(len(collected_nodes), 7) + self.assertEquals(self.app.logger.log_dict['warning'], []) + self.assertEquals(self.app.logger.get_increments(), []) - self.app.log_handoffs = False + # one error-limited primary node -> one handoff warning + self.app.log_handoffs = True self.app.logger = FakeLogger() - self.app.object_ring.max_more_nodes = 2 - partition, nodes = self.app.object_ring.get_nodes('account', - 'container', - 'object') + self.app.request_node_count = lambda r: 7 + self.app._error_limiting = {} # clear out errors + set_node_errors(self.app, object_ring._devs[0], 999, + last_error=(2 ** 63 - 1)) + collected_nodes = [] - for node in self.app.iter_nodes(self.app.object_ring, - partition): + for node in self.app.iter_nodes(object_ring, partition): collected_nodes.append(node) - self.assertEquals(len(collected_nodes), 5) - self.assertEquals(self.app.logger.log_dict['warning'], []) + self.assertEquals(len(collected_nodes), 7) + self.assertEquals(self.app.logger.log_dict['warning'], [ + (('Handoff requested (5)',), {})]) + self.assertEquals(self.app.logger.get_increments(), + ['handoff_count']) + + # two error-limited primary nodes -> two handoff warnings + self.app.log_handoffs = True + self.app.logger = FakeLogger() + self.app.request_node_count = lambda r: 7 + self.app._error_limiting = {} # clear out errors + for i in range(2): + set_node_errors(self.app, object_ring._devs[i], 999, + last_error=(2 ** 63 - 1)) + + collected_nodes = [] + for node in self.app.iter_nodes(object_ring, partition): + collected_nodes.append(node) + self.assertEquals(len(collected_nodes), 7) + self.assertEquals(self.app.logger.log_dict['warning'], [ + (('Handoff requested (5)',), {}), + (('Handoff requested (6)',), {})]) + self.assertEquals(self.app.logger.get_increments(), + ['handoff_count', + 'handoff_count']) + + # all error-limited primary nodes -> four handoff warnings, + # plus a handoff-all metric + self.app.log_handoffs = True + self.app.logger = FakeLogger() + self.app.request_node_count = lambda r: 10 + object_ring.set_replicas(4) # otherwise we run out of handoffs + self.app._error_limiting = {} # clear out errors + for i in range(4): + set_node_errors(self.app, object_ring._devs[i], 999, + last_error=(2 ** 63 - 1)) + + collected_nodes = [] + for node in self.app.iter_nodes(object_ring, partition): + collected_nodes.append(node) + self.assertEquals(len(collected_nodes), 10) + self.assertEquals(self.app.logger.log_dict['warning'], [ + (('Handoff requested (7)',), {}), + (('Handoff requested (8)',), {}), + (('Handoff requested (9)',), {}), + (('Handoff requested (10)',), {})]) + self.assertEquals(self.app.logger.get_increments(), + ['handoff_count', + 'handoff_count', + 'handoff_count', + 'handoff_count', + 'handoff_all_count']) + finally: - self.app.object_ring.max_more_nodes = 0 + object_ring.max_more_nodes = 0 def test_iter_nodes_calls_sort_nodes(self): with mock.patch.object(self.app, 'sort_nodes') as sort_nodes: - for node in self.app.iter_nodes(self.app.object_ring, 0): + object_ring = self.app.get_object_ring(None) + for node in self.app.iter_nodes(object_ring, 0): pass sort_nodes.assert_called_once_with( - self.app.object_ring.get_part_nodes(0)) + object_ring.get_part_nodes(0)) def test_iter_nodes_skips_error_limited(self): with mock.patch.object(self.app, 'sort_nodes', lambda n: n): - first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) - second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) + object_ring = self.app.get_object_ring(None) + first_nodes = list(self.app.iter_nodes(object_ring, 0)) + second_nodes = list(self.app.iter_nodes(object_ring, 0)) self.assertTrue(first_nodes[0] in second_nodes) self.app.error_limit(first_nodes[0], 'test') - second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) + second_nodes = list(self.app.iter_nodes(object_ring, 0)) self.assertTrue(first_nodes[0] not in second_nodes) def test_iter_nodes_gives_extra_if_error_limited_inline(self): + object_ring = self.app.get_object_ring(None) with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 6), - mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)): - first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) + mock.patch.object(object_ring, 'max_more_nodes', 99)): + first_nodes = list(self.app.iter_nodes(object_ring, 0)) second_nodes = [] - for node in self.app.iter_nodes(self.app.object_ring, 0): + for node in self.app.iter_nodes(object_ring, 0): if not second_nodes: self.app.error_limit(node, 'test') second_nodes.append(node) @@ -1894,12 +3532,14 @@ class TestObjectController(unittest.TestCase): self.assertEquals(len(second_nodes), 7) def test_iter_nodes_with_custom_node_iter(self): - node_list = [dict(id=n) for n in xrange(10)] + object_ring = self.app.get_object_ring(None) + node_list = [dict(id=n, ip='1.2.3.4', port=n, device='D') + for n in xrange(10)] with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 3)): - got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, + got_nodes = list(self.app.iter_nodes(object_ring, 0, node_iter=iter(node_list))) self.assertEqual(node_list[:3], got_nodes) @@ -1907,13 +3547,13 @@ class TestObjectController(unittest.TestCase): mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 1000000)): - got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, + got_nodes = list(self.app.iter_nodes(object_ring, 0, node_iter=iter(node_list))) self.assertEqual(node_list, got_nodes) def test_best_response_sets_headers(self): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object', headers=[{'X-Test': '1'}, @@ -1922,8 +3562,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers['X-Test'], '1') def test_best_response_sets_etag(self): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object') @@ -1956,8 +3596,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 200) @@ -1969,21 +3609,26 @@ class TestObjectController(unittest.TestCase): def test_error_limiting(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l + object_ring = controller.app.get_object_ring(None) self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), 200) - self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2) - self.assert_('last_error' in controller.app.object_ring.devs[0]) + self.assertEquals( + node_error_count(controller.app, object_ring.devs[0]), 2) + self.assert_(node_last_error(controller.app, object_ring.devs[0]) + is not None) for _junk in xrange(self.app.error_suppression_limit): self.assert_status_map(controller.HEAD, (200, 200, 503, 503, 503), 503) - self.assertEquals(controller.app.object_ring.devs[0]['errors'], - self.app.error_suppression_limit + 1) + self.assertEquals( + node_error_count(controller.app, object_ring.devs[0]), + self.app.error_suppression_limit + 1) self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 503) - self.assert_('last_error' in controller.app.object_ring.devs[0]) + self.assert_(node_last_error(controller.app, object_ring.devs[0]) + is not None) self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201, 201), 503) self.assert_status_map(controller.POST, @@ -1999,17 +3644,78 @@ class TestObjectController(unittest.TestCase): (200, 200, 200, 204, 204, 204), 503, raise_exc=True) + def test_error_limiting_survives_ring_reload(self): + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') + controller.app.sort_nodes = lambda l: l + object_ring = controller.app.get_object_ring(None) + self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), + 200) + self.assertEquals( + node_error_count(controller.app, object_ring.devs[0]), 2) + self.assert_(node_last_error(controller.app, object_ring.devs[0]) + is not None) + for _junk in xrange(self.app.error_suppression_limit): + self.assert_status_map(controller.HEAD, (200, 200, 503, 503, + 503), 503) + self.assertEquals( + node_error_count(controller.app, object_ring.devs[0]), + self.app.error_suppression_limit + 1) + + # wipe out any state in the ring + for policy in POLICIES: + policy.object_ring = FakeRing(base_port=3000) + + # and we still get an error, which proves that the + # error-limiting info survived a ring reload + self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), + 503) + + def test_PUT_error_limiting(self): + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') + controller.app.sort_nodes = lambda l: l + object_ring = controller.app.get_object_ring(None) + # acc con obj obj obj + self.assert_status_map(controller.PUT, (200, 200, 503, 200, 200), + 200) + + # 2, not 1, because assert_status_map() calls the method twice + odevs = object_ring.devs + self.assertEquals(node_error_count(controller.app, odevs[0]), 2) + self.assertEquals(node_error_count(controller.app, odevs[1]), 0) + self.assertEquals(node_error_count(controller.app, odevs[2]), 0) + self.assert_(node_last_error(controller.app, odevs[0]) is not None) + self.assert_(node_last_error(controller.app, odevs[1]) is None) + self.assert_(node_last_error(controller.app, odevs[2]) is None) + + def test_PUT_error_limiting_last_node(self): + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') + controller.app.sort_nodes = lambda l: l + object_ring = controller.app.get_object_ring(None) + # acc con obj obj obj + self.assert_status_map(controller.PUT, (200, 200, 200, 200, 503), + 200) + + # 2, not 1, because assert_status_map() calls the method twice + odevs = object_ring.devs + self.assertEquals(node_error_count(controller.app, odevs[0]), 0) + self.assertEquals(node_error_count(controller.app, odevs[1]), 0) + self.assertEquals(node_error_count(controller.app, odevs[2]), 2) + self.assert_(node_last_error(controller.app, odevs[0]) is None) + self.assert_(node_last_error(controller.app, odevs[1]) is None) + self.assert_(node_last_error(controller.app, odevs[2]) is not None) + def test_acc_or_con_missing_returns_404(self): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() - for dev in self.app.account_ring.devs.values(): - del dev['errors'] - del dev['last_error'] - for dev in self.app.container_ring.devs.values(): - del dev['errors'] - del dev['last_error'] - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + self.app._error_limiting = {} + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) @@ -2073,9 +3779,10 @@ class TestObjectController(unittest.TestCase): resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) - for dev in self.app.account_ring.devs.values(): - dev['errors'] = self.app.error_suppression_limit + 1 - dev['last_error'] = time.time() + for dev in self.app.account_ring.devs: + set_node_errors( + self.app, dev, self.app.error_suppression_limit + 1, + time.time()) set_http_connect(200) # acct [isn't actually called since everything # is error limited] @@ -2085,11 +3792,12 @@ class TestObjectController(unittest.TestCase): resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) - for dev in self.app.account_ring.devs.values(): - dev['errors'] = 0 - for dev in self.app.container_ring.devs.values(): - dev['errors'] = self.app.error_suppression_limit + 1 - dev['last_error'] = time.time() + for dev in self.app.account_ring.devs: + set_node_errors(self.app, dev, 0, last_error=None) + for dev in self.app.container_ring.devs: + set_node_errors(self.app, dev, + self.app.error_suppression_limit + 1, + time.time()) set_http_connect(200, 200) # acct cont [isn't actually called since # everything is error limited] @@ -2103,8 +3811,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): self.app.object_post_as_copy = False self.app.memcache = FakeMemcacheReturnsNone() - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', @@ -2124,8 +3832,8 @@ class TestObjectController(unittest.TestCase): def test_PUT_POST_as_copy_requires_container_exist(self): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) @@ -2142,8 +3850,8 @@ class TestObjectController(unittest.TestCase): def test_bad_metadata(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -2153,18 +3861,21 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Object-Meta-' + ('a' * - MAX_META_NAME_LENGTH): 'v'}) + req = Request.blank( + '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-' + ( + 'a' * constraints.MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Object-Meta-' + ('a' * - (MAX_META_NAME_LENGTH + 1)): 'v'}) + req = Request.blank( + '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'Content-Length': '0', + 'X-Object-Meta-' + ( + 'a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -2173,22 +3884,23 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * - MAX_META_VALUE_LENGTH}) + constraints.MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Object-Meta-Too-Long': 'a' * - (MAX_META_VALUE_LENGTH + 1)}) + req = Request.blank( + '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-Too-Long': 'a' * + (constraints.MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} - for x in xrange(MAX_META_COUNT): + for x in xrange(constraints.MAX_META_COUNT): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) @@ -2197,7 +3909,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(constraints.MAX_META_COUNT + 1): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) @@ -2207,17 +3919,17 @@ class TestObjectController(unittest.TestCase): set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} - header_value = 'a' * MAX_META_VALUE_LENGTH + header_value = 'a' * constraints.MAX_META_VALUE_LENGTH size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - \ - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < constraints.MAX_META_OVERALL_SIZE - 4 - \ + constraints.MAX_META_VALUE_LENGTH: + size += 4 + constraints.MAX_META_VALUE_LENGTH headers['X-Object-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if constraints.MAX_META_OVERALL_SIZE - size > 1: headers['X-Object-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size - 1) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) @@ -2225,7 +3937,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Object-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) @@ -2235,8 +3947,8 @@ class TestObjectController(unittest.TestCase): @contextmanager def controller_context(self, req, *args, **kwargs): _v, account, container, obj = utils.split_path(req.path, 4, 4, True) - controller = proxy_server.ObjectController(self.app, account, - container, obj) + controller = ReplicatedObjectController( + self.app, account, container, obj) self.app.update_request(req) self.app.memcache.store = {} with save_globals(): @@ -2263,6 +3975,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') + def test_basic_put_with_x_copy_from_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_basic_put_with_x_copy_from_across_container(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2274,6 +3999,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c2/o') + def test_basic_put_with_x_copy_from_across_container_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c2/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c2/o') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_copy_non_zero_content_length(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5', @@ -2284,6 +4022,17 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) + def test_copy_non_zero_content_length_with_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '5', + 'X-Copy-From': 'c/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 400) + def test_copy_with_slashes_in_x_copy_from(self): # extra source path parsing req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -2296,6 +4045,20 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + def test_copy_with_slashes_in_x_copy_from_and_account(self): + # extra source path parsing + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o/o2', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_copy_with_spaces_in_x_copy_from(self): # space in soure path req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -2308,6 +4071,20 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') + def test_copy_with_spaces_in_x_copy_from_and_account(self): + # space in soure path + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_copy_with_leading_slash_in_x_copy_from(self): # repeat tests with leading / req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -2320,6 +4097,20 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') + def test_copy_with_leading_slash_in_x_copy_from_and_account(self): + # repeat tests with leading / + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2331,6 +4122,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acc1 con1 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_copy_with_no_object_in_x_copy_from(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2346,6 +4150,22 @@ class TestObjectController(unittest.TestCase): raise self.fail('Invalid X-Copy-From did not raise ' 'client error') + def test_copy_with_no_object_in_x_copy_from_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: + try: + controller.PUT(req) + except HTTPException as resp: + self.assertEquals(resp.status_int // 100, 4) # client error + else: + raise self.fail('Invalid X-Copy-From did not raise ' + 'client error') + def test_copy_server_error_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2356,6 +4176,17 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 503) + def test_copy_server_error_reading_source_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 503, 503, 503) + # acct cont acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 503) + def test_copy_not_found_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2367,6 +4198,18 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) + def test_copy_not_found_reading_source_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + # not found + status_list = (200, 200, 200, 200, 404, 404, 404) + # acct cont acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 404) + def test_copy_with_some_missing_sources(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2377,6 +4220,17 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) + def test_copy_with_some_missing_sources_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + def test_copy_with_object_metadata(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', @@ -2392,16 +4246,33 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + def test_copy_with_object_metadata_and_account(self): + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay', + 'X-Copy-From-Account': 'a'}) + # test object metadata + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + + @_limit_max_file_size def test_copy_source_larger_than_max_file_size(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - # copy-from object is too large to fit in target object + class LargeResponseBody(object): def __len__(self): - return MAX_FILE_SIZE + 1 + return constraints.MAX_FILE_SIZE + 1 def __getitem__(self, key): return '' @@ -2415,7 +4286,10 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) self.app.memcache.store = {} - resp = controller.PUT(req) + try: + resp = controller.PUT(req) + except HTTPException as resp: + pass self.assertEquals(resp.status_int, 413) def test_basic_COPY(self): @@ -2429,6 +4303,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') + def test_basic_COPY_account(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o2', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_COPY_across_containers(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2451,6 +4338,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + def test_COPY_account_source_with_slashes_in_name(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_COPY_destination_leading_slash(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2462,6 +4362,19 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') + def test_COPY_account_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_COPY_source_with_slashes_destination_leading_slash(self): req = Request.blank('/v1/a/c/o/o2', environ={'REQUEST_METHOD': 'COPY'}, @@ -2473,14 +4386,35 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + def test_COPY_account_source_with_slashes_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.headers['x-copied-from-account'], 'a') + def test_COPY_no_object_in_destination(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c_o'}) status_list = [] # no requests needed with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEquals(resp.status_int, 412) + self.assertRaises(HTTPException, controller.COPY, req) + + def test_COPY_account_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o', + 'Destination-Account': 'a1'}) + status_list = [] # no requests needed + with self.controller_context(req, *status_list) as controller: + self.assertRaises(HTTPException, controller.COPY, req) def test_COPY_server_error_reading_source(self): req = Request.blank('/v1/a/c/o', @@ -2492,6 +4426,17 @@ class TestObjectController(unittest.TestCase): resp = controller.COPY(req) self.assertEquals(resp.status_int, 503) + def test_COPY_account_server_error_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 503, 503, 503) + # acct cont acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 503) + def test_COPY_not_found_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2502,6 +4447,17 @@ class TestObjectController(unittest.TestCase): resp = controller.COPY(req) self.assertEquals(resp.status_int, 404) + def test_COPY_account_not_found_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 404, 404, 404) + # acct cont acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 404) + def test_COPY_with_some_missing_sources(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2512,6 +4468,17 @@ class TestObjectController(unittest.TestCase): resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) + def test_COPY_account_with_some_missing_sources(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + def test_COPY_with_metadata(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2527,6 +4494,23 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + def test_COPY_account_with_metadata(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'X-Object-Meta-Ours': 'okay', + 'Destination-Account': 'a1'}) + status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), + 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + + @_limit_max_file_size def test_COPY_source_larger_than_max_file_size(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, @@ -2535,7 +4519,7 @@ class TestObjectController(unittest.TestCase): class LargeResponseBody(object): def __len__(self): - return MAX_FILE_SIZE + 1 + return constraints.MAX_FILE_SIZE + 1 def __getitem__(self, key): return '' @@ -2546,12 +4530,43 @@ class TestObjectController(unittest.TestCase): kwargs = dict(body=copy_from_obj_body) with self.controller_context(req, *status_list, **kwargs) as controller: - resp = controller.COPY(req) + try: + resp = controller.COPY(req) + except HTTPException as resp: + pass + self.assertEquals(resp.status_int, 413) + + @_limit_max_file_size + def test_COPY_account_source_larger_than_max_file_size(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + + class LargeResponseBody(object): + + def __len__(self): + return constraints.MAX_FILE_SIZE + 1 + + def __getitem__(self, key): + return '' + + copy_from_obj_body = LargeResponseBody() + status_list = (200, 200, 200, 200, 200) + # acct cont objc objc objc + kwargs = dict(body=copy_from_obj_body) + with self.controller_context(req, *status_list, + **kwargs) as controller: + try: + resp = controller.COPY(req) + except HTTPException as resp: + pass self.assertEquals(resp.status_int, 413) def test_COPY_newest(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) @@ -2567,30 +4582,86 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers['x-copied-from-last-modified'], '3') + def test_COPY_account_newest(self): + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + req.account = 'a' + controller.object_name = 'o' + set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, + #act cont acct cont objc objc objc obj obj obj + timestamps=('1', '1', '1', '1', '3', '2', '1', + '4', '4', '4')) + self.app.memcache.store = {} + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from-last-modified'], + '3') + def test_COPY_delete_at(self): with save_globals(): - given_headers = {} + backend_requests = [] - def fake_connect_put_node(nodes, part, path, headers, - logger_thread_locals): - given_headers.update(headers) + def capture_requests(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + backend_requests.append((method, path, headers)) - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o') - controller._connect_put_node = fake_connect_put_node - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') + set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, + give_connect=capture_requests) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) self.app.update_request(req) - controller.COPY(req) - self.assertEquals(given_headers.get('X-Delete-At'), '9876543210') - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) + resp = controller.COPY(req) + self.assertEqual(201, resp.status_int) # sanity + for method, path, given_headers in backend_requests: + if method != 'PUT': + continue + self.assertEquals(given_headers.get('X-Delete-At'), + '9876543210') + self.assertTrue('X-Delete-At-Host' in given_headers) + self.assertTrue('X-Delete-At-Device' in given_headers) + self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) + + def test_COPY_account_delete_at(self): + with save_globals(): + backend_requests = [] + + def capture_requests(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + backend_requests.append((method, path, headers)) + + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') + set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, + give_connect=capture_requests) + self.app.memcache.store = {} + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + + self.app.update_request(req) + resp = controller.COPY(req) + self.assertEqual(201, resp.status_int) # sanity + for method, path, given_headers in backend_requests: + if method != 'PUT': + continue + self.assertEquals(given_headers.get('X-Delete-At'), + '9876543210') + self.assertTrue('X-Delete-At-Host' in given_headers) + self.assertTrue('X-Delete-At-Device' in given_headers) + self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) def test_chunked_put(self): @@ -2615,8 +4686,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): set_http_connect(201, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Transfer-Encoding': 'chunked', @@ -2637,17 +4708,16 @@ class TestObjectController(unittest.TestCase): req.body_file = ChunkedFile(11) self.app.memcache.store = {} self.app.update_request(req) - try: - swift.proxy.controllers.obj.MAX_FILE_SIZE = 10 + + with mock.patch('swift.common.constraints.MAX_FILE_SIZE', 10): res = controller.PUT(req) self.assertEquals(res.status_int, 413) - finally: - swift.proxy.controllers.obj.MAX_FILE_SIZE = MAX_FILE_SIZE + @unpatch_policies def test_chunked_put_bad_version(self): # Check bad version (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n' @@ -2657,10 +4727,11 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_chunked_put_bad_path(self): # Check bad path (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n' @@ -2670,10 +4741,11 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_chunked_put_bad_utf8(self): # Check invalid utf-8 (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n' @@ -2684,10 +4756,11 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_chunked_put_bad_path_no_controller(self): # Check bad path, no controller (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n' @@ -2698,10 +4771,11 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_chunked_put_bad_method(self): # Check bad method (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n' @@ -2712,12 +4786,13 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 405' self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_chunked_put_unhandled_exception(self): # Check unhandled exception (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, - obj2srv) = _test_servers + obj2srv, obj3srv) = _test_servers (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets orig_update_request = prosrv.update_request def broken_update_request(*args, **kwargs): @@ -2735,12 +4810,13 @@ class TestObjectController(unittest.TestCase): self.assertEquals(headers[:len(exp)], exp) prosrv.update_request = orig_update_request + @unpatch_policies def test_chunked_put_head_account(self): # Head account, just a double check and really is here to test # the part Application.log_request that 'enforces' a # content_length on the response. (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' @@ -2752,6 +4828,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(headers[:len(exp)], exp) self.assert_('\r\nContent-Length: 0\r\n' in headers) + @unpatch_policies def test_chunked_put_utf8_all_the_way_down(self): # Test UTF-8 Unicode all the way through the system ustr = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xba \xe1\xbc\xb0\xce' \ @@ -2763,7 +4840,7 @@ class TestObjectController(unittest.TestCase): ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest' # Create ustr container (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' @@ -2795,7 +4872,7 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) - listing = simplejson.loads(fd.read()) + listing = json.loads(fd.read()) self.assert_(ustr.decode('utf8') in [l['name'] for l in listing]) # List account with ustr container (test xml) sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -2843,7 +4920,7 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) - listing = simplejson.loads(fd.read()) + listing = json.loads(fd.read()) self.assertEquals(listing[0]['name'], ustr.decode('utf8')) # List ustr container with ustr object (test xml) sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -2871,10 +4948,11 @@ class TestObjectController(unittest.TestCase): self.assert_('\r\nX-Object-Meta-%s: %s\r\n' % (quote(ustr_short).lower(), quote(ustr)) in headers) + @unpatch_policies def test_chunked_put_chunked_put(self): # Do chunked object put (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() # Also happens to assert that x-storage-token is taken as a @@ -2900,11 +4978,12 @@ class TestObjectController(unittest.TestCase): body = fd.read() self.assertEquals(body, 'oh hai123456789abcdef') + @unpatch_policies def test_version_manifest(self, oc='versions', vc='vers', o='name'): versions_to_create = 3 # Create a container for our versioned object testing (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, - obj2lis) = _test_sockets + obj2lis, obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() pre = quote('%03x' % len(o)) @@ -3108,18 +5187,22 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) - # make sure manifest files don't get versioned - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 0\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\nX-Object-Manifest: %s/foo_\r\n\r\n' - % (oc, vc, o)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEquals(headers[:len(exp)], exp) + # make sure dlo manifest files don't get versioned + for _junk in xrange(1, versions_to_create): + sleep(.01) # guarantee that the timestamp changes + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 0\r\n' + 'Content-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\nX-Object-Manifest: %s/%s/\r\n\r\n' + % (oc, o, oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + # Ensure we have no saved versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -3233,51 +5316,59 @@ class TestObjectController(unittest.TestCase): exp = 'HTTP/1.1 2' # 2xx response self.assertEquals(headers[:len(exp)], exp) + @unpatch_policies def test_version_manifest_utf8(self): oc = '0_oc_non_ascii\xc2\xa3' vc = '0_vc_non_ascii\xc2\xa3' o = '0_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_container(self): oc = '1_oc_non_ascii\xc2\xa3' vc = '1_vc_ascii' o = '1_o_ascii' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_version_container(self): oc = '2_oc_ascii' vc = '2_vc_non_ascii\xc2\xa3' o = '2_o_ascii' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_containers(self): oc = '3_oc_non_ascii\xc2\xa3' vc = '3_vc_non_ascii\xc2\xa3' o = '3_o_ascii' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_object(self): oc = '4_oc_ascii' vc = '4_vc_ascii' o = '4_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_version_container_utf_object(self): oc = '5_oc_ascii' vc = '5_vc_non_ascii\xc2\xa3' o = '5_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_version_manifest_utf8_container_utf_object(self): oc = '6_oc_non_ascii\xc2\xa3' vc = '6_vc_ascii' o = '6_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) + @unpatch_policies def test_conditional_range_get(self): - (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ - _test_sockets + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, + obj3lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) # make a container @@ -3325,8 +5416,8 @@ class TestObjectController(unittest.TestCase): def test_mismatched_etags(self): with save_globals(): # no etag supplied, object servers return success w/ diff values - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) self.app.update_request(req) @@ -3357,8 +5448,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.GET(req) self.assert_('accept-ranges' in resp.headers) @@ -3369,8 +5460,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assert_('accept-ranges' in resp.headers) @@ -3384,8 +5475,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o') req.environ['swift.authorize'] = authorize self.app.update_request(req) @@ -3400,8 +5491,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) @@ -3417,8 +5508,8 @@ class TestObjectController(unittest.TestCase): with save_globals(): self.app.object_post_as_copy = False set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') @@ -3435,8 +5526,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') @@ -3453,8 +5544,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') req.environ['swift.authorize'] = authorize @@ -3470,8 +5561,8 @@ class TestObjectController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c/o'}) @@ -3482,9 +5573,10 @@ class TestObjectController(unittest.TestCase): def test_POST_converts_delete_after_to_delete_at(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) + self.app.object_post_as_copy = False + controller = ReplicatedObjectController( + self.app, 'account', 'container', 'object') + set_http_connect(200, 200, 202, 202, 202) self.app.memcache.store = {} orig_time = time.time try: @@ -3498,194 +5590,133 @@ class TestObjectController(unittest.TestCase): self.assertEquals(res.status, '202 Fake') self.assertEquals(req.headers.get('x-delete-at'), str(int(t + 60))) - - self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', - 'object') - set_http_connect(200, 200, 202, 202, 202) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '60'}) - self.app.update_request(req) - res = controller.POST(req) - self.assertEquals(res.status, '202 Fake') - self.assertEquals(req.headers.get('x-delete-at'), - str(int(t + 60))) finally: time.time = orig_time - def test_POST_non_int_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '60.1'}) - self.app.update_request(req) - res = controller.POST(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('Non-integer X-Delete-After' in res.body) - - def test_POST_negative_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) - self.app.update_request(req) - res = controller.POST(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('X-Delete-At in past' in res.body) - - def test_POST_delete_at(self): - with save_globals(): - given_headers = {} - - def fake_make_requests(req, ring, part, method, path, headers, - query_string=''): - given_headers.update(headers[0]) - - self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - controller.make_requests = fake_make_requests - set_http_connect(200, 200) - self.app.memcache.store = {} - t = str(int(time.time() + 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - controller.POST(req) - self.assertEquals(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - t = str(int(time.time() + 100)) + '.1' - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.POST(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('Non-integer X-Delete-At' in resp.body) + @patch_policies([ + StoragePolicy(0, 'zero', False, object_ring=FakeRing()), + StoragePolicy(1, 'one', True, object_ring=FakeRing()) + ]) + def test_PUT_versioning_with_nonzero_default_policy(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() - t = str(int(time.time() - 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) + def test_connect(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + if method == "HEAD": + self.assertEquals(path, '/a/c/o.jpg') + self.assertNotEquals(None, + headers['X-Backend-Storage-Policy-Index']) + self.assertEquals(1, int(headers + ['X-Backend-Storage-Policy-Index'])) + + def fake_container_info(account, container, req): + return {'status': 200, 'sync_key': None, 'storage_policy': '1', + 'meta': {}, 'cors': {'allow_origin': None, + 'expose_headers': None, + 'max_age': None}, + 'sysmeta': {}, 'read_acl': None, 'object_count': None, + 'write_acl': None, 'versions': 'c-versions', + 'partition': 1, 'bytes': None, + 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, + 'id': 0, 'device': 'sda', 'port': 1000}, + {'zone': 1, 'ip': '10.0.0.1', 'region': 1, + 'id': 1, 'device': 'sdb', 'port': 1001}, + {'zone': 2, 'ip': '10.0.0.2', 'region': 0, + 'id': 2, 'device': 'sdc', 'port': 1002}]} + with save_globals(): + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') + + controller.container_info = fake_container_info + set_http_connect(200, 200, 200, # head: for the last version + 200, 200, 200, # get: for the last version + 201, 201, 201, # put: move the current version + 201, 201, 201, # put: save the new version + give_connect=test_connect) + req = Request.blank('/v1/a/c/o.jpg', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0'}) self.app.update_request(req) - resp = controller.POST(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('X-Delete-At in past' in resp.body) - - def test_PUT_converts_delete_after_to_delete_at(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) - self.app.memcache.store = {} - orig_time = time.time - try: - t = time.time() - time.time = lambda: t - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '60'}) - self.app.update_request(req) - res = controller.PUT(req) - self.assertEquals(res.status, '201 Fake') - self.assertEquals(req.headers.get('x-delete-at'), - str(int(t + 60))) - finally: - time.time = orig_time - - def test_PUT_non_int_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '60.1'}) - self.app.update_request(req) res = controller.PUT(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('Non-integer X-Delete-After' in res.body) - - def test_PUT_negative_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) + self.assertEquals(201, res.status_int) + + @patch_policies([ + StoragePolicy(0, 'zero', False, object_ring=FakeRing()), + StoragePolicy(1, 'one', True, object_ring=FakeRing()) + ]) + def test_cross_policy_DELETE_versioning(self): + # reset the router post patch_policies + self.app.obj_controller_router = proxy_server.ObjectControllerRouter() + requests = [] + + def capture_requests(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + requests.append((method, path, headers)) + + def fake_container_info(app, env, account, container, **kwargs): + info = {'status': 200, 'sync_key': None, 'storage_policy': None, + 'meta': {}, 'cors': {'allow_origin': None, + 'expose_headers': None, + 'max_age': None}, + 'sysmeta': {}, 'read_acl': None, 'object_count': None, + 'write_acl': None, 'versions': None, + 'partition': 1, 'bytes': None, + 'nodes': [{'zone': 0, 'ip': '10.0.0.0', 'region': 0, + 'id': 0, 'device': 'sda', 'port': 1000}, + {'zone': 1, 'ip': '10.0.0.1', 'region': 1, + 'id': 1, 'device': 'sdb', 'port': 1001}, + {'zone': 2, 'ip': '10.0.0.2', 'region': 0, + 'id': 2, 'device': 'sdc', 'port': 1002}]} + if container == 'c': + info['storage_policy'] = '1' + info['versions'] = 'c-versions' + elif container == 'c-versions': + info['storage_policy'] = '0' + else: + self.fail('Unexpected call to get_info for %r' % container) + return info + container_listing = json.dumps([{'name': 'old_version'}]) + with save_globals(): + resp_status = ( + 200, 200, # listings for versions container + 200, 200, 200, # get: for the last version + 201, 201, 201, # put: move the last version + 200, 200, 200, # delete: for the last version + ) + body_iter = iter([container_listing] + [ + '' for x in range(len(resp_status) - 1)]) + set_http_connect(*resp_status, body_iter=body_iter, + give_connect=capture_requests) + req = Request.blank('/v1/a/c/current_version', method='DELETE') self.app.update_request(req) - res = controller.PUT(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('X-Delete-At in past' in res.body) - - def test_PUT_delete_at(self): - with save_globals(): - given_headers = {} - - def fake_connect_put_node(nodes, part, path, headers, - logger_thread_locals): - given_headers.update(headers) - - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - controller._connect_put_node = fake_connect_put_node - set_http_connect(200, 200) self.app.memcache.store = {} - t = str(int(time.time() + 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - controller.PUT(req) - self.assertEquals(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - t = str(int(time.time() + 100)) + '.1' - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('Non-integer X-Delete-At' in resp.body) - - t = str(int(time.time() - 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('X-Delete-At in past' in resp.body) - + with mock.patch('swift.proxy.controllers.base.get_info', + fake_container_info): + resp = self.app.handle_request(req) + self.assertEquals(200, resp.status_int) + expected = [('GET', '/a/c-versions')] * 2 + \ + [('GET', '/a/c-versions/old_version')] * 3 + \ + [('PUT', '/a/c/current_version')] * 3 + \ + [('DELETE', '/a/c-versions/old_version')] * 3 + self.assertEqual(expected, [(m, p) for m, p, h in requests]) + for method, path, headers in requests: + if 'current_version' in path: + expected_storage_policy = 1 + elif 'old_version' in path: + expected_storage_policy = 0 + else: + continue + storage_policy_index = \ + int(headers['X-Backend-Storage-Policy-Index']) + self.assertEqual( + expected_storage_policy, storage_policy_index, + 'Unexpected %s request for %s ' + 'with storage policy index %s' % ( + method, path, storage_policy_index)) + + @unpatch_policies def test_leak_1(self): _request_instances = weakref.WeakKeyDictionary() _orig_init = Request.__init__ @@ -3747,8 +5778,8 @@ class TestObjectController(unittest.TestCase): def test_OPTIONS(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', - 'c', 'o.jpg') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o.jpg') def my_empty_container_info(*args): return {} @@ -3855,7 +5886,8 @@ class TestObjectController(unittest.TestCase): def test_CORS_valid(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { @@ -3908,7 +5940,8 @@ class TestObjectController(unittest.TestCase): def test_CORS_valid_with_obj_headers(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { @@ -3969,20 +6002,21 @@ class TestObjectController(unittest.TestCase): def test_PUT_x_container_headers_with_equal_replicas(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdc'}]) def test_PUT_x_container_headers_with_fewer_container_replicas(self): @@ -3990,7 +6024,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT @@ -3998,10 +6033,10 @@ class TestObjectController(unittest.TestCase): self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdb'}, {'X-Container-Host': None, 'X-Container-Partition': None, @@ -4012,7 +6047,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT @@ -4020,13 +6056,13 @@ class TestObjectController(unittest.TestCase): self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdc'}]) def test_POST_x_container_headers_with_more_container_replicas(self): @@ -4036,7 +6072,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'application/stuff'}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.POST, req, 200, 200, 200, 200, 200) # HEAD HEAD POST POST POST @@ -4044,13 +6081,13 @@ class TestObjectController(unittest.TestCase): self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdc'}]) def test_DELETE_x_container_headers_with_more_container_replicas(self): @@ -4059,20 +6096,21 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'Content-Type': 'application/stuff'}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.DELETE, req, 200, 200, 200, 200, 200) # HEAD HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': '1', + 'X-Container-Partition': '0', 'X-Container-Device': 'sdc'} ]) @@ -4081,15 +6119,15 @@ class TestObjectController(unittest.TestCase): self.app.container_ring.set_replicas(2) delete_at_timestamp = int(time.time()) + 100000 - delete_at_container = str( - delete_at_timestamp / - self.app.expiring_objects_container_divisor * - self.app.expiring_objects_container_divisor) + delete_at_container = utils.get_expirer_container( + delete_at_timestamp, self.app.expiring_objects_container_divisor, + 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': '0', 'X-Delete-At': str(delete_at_timestamp)}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT @@ -4099,11 +6137,11 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000', 'X-Delete-At-Container': delete_at_container, - 'X-Delete-At-Partition': '1', + 'X-Delete-At-Partition': '0', 'X-Delete-At-Device': 'sda'}, {'X-Delete-At-Host': '10.0.0.1:1001', 'X-Delete-At-Container': delete_at_container, - 'X-Delete-At-Partition': '1', + 'X-Delete-At-Partition': '0', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': None, 'X-Delete-At-Container': None, @@ -4118,15 +6156,15 @@ class TestObjectController(unittest.TestCase): self.app.expiring_objects_container_divisor = 60 delete_at_timestamp = int(time.time()) + 100000 - delete_at_container = str( - delete_at_timestamp / - self.app.expiring_objects_container_divisor * - self.app.expiring_objects_container_divisor) + delete_at_container = utils.get_expirer_container( + delete_at_timestamp, self.app.expiring_objects_container_divisor, + 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': 0, 'X-Delete-At': str(delete_at_timestamp)}) - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + controller = ReplicatedObjectController( + self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT @@ -4135,38 +6173,483 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Delete-At-Container': delete_at_container, - 'X-Delete-At-Partition': '1', + 'X-Delete-At-Partition': '0', 'X-Delete-At-Device': 'sda,sdd'}, {'X-Delete-At-Host': '10.0.0.1:1001', 'X-Delete-At-Container': delete_at_container, - 'X-Delete-At-Partition': '1', + 'X-Delete-At-Partition': '0', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': '10.0.0.2:1002', 'X-Delete-At-Container': delete_at_container, - 'X-Delete-At-Partition': '1', + 'X-Delete-At-Partition': '0', 'X-Delete-At-Device': 'sdc'} ]) +class TestECMismatchedFA(unittest.TestCase): + def tearDown(self): + prosrv = _test_servers[0] + # don't leak error limits and poison other tests + prosrv._error_limiting = {} + + def test_mixing_different_objects_fragment_archives(self): + (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, + obj2srv, obj3srv) = _test_servers + ec_policy = POLICIES[3] + + @public + def bad_disk(req): + return Response(status=507, body="borken") + + ensure_container = Request.blank( + "/v1/a/ec-crazytown", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Storage-Policy": "ec", "X-Auth-Token": "t"}) + resp = ensure_container.get_response(prosrv) + self.assertTrue(resp.status_int in (201, 202)) + + obj1 = "first version..." + put_req1 = Request.blank( + "/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Auth-Token": "t"}) + put_req1.body = obj1 + + obj2 = u"versión segundo".encode("utf-8") + put_req2 = Request.blank( + "/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "PUT"}, + headers={"X-Auth-Token": "t"}) + put_req2.body = obj2 + + # pyeclib has checks for unequal-length; we don't want to trip those + self.assertEqual(len(obj1), len(obj2)) + + # Servers obj1 and obj2 will have the first version of the object + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj3srv, 'PUT', bad_disk), + mock.patch( + 'swift.common.storage_policy.ECStoragePolicy.quorum')): + type(ec_policy).quorum = mock.PropertyMock(return_value=2) + resp = put_req1.get_response(prosrv) + self.assertEqual(resp.status_int, 201) + + # Server obj3 (and, in real life, some handoffs) will have the + # second version of the object. + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj1srv, 'PUT', bad_disk), + mock.patch.object(obj2srv, 'PUT', bad_disk), + mock.patch( + 'swift.common.storage_policy.ECStoragePolicy.quorum'), + mock.patch( + 'swift.proxy.controllers.base.Controller._quorum_size', + lambda *a, **kw: 1)): + type(ec_policy).quorum = mock.PropertyMock(return_value=1) + resp = put_req2.get_response(prosrv) + self.assertEqual(resp.status_int, 201) + + # A GET that only sees 1 fragment archive should fail + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with nested( + mock.patch.object(obj1srv, 'GET', bad_disk), + mock.patch.object(obj2srv, 'GET', bad_disk)): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 503) + + # A GET that sees 2 matching FAs will work + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with mock.patch.object(obj3srv, 'GET', bad_disk): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.body, obj1) + + # A GET that sees 2 mismatching FAs will fail + get_req = Request.blank("/v1/a/ec-crazytown/obj", + environ={"REQUEST_METHOD": "GET"}, + headers={"X-Auth-Token": "t"}) + prosrv._error_limiting = {} + with mock.patch.object(obj2srv, 'GET', bad_disk): + resp = get_req.get_response(prosrv) + self.assertEqual(resp.status_int, 503) + + +class TestObjectECRangedGET(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application( + None, FakeMemcache(), + logger=debug_logger('proxy-ut'), + account_ring=FakeRing(), + container_ring=FakeRing()) + + @classmethod + def setUpClass(cls): + cls.obj_name = 'range-get-test' + cls.tiny_obj_name = 'range-get-test-tiny' + cls.aligned_obj_name = 'range-get-test-aligned' + + # Note: only works if called with unpatched policies + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: 0\r\n' + 'X-Storage-Token: t\r\n' + 'X-Storage-Policy: ec\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 2' + assert headers[:len(exp)] == exp, "container PUT failed" + + seg_size = POLICIES.get_by_name("ec").ec_segment_size + cls.seg_size = seg_size + # EC segment size is 4 KiB, hence this gives 4 segments, which we + # then verify with a quick sanity check + cls.obj = ' my hovercraft is full of eels '.join( + str(s) for s in range(431)) + assert seg_size * 4 > len(cls.obj) > seg_size * 3, \ + "object is wrong number of segments" + + cls.tiny_obj = 'tiny, tiny object' + assert len(cls.tiny_obj) < seg_size, "tiny_obj too large" + + cls.aligned_obj = "".join( + "abcdEFGHijkl%04d" % x for x in range(512)) + assert len(cls.aligned_obj) % seg_size == 0, "aligned obj not aligned" + + for obj_name, obj in ((cls.obj_name, cls.obj), + (cls.tiny_obj_name, cls.tiny_obj), + (cls.aligned_obj_name, cls.aligned_obj)): + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/ec-con/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'Content-Length: %d\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (obj_name, len(obj), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + assert headers[:len(exp)] == exp, \ + "object PUT failed %s" % obj_name + + def _get_obj(self, range_value, obj_name=None): + if obj_name is None: + obj_name = self.obj_name + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/ec-con/%s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Range: %s\r\n' + '\r\n' % (obj_name, range_value)) + fd.flush() + headers = readuntil2crlfs(fd) + # e.g. "HTTP/1.1 206 Partial Content\r\n..." + status_code = int(headers[9:12]) + headers = parse_headers_string(headers) + + gotten_obj = '' + while True: + buf = fd.read(64) + if not buf: + break + gotten_obj += buf + + return (status_code, headers, gotten_obj) + + def test_unaligned(self): + # One segment's worth of data, but straddling two segment boundaries + # (so it has data from three segments) + status, headers, gotten_obj = self._get_obj("bytes=3783-7878") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 3783-7878/14513") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.obj[3783:7879]) + + def test_aligned_left(self): + # First byte is aligned to a segment boundary, last byte is not + status, headers, gotten_obj = self._get_obj("bytes=0-5500") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "5501") + self.assertEqual(headers['Content-Range'], "bytes 0-5500/14513") + self.assertEqual(len(gotten_obj), 5501) + self.assertEqual(gotten_obj, self.obj[:5501]) + + def test_aligned_range(self): + # Ranged GET that wants exactly one segment + status, headers, gotten_obj = self._get_obj("bytes=4096-8191") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 4096-8191/14513") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.obj[4096:8192]) + + def test_aligned_range_end(self): + # Ranged GET that wants exactly the last segment + status, headers, gotten_obj = self._get_obj("bytes=12288-14512") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "2225") + self.assertEqual(headers['Content-Range'], "bytes 12288-14512/14513") + self.assertEqual(len(gotten_obj), 2225) + self.assertEqual(gotten_obj, self.obj[12288:]) + + def test_aligned_range_aligned_obj(self): + # Ranged GET that wants exactly the last segment, which is full-size + status, headers, gotten_obj = self._get_obj("bytes=4096-8191", + self.aligned_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "4096") + self.assertEqual(headers['Content-Range'], "bytes 4096-8191/8192") + self.assertEqual(len(gotten_obj), 4096) + self.assertEqual(gotten_obj, self.aligned_obj[4096:8192]) + + def test_byte_0(self): + # Just the first byte, but it's index 0, so that's easy to get wrong + status, headers, gotten_obj = self._get_obj("bytes=0-0") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], "1") + self.assertEqual(headers['Content-Range'], "bytes 0-0/14513") + self.assertEqual(gotten_obj, self.obj[0]) + + def test_unsatisfiable(self): + # Goes just one byte too far off the end of the object, so it's + # unsatisfiable + status, _junk, _junk = self._get_obj( + "bytes=%d-%d" % (len(self.obj), len(self.obj) + 100)) + self.assertEqual(status, 416) + + def test_off_end(self): + # Ranged GET that's mostly off the end of the object, but overlaps + # it in just the last byte + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (len(self.obj) - 1, len(self.obj) + 100)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '1') + self.assertEqual(headers['Content-Range'], 'bytes 14512-14512/14513') + self.assertEqual(gotten_obj, self.obj[-1]) + + def test_aligned_off_end(self): + # Ranged GET that starts on a segment boundary but asks for a whole lot + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (8192, len(self.obj) + 100)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '6321') + self.assertEqual(headers['Content-Range'], 'bytes 8192-14512/14513') + self.assertEqual(gotten_obj, self.obj[8192:]) + + def test_way_off_end(self): + # Ranged GET that's mostly off the end of the object, but overlaps + # it in just the last byte, and wants multiple segments' worth off + # the end + status, headers, gotten_obj = self._get_obj( + "bytes=%d-%d" % (len(self.obj) - 1, len(self.obj) * 1000)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '1') + self.assertEqual(headers['Content-Range'], 'bytes 14512-14512/14513') + self.assertEqual(gotten_obj, self.obj[-1]) + + def test_boundaries(self): + # Wants the last byte of segment 1 + the first byte of segment 2 + status, headers, gotten_obj = self._get_obj("bytes=4095-4096") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '2') + self.assertEqual(headers['Content-Range'], 'bytes 4095-4096/14513') + self.assertEqual(gotten_obj, self.obj[4095:4097]) + + def test_until_end(self): + # Wants the last byte of segment 1 + the rest + status, headers, gotten_obj = self._get_obj("bytes=4095-") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '10418') + self.assertEqual(headers['Content-Range'], 'bytes 4095-14512/14513') + self.assertEqual(gotten_obj, self.obj[4095:]) + + def test_small_suffix(self): + # Small range-suffix GET: the last 100 bytes (less than one segment) + status, headers, gotten_obj = self._get_obj("bytes=-100") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '100') + self.assertEqual(headers['Content-Range'], 'bytes 14413-14512/14513') + self.assertEqual(len(gotten_obj), 100) + self.assertEqual(gotten_obj, self.obj[-100:]) + + def test_small_suffix_aligned(self): + # Small range-suffix GET: the last 100 bytes, last segment is + # full-size + status, headers, gotten_obj = self._get_obj("bytes=-100", + self.aligned_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '100') + self.assertEqual(headers['Content-Range'], 'bytes 8092-8191/8192') + self.assertEqual(len(gotten_obj), 100) + + def test_suffix_two_segs(self): + # Ask for enough data that we need the last two segments. The last + # segment is short, though, so this ensures we compensate for that. + # + # Note that the total range size is less than one full-size segment. + suffix_len = len(self.obj) % self.seg_size + 1 + + status, headers, gotten_obj = self._get_obj("bytes=-%d" % suffix_len) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], str(suffix_len)) + self.assertEqual(headers['Content-Range'], + 'bytes %d-%d/%d' % (len(self.obj) - suffix_len, + len(self.obj) - 1, + len(self.obj))) + self.assertEqual(len(gotten_obj), suffix_len) + + def test_large_suffix(self): + # Large range-suffix GET: the last 5000 bytes (more than one segment) + status, headers, gotten_obj = self._get_obj("bytes=-5000") + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '5000') + self.assertEqual(headers['Content-Range'], 'bytes 9513-14512/14513') + self.assertEqual(len(gotten_obj), 5000) + self.assertEqual(gotten_obj, self.obj[-5000:]) + + def test_overlarge_suffix(self): + # The last N+1 bytes of an N-byte object + status, headers, gotten_obj = self._get_obj( + "bytes=-%d" % (len(self.obj) + 1)) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '14513') + self.assertEqual(headers['Content-Range'], 'bytes 0-14512/14513') + self.assertEqual(len(gotten_obj), len(self.obj)) + self.assertEqual(gotten_obj, self.obj) + + def test_small_suffix_tiny_object(self): + status, headers, gotten_obj = self._get_obj( + "bytes=-5", self.tiny_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '5') + self.assertEqual(headers['Content-Range'], 'bytes 12-16/17') + self.assertEqual(gotten_obj, self.tiny_obj[12:]) + + def test_overlarge_suffix_tiny_object(self): + status, headers, gotten_obj = self._get_obj( + "bytes=-1234567890", self.tiny_obj_name) + self.assertEqual(status, 206) + self.assertEqual(headers['Content-Length'], '17') + self.assertEqual(headers['Content-Range'], 'bytes 0-16/17') + self.assertEqual(len(gotten_obj), len(self.tiny_obj)) + self.assertEqual(gotten_obj, self.tiny_obj) + + +@patch_policies([ + StoragePolicy(0, 'zero', True, object_ring=FakeRing(base_port=3000)), + StoragePolicy(1, 'one', False, object_ring=FakeRing(base_port=3000)), + StoragePolicy(2, 'two', False, True, object_ring=FakeRing(base_port=3000)) +]) class TestContainerController(unittest.TestCase): "Test swift.proxy_server.ContainerController" def setUp(self): - self.app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing(), - logger=FakeLogger()) + self.app = proxy_server.Application( + None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(base_port=2000), + logger=debug_logger()) + + def test_convert_policy_to_index(self): + controller = swift.proxy.controllers.ContainerController(self.app, + 'a', 'c') + expected = { + 'zero': 0, + 'ZeRo': 0, + 'one': 1, + 'OnE': 1, + } + for name, index in expected.items(): + req = Request.blank('/a/c', headers={'Content-Length': '0', + 'Content-Type': 'text/plain', + 'X-Storage-Policy': name}) + self.assertEqual(controller._convert_policy_to_index(req), index) + # default test + req = Request.blank('/a/c', headers={'Content-Length': '0', + 'Content-Type': 'text/plain'}) + self.assertEqual(controller._convert_policy_to_index(req), None) + # negative test + req = Request.blank('/a/c', headers={'Content-Length': '0', + 'Content-Type': 'text/plain', + 'X-Storage-Policy': 'nada'}) + self.assertRaises(HTTPException, controller._convert_policy_to_index, + req) + # storage policy two is deprecated + req = Request.blank('/a/c', headers={'Content-Length': '0', + 'Content-Type': 'text/plain', + 'X-Storage-Policy': 'two'}) + self.assertRaises(HTTPException, controller._convert_policy_to_index, + req) + + def test_convert_index_to_name(self): + policy = random.choice(list(POLICIES)) + req = Request.blank('/v1/a/c') + with mocked_http_conn( + 200, 200, + headers={'X-Backend-Storage-Policy-Index': int(policy)}, + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Storage-Policy'], policy.name) + + def test_no_convert_index_to_name_when_container_not_found(self): + policy = random.choice(list(POLICIES)) + req = Request.blank('/v1/a/c') + with mocked_http_conn( + 200, 404, 404, 404, + headers={'X-Backend-Storage-Policy-Index': + int(policy)}) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 404) + self.assertEqual(resp.headers['X-Storage-Policy'], None) + + def test_error_convert_index_to_name(self): + req = Request.blank('/v1/a/c') + with mocked_http_conn( + 200, 200, + headers={'X-Backend-Storage-Policy-Index': '-1'}) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Storage-Policy'], None) + error_lines = self.app.logger.get_lines_for_level('error') + self.assertEqual(2, len(error_lines)) + for msg in error_lines: + expected = "Could not translate " \ + "X-Backend-Storage-Policy-Index ('-1')" + self.assertTrue(expected in msg) def test_transfer_headers(self): src_headers = {'x-remove-versions-location': 'x', - 'x-container-read': '*:user'} + 'x-container-read': '*:user', + 'x-remove-container-sync-key': 'x'} dst_headers = {'x-versions-location': 'backup'} controller = swift.proxy.controllers.ContainerController(self.app, 'a', 'c') controller.transfer_headers(src_headers, dst_headers) expected_headers = {'x-versions-location': '', - 'x-container-read': '*:user'} + 'x-container-read': '*:user', + 'x-container-sync-key': ''} self.assertEqual(dst_headers, expected_headers) def assert_status_map(self, method, statuses, expected, @@ -4249,22 +6732,78 @@ class TestContainerController(unittest.TestCase): # return 200 and cache 200 for and container test_status_map((200, 200, 404, 404), 200, 200, 200) test_status_map((200, 200, 500, 404), 200, 200, 200) - # return 304 dont cache container + # return 304 don't cache container test_status_map((200, 304, 500, 404), 304, None, 200) # return 404 and cache 404 for container test_status_map((200, 404, 404, 404), 404, 404, 200) test_status_map((200, 404, 404, 500), 404, 404, 200) - # return 503, dont cache container + # return 503, don't cache container test_status_map((200, 500, 500, 500), 503, None, 200) self.assertFalse(self.app.account_autocreate) # In all the following tests cache 404 for account - # return 404 (as account is not found) and dont cache container + # return 404 (as account is not found) and don't cache container test_status_map((404, 404, 404), 404, None, 404) # This should make no difference self.app.account_autocreate = True test_status_map((404, 404, 404), 404, None, 404) + def test_PUT_policy_headers(self): + backend_requests = [] + + def capture_requests(ipaddr, port, device, partition, method, + path, headers=None, query_string=None): + if method == 'PUT': + backend_requests.append(headers) + + def test_policy(requested_policy): + with save_globals(): + mock_conn = set_http_connect(200, 201, 201, 201, + give_connect=capture_requests) + self.app.memcache.store = {} + req = Request.blank('/v1/a/test', method='PUT', + headers={'Content-Length': 0}) + if requested_policy: + expected_policy = requested_policy + req.headers['X-Storage-Policy'] = policy.name + else: + expected_policy = POLICIES.default + res = req.get_response(self.app) + if expected_policy.is_deprecated: + self.assertEquals(res.status_int, 400) + self.assertEqual(0, len(backend_requests)) + expected = 'is deprecated' + self.assertTrue(expected in res.body, + '%r did not include %r' % ( + res.body, expected)) + return + self.assertEquals(res.status_int, 201) + self.assertEqual( + expected_policy.object_ring.replicas, + len(backend_requests)) + for headers in backend_requests: + if not requested_policy: + self.assertFalse('X-Backend-Storage-Policy-Index' in + headers) + self.assertTrue( + 'X-Backend-Storage-Policy-Default' in headers) + self.assertEqual( + int(expected_policy), + int(headers['X-Backend-Storage-Policy-Default'])) + else: + self.assertTrue('X-Backend-Storage-Policy-Index' in + headers) + self.assertEqual(int(headers + ['X-Backend-Storage-Policy-Index']), + int(policy)) + # make sure all mocked responses are consumed + self.assertRaises(StopIteration, mock_conn.code_iter.next) + + test_policy(None) # no policy header + for policy in POLICIES: + backend_requests = [] # reset backend requests + test_policy(policy) + def test_PUT(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', @@ -4323,6 +6862,47 @@ class TestContainerController(unittest.TestCase): 503, 201, 201), # put container success 201, missing_container=True) + def test_PUT_autocreate_account_with_sysmeta(self): + # x-account-sysmeta headers in a container PUT request should be + # transferred to the account autocreate PUT request + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + + def test_status_map(statuses, expected, headers=None, **kwargs): + set_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/v1/a/c', {}, headers=headers) + req.content_length = 0 + self.app.update_request(req) + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + + self.app.account_autocreate = True + calls = [] + callback = _make_callback_func(calls) + key, value = 'X-Account-Sysmeta-Blah', 'something' + headers = {key: value} + + # all goes according to plan + test_status_map( + (404, 404, 404, # account_info fails on 404 + 201, 201, 201, # PUT account + 200, # account_info success + 201, 201, 201), # put container success + 201, missing_container=True, + headers=headers, + give_connect=callback) + + self.assertEqual(10, len(calls)) + for call in calls[3:6]: + self.assertEqual('/account', call['path']) + self.assertTrue(key in call['headers'], + '%s call, key %s missing in headers %s' % + (call['method'], key, call['headers'])) + self.assertEqual(value, call['headers'][key]) + def test_POST(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', @@ -4359,7 +6939,14 @@ class TestContainerController(unittest.TestCase): self.app.max_containers_per_account = 12345 controller = proxy_server.ContainerController(self.app, 'account', 'container') - self.assert_status_map(controller.PUT, (201, 201, 201), 403, + self.assert_status_map(controller.PUT, + (200, 200, 201, 201, 201), 201, + missing_container=True) + + controller = proxy_server.ContainerController(self.app, 'account', + 'container_new') + + self.assert_status_map(controller.PUT, (200, 404, 404, 404), 403, missing_container=True) self.app.max_containers_per_account = 12345 @@ -4372,7 +6959,7 @@ class TestContainerController(unittest.TestCase): def test_PUT_max_container_name_length(self): with save_globals(): - limit = MAX_CONTAINER_NAME_LENGTH + limit = constraints.MAX_CONTAINER_NAME_LENGTH controller = proxy_server.ContainerController(self.app, 'account', '1' * limit) self.assert_status_map(controller.PUT, @@ -4398,9 +6985,7 @@ class TestContainerController(unittest.TestCase): for meth in ('DELETE', 'PUT'): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() - for dev in self.app.account_ring.devs.values(): - del dev['errors'] - del dev['last_error'] + self.app._error_limiting = {} controller = proxy_server.ContainerController(self.app, 'account', 'container') @@ -4437,9 +7022,10 @@ class TestContainerController(unittest.TestCase): resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) - for dev in self.app.account_ring.devs.values(): - dev['errors'] = self.app.error_suppression_limit + 1 - dev['last_error'] = time.time() + for dev in self.app.account_ring.devs: + set_node_errors(self.app, dev, + self.app.error_suppression_limit + 1, + time.time()) set_http_connect(200, 200, 200, 200, 200, 200) # Make sure it is a blank request wthout env caching req = Request.blank('/v1/a/c', @@ -4477,19 +7063,26 @@ class TestContainerController(unittest.TestCase): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') + container_ring = controller.app.container_ring controller.app.sort_nodes = lambda l: l self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, missing_container=False) + self.assertEquals( - controller.app.container_ring.devs[0]['errors'], 2) - self.assert_('last_error' in controller.app.container_ring.devs[0]) + node_error_count(controller.app, container_ring.devs[0]), 2) + self.assert_( + node_last_error(controller.app, container_ring.devs[0]) + is not None) for _junk in xrange(self.app.error_suppression_limit): self.assert_status_map(controller.HEAD, (200, 503, 503, 503), 503) - self.assertEquals(controller.app.container_ring.devs[0]['errors'], - self.app.error_suppression_limit + 1) + self.assertEquals( + node_error_count(controller.app, container_ring.devs[0]), + self.app.error_suppression_limit + 1) self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503) - self.assert_('last_error' in controller.app.container_ring.devs[0]) + self.assert_( + node_last_error(controller.app, container_ring.devs[0]) + is not None) self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503, missing_container=True) self.assert_status_map(controller.DELETE, @@ -4606,14 +7199,15 @@ class TestContainerController(unittest.TestCase): set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + - ('a' * MAX_META_NAME_LENGTH): 'v'}) + ('a' * constraints.MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, - headers={'X-Container-Meta-' + - ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + req = Request.blank( + '/v1/a/c', environ={'REQUEST_METHOD': method}, + headers={'X-Container-Meta-' + + ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -4621,21 +7215,21 @@ class TestContainerController(unittest.TestCase): set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': - 'a' * MAX_META_VALUE_LENGTH}) + 'a' * constraints.MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': - 'a' * (MAX_META_VALUE_LENGTH + 1)}) + 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} - for x in xrange(MAX_META_COUNT): + for x in xrange(constraints.MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) @@ -4644,7 +7238,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(constraints.MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) @@ -4654,16 +7248,17 @@ class TestContainerController(unittest.TestCase): set_http_connect(201, 201, 201) headers = {} - header_value = 'a' * MAX_META_VALUE_LENGTH + header_value = 'a' * constraints.MAX_META_VALUE_LENGTH size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < (constraints.MAX_META_OVERALL_SIZE - 4 + - constraints.MAX_META_VALUE_LENGTH): + size += 4 + constraints.MAX_META_VALUE_LENGTH headers['X-Container-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if constraints.MAX_META_OVERALL_SIZE - size > 1: headers['X-Container-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size - 1) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) @@ -4671,7 +7266,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Container-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) @@ -4781,6 +7376,78 @@ class TestContainerController(unittest.TestCase): controller.HEAD(req) self.assert_(called[0]) + def test_unauthorized_requests_when_account_not_found(self): + # verify unauthorized container requests always return response + # from swift.authorize + called = [0, 0] + + def authorize(req): + called[0] += 1 + return HTTPUnauthorized(request=req) + + def account_info(*args): + called[1] += 1 + return None, None, None + + def _do_test(method): + with save_globals(): + swift.proxy.controllers.Controller.account_info = account_info + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing()) + set_http_connect(201, 201, 201) + req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = app.handle_request(req) + return res + + for method in ('PUT', 'POST', 'DELETE'): + # no delay_denial on method, expect one call to authorize + called = [0, 0] + res = _do_test(method) + self.assertEqual(401, res.status_int) + self.assertEqual([1, 0], called) + + for method in ('HEAD', 'GET'): + # delay_denial on method, expect two calls to authorize + called = [0, 0] + res = _do_test(method) + self.assertEqual(401, res.status_int) + self.assertEqual([2, 1], called) + + def test_authorized_requests_when_account_not_found(self): + # verify authorized container requests always return 404 when + # account not found + called = [0, 0] + + def authorize(req): + called[0] += 1 + + def account_info(*args): + called[1] += 1 + return None, None, None + + def _do_test(method): + with save_globals(): + swift.proxy.controllers.Controller.account_info = account_info + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing()) + set_http_connect(201, 201, 201) + req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = app.handle_request(req) + return res + + for method in ('PUT', 'POST', 'DELETE', 'HEAD', 'GET'): + # expect one call to authorize + called = [0, 0] + res = _do_test(method) + self.assertEqual(404, res.status_int) + self.assertEqual([1, 1], called) + def test_OPTIONS_get_info_drops_origin(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') @@ -5008,10 +7675,10 @@ class TestContainerController(unittest.TestCase): 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, @@ -5028,13 +7695,13 @@ class TestContainerController(unittest.TestCase): 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdc'} ]) @@ -5048,10 +7715,10 @@ class TestContainerController(unittest.TestCase): 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, @@ -5068,13 +7735,13 @@ class TestContainerController(unittest.TestCase): 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', - 'X-Account-Partition': '1', + 'X-Account-Partition': '0', 'X-Account-Device': 'sdc'} ]) @@ -5087,7 +7754,7 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/v1/a/c', method='PUT', headers={'': ''}) with save_globals(): - new_connect = set_http_connect(200, # account existance check + new_connect = set_http_connect(200, # account existence check 201, 201, 201, give_connect=capture_timestamps) resp = self.app.handle_request(req) @@ -5096,7 +7763,7 @@ class TestContainerController(unittest.TestCase): self.assertRaises(StopIteration, new_connect.code_iter.next) self.assertEqual(2, resp.status_int // 100) - timestamps.pop(0) # account existance check + timestamps.pop(0) # account existence check self.assertEqual(3, len(timestamps)) for timestamp in timestamps: self.assertEqual(timestamp, timestamps[0]) @@ -5112,7 +7779,7 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''}) self.app.update_request(req) with save_globals(): - new_connect = set_http_connect(200, # account existance check + new_connect = set_http_connect(200, # account existence check 201, 201, 201, give_connect=capture_timestamps) resp = self.app.handle_request(req) @@ -5121,7 +7788,7 @@ class TestContainerController(unittest.TestCase): self.assertRaises(StopIteration, new_connect.code_iter.next) self.assertEqual(2, resp.status_int // 100) - timestamps.pop(0) # account existance check + timestamps.pop(0) # account existence check self.assertEqual(3, len(timestamps)) for timestamp in timestamps: self.assertEqual(timestamp, timestamps[0]) @@ -5141,18 +7808,20 @@ class TestContainerController(unittest.TestCase): self.assert_(got_exc) +@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestAccountController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing) + container_ring=FakeRing()) - def assert_status_map(self, method, statuses, expected, env_expected=None): + def assert_status_map(self, method, statuses, expected, env_expected=None, + headers=None, **kwargs): + headers = headers or {} with save_globals(): - set_http_connect(*statuses) - req = Request.blank('/v1/a', {}) + set_http_connect(*statuses, **kwargs) + req = Request.blank('/v1/a', {}, headers=headers) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -5311,9 +7980,36 @@ class TestAccountController(unittest.TestCase): controller.POST, (404, 404, 404, 403, 403, 403, 400, 400, 400), 400) + def test_POST_autocreate_with_sysmeta(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + # first test with autocreate being False + self.assertFalse(self.app.account_autocreate) + self.assert_status_map(controller.POST, + (404, 404, 404), 404) + # next turn it on and test account being created than updated + controller.app.account_autocreate = True + calls = [] + callback = _make_callback_func(calls) + key, value = 'X-Account-Sysmeta-Blah', 'something' + headers = {key: value} + self.assert_status_map( + controller.POST, + (404, 404, 404, 202, 202, 202, 201, 201, 201), 201, + # POST , autocreate PUT, POST again + headers=headers, + give_connect=callback) + self.assertEqual(9, len(calls)) + for call in calls: + self.assertTrue(key in call['headers'], + '%s call, key %s missing in headers %s' % + (call['method'], key, call['headers'])) + self.assertEqual(value, call['headers'][key]) + def test_connection_refused(self): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 # can't connect on this port controller = proxy_server.AccountController(self.app, 'account') @@ -5324,7 +8020,7 @@ class TestAccountController(unittest.TestCase): def test_other_socket_error(self): self.app.account_ring.get_nodes('account') - for dev in self.app.account_ring.devs.values(): + for dev in self.app.account_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = -1 # invalid port number controller = proxy_server.AccountController(self.app, 'account') @@ -5377,7 +8073,7 @@ class TestAccountController(unittest.TestCase): def test_PUT_max_account_name_length(self): with save_globals(): self.app.allow_account_management = True - limit = MAX_ACCOUNT_NAME_LENGTH + limit = constraints.MAX_ACCOUNT_NAME_LENGTH controller = proxy_server.AccountController(self.app, '1' * limit) self.assert_status_map(controller.PUT, (201, 201, 201), 201) controller = proxy_server.AccountController( @@ -5392,6 +8088,12 @@ class TestAccountController(unittest.TestCase): self.assert_status_map(controller.PUT, (201, -1, -1), 503) self.assert_status_map(controller.PUT, (503, 503, -1), 503) + def test_PUT_status(self): + with save_globals(): + self.app.allow_account_management = True + controller = proxy_server.AccountController(self.app, 'account') + self.assert_status_map(controller.PUT, (201, 201, 202), 202) + def test_PUT_metadata(self): self.metadata_helper('PUT') @@ -5452,14 +8154,15 @@ class TestAccountController(unittest.TestCase): set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-' + - ('a' * MAX_META_NAME_LENGTH): 'v'}) + ('a' * constraints.MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, - headers={'X-Account-Meta-' + - ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + req = Request.blank( + '/v1/a/c', environ={'REQUEST_METHOD': method}, + headers={'X-Account-Meta-' + + ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -5467,21 +8170,21 @@ class TestAccountController(unittest.TestCase): set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-Too-Long': - 'a' * MAX_META_VALUE_LENGTH}) + 'a' * constraints.MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-Too-Long': - 'a' * (MAX_META_VALUE_LENGTH + 1)}) + 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} - for x in xrange(MAX_META_COUNT): + for x in xrange(constraints.MAX_META_COUNT): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) @@ -5490,7 +8193,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(constraints.MAX_META_COUNT + 1): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) @@ -5500,16 +8203,17 @@ class TestAccountController(unittest.TestCase): set_http_connect(201, 201, 201) headers = {} - header_value = 'a' * MAX_META_VALUE_LENGTH + header_value = 'a' * constraints.MAX_META_VALUE_LENGTH size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < (constraints.MAX_META_OVERALL_SIZE - 4 + - constraints.MAX_META_VALUE_LENGTH): + size += 4 + constraints.MAX_META_VALUE_LENGTH headers['X-Account-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if constraints.MAX_META_OVERALL_SIZE - size > 1: headers['X-Account-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size - 1) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) @@ -5517,7 +8221,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Account-Meta-a'] = \ - 'a' * (MAX_META_OVERALL_SIZE - size) + 'a' * (constraints.MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) @@ -5569,6 +8273,7 @@ class TestAccountController(unittest.TestCase): test_status_map((204, 500, 404), 400) +@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestAccountControllerFakeGetResponse(unittest.TestCase): """ Test all the faked-out GET responses for accounts that don't exist. They @@ -5578,8 +8283,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): conf = {'account_autocreate': 'yes'} self.app = proxy_server.Application(conf, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing) + container_ring=FakeRing()) self.app.memcache = FakeMemcacheReturnsNone() def test_GET_autocreate_accept_json(self): @@ -5670,7 +8374,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), object_ring=FakeRing()) + container_ring=FakeRing()) with save_globals(): # Mock account server will provide privileged information (ACLs) @@ -5745,7 +8449,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), object_ring=FakeRing()) + container_ring=FakeRing()) app.allow_account_management = True ext_header = 'x-account-access-control' @@ -5841,10 +8545,6 @@ class FakeObjectController(object): return -class Stub(object): - pass - - class TestProxyObjectPerformance(unittest.TestCase): def setUp(self): @@ -5904,6 +8604,11 @@ class TestProxyObjectPerformance(unittest.TestCase): print "Run %02d took %07.03f" % (i, end - start) +@patch_policies([StoragePolicy(0, 'migrated', object_ring=FakeRing()), + StoragePolicy(1, 'ernie', True, object_ring=FakeRing()), + StoragePolicy(2, 'deprecated', is_deprecated=True, + object_ring=FakeRing()), + StoragePolicy(3, 'bert', object_ring=FakeRing())]) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} @@ -5912,23 +8617,47 @@ class TestSwiftInfo(unittest.TestCase): def test_registered_defaults(self): proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing(), - object_ring=FakeRing) + container_ring=FakeRing()) si = utils.get_swift_info()['swift'] self.assertTrue('version' in si) - self.assertEqual(si['max_file_size'], MAX_FILE_SIZE) - self.assertEqual(si['max_meta_name_length'], MAX_META_NAME_LENGTH) - self.assertEqual(si['max_meta_value_length'], MAX_META_VALUE_LENGTH) - self.assertEqual(si['max_meta_count'], MAX_META_COUNT) - self.assertEqual(si['account_listing_limit'], ACCOUNT_LISTING_LIMIT) + self.assertEqual(si['max_file_size'], constraints.MAX_FILE_SIZE) + self.assertEqual(si['max_meta_name_length'], + constraints.MAX_META_NAME_LENGTH) + self.assertEqual(si['max_meta_value_length'], + constraints.MAX_META_VALUE_LENGTH) + self.assertEqual(si['max_meta_count'], constraints.MAX_META_COUNT) + self.assertEqual(si['max_header_size'], constraints.MAX_HEADER_SIZE) + self.assertEqual(si['max_meta_overall_size'], + constraints.MAX_META_OVERALL_SIZE) + self.assertEqual(si['account_listing_limit'], + constraints.ACCOUNT_LISTING_LIMIT) self.assertEqual(si['container_listing_limit'], - CONTAINER_LISTING_LIMIT) + constraints.CONTAINER_LISTING_LIMIT) self.assertEqual(si['max_account_name_length'], - MAX_ACCOUNT_NAME_LENGTH) + constraints.MAX_ACCOUNT_NAME_LENGTH) self.assertEqual(si['max_container_name_length'], - MAX_CONTAINER_NAME_LENGTH) - self.assertEqual(si['max_object_name_length'], MAX_OBJECT_NAME_LENGTH) + constraints.MAX_CONTAINER_NAME_LENGTH) + self.assertEqual(si['max_object_name_length'], + constraints.MAX_OBJECT_NAME_LENGTH) + self.assertTrue('strict_cors_mode' in si) + self.assertEqual(si['allow_account_management'], False) + self.assertEqual(si['account_autocreate'], False) + # This setting is by default excluded by disallowed_sections + self.assertEqual(si['valid_api_versions'], + constraints.VALID_API_VERSIONS) + # this next test is deliberately brittle in order to alert if + # other items are added to swift info + self.assertEqual(len(si), 18) + + self.assertTrue('policies' in si) + sorted_pols = sorted(si['policies'], key=operator.itemgetter('name')) + self.assertEqual(len(sorted_pols), 3) + for policy in sorted_pols: + self.assertNotEquals(policy['name'], 'deprecated') + self.assertEqual(sorted_pols[0]['name'], 'bert') + self.assertEqual(sorted_pols[1]['name'], 'ernie') + self.assertEqual(sorted_pols[2]['name'], 'migrated') if __name__ == '__main__': |